1// SPDX-License-Identifier: GPL-2.0
2#include <linux/kernel.h>
3#include <linux/printk.h>
4#include <linux/crypto.h>
5#include <linux/rtnetlink.h>
6
7#include <crypto/aead.h>
8#include <crypto/authenc.h>
9#include <crypto/des.h>
10#include <crypto/internal/aead.h>
11#include <crypto/scatterwalk.h>
12#include <crypto/gcm.h>
13
14#include "nitrox_dev.h"
15#include "nitrox_common.h"
16#include "nitrox_req.h"
17
18#define GCM_AES_SALT_SIZE	4
19
20union gph_p3 {
21	struct {
22#ifdef __BIG_ENDIAN_BITFIELD
23		u16 iv_offset : 8;
24		u16 auth_offset	: 8;
25#else
26		u16 auth_offset	: 8;
27		u16 iv_offset : 8;
28#endif
29	};
30	u16 param;
31};
32
33static int nitrox_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key,
34				 unsigned int keylen)
35{
36	int aes_keylen;
37	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
38	struct flexi_crypto_context *fctx;
39	union fc_ctx_flags flags;
40
41	aes_keylen = flexi_aes_keylen(keylen);
42	if (aes_keylen < 0)
43		return -EINVAL;
44
45	/* fill crypto context */
46	fctx = nctx->u.fctx;
47	flags.fu = be64_to_cpu(fctx->flags.f);
48	flags.w0.aes_keylen = aes_keylen;
49	fctx->flags.f = cpu_to_be64(flags.fu);
50
51	/* copy enc key to context */
52	memset(&fctx->crypto, 0, sizeof(fctx->crypto));
53	memcpy(fctx->crypto.u.key, key, keylen);
54
55	return 0;
56}
57
58static int nitrox_aead_setauthsize(struct crypto_aead *aead,
59				   unsigned int authsize)
60{
61	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
62	struct flexi_crypto_context *fctx = nctx->u.fctx;
63	union fc_ctx_flags flags;
64
65	flags.fu = be64_to_cpu(fctx->flags.f);
66	flags.w0.mac_len = authsize;
67	fctx->flags.f = cpu_to_be64(flags.fu);
68
69	aead->authsize = authsize;
70
71	return 0;
72}
73
74static int nitrox_aes_gcm_setauthsize(struct crypto_aead *aead,
75				      unsigned int authsize)
76{
77	switch (authsize) {
78	case 4:
79	case 8:
80	case 12:
81	case 13:
82	case 14:
83	case 15:
84	case 16:
85		break;
86	default:
87		return -EINVAL;
88	}
89
90	return nitrox_aead_setauthsize(aead, authsize);
91}
92
93static int alloc_src_sglist(struct nitrox_kcrypt_request *nkreq,
94			    struct scatterlist *src, char *iv, int ivsize,
95			    int buflen)
96{
97	int nents = sg_nents_for_len(src, buflen);
98	int ret;
99
100	if (nents < 0)
101		return nents;
102
103	/* IV entry */
104	nents += 1;
105	/* Allocate buffer to hold IV and input scatterlist array */
106	ret = alloc_src_req_buf(nkreq, nents, ivsize);
107	if (ret)
108		return ret;
109
110	nitrox_creq_copy_iv(nkreq->src, iv, ivsize);
111	nitrox_creq_set_src_sg(nkreq, nents, ivsize, src, buflen);
112
113	return 0;
114}
115
116static int alloc_dst_sglist(struct nitrox_kcrypt_request *nkreq,
117			    struct scatterlist *dst, int ivsize, int buflen)
118{
119	int nents = sg_nents_for_len(dst, buflen);
120	int ret;
121
122	if (nents < 0)
123		return nents;
124
125	/* IV, ORH, COMPLETION entries */
126	nents += 3;
127	/* Allocate buffer to hold ORH, COMPLETION and output scatterlist
128	 * array
129	 */
130	ret = alloc_dst_req_buf(nkreq, nents);
131	if (ret)
132		return ret;
133
134	nitrox_creq_set_orh(nkreq);
135	nitrox_creq_set_comp(nkreq);
136	nitrox_creq_set_dst_sg(nkreq, nents, ivsize, dst, buflen);
137
138	return 0;
139}
140
141static void free_src_sglist(struct nitrox_kcrypt_request *nkreq)
142{
143	kfree(nkreq->src);
144}
145
146static void free_dst_sglist(struct nitrox_kcrypt_request *nkreq)
147{
148	kfree(nkreq->dst);
149}
150
151static int nitrox_set_creq(struct nitrox_aead_rctx *rctx)
152{
153	struct se_crypto_request *creq = &rctx->nkreq.creq;
154	union gph_p3 param3;
155	int ret;
156
157	creq->flags = rctx->flags;
158	creq->gfp = (rctx->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL :
159							       GFP_ATOMIC;
160
161	creq->ctrl.value = 0;
162	creq->opcode = FLEXI_CRYPTO_ENCRYPT_HMAC;
163	creq->ctrl.s.arg = rctx->ctrl_arg;
164
165	creq->gph.param0 = cpu_to_be16(rctx->cryptlen);
166	creq->gph.param1 = cpu_to_be16(rctx->cryptlen + rctx->assoclen);
167	creq->gph.param2 = cpu_to_be16(rctx->ivsize + rctx->assoclen);
168	param3.iv_offset = 0;
169	param3.auth_offset = rctx->ivsize;
170	creq->gph.param3 = cpu_to_be16(param3.param);
171
172	creq->ctx_handle = rctx->ctx_handle;
173	creq->ctrl.s.ctxl = sizeof(struct flexi_crypto_context);
174
175	ret = alloc_src_sglist(&rctx->nkreq, rctx->src, rctx->iv, rctx->ivsize,
176			       rctx->srclen);
177	if (ret)
178		return ret;
179
180	ret = alloc_dst_sglist(&rctx->nkreq, rctx->dst, rctx->ivsize,
181			       rctx->dstlen);
182	if (ret) {
183		free_src_sglist(&rctx->nkreq);
184		return ret;
185	}
186
187	return 0;
188}
189
190static void nitrox_aead_callback(void *arg, int err)
191{
192	struct aead_request *areq = arg;
193	struct nitrox_aead_rctx *rctx = aead_request_ctx(areq);
194
195	free_src_sglist(&rctx->nkreq);
196	free_dst_sglist(&rctx->nkreq);
197	if (err) {
198		pr_err_ratelimited("request failed status 0x%0x\n", err);
199		err = -EINVAL;
200	}
201
202	aead_request_complete(areq, err);
203}
204
205static inline bool nitrox_aes_gcm_assoclen_supported(unsigned int assoclen)
206{
207	if (assoclen <= 512)
208		return true;
209
210	return false;
211}
212
213static int nitrox_aes_gcm_enc(struct aead_request *areq)
214{
215	struct crypto_aead *aead = crypto_aead_reqtfm(areq);
216	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
217	struct nitrox_aead_rctx *rctx = aead_request_ctx(areq);
218	struct se_crypto_request *creq = &rctx->nkreq.creq;
219	struct flexi_crypto_context *fctx = nctx->u.fctx;
220	int ret;
221
222	if (!nitrox_aes_gcm_assoclen_supported(areq->assoclen))
223		return -EINVAL;
224
225	memcpy(fctx->crypto.iv, areq->iv, GCM_AES_SALT_SIZE);
226
227	rctx->cryptlen = areq->cryptlen;
228	rctx->assoclen = areq->assoclen;
229	rctx->srclen = areq->assoclen + areq->cryptlen;
230	rctx->dstlen = rctx->srclen + aead->authsize;
231	rctx->iv = &areq->iv[GCM_AES_SALT_SIZE];
232	rctx->ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE;
233	rctx->flags = areq->base.flags;
234	rctx->ctx_handle = nctx->u.ctx_handle;
235	rctx->src = areq->src;
236	rctx->dst = areq->dst;
237	rctx->ctrl_arg = ENCRYPT;
238	ret = nitrox_set_creq(rctx);
239	if (ret)
240		return ret;
241
242	/* send the crypto request */
243	return nitrox_process_se_request(nctx->ndev, creq, nitrox_aead_callback,
244					 areq);
245}
246
247static int nitrox_aes_gcm_dec(struct aead_request *areq)
248{
249	struct crypto_aead *aead = crypto_aead_reqtfm(areq);
250	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
251	struct nitrox_aead_rctx *rctx = aead_request_ctx(areq);
252	struct se_crypto_request *creq = &rctx->nkreq.creq;
253	struct flexi_crypto_context *fctx = nctx->u.fctx;
254	int ret;
255
256	if (!nitrox_aes_gcm_assoclen_supported(areq->assoclen))
257		return -EINVAL;
258
259	memcpy(fctx->crypto.iv, areq->iv, GCM_AES_SALT_SIZE);
260
261	rctx->cryptlen = areq->cryptlen - aead->authsize;
262	rctx->assoclen = areq->assoclen;
263	rctx->srclen = areq->cryptlen + areq->assoclen;
264	rctx->dstlen = rctx->srclen - aead->authsize;
265	rctx->iv = &areq->iv[GCM_AES_SALT_SIZE];
266	rctx->ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE;
267	rctx->flags = areq->base.flags;
268	rctx->ctx_handle = nctx->u.ctx_handle;
269	rctx->src = areq->src;
270	rctx->dst = areq->dst;
271	rctx->ctrl_arg = DECRYPT;
272	ret = nitrox_set_creq(rctx);
273	if (ret)
274		return ret;
275
276	/* send the crypto request */
277	return nitrox_process_se_request(nctx->ndev, creq, nitrox_aead_callback,
278					 areq);
279}
280
281static int nitrox_aead_init(struct crypto_aead *aead)
282{
283	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
284	struct crypto_ctx_hdr *chdr;
285
286	/* get the first device */
287	nctx->ndev = nitrox_get_first_device();
288	if (!nctx->ndev)
289		return -ENODEV;
290
291	/* allocate nitrox crypto context */
292	chdr = crypto_alloc_context(nctx->ndev);
293	if (!chdr) {
294		nitrox_put_device(nctx->ndev);
295		return -ENOMEM;
296	}
297	nctx->chdr = chdr;
298	nctx->u.ctx_handle = (uintptr_t)((u8 *)chdr->vaddr +
299					 sizeof(struct ctx_hdr));
300	nctx->u.fctx->flags.f = 0;
301
302	return 0;
303}
304
305static int nitrox_gcm_common_init(struct crypto_aead *aead)
306{
307	int ret;
308	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
309	union fc_ctx_flags *flags;
310
311	ret = nitrox_aead_init(aead);
312	if (ret)
313		return ret;
314
315	flags = &nctx->u.fctx->flags;
316	flags->w0.cipher_type = CIPHER_AES_GCM;
317	flags->w0.hash_type = AUTH_NULL;
318	flags->w0.iv_source = IV_FROM_DPTR;
319	/* ask microcode to calculate ipad/opad */
320	flags->w0.auth_input_type = 1;
321	flags->f = cpu_to_be64(flags->fu);
322
323	return 0;
324}
325
326static int nitrox_aes_gcm_init(struct crypto_aead *aead)
327{
328	int ret;
329
330	ret = nitrox_gcm_common_init(aead);
331	if (ret)
332		return ret;
333
334	crypto_aead_set_reqsize(aead,
335				sizeof(struct aead_request) +
336					sizeof(struct nitrox_aead_rctx));
337
338	return 0;
339}
340
341static void nitrox_aead_exit(struct crypto_aead *aead)
342{
343	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
344
345	/* free the nitrox crypto context */
346	if (nctx->u.ctx_handle) {
347		struct flexi_crypto_context *fctx = nctx->u.fctx;
348
349		memzero_explicit(&fctx->crypto, sizeof(struct crypto_keys));
350		memzero_explicit(&fctx->auth, sizeof(struct auth_keys));
351		crypto_free_context((void *)nctx->chdr);
352	}
353	nitrox_put_device(nctx->ndev);
354
355	nctx->u.ctx_handle = 0;
356	nctx->ndev = NULL;
357}
358
359static int nitrox_rfc4106_setkey(struct crypto_aead *aead, const u8 *key,
360				 unsigned int keylen)
361{
362	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
363	struct flexi_crypto_context *fctx = nctx->u.fctx;
364	int ret;
365
366	if (keylen < GCM_AES_SALT_SIZE)
367		return -EINVAL;
368
369	keylen -= GCM_AES_SALT_SIZE;
370	ret = nitrox_aes_gcm_setkey(aead, key, keylen);
371	if (ret)
372		return ret;
373
374	memcpy(fctx->crypto.iv, key + keylen, GCM_AES_SALT_SIZE);
375	return 0;
376}
377
378static int nitrox_rfc4106_setauthsize(struct crypto_aead *aead,
379				      unsigned int authsize)
380{
381	switch (authsize) {
382	case 8:
383	case 12:
384	case 16:
385		break;
386	default:
387		return -EINVAL;
388	}
389
390	return nitrox_aead_setauthsize(aead, authsize);
391}
392
393static int nitrox_rfc4106_set_aead_rctx_sglist(struct aead_request *areq)
394{
395	struct nitrox_rfc4106_rctx *rctx = aead_request_ctx_dma(areq);
396	struct nitrox_aead_rctx *aead_rctx = &rctx->base;
397	unsigned int assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE;
398	struct scatterlist *sg;
399
400	if (areq->assoclen != 16 && areq->assoclen != 20)
401		return -EINVAL;
402
403	scatterwalk_map_and_copy(rctx->assoc, areq->src, 0, assoclen, 0);
404	sg_init_table(rctx->src, 3);
405	sg_set_buf(rctx->src, rctx->assoc, assoclen);
406	sg = scatterwalk_ffwd(rctx->src + 1, areq->src, areq->assoclen);
407	if (sg != rctx->src + 1)
408		sg_chain(rctx->src, 2, sg);
409
410	if (areq->src != areq->dst) {
411		sg_init_table(rctx->dst, 3);
412		sg_set_buf(rctx->dst, rctx->assoc, assoclen);
413		sg = scatterwalk_ffwd(rctx->dst + 1, areq->dst, areq->assoclen);
414		if (sg != rctx->dst + 1)
415			sg_chain(rctx->dst, 2, sg);
416	}
417
418	aead_rctx->src = rctx->src;
419	aead_rctx->dst = (areq->src == areq->dst) ? rctx->src : rctx->dst;
420
421	return 0;
422}
423
424static void nitrox_rfc4106_callback(void *arg, int err)
425{
426	struct aead_request *areq = arg;
427	struct nitrox_rfc4106_rctx *rctx = aead_request_ctx_dma(areq);
428	struct nitrox_kcrypt_request *nkreq = &rctx->base.nkreq;
429
430	free_src_sglist(nkreq);
431	free_dst_sglist(nkreq);
432	if (err) {
433		pr_err_ratelimited("request failed status 0x%0x\n", err);
434		err = -EINVAL;
435	}
436
437	aead_request_complete(areq, err);
438}
439
440static int nitrox_rfc4106_enc(struct aead_request *areq)
441{
442	struct crypto_aead *aead = crypto_aead_reqtfm(areq);
443	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
444	struct nitrox_rfc4106_rctx *rctx = aead_request_ctx_dma(areq);
445	struct nitrox_aead_rctx *aead_rctx = &rctx->base;
446	struct se_crypto_request *creq = &aead_rctx->nkreq.creq;
447	int ret;
448
449	aead_rctx->cryptlen = areq->cryptlen;
450	aead_rctx->assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE;
451	aead_rctx->srclen = aead_rctx->assoclen + aead_rctx->cryptlen;
452	aead_rctx->dstlen = aead_rctx->srclen + aead->authsize;
453	aead_rctx->iv = areq->iv;
454	aead_rctx->ivsize = GCM_RFC4106_IV_SIZE;
455	aead_rctx->flags = areq->base.flags;
456	aead_rctx->ctx_handle = nctx->u.ctx_handle;
457	aead_rctx->ctrl_arg = ENCRYPT;
458
459	ret = nitrox_rfc4106_set_aead_rctx_sglist(areq);
460	if (ret)
461		return ret;
462
463	ret = nitrox_set_creq(aead_rctx);
464	if (ret)
465		return ret;
466
467	/* send the crypto request */
468	return nitrox_process_se_request(nctx->ndev, creq,
469					 nitrox_rfc4106_callback, areq);
470}
471
472static int nitrox_rfc4106_dec(struct aead_request *areq)
473{
474	struct crypto_aead *aead = crypto_aead_reqtfm(areq);
475	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx_dma(aead);
476	struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq);
477	struct nitrox_aead_rctx *aead_rctx = &rctx->base;
478	struct se_crypto_request *creq = &aead_rctx->nkreq.creq;
479	int ret;
480
481	aead_rctx->cryptlen = areq->cryptlen - aead->authsize;
482	aead_rctx->assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE;
483	aead_rctx->srclen =
484		areq->cryptlen - GCM_RFC4106_IV_SIZE + areq->assoclen;
485	aead_rctx->dstlen = aead_rctx->srclen - aead->authsize;
486	aead_rctx->iv = areq->iv;
487	aead_rctx->ivsize = GCM_RFC4106_IV_SIZE;
488	aead_rctx->flags = areq->base.flags;
489	aead_rctx->ctx_handle = nctx->u.ctx_handle;
490	aead_rctx->ctrl_arg = DECRYPT;
491
492	ret = nitrox_rfc4106_set_aead_rctx_sglist(areq);
493	if (ret)
494		return ret;
495
496	ret = nitrox_set_creq(aead_rctx);
497	if (ret)
498		return ret;
499
500	/* send the crypto request */
501	return nitrox_process_se_request(nctx->ndev, creq,
502					 nitrox_rfc4106_callback, areq);
503}
504
505static int nitrox_rfc4106_init(struct crypto_aead *aead)
506{
507	int ret;
508
509	ret = nitrox_gcm_common_init(aead);
510	if (ret)
511		return ret;
512
513	crypto_aead_set_reqsize_dma(aead, sizeof(struct aead_request) +
514					  sizeof(struct nitrox_rfc4106_rctx));
515
516	return 0;
517}
518
519static struct aead_alg nitrox_aeads[] = { {
520	.base = {
521		.cra_name = "gcm(aes)",
522		.cra_driver_name = "n5_aes_gcm",
523		.cra_priority = PRIO,
524		.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
525		.cra_blocksize = 1,
526		.cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
527		.cra_alignmask = 0,
528		.cra_module = THIS_MODULE,
529	},
530	.setkey = nitrox_aes_gcm_setkey,
531	.setauthsize = nitrox_aes_gcm_setauthsize,
532	.encrypt = nitrox_aes_gcm_enc,
533	.decrypt = nitrox_aes_gcm_dec,
534	.init = nitrox_aes_gcm_init,
535	.exit = nitrox_aead_exit,
536	.ivsize = GCM_AES_IV_SIZE,
537	.maxauthsize = AES_BLOCK_SIZE,
538}, {
539	.base = {
540		.cra_name = "rfc4106(gcm(aes))",
541		.cra_driver_name = "n5_rfc4106",
542		.cra_priority = PRIO,
543		.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
544		.cra_blocksize = 1,
545		.cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
546		.cra_alignmask = 0,
547		.cra_module = THIS_MODULE,
548	},
549	.setkey = nitrox_rfc4106_setkey,
550	.setauthsize = nitrox_rfc4106_setauthsize,
551	.encrypt = nitrox_rfc4106_enc,
552	.decrypt = nitrox_rfc4106_dec,
553	.init = nitrox_rfc4106_init,
554	.exit = nitrox_aead_exit,
555	.ivsize = GCM_RFC4106_IV_SIZE,
556	.maxauthsize = AES_BLOCK_SIZE,
557} };
558
559int nitrox_register_aeads(void)
560{
561	return crypto_register_aeads(nitrox_aeads, ARRAY_SIZE(nitrox_aeads));
562}
563
564void nitrox_unregister_aeads(void)
565{
566	crypto_unregister_aeads(nitrox_aeads, ARRAY_SIZE(nitrox_aeads));
567}
568