1// SPDX-License-Identifier: GPL-2.0
2#include <linux/kernel.h>
3#include <linux/printk.h>
4#include <linux/crypto.h>
5#include <linux/rtnetlink.h>
6
7#include <crypto/aead.h>
8#include <crypto/authenc.h>
9#include <crypto/des.h>
10#include <crypto/sha.h>
11#include <crypto/internal/aead.h>
12#include <crypto/scatterwalk.h>
13#include <crypto/gcm.h>
14
15#include "nitrox_dev.h"
16#include "nitrox_common.h"
17#include "nitrox_req.h"
18
19#define GCM_AES_SALT_SIZE	4
20
21union gph_p3 {
22	struct {
23#ifdef __BIG_ENDIAN_BITFIELD
24		u16 iv_offset : 8;
25		u16 auth_offset	: 8;
26#else
27		u16 auth_offset	: 8;
28		u16 iv_offset : 8;
29#endif
30	};
31	u16 param;
32};
33
34static int nitrox_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key,
35				 unsigned int keylen)
36{
37	int aes_keylen;
38	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
39	struct flexi_crypto_context *fctx;
40	union fc_ctx_flags flags;
41
42	aes_keylen = flexi_aes_keylen(keylen);
43	if (aes_keylen < 0)
44		return -EINVAL;
45
46	/* fill crypto context */
47	fctx = nctx->u.fctx;
48	flags.f = be64_to_cpu(fctx->flags.f);
49	flags.w0.aes_keylen = aes_keylen;
50	fctx->flags.f = cpu_to_be64(flags.f);
51
52	/* copy enc key to context */
53	memset(&fctx->crypto, 0, sizeof(fctx->crypto));
54	memcpy(fctx->crypto.u.key, key, keylen);
55
56	return 0;
57}
58
59static int nitrox_aead_setauthsize(struct crypto_aead *aead,
60				   unsigned int authsize)
61{
62	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
63	struct flexi_crypto_context *fctx = nctx->u.fctx;
64	union fc_ctx_flags flags;
65
66	flags.f = be64_to_cpu(fctx->flags.f);
67	flags.w0.mac_len = authsize;
68	fctx->flags.f = cpu_to_be64(flags.f);
69
70	aead->authsize = authsize;
71
72	return 0;
73}
74
75static int nitrox_aes_gcm_setauthsize(struct crypto_aead *aead,
76				      unsigned int authsize)
77{
78	switch (authsize) {
79	case 4:
80	case 8:
81	case 12:
82	case 13:
83	case 14:
84	case 15:
85	case 16:
86		break;
87	default:
88		return -EINVAL;
89	}
90
91	return nitrox_aead_setauthsize(aead, authsize);
92}
93
94static int alloc_src_sglist(struct nitrox_kcrypt_request *nkreq,
95			    struct scatterlist *src, char *iv, int ivsize,
96			    int buflen)
97{
98	int nents = sg_nents_for_len(src, buflen);
99	int ret;
100
101	if (nents < 0)
102		return nents;
103
104	/* IV entry */
105	nents += 1;
106	/* Allocate buffer to hold IV and input scatterlist array */
107	ret = alloc_src_req_buf(nkreq, nents, ivsize);
108	if (ret)
109		return ret;
110
111	nitrox_creq_copy_iv(nkreq->src, iv, ivsize);
112	nitrox_creq_set_src_sg(nkreq, nents, ivsize, src, buflen);
113
114	return 0;
115}
116
117static int alloc_dst_sglist(struct nitrox_kcrypt_request *nkreq,
118			    struct scatterlist *dst, int ivsize, int buflen)
119{
120	int nents = sg_nents_for_len(dst, buflen);
121	int ret;
122
123	if (nents < 0)
124		return nents;
125
126	/* IV, ORH, COMPLETION entries */
127	nents += 3;
128	/* Allocate buffer to hold ORH, COMPLETION and output scatterlist
129	 * array
130	 */
131	ret = alloc_dst_req_buf(nkreq, nents);
132	if (ret)
133		return ret;
134
135	nitrox_creq_set_orh(nkreq);
136	nitrox_creq_set_comp(nkreq);
137	nitrox_creq_set_dst_sg(nkreq, nents, ivsize, dst, buflen);
138
139	return 0;
140}
141
142static void free_src_sglist(struct nitrox_kcrypt_request *nkreq)
143{
144	kfree(nkreq->src);
145}
146
147static void free_dst_sglist(struct nitrox_kcrypt_request *nkreq)
148{
149	kfree(nkreq->dst);
150}
151
152static int nitrox_set_creq(struct nitrox_aead_rctx *rctx)
153{
154	struct se_crypto_request *creq = &rctx->nkreq.creq;
155	union gph_p3 param3;
156	int ret;
157
158	creq->flags = rctx->flags;
159	creq->gfp = (rctx->flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? GFP_KERNEL :
160							       GFP_ATOMIC;
161
162	creq->ctrl.value = 0;
163	creq->opcode = FLEXI_CRYPTO_ENCRYPT_HMAC;
164	creq->ctrl.s.arg = rctx->ctrl_arg;
165
166	creq->gph.param0 = cpu_to_be16(rctx->cryptlen);
167	creq->gph.param1 = cpu_to_be16(rctx->cryptlen + rctx->assoclen);
168	creq->gph.param2 = cpu_to_be16(rctx->ivsize + rctx->assoclen);
169	param3.iv_offset = 0;
170	param3.auth_offset = rctx->ivsize;
171	creq->gph.param3 = cpu_to_be16(param3.param);
172
173	creq->ctx_handle = rctx->ctx_handle;
174	creq->ctrl.s.ctxl = sizeof(struct flexi_crypto_context);
175
176	ret = alloc_src_sglist(&rctx->nkreq, rctx->src, rctx->iv, rctx->ivsize,
177			       rctx->srclen);
178	if (ret)
179		return ret;
180
181	ret = alloc_dst_sglist(&rctx->nkreq, rctx->dst, rctx->ivsize,
182			       rctx->dstlen);
183	if (ret) {
184		free_src_sglist(&rctx->nkreq);
185		return ret;
186	}
187
188	return 0;
189}
190
191static void nitrox_aead_callback(void *arg, int err)
192{
193	struct aead_request *areq = arg;
194	struct nitrox_aead_rctx *rctx = aead_request_ctx(areq);
195
196	free_src_sglist(&rctx->nkreq);
197	free_dst_sglist(&rctx->nkreq);
198	if (err) {
199		pr_err_ratelimited("request failed status 0x%0x\n", err);
200		err = -EINVAL;
201	}
202
203	areq->base.complete(&areq->base, err);
204}
205
206static inline bool nitrox_aes_gcm_assoclen_supported(unsigned int assoclen)
207{
208	if (assoclen <= 512)
209		return true;
210
211	return false;
212}
213
214static int nitrox_aes_gcm_enc(struct aead_request *areq)
215{
216	struct crypto_aead *aead = crypto_aead_reqtfm(areq);
217	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
218	struct nitrox_aead_rctx *rctx = aead_request_ctx(areq);
219	struct se_crypto_request *creq = &rctx->nkreq.creq;
220	struct flexi_crypto_context *fctx = nctx->u.fctx;
221	int ret;
222
223	if (!nitrox_aes_gcm_assoclen_supported(areq->assoclen))
224		return -EINVAL;
225
226	memcpy(fctx->crypto.iv, areq->iv, GCM_AES_SALT_SIZE);
227
228	rctx->cryptlen = areq->cryptlen;
229	rctx->assoclen = areq->assoclen;
230	rctx->srclen = areq->assoclen + areq->cryptlen;
231	rctx->dstlen = rctx->srclen + aead->authsize;
232	rctx->iv = &areq->iv[GCM_AES_SALT_SIZE];
233	rctx->ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE;
234	rctx->flags = areq->base.flags;
235	rctx->ctx_handle = nctx->u.ctx_handle;
236	rctx->src = areq->src;
237	rctx->dst = areq->dst;
238	rctx->ctrl_arg = ENCRYPT;
239	ret = nitrox_set_creq(rctx);
240	if (ret)
241		return ret;
242
243	/* send the crypto request */
244	return nitrox_process_se_request(nctx->ndev, creq, nitrox_aead_callback,
245					 areq);
246}
247
248static int nitrox_aes_gcm_dec(struct aead_request *areq)
249{
250	struct crypto_aead *aead = crypto_aead_reqtfm(areq);
251	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
252	struct nitrox_aead_rctx *rctx = aead_request_ctx(areq);
253	struct se_crypto_request *creq = &rctx->nkreq.creq;
254	struct flexi_crypto_context *fctx = nctx->u.fctx;
255	int ret;
256
257	if (!nitrox_aes_gcm_assoclen_supported(areq->assoclen))
258		return -EINVAL;
259
260	memcpy(fctx->crypto.iv, areq->iv, GCM_AES_SALT_SIZE);
261
262	rctx->cryptlen = areq->cryptlen - aead->authsize;
263	rctx->assoclen = areq->assoclen;
264	rctx->srclen = areq->cryptlen + areq->assoclen;
265	rctx->dstlen = rctx->srclen - aead->authsize;
266	rctx->iv = &areq->iv[GCM_AES_SALT_SIZE];
267	rctx->ivsize = GCM_AES_IV_SIZE - GCM_AES_SALT_SIZE;
268	rctx->flags = areq->base.flags;
269	rctx->ctx_handle = nctx->u.ctx_handle;
270	rctx->src = areq->src;
271	rctx->dst = areq->dst;
272	rctx->ctrl_arg = DECRYPT;
273	ret = nitrox_set_creq(rctx);
274	if (ret)
275		return ret;
276
277	/* send the crypto request */
278	return nitrox_process_se_request(nctx->ndev, creq, nitrox_aead_callback,
279					 areq);
280}
281
282static int nitrox_aead_init(struct crypto_aead *aead)
283{
284	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
285	struct crypto_ctx_hdr *chdr;
286
287	/* get the first device */
288	nctx->ndev = nitrox_get_first_device();
289	if (!nctx->ndev)
290		return -ENODEV;
291
292	/* allocate nitrox crypto context */
293	chdr = crypto_alloc_context(nctx->ndev);
294	if (!chdr) {
295		nitrox_put_device(nctx->ndev);
296		return -ENOMEM;
297	}
298	nctx->chdr = chdr;
299	nctx->u.ctx_handle = (uintptr_t)((u8 *)chdr->vaddr +
300					 sizeof(struct ctx_hdr));
301	nctx->u.fctx->flags.f = 0;
302
303	return 0;
304}
305
306static int nitrox_gcm_common_init(struct crypto_aead *aead)
307{
308	int ret;
309	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
310	union fc_ctx_flags *flags;
311
312	ret = nitrox_aead_init(aead);
313	if (ret)
314		return ret;
315
316	flags = &nctx->u.fctx->flags;
317	flags->w0.cipher_type = CIPHER_AES_GCM;
318	flags->w0.hash_type = AUTH_NULL;
319	flags->w0.iv_source = IV_FROM_DPTR;
320	/* ask microcode to calculate ipad/opad */
321	flags->w0.auth_input_type = 1;
322	flags->f = be64_to_cpu(flags->f);
323
324	return 0;
325}
326
327static int nitrox_aes_gcm_init(struct crypto_aead *aead)
328{
329	int ret;
330
331	ret = nitrox_gcm_common_init(aead);
332	if (ret)
333		return ret;
334
335	crypto_aead_set_reqsize(aead,
336				sizeof(struct aead_request) +
337					sizeof(struct nitrox_aead_rctx));
338
339	return 0;
340}
341
342static void nitrox_aead_exit(struct crypto_aead *aead)
343{
344	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
345
346	/* free the nitrox crypto context */
347	if (nctx->u.ctx_handle) {
348		struct flexi_crypto_context *fctx = nctx->u.fctx;
349
350		memzero_explicit(&fctx->crypto, sizeof(struct crypto_keys));
351		memzero_explicit(&fctx->auth, sizeof(struct auth_keys));
352		crypto_free_context((void *)nctx->chdr);
353	}
354	nitrox_put_device(nctx->ndev);
355
356	nctx->u.ctx_handle = 0;
357	nctx->ndev = NULL;
358}
359
360static int nitrox_rfc4106_setkey(struct crypto_aead *aead, const u8 *key,
361				 unsigned int keylen)
362{
363	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
364	struct flexi_crypto_context *fctx = nctx->u.fctx;
365	int ret;
366
367	if (keylen < GCM_AES_SALT_SIZE)
368		return -EINVAL;
369
370	keylen -= GCM_AES_SALT_SIZE;
371	ret = nitrox_aes_gcm_setkey(aead, key, keylen);
372	if (ret)
373		return ret;
374
375	memcpy(fctx->crypto.iv, key + keylen, GCM_AES_SALT_SIZE);
376	return 0;
377}
378
379static int nitrox_rfc4106_setauthsize(struct crypto_aead *aead,
380				      unsigned int authsize)
381{
382	switch (authsize) {
383	case 8:
384	case 12:
385	case 16:
386		break;
387	default:
388		return -EINVAL;
389	}
390
391	return nitrox_aead_setauthsize(aead, authsize);
392}
393
394static int nitrox_rfc4106_set_aead_rctx_sglist(struct aead_request *areq)
395{
396	struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq);
397	struct nitrox_aead_rctx *aead_rctx = &rctx->base;
398	unsigned int assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE;
399	struct scatterlist *sg;
400
401	if (areq->assoclen != 16 && areq->assoclen != 20)
402		return -EINVAL;
403
404	scatterwalk_map_and_copy(rctx->assoc, areq->src, 0, assoclen, 0);
405	sg_init_table(rctx->src, 3);
406	sg_set_buf(rctx->src, rctx->assoc, assoclen);
407	sg = scatterwalk_ffwd(rctx->src + 1, areq->src, areq->assoclen);
408	if (sg != rctx->src + 1)
409		sg_chain(rctx->src, 2, sg);
410
411	if (areq->src != areq->dst) {
412		sg_init_table(rctx->dst, 3);
413		sg_set_buf(rctx->dst, rctx->assoc, assoclen);
414		sg = scatterwalk_ffwd(rctx->dst + 1, areq->dst, areq->assoclen);
415		if (sg != rctx->dst + 1)
416			sg_chain(rctx->dst, 2, sg);
417	}
418
419	aead_rctx->src = rctx->src;
420	aead_rctx->dst = (areq->src == areq->dst) ? rctx->src : rctx->dst;
421
422	return 0;
423}
424
425static void nitrox_rfc4106_callback(void *arg, int err)
426{
427	struct aead_request *areq = arg;
428	struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq);
429	struct nitrox_kcrypt_request *nkreq = &rctx->base.nkreq;
430
431	free_src_sglist(nkreq);
432	free_dst_sglist(nkreq);
433	if (err) {
434		pr_err_ratelimited("request failed status 0x%0x\n", err);
435		err = -EINVAL;
436	}
437
438	areq->base.complete(&areq->base, err);
439}
440
441static int nitrox_rfc4106_enc(struct aead_request *areq)
442{
443	struct crypto_aead *aead = crypto_aead_reqtfm(areq);
444	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
445	struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq);
446	struct nitrox_aead_rctx *aead_rctx = &rctx->base;
447	struct se_crypto_request *creq = &aead_rctx->nkreq.creq;
448	int ret;
449
450	aead_rctx->cryptlen = areq->cryptlen;
451	aead_rctx->assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE;
452	aead_rctx->srclen = aead_rctx->assoclen + aead_rctx->cryptlen;
453	aead_rctx->dstlen = aead_rctx->srclen + aead->authsize;
454	aead_rctx->iv = areq->iv;
455	aead_rctx->ivsize = GCM_RFC4106_IV_SIZE;
456	aead_rctx->flags = areq->base.flags;
457	aead_rctx->ctx_handle = nctx->u.ctx_handle;
458	aead_rctx->ctrl_arg = ENCRYPT;
459
460	ret = nitrox_rfc4106_set_aead_rctx_sglist(areq);
461	if (ret)
462		return ret;
463
464	ret = nitrox_set_creq(aead_rctx);
465	if (ret)
466		return ret;
467
468	/* send the crypto request */
469	return nitrox_process_se_request(nctx->ndev, creq,
470					 nitrox_rfc4106_callback, areq);
471}
472
473static int nitrox_rfc4106_dec(struct aead_request *areq)
474{
475	struct crypto_aead *aead = crypto_aead_reqtfm(areq);
476	struct nitrox_crypto_ctx *nctx = crypto_aead_ctx(aead);
477	struct nitrox_rfc4106_rctx *rctx = aead_request_ctx(areq);
478	struct nitrox_aead_rctx *aead_rctx = &rctx->base;
479	struct se_crypto_request *creq = &aead_rctx->nkreq.creq;
480	int ret;
481
482	aead_rctx->cryptlen = areq->cryptlen - aead->authsize;
483	aead_rctx->assoclen = areq->assoclen - GCM_RFC4106_IV_SIZE;
484	aead_rctx->srclen =
485		areq->cryptlen - GCM_RFC4106_IV_SIZE + areq->assoclen;
486	aead_rctx->dstlen = aead_rctx->srclen - aead->authsize;
487	aead_rctx->iv = areq->iv;
488	aead_rctx->ivsize = GCM_RFC4106_IV_SIZE;
489	aead_rctx->flags = areq->base.flags;
490	aead_rctx->ctx_handle = nctx->u.ctx_handle;
491	aead_rctx->ctrl_arg = DECRYPT;
492
493	ret = nitrox_rfc4106_set_aead_rctx_sglist(areq);
494	if (ret)
495		return ret;
496
497	ret = nitrox_set_creq(aead_rctx);
498	if (ret)
499		return ret;
500
501	/* send the crypto request */
502	return nitrox_process_se_request(nctx->ndev, creq,
503					 nitrox_rfc4106_callback, areq);
504}
505
506static int nitrox_rfc4106_init(struct crypto_aead *aead)
507{
508	int ret;
509
510	ret = nitrox_gcm_common_init(aead);
511	if (ret)
512		return ret;
513
514	crypto_aead_set_reqsize(aead, sizeof(struct aead_request) +
515				sizeof(struct nitrox_rfc4106_rctx));
516
517	return 0;
518}
519
520static struct aead_alg nitrox_aeads[] = { {
521	.base = {
522		.cra_name = "gcm(aes)",
523		.cra_driver_name = "n5_aes_gcm",
524		.cra_priority = PRIO,
525		.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
526		.cra_blocksize = 1,
527		.cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
528		.cra_alignmask = 0,
529		.cra_module = THIS_MODULE,
530	},
531	.setkey = nitrox_aes_gcm_setkey,
532	.setauthsize = nitrox_aes_gcm_setauthsize,
533	.encrypt = nitrox_aes_gcm_enc,
534	.decrypt = nitrox_aes_gcm_dec,
535	.init = nitrox_aes_gcm_init,
536	.exit = nitrox_aead_exit,
537	.ivsize = GCM_AES_IV_SIZE,
538	.maxauthsize = AES_BLOCK_SIZE,
539}, {
540	.base = {
541		.cra_name = "rfc4106(gcm(aes))",
542		.cra_driver_name = "n5_rfc4106",
543		.cra_priority = PRIO,
544		.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
545		.cra_blocksize = 1,
546		.cra_ctxsize = sizeof(struct nitrox_crypto_ctx),
547		.cra_alignmask = 0,
548		.cra_module = THIS_MODULE,
549	},
550	.setkey = nitrox_rfc4106_setkey,
551	.setauthsize = nitrox_rfc4106_setauthsize,
552	.encrypt = nitrox_rfc4106_enc,
553	.decrypt = nitrox_rfc4106_dec,
554	.init = nitrox_rfc4106_init,
555	.exit = nitrox_aead_exit,
556	.ivsize = GCM_RFC4106_IV_SIZE,
557	.maxauthsize = AES_BLOCK_SIZE,
558} };
559
560int nitrox_register_aeads(void)
561{
562	return crypto_register_aeads(nitrox_aeads, ARRAY_SIZE(nitrox_aeads));
563}
564
565void nitrox_unregister_aeads(void)
566{
567	crypto_unregister_aeads(nitrox_aeads, ARRAY_SIZE(nitrox_aeads));
568}
569