1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Glue Code for x86_64/AVX2/AES-NI assembler optimized version of Camellia
4 *
5 * Copyright © 2013 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
6 */
7
8#include <asm/crypto/camellia.h>
9#include <asm/crypto/glue_helper.h>
10#include <crypto/algapi.h>
11#include <crypto/internal/simd.h>
12#include <crypto/xts.h>
13#include <linux/crypto.h>
14#include <linux/err.h>
15#include <linux/module.h>
16#include <linux/types.h>
17
18#define CAMELLIA_AESNI_PARALLEL_BLOCKS 16
19#define CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS 32
20
21/* 32-way AVX2/AES-NI parallel cipher functions */
22asmlinkage void camellia_ecb_enc_32way(const void *ctx, u8 *dst, const u8 *src);
23asmlinkage void camellia_ecb_dec_32way(const void *ctx, u8 *dst, const u8 *src);
24
25asmlinkage void camellia_cbc_dec_32way(const void *ctx, u8 *dst, const u8 *src);
26asmlinkage void camellia_ctr_32way(const void *ctx, u8 *dst, const u8 *src,
27				   le128 *iv);
28
29asmlinkage void camellia_xts_enc_32way(const void *ctx, u8 *dst, const u8 *src,
30				       le128 *iv);
31asmlinkage void camellia_xts_dec_32way(const void *ctx, u8 *dst, const u8 *src,
32				       le128 *iv);
33
34static const struct common_glue_ctx camellia_enc = {
35	.num_funcs = 4,
36	.fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
37
38	.funcs = { {
39		.num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
40		.fn_u = { .ecb = camellia_ecb_enc_32way }
41	}, {
42		.num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
43		.fn_u = { .ecb = camellia_ecb_enc_16way }
44	}, {
45		.num_blocks = 2,
46		.fn_u = { .ecb = camellia_enc_blk_2way }
47	}, {
48		.num_blocks = 1,
49		.fn_u = { .ecb = camellia_enc_blk }
50	} }
51};
52
53static const struct common_glue_ctx camellia_ctr = {
54	.num_funcs = 4,
55	.fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
56
57	.funcs = { {
58		.num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
59		.fn_u = { .ctr = camellia_ctr_32way }
60	}, {
61		.num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
62		.fn_u = { .ctr = camellia_ctr_16way }
63	}, {
64		.num_blocks = 2,
65		.fn_u = { .ctr = camellia_crypt_ctr_2way }
66	}, {
67		.num_blocks = 1,
68		.fn_u = { .ctr = camellia_crypt_ctr }
69	} }
70};
71
72static const struct common_glue_ctx camellia_enc_xts = {
73	.num_funcs = 3,
74	.fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
75
76	.funcs = { {
77		.num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
78		.fn_u = { .xts = camellia_xts_enc_32way }
79	}, {
80		.num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
81		.fn_u = { .xts = camellia_xts_enc_16way }
82	}, {
83		.num_blocks = 1,
84		.fn_u = { .xts = camellia_xts_enc }
85	} }
86};
87
88static const struct common_glue_ctx camellia_dec = {
89	.num_funcs = 4,
90	.fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
91
92	.funcs = { {
93		.num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
94		.fn_u = { .ecb = camellia_ecb_dec_32way }
95	}, {
96		.num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
97		.fn_u = { .ecb = camellia_ecb_dec_16way }
98	}, {
99		.num_blocks = 2,
100		.fn_u = { .ecb = camellia_dec_blk_2way }
101	}, {
102		.num_blocks = 1,
103		.fn_u = { .ecb = camellia_dec_blk }
104	} }
105};
106
107static const struct common_glue_ctx camellia_dec_cbc = {
108	.num_funcs = 4,
109	.fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
110
111	.funcs = { {
112		.num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
113		.fn_u = { .cbc = camellia_cbc_dec_32way }
114	}, {
115		.num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
116		.fn_u = { .cbc = camellia_cbc_dec_16way }
117	}, {
118		.num_blocks = 2,
119		.fn_u = { .cbc = camellia_decrypt_cbc_2way }
120	}, {
121		.num_blocks = 1,
122		.fn_u = { .cbc = camellia_dec_blk }
123	} }
124};
125
126static const struct common_glue_ctx camellia_dec_xts = {
127	.num_funcs = 3,
128	.fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
129
130	.funcs = { {
131		.num_blocks = CAMELLIA_AESNI_AVX2_PARALLEL_BLOCKS,
132		.fn_u = { .xts = camellia_xts_dec_32way }
133	}, {
134		.num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
135		.fn_u = { .xts = camellia_xts_dec_16way }
136	}, {
137		.num_blocks = 1,
138		.fn_u = { .xts = camellia_xts_dec }
139	} }
140};
141
142static int camellia_setkey(struct crypto_skcipher *tfm, const u8 *key,
143			   unsigned int keylen)
144{
145	return __camellia_setkey(crypto_skcipher_ctx(tfm), key, keylen);
146}
147
148static int ecb_encrypt(struct skcipher_request *req)
149{
150	return glue_ecb_req_128bit(&camellia_enc, req);
151}
152
153static int ecb_decrypt(struct skcipher_request *req)
154{
155	return glue_ecb_req_128bit(&camellia_dec, req);
156}
157
158static int cbc_encrypt(struct skcipher_request *req)
159{
160	return glue_cbc_encrypt_req_128bit(camellia_enc_blk, req);
161}
162
163static int cbc_decrypt(struct skcipher_request *req)
164{
165	return glue_cbc_decrypt_req_128bit(&camellia_dec_cbc, req);
166}
167
168static int ctr_crypt(struct skcipher_request *req)
169{
170	return glue_ctr_req_128bit(&camellia_ctr, req);
171}
172
173static int xts_encrypt(struct skcipher_request *req)
174{
175	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
176	struct camellia_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
177
178	return glue_xts_req_128bit(&camellia_enc_xts, req, camellia_enc_blk,
179				   &ctx->tweak_ctx, &ctx->crypt_ctx, false);
180}
181
182static int xts_decrypt(struct skcipher_request *req)
183{
184	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
185	struct camellia_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
186
187	return glue_xts_req_128bit(&camellia_dec_xts, req, camellia_enc_blk,
188				   &ctx->tweak_ctx, &ctx->crypt_ctx, true);
189}
190
191static struct skcipher_alg camellia_algs[] = {
192	{
193		.base.cra_name		= "__ecb(camellia)",
194		.base.cra_driver_name	= "__ecb-camellia-aesni-avx2",
195		.base.cra_priority	= 500,
196		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
197		.base.cra_blocksize	= CAMELLIA_BLOCK_SIZE,
198		.base.cra_ctxsize	= sizeof(struct camellia_ctx),
199		.base.cra_module	= THIS_MODULE,
200		.min_keysize		= CAMELLIA_MIN_KEY_SIZE,
201		.max_keysize		= CAMELLIA_MAX_KEY_SIZE,
202		.setkey			= camellia_setkey,
203		.encrypt		= ecb_encrypt,
204		.decrypt		= ecb_decrypt,
205	}, {
206		.base.cra_name		= "__cbc(camellia)",
207		.base.cra_driver_name	= "__cbc-camellia-aesni-avx2",
208		.base.cra_priority	= 500,
209		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
210		.base.cra_blocksize	= CAMELLIA_BLOCK_SIZE,
211		.base.cra_ctxsize	= sizeof(struct camellia_ctx),
212		.base.cra_module	= THIS_MODULE,
213		.min_keysize		= CAMELLIA_MIN_KEY_SIZE,
214		.max_keysize		= CAMELLIA_MAX_KEY_SIZE,
215		.ivsize			= CAMELLIA_BLOCK_SIZE,
216		.setkey			= camellia_setkey,
217		.encrypt		= cbc_encrypt,
218		.decrypt		= cbc_decrypt,
219	}, {
220		.base.cra_name		= "__ctr(camellia)",
221		.base.cra_driver_name	= "__ctr-camellia-aesni-avx2",
222		.base.cra_priority	= 500,
223		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
224		.base.cra_blocksize	= 1,
225		.base.cra_ctxsize	= sizeof(struct camellia_ctx),
226		.base.cra_module	= THIS_MODULE,
227		.min_keysize		= CAMELLIA_MIN_KEY_SIZE,
228		.max_keysize		= CAMELLIA_MAX_KEY_SIZE,
229		.ivsize			= CAMELLIA_BLOCK_SIZE,
230		.chunksize		= CAMELLIA_BLOCK_SIZE,
231		.setkey			= camellia_setkey,
232		.encrypt		= ctr_crypt,
233		.decrypt		= ctr_crypt,
234	}, {
235		.base.cra_name		= "__xts(camellia)",
236		.base.cra_driver_name	= "__xts-camellia-aesni-avx2",
237		.base.cra_priority	= 500,
238		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
239		.base.cra_blocksize	= CAMELLIA_BLOCK_SIZE,
240		.base.cra_ctxsize	= sizeof(struct camellia_xts_ctx),
241		.base.cra_module	= THIS_MODULE,
242		.min_keysize		= 2 * CAMELLIA_MIN_KEY_SIZE,
243		.max_keysize		= 2 * CAMELLIA_MAX_KEY_SIZE,
244		.ivsize			= CAMELLIA_BLOCK_SIZE,
245		.setkey			= xts_camellia_setkey,
246		.encrypt		= xts_encrypt,
247		.decrypt		= xts_decrypt,
248	},
249};
250
251static struct simd_skcipher_alg *camellia_simd_algs[ARRAY_SIZE(camellia_algs)];
252
253static int __init camellia_aesni_init(void)
254{
255	const char *feature_name;
256
257	if (!boot_cpu_has(X86_FEATURE_AVX) ||
258	    !boot_cpu_has(X86_FEATURE_AVX2) ||
259	    !boot_cpu_has(X86_FEATURE_AES) ||
260	    !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
261		pr_info("AVX2 or AES-NI instructions are not detected.\n");
262		return -ENODEV;
263	}
264
265	if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
266				&feature_name)) {
267		pr_info("CPU feature '%s' is not supported.\n", feature_name);
268		return -ENODEV;
269	}
270
271	return simd_register_skciphers_compat(camellia_algs,
272					      ARRAY_SIZE(camellia_algs),
273					      camellia_simd_algs);
274}
275
276static void __exit camellia_aesni_fini(void)
277{
278	simd_unregister_skciphers(camellia_algs, ARRAY_SIZE(camellia_algs),
279				  camellia_simd_algs);
280}
281
282module_init(camellia_aesni_init);
283module_exit(camellia_aesni_fini);
284
285MODULE_LICENSE("GPL");
286MODULE_DESCRIPTION("Camellia Cipher Algorithm, AES-NI/AVX2 optimized");
287MODULE_ALIAS_CRYPTO("camellia");
288MODULE_ALIAS_CRYPTO("camellia-asm");
289