1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Glue Code for x86_64/AVX/AES-NI assembler optimized version of Camellia
4 *
5 * Copyright © 2012-2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
6 */
7
8#include <asm/crypto/camellia.h>
9#include <asm/crypto/glue_helper.h>
10#include <crypto/algapi.h>
11#include <crypto/internal/simd.h>
12#include <crypto/xts.h>
13#include <linux/crypto.h>
14#include <linux/err.h>
15#include <linux/module.h>
16#include <linux/types.h>
17
18#define CAMELLIA_AESNI_PARALLEL_BLOCKS 16
19
20/* 16-way parallel cipher functions (avx/aes-ni) */
21asmlinkage void camellia_ecb_enc_16way(const void *ctx, u8 *dst, const u8 *src);
22EXPORT_SYMBOL_GPL(camellia_ecb_enc_16way);
23
24asmlinkage void camellia_ecb_dec_16way(const void *ctx, u8 *dst, const u8 *src);
25EXPORT_SYMBOL_GPL(camellia_ecb_dec_16way);
26
27asmlinkage void camellia_cbc_dec_16way(const void *ctx, u8 *dst, const u8 *src);
28EXPORT_SYMBOL_GPL(camellia_cbc_dec_16way);
29
30asmlinkage void camellia_ctr_16way(const void *ctx, u8 *dst, const u8 *src,
31				   le128 *iv);
32EXPORT_SYMBOL_GPL(camellia_ctr_16way);
33
34asmlinkage void camellia_xts_enc_16way(const void *ctx, u8 *dst, const u8 *src,
35				       le128 *iv);
36EXPORT_SYMBOL_GPL(camellia_xts_enc_16way);
37
38asmlinkage void camellia_xts_dec_16way(const void *ctx, u8 *dst, const u8 *src,
39				       le128 *iv);
40EXPORT_SYMBOL_GPL(camellia_xts_dec_16way);
41
42void camellia_xts_enc(const void *ctx, u8 *dst, const u8 *src, le128 *iv)
43{
44	glue_xts_crypt_128bit_one(ctx, dst, src, iv, camellia_enc_blk);
45}
46EXPORT_SYMBOL_GPL(camellia_xts_enc);
47
48void camellia_xts_dec(const void *ctx, u8 *dst, const u8 *src, le128 *iv)
49{
50	glue_xts_crypt_128bit_one(ctx, dst, src, iv, camellia_dec_blk);
51}
52EXPORT_SYMBOL_GPL(camellia_xts_dec);
53
54static const struct common_glue_ctx camellia_enc = {
55	.num_funcs = 3,
56	.fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
57
58	.funcs = { {
59		.num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
60		.fn_u = { .ecb = camellia_ecb_enc_16way }
61	}, {
62		.num_blocks = 2,
63		.fn_u = { .ecb = camellia_enc_blk_2way }
64	}, {
65		.num_blocks = 1,
66		.fn_u = { .ecb = camellia_enc_blk }
67	} }
68};
69
70static const struct common_glue_ctx camellia_ctr = {
71	.num_funcs = 3,
72	.fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
73
74	.funcs = { {
75		.num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
76		.fn_u = { .ctr = camellia_ctr_16way }
77	}, {
78		.num_blocks = 2,
79		.fn_u = { .ctr = camellia_crypt_ctr_2way }
80	}, {
81		.num_blocks = 1,
82		.fn_u = { .ctr = camellia_crypt_ctr }
83	} }
84};
85
86static const struct common_glue_ctx camellia_enc_xts = {
87	.num_funcs = 2,
88	.fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
89
90	.funcs = { {
91		.num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
92		.fn_u = { .xts = camellia_xts_enc_16way }
93	}, {
94		.num_blocks = 1,
95		.fn_u = { .xts = camellia_xts_enc }
96	} }
97};
98
99static const struct common_glue_ctx camellia_dec = {
100	.num_funcs = 3,
101	.fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
102
103	.funcs = { {
104		.num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
105		.fn_u = { .ecb = camellia_ecb_dec_16way }
106	}, {
107		.num_blocks = 2,
108		.fn_u = { .ecb = camellia_dec_blk_2way }
109	}, {
110		.num_blocks = 1,
111		.fn_u = { .ecb = camellia_dec_blk }
112	} }
113};
114
115static const struct common_glue_ctx camellia_dec_cbc = {
116	.num_funcs = 3,
117	.fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
118
119	.funcs = { {
120		.num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
121		.fn_u = { .cbc = camellia_cbc_dec_16way }
122	}, {
123		.num_blocks = 2,
124		.fn_u = { .cbc = camellia_decrypt_cbc_2way }
125	}, {
126		.num_blocks = 1,
127		.fn_u = { .cbc = camellia_dec_blk }
128	} }
129};
130
131static const struct common_glue_ctx camellia_dec_xts = {
132	.num_funcs = 2,
133	.fpu_blocks_limit = CAMELLIA_AESNI_PARALLEL_BLOCKS,
134
135	.funcs = { {
136		.num_blocks = CAMELLIA_AESNI_PARALLEL_BLOCKS,
137		.fn_u = { .xts = camellia_xts_dec_16way }
138	}, {
139		.num_blocks = 1,
140		.fn_u = { .xts = camellia_xts_dec }
141	} }
142};
143
144static int camellia_setkey(struct crypto_skcipher *tfm, const u8 *key,
145			   unsigned int keylen)
146{
147	return __camellia_setkey(crypto_skcipher_ctx(tfm), key, keylen);
148}
149
150static int ecb_encrypt(struct skcipher_request *req)
151{
152	return glue_ecb_req_128bit(&camellia_enc, req);
153}
154
155static int ecb_decrypt(struct skcipher_request *req)
156{
157	return glue_ecb_req_128bit(&camellia_dec, req);
158}
159
160static int cbc_encrypt(struct skcipher_request *req)
161{
162	return glue_cbc_encrypt_req_128bit(camellia_enc_blk, req);
163}
164
165static int cbc_decrypt(struct skcipher_request *req)
166{
167	return glue_cbc_decrypt_req_128bit(&camellia_dec_cbc, req);
168}
169
170static int ctr_crypt(struct skcipher_request *req)
171{
172	return glue_ctr_req_128bit(&camellia_ctr, req);
173}
174
175int xts_camellia_setkey(struct crypto_skcipher *tfm, const u8 *key,
176			unsigned int keylen)
177{
178	struct camellia_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
179	int err;
180
181	err = xts_verify_key(tfm, key, keylen);
182	if (err)
183		return err;
184
185	/* first half of xts-key is for crypt */
186	err = __camellia_setkey(&ctx->crypt_ctx, key, keylen / 2);
187	if (err)
188		return err;
189
190	/* second half of xts-key is for tweak */
191	return __camellia_setkey(&ctx->tweak_ctx, key + keylen / 2, keylen / 2);
192}
193EXPORT_SYMBOL_GPL(xts_camellia_setkey);
194
195static int xts_encrypt(struct skcipher_request *req)
196{
197	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
198	struct camellia_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
199
200	return glue_xts_req_128bit(&camellia_enc_xts, req, camellia_enc_blk,
201				   &ctx->tweak_ctx, &ctx->crypt_ctx, false);
202}
203
204static int xts_decrypt(struct skcipher_request *req)
205{
206	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
207	struct camellia_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
208
209	return glue_xts_req_128bit(&camellia_dec_xts, req, camellia_enc_blk,
210				   &ctx->tweak_ctx, &ctx->crypt_ctx, true);
211}
212
213static struct skcipher_alg camellia_algs[] = {
214	{
215		.base.cra_name		= "__ecb(camellia)",
216		.base.cra_driver_name	= "__ecb-camellia-aesni",
217		.base.cra_priority	= 400,
218		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
219		.base.cra_blocksize	= CAMELLIA_BLOCK_SIZE,
220		.base.cra_ctxsize	= sizeof(struct camellia_ctx),
221		.base.cra_module	= THIS_MODULE,
222		.min_keysize		= CAMELLIA_MIN_KEY_SIZE,
223		.max_keysize		= CAMELLIA_MAX_KEY_SIZE,
224		.setkey			= camellia_setkey,
225		.encrypt		= ecb_encrypt,
226		.decrypt		= ecb_decrypt,
227	}, {
228		.base.cra_name		= "__cbc(camellia)",
229		.base.cra_driver_name	= "__cbc-camellia-aesni",
230		.base.cra_priority	= 400,
231		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
232		.base.cra_blocksize	= CAMELLIA_BLOCK_SIZE,
233		.base.cra_ctxsize	= sizeof(struct camellia_ctx),
234		.base.cra_module	= THIS_MODULE,
235		.min_keysize		= CAMELLIA_MIN_KEY_SIZE,
236		.max_keysize		= CAMELLIA_MAX_KEY_SIZE,
237		.ivsize			= CAMELLIA_BLOCK_SIZE,
238		.setkey			= camellia_setkey,
239		.encrypt		= cbc_encrypt,
240		.decrypt		= cbc_decrypt,
241	}, {
242		.base.cra_name		= "__ctr(camellia)",
243		.base.cra_driver_name	= "__ctr-camellia-aesni",
244		.base.cra_priority	= 400,
245		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
246		.base.cra_blocksize	= 1,
247		.base.cra_ctxsize	= sizeof(struct camellia_ctx),
248		.base.cra_module	= THIS_MODULE,
249		.min_keysize		= CAMELLIA_MIN_KEY_SIZE,
250		.max_keysize		= CAMELLIA_MAX_KEY_SIZE,
251		.ivsize			= CAMELLIA_BLOCK_SIZE,
252		.chunksize		= CAMELLIA_BLOCK_SIZE,
253		.setkey			= camellia_setkey,
254		.encrypt		= ctr_crypt,
255		.decrypt		= ctr_crypt,
256	}, {
257		.base.cra_name		= "__xts(camellia)",
258		.base.cra_driver_name	= "__xts-camellia-aesni",
259		.base.cra_priority	= 400,
260		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
261		.base.cra_blocksize	= CAMELLIA_BLOCK_SIZE,
262		.base.cra_ctxsize	= sizeof(struct camellia_xts_ctx),
263		.base.cra_module	= THIS_MODULE,
264		.min_keysize		= 2 * CAMELLIA_MIN_KEY_SIZE,
265		.max_keysize		= 2 * CAMELLIA_MAX_KEY_SIZE,
266		.ivsize			= CAMELLIA_BLOCK_SIZE,
267		.setkey			= xts_camellia_setkey,
268		.encrypt		= xts_encrypt,
269		.decrypt		= xts_decrypt,
270	},
271};
272
273static struct simd_skcipher_alg *camellia_simd_algs[ARRAY_SIZE(camellia_algs)];
274
275static int __init camellia_aesni_init(void)
276{
277	const char *feature_name;
278
279	if (!boot_cpu_has(X86_FEATURE_AVX) ||
280	    !boot_cpu_has(X86_FEATURE_AES) ||
281	    !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
282		pr_info("AVX or AES-NI instructions are not detected.\n");
283		return -ENODEV;
284	}
285
286	if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
287				&feature_name)) {
288		pr_info("CPU feature '%s' is not supported.\n", feature_name);
289		return -ENODEV;
290	}
291
292	return simd_register_skciphers_compat(camellia_algs,
293					      ARRAY_SIZE(camellia_algs),
294					      camellia_simd_algs);
295}
296
297static void __exit camellia_aesni_fini(void)
298{
299	simd_unregister_skciphers(camellia_algs, ARRAY_SIZE(camellia_algs),
300				  camellia_simd_algs);
301}
302
303module_init(camellia_aesni_init);
304module_exit(camellia_aesni_fini);
305
306MODULE_LICENSE("GPL");
307MODULE_DESCRIPTION("Camellia Cipher Algorithm, AES-NI/AVX optimized");
308MODULE_ALIAS_CRYPTO("camellia");
309MODULE_ALIAS_CRYPTO("camellia-asm");
310