1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Glue Code for x86_64/AVX2 assembler optimized version of Serpent
4 *
5 * Copyright © 2012-2013 Jussi Kivilinna <jussi.kivilinna@mbnet.fi>
6 */
7
8#include <linux/module.h>
9#include <linux/types.h>
10#include <linux/crypto.h>
11#include <linux/err.h>
12#include <crypto/algapi.h>
13#include <crypto/internal/simd.h>
14#include <crypto/serpent.h>
15#include <crypto/xts.h>
16#include <asm/crypto/glue_helper.h>
17#include <asm/crypto/serpent-avx.h>
18
19#define SERPENT_AVX2_PARALLEL_BLOCKS 16
20
21/* 16-way AVX2 parallel cipher functions */
22asmlinkage void serpent_ecb_enc_16way(const void *ctx, u8 *dst, const u8 *src);
23asmlinkage void serpent_ecb_dec_16way(const void *ctx, u8 *dst, const u8 *src);
24asmlinkage void serpent_cbc_dec_16way(const void *ctx, u8 *dst, const u8 *src);
25
26asmlinkage void serpent_ctr_16way(const void *ctx, u8 *dst, const u8 *src,
27				  le128 *iv);
28asmlinkage void serpent_xts_enc_16way(const void *ctx, u8 *dst, const u8 *src,
29				      le128 *iv);
30asmlinkage void serpent_xts_dec_16way(const void *ctx, u8 *dst, const u8 *src,
31				      le128 *iv);
32
33static int serpent_setkey_skcipher(struct crypto_skcipher *tfm,
34				   const u8 *key, unsigned int keylen)
35{
36	return __serpent_setkey(crypto_skcipher_ctx(tfm), key, keylen);
37}
38
39static const struct common_glue_ctx serpent_enc = {
40	.num_funcs = 3,
41	.fpu_blocks_limit = 8,
42
43	.funcs = { {
44		.num_blocks = 16,
45		.fn_u = { .ecb = serpent_ecb_enc_16way }
46	}, {
47		.num_blocks = 8,
48		.fn_u = { .ecb = serpent_ecb_enc_8way_avx }
49	}, {
50		.num_blocks = 1,
51		.fn_u = { .ecb = __serpent_encrypt }
52	} }
53};
54
55static const struct common_glue_ctx serpent_ctr = {
56	.num_funcs = 3,
57	.fpu_blocks_limit = 8,
58
59	.funcs = { {
60		.num_blocks = 16,
61		.fn_u = { .ctr = serpent_ctr_16way }
62	},  {
63		.num_blocks = 8,
64		.fn_u = { .ctr = serpent_ctr_8way_avx }
65	}, {
66		.num_blocks = 1,
67		.fn_u = { .ctr = __serpent_crypt_ctr }
68	} }
69};
70
71static const struct common_glue_ctx serpent_enc_xts = {
72	.num_funcs = 3,
73	.fpu_blocks_limit = 8,
74
75	.funcs = { {
76		.num_blocks = 16,
77		.fn_u = { .xts = serpent_xts_enc_16way }
78	}, {
79		.num_blocks = 8,
80		.fn_u = { .xts = serpent_xts_enc_8way_avx }
81	}, {
82		.num_blocks = 1,
83		.fn_u = { .xts = serpent_xts_enc }
84	} }
85};
86
87static const struct common_glue_ctx serpent_dec = {
88	.num_funcs = 3,
89	.fpu_blocks_limit = 8,
90
91	.funcs = { {
92		.num_blocks = 16,
93		.fn_u = { .ecb = serpent_ecb_dec_16way }
94	}, {
95		.num_blocks = 8,
96		.fn_u = { .ecb = serpent_ecb_dec_8way_avx }
97	}, {
98		.num_blocks = 1,
99		.fn_u = { .ecb = __serpent_decrypt }
100	} }
101};
102
103static const struct common_glue_ctx serpent_dec_cbc = {
104	.num_funcs = 3,
105	.fpu_blocks_limit = 8,
106
107	.funcs = { {
108		.num_blocks = 16,
109		.fn_u = { .cbc = serpent_cbc_dec_16way }
110	}, {
111		.num_blocks = 8,
112		.fn_u = { .cbc = serpent_cbc_dec_8way_avx }
113	}, {
114		.num_blocks = 1,
115		.fn_u = { .cbc = __serpent_decrypt }
116	} }
117};
118
119static const struct common_glue_ctx serpent_dec_xts = {
120	.num_funcs = 3,
121	.fpu_blocks_limit = 8,
122
123	.funcs = { {
124		.num_blocks = 16,
125		.fn_u = { .xts = serpent_xts_dec_16way }
126	}, {
127		.num_blocks = 8,
128		.fn_u = { .xts = serpent_xts_dec_8way_avx }
129	}, {
130		.num_blocks = 1,
131		.fn_u = { .xts = serpent_xts_dec }
132	} }
133};
134
135static int ecb_encrypt(struct skcipher_request *req)
136{
137	return glue_ecb_req_128bit(&serpent_enc, req);
138}
139
140static int ecb_decrypt(struct skcipher_request *req)
141{
142	return glue_ecb_req_128bit(&serpent_dec, req);
143}
144
145static int cbc_encrypt(struct skcipher_request *req)
146{
147	return glue_cbc_encrypt_req_128bit(__serpent_encrypt, req);
148}
149
150static int cbc_decrypt(struct skcipher_request *req)
151{
152	return glue_cbc_decrypt_req_128bit(&serpent_dec_cbc, req);
153}
154
155static int ctr_crypt(struct skcipher_request *req)
156{
157	return glue_ctr_req_128bit(&serpent_ctr, req);
158}
159
160static int xts_encrypt(struct skcipher_request *req)
161{
162	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
163	struct serpent_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
164
165	return glue_xts_req_128bit(&serpent_enc_xts, req,
166				   __serpent_encrypt, &ctx->tweak_ctx,
167				   &ctx->crypt_ctx, false);
168}
169
170static int xts_decrypt(struct skcipher_request *req)
171{
172	struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
173	struct serpent_xts_ctx *ctx = crypto_skcipher_ctx(tfm);
174
175	return glue_xts_req_128bit(&serpent_dec_xts, req,
176				   __serpent_encrypt, &ctx->tweak_ctx,
177				   &ctx->crypt_ctx, true);
178}
179
180static struct skcipher_alg serpent_algs[] = {
181	{
182		.base.cra_name		= "__ecb(serpent)",
183		.base.cra_driver_name	= "__ecb-serpent-avx2",
184		.base.cra_priority	= 600,
185		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
186		.base.cra_blocksize	= SERPENT_BLOCK_SIZE,
187		.base.cra_ctxsize	= sizeof(struct serpent_ctx),
188		.base.cra_module	= THIS_MODULE,
189		.min_keysize		= SERPENT_MIN_KEY_SIZE,
190		.max_keysize		= SERPENT_MAX_KEY_SIZE,
191		.setkey			= serpent_setkey_skcipher,
192		.encrypt		= ecb_encrypt,
193		.decrypt		= ecb_decrypt,
194	}, {
195		.base.cra_name		= "__cbc(serpent)",
196		.base.cra_driver_name	= "__cbc-serpent-avx2",
197		.base.cra_priority	= 600,
198		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
199		.base.cra_blocksize	= SERPENT_BLOCK_SIZE,
200		.base.cra_ctxsize	= sizeof(struct serpent_ctx),
201		.base.cra_module	= THIS_MODULE,
202		.min_keysize		= SERPENT_MIN_KEY_SIZE,
203		.max_keysize		= SERPENT_MAX_KEY_SIZE,
204		.ivsize			= SERPENT_BLOCK_SIZE,
205		.setkey			= serpent_setkey_skcipher,
206		.encrypt		= cbc_encrypt,
207		.decrypt		= cbc_decrypt,
208	}, {
209		.base.cra_name		= "__ctr(serpent)",
210		.base.cra_driver_name	= "__ctr-serpent-avx2",
211		.base.cra_priority	= 600,
212		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
213		.base.cra_blocksize	= 1,
214		.base.cra_ctxsize	= sizeof(struct serpent_ctx),
215		.base.cra_module	= THIS_MODULE,
216		.min_keysize		= SERPENT_MIN_KEY_SIZE,
217		.max_keysize		= SERPENT_MAX_KEY_SIZE,
218		.ivsize			= SERPENT_BLOCK_SIZE,
219		.chunksize		= SERPENT_BLOCK_SIZE,
220		.setkey			= serpent_setkey_skcipher,
221		.encrypt		= ctr_crypt,
222		.decrypt		= ctr_crypt,
223	}, {
224		.base.cra_name		= "__xts(serpent)",
225		.base.cra_driver_name	= "__xts-serpent-avx2",
226		.base.cra_priority	= 600,
227		.base.cra_flags		= CRYPTO_ALG_INTERNAL,
228		.base.cra_blocksize	= SERPENT_BLOCK_SIZE,
229		.base.cra_ctxsize	= sizeof(struct serpent_xts_ctx),
230		.base.cra_module	= THIS_MODULE,
231		.min_keysize		= 2 * SERPENT_MIN_KEY_SIZE,
232		.max_keysize		= 2 * SERPENT_MAX_KEY_SIZE,
233		.ivsize			= SERPENT_BLOCK_SIZE,
234		.setkey			= xts_serpent_setkey,
235		.encrypt		= xts_encrypt,
236		.decrypt		= xts_decrypt,
237	},
238};
239
240static struct simd_skcipher_alg *serpent_simd_algs[ARRAY_SIZE(serpent_algs)];
241
242static int __init init(void)
243{
244	const char *feature_name;
245
246	if (!boot_cpu_has(X86_FEATURE_AVX2) || !boot_cpu_has(X86_FEATURE_OSXSAVE)) {
247		pr_info("AVX2 instructions are not detected.\n");
248		return -ENODEV;
249	}
250	if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM,
251				&feature_name)) {
252		pr_info("CPU feature '%s' is not supported.\n", feature_name);
253		return -ENODEV;
254	}
255
256	return simd_register_skciphers_compat(serpent_algs,
257					      ARRAY_SIZE(serpent_algs),
258					      serpent_simd_algs);
259}
260
261static void __exit fini(void)
262{
263	simd_unregister_skciphers(serpent_algs, ARRAY_SIZE(serpent_algs),
264				  serpent_simd_algs);
265}
266
267module_init(init);
268module_exit(fini);
269
270MODULE_LICENSE("GPL");
271MODULE_DESCRIPTION("Serpent Cipher Algorithm, AVX2 optimized");
272MODULE_ALIAS_CRYPTO("serpent");
273MODULE_ALIAS_CRYPTO("serpent-asm");
274