1/*
2 * Cryptographic API.
3 *
4 * Glue code for the SHA256 Secure Hash Algorithm assembler
5 * implementation using supplemental SSE3 / AVX / AVX2 instructions.
6 *
7 * This file is based on sha256_generic.c
8 *
9 * Copyright (C) 2013 Intel Corporation.
10 *
11 * Author:
12 *     Tim Chen <tim.c.chen@linux.intel.com>
13 *
14 * This program is free software; you can redistribute it and/or modify it
15 * under the terms of the GNU General Public License as published by the Free
16 * Software Foundation; either version 2 of the License, or (at your option)
17 * any later version.
18 *
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
20 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
21 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
22 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
23 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
24 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
25 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
26 * SOFTWARE.
27 */
28
29
30#define pr_fmt(fmt)	KBUILD_MODNAME ": " fmt
31
32#include <crypto/internal/hash.h>
33#include <crypto/internal/simd.h>
34#include <linux/init.h>
35#include <linux/module.h>
36#include <linux/mm.h>
37#include <linux/types.h>
38#include <crypto/sha2.h>
39#include <crypto/sha256_base.h>
40#include <linux/string.h>
41#include <asm/cpu_device_id.h>
42#include <asm/simd.h>
43
44asmlinkage void sha256_transform_ssse3(struct sha256_state *state,
45				       const u8 *data, int blocks);
46
47static const struct x86_cpu_id module_cpu_ids[] = {
48	X86_MATCH_FEATURE(X86_FEATURE_AVX2, NULL),
49	X86_MATCH_FEATURE(X86_FEATURE_AVX, NULL),
50	X86_MATCH_FEATURE(X86_FEATURE_SSSE3, NULL),
51	{}
52};
53MODULE_DEVICE_TABLE(x86cpu, module_cpu_ids);
54
55static int _sha256_update(struct shash_desc *desc, const u8 *data,
56			  unsigned int len, sha256_block_fn *sha256_xform)
57{
58	struct sha256_state *sctx = shash_desc_ctx(desc);
59
60	if (!crypto_simd_usable() ||
61	    (sctx->count % SHA256_BLOCK_SIZE) + len < SHA256_BLOCK_SIZE)
62		return crypto_sha256_update(desc, data, len);
63
64	/*
65	 * Make sure struct sha256_state begins directly with the SHA256
66	 * 256-bit internal state, as this is what the asm functions expect.
67	 */
68	BUILD_BUG_ON(offsetof(struct sha256_state, state) != 0);
69
70	kernel_fpu_begin();
71	sha256_base_do_update(desc, data, len, sha256_xform);
72	kernel_fpu_end();
73
74	return 0;
75}
76
77static int sha256_finup(struct shash_desc *desc, const u8 *data,
78	      unsigned int len, u8 *out, sha256_block_fn *sha256_xform)
79{
80	if (!crypto_simd_usable())
81		return crypto_sha256_finup(desc, data, len, out);
82
83	kernel_fpu_begin();
84	if (len)
85		sha256_base_do_update(desc, data, len, sha256_xform);
86	sha256_base_do_finalize(desc, sha256_xform);
87	kernel_fpu_end();
88
89	return sha256_base_finish(desc, out);
90}
91
92static int sha256_ssse3_update(struct shash_desc *desc, const u8 *data,
93			 unsigned int len)
94{
95	return _sha256_update(desc, data, len, sha256_transform_ssse3);
96}
97
98static int sha256_ssse3_finup(struct shash_desc *desc, const u8 *data,
99	      unsigned int len, u8 *out)
100{
101	return sha256_finup(desc, data, len, out, sha256_transform_ssse3);
102}
103
104/* Add padding and return the message digest. */
105static int sha256_ssse3_final(struct shash_desc *desc, u8 *out)
106{
107	return sha256_ssse3_finup(desc, NULL, 0, out);
108}
109
110static struct shash_alg sha256_ssse3_algs[] = { {
111	.digestsize	=	SHA256_DIGEST_SIZE,
112	.init		=	sha256_base_init,
113	.update		=	sha256_ssse3_update,
114	.final		=	sha256_ssse3_final,
115	.finup		=	sha256_ssse3_finup,
116	.descsize	=	sizeof(struct sha256_state),
117	.base		=	{
118		.cra_name	=	"sha256",
119		.cra_driver_name =	"sha256-ssse3",
120		.cra_priority	=	150,
121		.cra_blocksize	=	SHA256_BLOCK_SIZE,
122		.cra_module	=	THIS_MODULE,
123	}
124}, {
125	.digestsize	=	SHA224_DIGEST_SIZE,
126	.init		=	sha224_base_init,
127	.update		=	sha256_ssse3_update,
128	.final		=	sha256_ssse3_final,
129	.finup		=	sha256_ssse3_finup,
130	.descsize	=	sizeof(struct sha256_state),
131	.base		=	{
132		.cra_name	=	"sha224",
133		.cra_driver_name =	"sha224-ssse3",
134		.cra_priority	=	150,
135		.cra_blocksize	=	SHA224_BLOCK_SIZE,
136		.cra_module	=	THIS_MODULE,
137	}
138} };
139
140static int register_sha256_ssse3(void)
141{
142	if (boot_cpu_has(X86_FEATURE_SSSE3))
143		return crypto_register_shashes(sha256_ssse3_algs,
144				ARRAY_SIZE(sha256_ssse3_algs));
145	return 0;
146}
147
148static void unregister_sha256_ssse3(void)
149{
150	if (boot_cpu_has(X86_FEATURE_SSSE3))
151		crypto_unregister_shashes(sha256_ssse3_algs,
152				ARRAY_SIZE(sha256_ssse3_algs));
153}
154
155asmlinkage void sha256_transform_avx(struct sha256_state *state,
156				     const u8 *data, int blocks);
157
158static int sha256_avx_update(struct shash_desc *desc, const u8 *data,
159			 unsigned int len)
160{
161	return _sha256_update(desc, data, len, sha256_transform_avx);
162}
163
164static int sha256_avx_finup(struct shash_desc *desc, const u8 *data,
165		      unsigned int len, u8 *out)
166{
167	return sha256_finup(desc, data, len, out, sha256_transform_avx);
168}
169
170static int sha256_avx_final(struct shash_desc *desc, u8 *out)
171{
172	return sha256_avx_finup(desc, NULL, 0, out);
173}
174
175static struct shash_alg sha256_avx_algs[] = { {
176	.digestsize	=	SHA256_DIGEST_SIZE,
177	.init		=	sha256_base_init,
178	.update		=	sha256_avx_update,
179	.final		=	sha256_avx_final,
180	.finup		=	sha256_avx_finup,
181	.descsize	=	sizeof(struct sha256_state),
182	.base		=	{
183		.cra_name	=	"sha256",
184		.cra_driver_name =	"sha256-avx",
185		.cra_priority	=	160,
186		.cra_blocksize	=	SHA256_BLOCK_SIZE,
187		.cra_module	=	THIS_MODULE,
188	}
189}, {
190	.digestsize	=	SHA224_DIGEST_SIZE,
191	.init		=	sha224_base_init,
192	.update		=	sha256_avx_update,
193	.final		=	sha256_avx_final,
194	.finup		=	sha256_avx_finup,
195	.descsize	=	sizeof(struct sha256_state),
196	.base		=	{
197		.cra_name	=	"sha224",
198		.cra_driver_name =	"sha224-avx",
199		.cra_priority	=	160,
200		.cra_blocksize	=	SHA224_BLOCK_SIZE,
201		.cra_module	=	THIS_MODULE,
202	}
203} };
204
205static bool avx_usable(void)
206{
207	if (!cpu_has_xfeatures(XFEATURE_MASK_SSE | XFEATURE_MASK_YMM, NULL)) {
208		if (boot_cpu_has(X86_FEATURE_AVX))
209			pr_info("AVX detected but unusable.\n");
210		return false;
211	}
212
213	return true;
214}
215
216static int register_sha256_avx(void)
217{
218	if (avx_usable())
219		return crypto_register_shashes(sha256_avx_algs,
220				ARRAY_SIZE(sha256_avx_algs));
221	return 0;
222}
223
224static void unregister_sha256_avx(void)
225{
226	if (avx_usable())
227		crypto_unregister_shashes(sha256_avx_algs,
228				ARRAY_SIZE(sha256_avx_algs));
229}
230
231asmlinkage void sha256_transform_rorx(struct sha256_state *state,
232				      const u8 *data, int blocks);
233
234static int sha256_avx2_update(struct shash_desc *desc, const u8 *data,
235			 unsigned int len)
236{
237	return _sha256_update(desc, data, len, sha256_transform_rorx);
238}
239
240static int sha256_avx2_finup(struct shash_desc *desc, const u8 *data,
241		      unsigned int len, u8 *out)
242{
243	return sha256_finup(desc, data, len, out, sha256_transform_rorx);
244}
245
246static int sha256_avx2_final(struct shash_desc *desc, u8 *out)
247{
248	return sha256_avx2_finup(desc, NULL, 0, out);
249}
250
251static struct shash_alg sha256_avx2_algs[] = { {
252	.digestsize	=	SHA256_DIGEST_SIZE,
253	.init		=	sha256_base_init,
254	.update		=	sha256_avx2_update,
255	.final		=	sha256_avx2_final,
256	.finup		=	sha256_avx2_finup,
257	.descsize	=	sizeof(struct sha256_state),
258	.base		=	{
259		.cra_name	=	"sha256",
260		.cra_driver_name =	"sha256-avx2",
261		.cra_priority	=	170,
262		.cra_blocksize	=	SHA256_BLOCK_SIZE,
263		.cra_module	=	THIS_MODULE,
264	}
265}, {
266	.digestsize	=	SHA224_DIGEST_SIZE,
267	.init		=	sha224_base_init,
268	.update		=	sha256_avx2_update,
269	.final		=	sha256_avx2_final,
270	.finup		=	sha256_avx2_finup,
271	.descsize	=	sizeof(struct sha256_state),
272	.base		=	{
273		.cra_name	=	"sha224",
274		.cra_driver_name =	"sha224-avx2",
275		.cra_priority	=	170,
276		.cra_blocksize	=	SHA224_BLOCK_SIZE,
277		.cra_module	=	THIS_MODULE,
278	}
279} };
280
281static bool avx2_usable(void)
282{
283	if (avx_usable() && boot_cpu_has(X86_FEATURE_AVX2) &&
284		    boot_cpu_has(X86_FEATURE_BMI2))
285		return true;
286
287	return false;
288}
289
290static int register_sha256_avx2(void)
291{
292	if (avx2_usable())
293		return crypto_register_shashes(sha256_avx2_algs,
294				ARRAY_SIZE(sha256_avx2_algs));
295	return 0;
296}
297
298static void unregister_sha256_avx2(void)
299{
300	if (avx2_usable())
301		crypto_unregister_shashes(sha256_avx2_algs,
302				ARRAY_SIZE(sha256_avx2_algs));
303}
304
305#ifdef CONFIG_AS_SHA256_NI
306asmlinkage void sha256_ni_transform(struct sha256_state *digest,
307				    const u8 *data, int rounds);
308
309static int sha256_ni_update(struct shash_desc *desc, const u8 *data,
310			 unsigned int len)
311{
312	return _sha256_update(desc, data, len, sha256_ni_transform);
313}
314
315static int sha256_ni_finup(struct shash_desc *desc, const u8 *data,
316		      unsigned int len, u8 *out)
317{
318	return sha256_finup(desc, data, len, out, sha256_ni_transform);
319}
320
321static int sha256_ni_final(struct shash_desc *desc, u8 *out)
322{
323	return sha256_ni_finup(desc, NULL, 0, out);
324}
325
326static struct shash_alg sha256_ni_algs[] = { {
327	.digestsize	=	SHA256_DIGEST_SIZE,
328	.init		=	sha256_base_init,
329	.update		=	sha256_ni_update,
330	.final		=	sha256_ni_final,
331	.finup		=	sha256_ni_finup,
332	.descsize	=	sizeof(struct sha256_state),
333	.base		=	{
334		.cra_name	=	"sha256",
335		.cra_driver_name =	"sha256-ni",
336		.cra_priority	=	250,
337		.cra_blocksize	=	SHA256_BLOCK_SIZE,
338		.cra_module	=	THIS_MODULE,
339	}
340}, {
341	.digestsize	=	SHA224_DIGEST_SIZE,
342	.init		=	sha224_base_init,
343	.update		=	sha256_ni_update,
344	.final		=	sha256_ni_final,
345	.finup		=	sha256_ni_finup,
346	.descsize	=	sizeof(struct sha256_state),
347	.base		=	{
348		.cra_name	=	"sha224",
349		.cra_driver_name =	"sha224-ni",
350		.cra_priority	=	250,
351		.cra_blocksize	=	SHA224_BLOCK_SIZE,
352		.cra_module	=	THIS_MODULE,
353	}
354} };
355
356static int register_sha256_ni(void)
357{
358	if (boot_cpu_has(X86_FEATURE_SHA_NI))
359		return crypto_register_shashes(sha256_ni_algs,
360				ARRAY_SIZE(sha256_ni_algs));
361	return 0;
362}
363
364static void unregister_sha256_ni(void)
365{
366	if (boot_cpu_has(X86_FEATURE_SHA_NI))
367		crypto_unregister_shashes(sha256_ni_algs,
368				ARRAY_SIZE(sha256_ni_algs));
369}
370
371#else
372static inline int register_sha256_ni(void) { return 0; }
373static inline void unregister_sha256_ni(void) { }
374#endif
375
376static int __init sha256_ssse3_mod_init(void)
377{
378	if (!x86_match_cpu(module_cpu_ids))
379		return -ENODEV;
380
381	if (register_sha256_ssse3())
382		goto fail;
383
384	if (register_sha256_avx()) {
385		unregister_sha256_ssse3();
386		goto fail;
387	}
388
389	if (register_sha256_avx2()) {
390		unregister_sha256_avx();
391		unregister_sha256_ssse3();
392		goto fail;
393	}
394
395	if (register_sha256_ni()) {
396		unregister_sha256_avx2();
397		unregister_sha256_avx();
398		unregister_sha256_ssse3();
399		goto fail;
400	}
401
402	return 0;
403fail:
404	return -ENODEV;
405}
406
407static void __exit sha256_ssse3_mod_fini(void)
408{
409	unregister_sha256_ni();
410	unregister_sha256_avx2();
411	unregister_sha256_avx();
412	unregister_sha256_ssse3();
413}
414
415module_init(sha256_ssse3_mod_init);
416module_exit(sha256_ssse3_mod_fini);
417
418MODULE_LICENSE("GPL");
419MODULE_DESCRIPTION("SHA256 Secure Hash Algorithm, Supplemental SSE3 accelerated");
420
421MODULE_ALIAS_CRYPTO("sha256");
422MODULE_ALIAS_CRYPTO("sha256-ssse3");
423MODULE_ALIAS_CRYPTO("sha256-avx");
424MODULE_ALIAS_CRYPTO("sha256-avx2");
425MODULE_ALIAS_CRYPTO("sha224");
426MODULE_ALIAS_CRYPTO("sha224-ssse3");
427MODULE_ALIAS_CRYPTO("sha224-avx");
428MODULE_ALIAS_CRYPTO("sha224-avx2");
429#ifdef CONFIG_AS_SHA256_NI
430MODULE_ALIAS_CRYPTO("sha256-ni");
431MODULE_ALIAS_CRYPTO("sha224-ni");
432#endif
433