1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Linux/arm64 port of the OpenSSL SHA256 implementation for AArch64
4 *
5 * Copyright (c) 2016 Linaro Ltd. <ard.biesheuvel@linaro.org>
6 */
7
8#include <asm/hwcap.h>
9#include <asm/neon.h>
10#include <asm/simd.h>
11#include <crypto/internal/hash.h>
12#include <crypto/internal/simd.h>
13#include <crypto/sha.h>
14#include <crypto/sha256_base.h>
15#include <linux/types.h>
16#include <linux/string.h>
17
18MODULE_DESCRIPTION("SHA-224/SHA-256 secure hash for arm64");
19MODULE_AUTHOR("Andy Polyakov <appro@openssl.org>");
20MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
21MODULE_LICENSE("GPL v2");
22MODULE_ALIAS_CRYPTO("sha224");
23MODULE_ALIAS_CRYPTO("sha256");
24
25asmlinkage void sha256_block_data_order(u32 *digest, const void *data,
26					unsigned int num_blks);
27EXPORT_SYMBOL(sha256_block_data_order);
28
29static void __sha256_block_data_order(struct sha256_state *sst, u8 const *src,
30				      int blocks)
31{
32	sha256_block_data_order(sst->state, src, blocks);
33}
34
35asmlinkage void sha256_block_neon(u32 *digest, const void *data,
36				  unsigned int num_blks);
37
38static void __sha256_block_neon(struct sha256_state *sst, u8 const *src,
39				int blocks)
40{
41	sha256_block_neon(sst->state, src, blocks);
42}
43
44static int crypto_sha256_arm64_update(struct shash_desc *desc, const u8 *data,
45				      unsigned int len)
46{
47	return sha256_base_do_update(desc, data, len,
48				     __sha256_block_data_order);
49}
50
51static int crypto_sha256_arm64_finup(struct shash_desc *desc, const u8 *data,
52				     unsigned int len, u8 *out)
53{
54	if (len)
55		sha256_base_do_update(desc, data, len,
56				      __sha256_block_data_order);
57	sha256_base_do_finalize(desc, __sha256_block_data_order);
58
59	return sha256_base_finish(desc, out);
60}
61
62static int crypto_sha256_arm64_final(struct shash_desc *desc, u8 *out)
63{
64	return crypto_sha256_arm64_finup(desc, NULL, 0, out);
65}
66
67static struct shash_alg algs[] = { {
68	.digestsize		= SHA256_DIGEST_SIZE,
69	.init			= sha256_base_init,
70	.update			= crypto_sha256_arm64_update,
71	.final			= crypto_sha256_arm64_final,
72	.finup			= crypto_sha256_arm64_finup,
73	.descsize		= sizeof(struct sha256_state),
74	.base.cra_name		= "sha256",
75	.base.cra_driver_name	= "sha256-arm64",
76	.base.cra_priority	= 125,
77	.base.cra_blocksize	= SHA256_BLOCK_SIZE,
78	.base.cra_module	= THIS_MODULE,
79}, {
80	.digestsize		= SHA224_DIGEST_SIZE,
81	.init			= sha224_base_init,
82	.update			= crypto_sha256_arm64_update,
83	.final			= crypto_sha256_arm64_final,
84	.finup			= crypto_sha256_arm64_finup,
85	.descsize		= sizeof(struct sha256_state),
86	.base.cra_name		= "sha224",
87	.base.cra_driver_name	= "sha224-arm64",
88	.base.cra_priority	= 125,
89	.base.cra_blocksize	= SHA224_BLOCK_SIZE,
90	.base.cra_module	= THIS_MODULE,
91} };
92
93static int sha256_update_neon(struct shash_desc *desc, const u8 *data,
94			      unsigned int len)
95{
96	struct sha256_state *sctx = shash_desc_ctx(desc);
97
98	if (!crypto_simd_usable())
99		return sha256_base_do_update(desc, data, len,
100				__sha256_block_data_order);
101
102	while (len > 0) {
103		unsigned int chunk = len;
104
105		/*
106		 * Don't hog the CPU for the entire time it takes to process all
107		 * input when running on a preemptible kernel, but process the
108		 * data block by block instead.
109		 */
110		if (IS_ENABLED(CONFIG_PREEMPTION) &&
111		    chunk + sctx->count % SHA256_BLOCK_SIZE > SHA256_BLOCK_SIZE)
112			chunk = SHA256_BLOCK_SIZE -
113				sctx->count % SHA256_BLOCK_SIZE;
114
115		kernel_neon_begin();
116		sha256_base_do_update(desc, data, chunk, __sha256_block_neon);
117		kernel_neon_end();
118		data += chunk;
119		len -= chunk;
120	}
121	return 0;
122}
123
124static int sha256_finup_neon(struct shash_desc *desc, const u8 *data,
125			     unsigned int len, u8 *out)
126{
127	if (!crypto_simd_usable()) {
128		if (len)
129			sha256_base_do_update(desc, data, len,
130				__sha256_block_data_order);
131		sha256_base_do_finalize(desc, __sha256_block_data_order);
132	} else {
133		if (len)
134			sha256_update_neon(desc, data, len);
135		kernel_neon_begin();
136		sha256_base_do_finalize(desc, __sha256_block_neon);
137		kernel_neon_end();
138	}
139	return sha256_base_finish(desc, out);
140}
141
142static int sha256_final_neon(struct shash_desc *desc, u8 *out)
143{
144	return sha256_finup_neon(desc, NULL, 0, out);
145}
146
147static struct shash_alg neon_algs[] = { {
148	.digestsize		= SHA256_DIGEST_SIZE,
149	.init			= sha256_base_init,
150	.update			= sha256_update_neon,
151	.final			= sha256_final_neon,
152	.finup			= sha256_finup_neon,
153	.descsize		= sizeof(struct sha256_state),
154	.base.cra_name		= "sha256",
155	.base.cra_driver_name	= "sha256-arm64-neon",
156	.base.cra_priority	= 150,
157	.base.cra_blocksize	= SHA256_BLOCK_SIZE,
158	.base.cra_module	= THIS_MODULE,
159}, {
160	.digestsize		= SHA224_DIGEST_SIZE,
161	.init			= sha224_base_init,
162	.update			= sha256_update_neon,
163	.final			= sha256_final_neon,
164	.finup			= sha256_finup_neon,
165	.descsize		= sizeof(struct sha256_state),
166	.base.cra_name		= "sha224",
167	.base.cra_driver_name	= "sha224-arm64-neon",
168	.base.cra_priority	= 150,
169	.base.cra_blocksize	= SHA224_BLOCK_SIZE,
170	.base.cra_module	= THIS_MODULE,
171} };
172
173static int __init sha256_mod_init(void)
174{
175	int ret = crypto_register_shashes(algs, ARRAY_SIZE(algs));
176	if (ret)
177		return ret;
178
179	if (cpu_have_named_feature(ASIMD)) {
180		ret = crypto_register_shashes(neon_algs, ARRAY_SIZE(neon_algs));
181		if (ret)
182			crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
183	}
184	return ret;
185}
186
187static void __exit sha256_mod_fini(void)
188{
189	if (cpu_have_named_feature(ASIMD))
190		crypto_unregister_shashes(neon_algs, ARRAY_SIZE(neon_algs));
191	crypto_unregister_shashes(algs, ARRAY_SIZE(algs));
192}
193
194module_init(sha256_mod_init);
195module_exit(sha256_mod_fini);
196