1// SPDX-License-Identifier: GPL-2.0
2/*
3 * crc32.c - CRC32 and CRC32C using LoongArch crc* instructions
4 *
5 * Module based on mips/crypto/crc32-mips.c
6 *
7 * Copyright (C) 2014 Linaro Ltd <yazen.ghannam@linaro.org>
8 * Copyright (C) 2018 MIPS Tech, LLC
9 * Copyright (C) 2020-2023 Loongson Technology Corporation Limited
10 */
11
12#include <linux/module.h>
13#include <crypto/internal/hash.h>
14
15#include <asm/cpu-features.h>
16#include <asm/unaligned.h>
17
18#define _CRC32(crc, value, size, type)			\
19do {							\
20	__asm__ __volatile__(				\
21		#type ".w." #size ".w" " %0, %1, %0\n\t"\
22		: "+r" (crc)				\
23		: "r" (value)				\
24		: "memory");				\
25} while (0)
26
27#define CRC32(crc, value, size)		_CRC32(crc, value, size, crc)
28#define CRC32C(crc, value, size)	_CRC32(crc, value, size, crcc)
29
30static u32 crc32_loongarch_hw(u32 crc_, const u8 *p, unsigned int len)
31{
32	u32 crc = crc_;
33
34	while (len >= sizeof(u64)) {
35		u64 value = get_unaligned_le64(p);
36
37		CRC32(crc, value, d);
38		p += sizeof(u64);
39		len -= sizeof(u64);
40	}
41
42	if (len & sizeof(u32)) {
43		u32 value = get_unaligned_le32(p);
44
45		CRC32(crc, value, w);
46		p += sizeof(u32);
47		len -= sizeof(u32);
48	}
49
50	if (len & sizeof(u16)) {
51		u16 value = get_unaligned_le16(p);
52
53		CRC32(crc, value, h);
54		p += sizeof(u16);
55	}
56
57	if (len & sizeof(u8)) {
58		u8 value = *p++;
59
60		CRC32(crc, value, b);
61	}
62
63	return crc;
64}
65
66static u32 crc32c_loongarch_hw(u32 crc_, const u8 *p, unsigned int len)
67{
68	u32 crc = crc_;
69
70	while (len >= sizeof(u64)) {
71		u64 value = get_unaligned_le64(p);
72
73		CRC32C(crc, value, d);
74		p += sizeof(u64);
75		len -= sizeof(u64);
76	}
77
78	if (len & sizeof(u32)) {
79		u32 value = get_unaligned_le32(p);
80
81		CRC32C(crc, value, w);
82		p += sizeof(u32);
83		len -= sizeof(u32);
84	}
85
86	if (len & sizeof(u16)) {
87		u16 value = get_unaligned_le16(p);
88
89		CRC32C(crc, value, h);
90		p += sizeof(u16);
91	}
92
93	if (len & sizeof(u8)) {
94		u8 value = *p++;
95
96		CRC32C(crc, value, b);
97	}
98
99	return crc;
100}
101
102#define CHKSUM_BLOCK_SIZE	1
103#define CHKSUM_DIGEST_SIZE	4
104
105struct chksum_ctx {
106	u32 key;
107};
108
109struct chksum_desc_ctx {
110	u32 crc;
111};
112
113static int chksum_init(struct shash_desc *desc)
114{
115	struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
116	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
117
118	ctx->crc = mctx->key;
119
120	return 0;
121}
122
123/*
124 * Setting the seed allows arbitrary accumulators and flexible XOR policy
125 * If your algorithm starts with ~0, then XOR with ~0 before you set the seed.
126 */
127static int chksum_setkey(struct crypto_shash *tfm, const u8 *key, unsigned int keylen)
128{
129	struct chksum_ctx *mctx = crypto_shash_ctx(tfm);
130
131	if (keylen != sizeof(mctx->key))
132		return -EINVAL;
133
134	mctx->key = get_unaligned_le32(key);
135
136	return 0;
137}
138
139static int chksum_update(struct shash_desc *desc, const u8 *data, unsigned int length)
140{
141	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
142
143	ctx->crc = crc32_loongarch_hw(ctx->crc, data, length);
144	return 0;
145}
146
147static int chksumc_update(struct shash_desc *desc, const u8 *data, unsigned int length)
148{
149	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
150
151	ctx->crc = crc32c_loongarch_hw(ctx->crc, data, length);
152	return 0;
153}
154
155static int chksum_final(struct shash_desc *desc, u8 *out)
156{
157	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
158
159	put_unaligned_le32(ctx->crc, out);
160	return 0;
161}
162
163static int chksumc_final(struct shash_desc *desc, u8 *out)
164{
165	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
166
167	put_unaligned_le32(~ctx->crc, out);
168	return 0;
169}
170
171static int __chksum_finup(u32 crc, const u8 *data, unsigned int len, u8 *out)
172{
173	put_unaligned_le32(crc32_loongarch_hw(crc, data, len), out);
174	return 0;
175}
176
177static int __chksumc_finup(u32 crc, const u8 *data, unsigned int len, u8 *out)
178{
179	put_unaligned_le32(~crc32c_loongarch_hw(crc, data, len), out);
180	return 0;
181}
182
183static int chksum_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out)
184{
185	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
186
187	return __chksum_finup(ctx->crc, data, len, out);
188}
189
190static int chksumc_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out)
191{
192	struct chksum_desc_ctx *ctx = shash_desc_ctx(desc);
193
194	return __chksumc_finup(ctx->crc, data, len, out);
195}
196
197static int chksum_digest(struct shash_desc *desc, const u8 *data, unsigned int length, u8 *out)
198{
199	struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
200
201	return __chksum_finup(mctx->key, data, length, out);
202}
203
204static int chksumc_digest(struct shash_desc *desc, const u8 *data, unsigned int length, u8 *out)
205{
206	struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm);
207
208	return __chksumc_finup(mctx->key, data, length, out);
209}
210
211static int chksum_cra_init(struct crypto_tfm *tfm)
212{
213	struct chksum_ctx *mctx = crypto_tfm_ctx(tfm);
214
215	mctx->key = 0;
216	return 0;
217}
218
219static int chksumc_cra_init(struct crypto_tfm *tfm)
220{
221	struct chksum_ctx *mctx = crypto_tfm_ctx(tfm);
222
223	mctx->key = ~0;
224	return 0;
225}
226
227static struct shash_alg crc32_alg = {
228	.digestsize		=	CHKSUM_DIGEST_SIZE,
229	.setkey			=	chksum_setkey,
230	.init			=	chksum_init,
231	.update			=	chksum_update,
232	.final			=	chksum_final,
233	.finup			=	chksum_finup,
234	.digest			=	chksum_digest,
235	.descsize		=	sizeof(struct chksum_desc_ctx),
236	.base			=	{
237		.cra_name		=	"crc32",
238		.cra_driver_name	=	"crc32-loongarch",
239		.cra_priority		=	300,
240		.cra_flags		=	CRYPTO_ALG_OPTIONAL_KEY,
241		.cra_blocksize		=	CHKSUM_BLOCK_SIZE,
242		.cra_alignmask		=	0,
243		.cra_ctxsize		=	sizeof(struct chksum_ctx),
244		.cra_module		=	THIS_MODULE,
245		.cra_init		=	chksum_cra_init,
246	}
247};
248
249static struct shash_alg crc32c_alg = {
250	.digestsize		=	CHKSUM_DIGEST_SIZE,
251	.setkey			=	chksum_setkey,
252	.init			=	chksum_init,
253	.update			=	chksumc_update,
254	.final			=	chksumc_final,
255	.finup			=	chksumc_finup,
256	.digest			=	chksumc_digest,
257	.descsize		=	sizeof(struct chksum_desc_ctx),
258	.base			=	{
259		.cra_name		=	"crc32c",
260		.cra_driver_name	=	"crc32c-loongarch",
261		.cra_priority		=	300,
262		.cra_flags		=	CRYPTO_ALG_OPTIONAL_KEY,
263		.cra_blocksize		=	CHKSUM_BLOCK_SIZE,
264		.cra_alignmask		=	0,
265		.cra_ctxsize		=	sizeof(struct chksum_ctx),
266		.cra_module		=	THIS_MODULE,
267		.cra_init		=	chksumc_cra_init,
268	}
269};
270
271static int __init crc32_mod_init(void)
272{
273	int err;
274
275	if (!cpu_has(CPU_FEATURE_CRC32))
276		return 0;
277
278	err = crypto_register_shash(&crc32_alg);
279	if (err)
280		return err;
281
282	err = crypto_register_shash(&crc32c_alg);
283	if (err)
284		return err;
285
286	return 0;
287}
288
289static void __exit crc32_mod_exit(void)
290{
291	if (!cpu_has(CPU_FEATURE_CRC32))
292		return;
293
294	crypto_unregister_shash(&crc32_alg);
295	crypto_unregister_shash(&crc32c_alg);
296}
297
298module_init(crc32_mod_init);
299module_exit(crc32_mod_exit);
300
301MODULE_AUTHOR("Min Zhou <zhoumin@loongson.cn>");
302MODULE_AUTHOR("Huacai Chen <chenhuacai@loongson.cn>");
303MODULE_DESCRIPTION("CRC32 and CRC32C using LoongArch crc* instructions");
304MODULE_LICENSE("GPL v2");
305