1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Glue code for POLYVAL using ARMv8 Crypto Extensions
4 *
5 * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi>
6 * Copyright (c) 2009 Intel Corp.
7 *   Author: Huang Ying <ying.huang@intel.com>
8 * Copyright 2021 Google LLC
9 */
10
11/*
12 * Glue code based on ghash-clmulni-intel_glue.c.
13 *
14 * This implementation of POLYVAL uses montgomery multiplication accelerated by
15 * ARMv8 Crypto Extensions instructions to implement the finite field operations.
16 */
17
18#include <crypto/algapi.h>
19#include <crypto/internal/hash.h>
20#include <crypto/internal/simd.h>
21#include <crypto/polyval.h>
22#include <linux/crypto.h>
23#include <linux/init.h>
24#include <linux/kernel.h>
25#include <linux/module.h>
26#include <linux/cpufeature.h>
27#include <asm/neon.h>
28#include <asm/simd.h>
29
30#define NUM_KEY_POWERS	8
31
32struct polyval_tfm_ctx {
33	/*
34	 * These powers must be in the order h^8, ..., h^1.
35	 */
36	u8 key_powers[NUM_KEY_POWERS][POLYVAL_BLOCK_SIZE];
37};
38
39struct polyval_desc_ctx {
40	u8 buffer[POLYVAL_BLOCK_SIZE];
41	u32 bytes;
42};
43
44asmlinkage void pmull_polyval_update(const struct polyval_tfm_ctx *keys,
45	const u8 *in, size_t nblocks, u8 *accumulator);
46asmlinkage void pmull_polyval_mul(u8 *op1, const u8 *op2);
47
48static void internal_polyval_update(const struct polyval_tfm_ctx *keys,
49	const u8 *in, size_t nblocks, u8 *accumulator)
50{
51	if (likely(crypto_simd_usable())) {
52		kernel_neon_begin();
53		pmull_polyval_update(keys, in, nblocks, accumulator);
54		kernel_neon_end();
55	} else {
56		polyval_update_non4k(keys->key_powers[NUM_KEY_POWERS-1], in,
57			nblocks, accumulator);
58	}
59}
60
61static void internal_polyval_mul(u8 *op1, const u8 *op2)
62{
63	if (likely(crypto_simd_usable())) {
64		kernel_neon_begin();
65		pmull_polyval_mul(op1, op2);
66		kernel_neon_end();
67	} else {
68		polyval_mul_non4k(op1, op2);
69	}
70}
71
72static int polyval_arm64_setkey(struct crypto_shash *tfm,
73			const u8 *key, unsigned int keylen)
74{
75	struct polyval_tfm_ctx *tctx = crypto_shash_ctx(tfm);
76	int i;
77
78	if (keylen != POLYVAL_BLOCK_SIZE)
79		return -EINVAL;
80
81	memcpy(tctx->key_powers[NUM_KEY_POWERS-1], key, POLYVAL_BLOCK_SIZE);
82
83	for (i = NUM_KEY_POWERS-2; i >= 0; i--) {
84		memcpy(tctx->key_powers[i], key, POLYVAL_BLOCK_SIZE);
85		internal_polyval_mul(tctx->key_powers[i],
86				     tctx->key_powers[i+1]);
87	}
88
89	return 0;
90}
91
92static int polyval_arm64_init(struct shash_desc *desc)
93{
94	struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);
95
96	memset(dctx, 0, sizeof(*dctx));
97
98	return 0;
99}
100
101static int polyval_arm64_update(struct shash_desc *desc,
102			 const u8 *src, unsigned int srclen)
103{
104	struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);
105	const struct polyval_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
106	u8 *pos;
107	unsigned int nblocks;
108	unsigned int n;
109
110	if (dctx->bytes) {
111		n = min(srclen, dctx->bytes);
112		pos = dctx->buffer + POLYVAL_BLOCK_SIZE - dctx->bytes;
113
114		dctx->bytes -= n;
115		srclen -= n;
116
117		while (n--)
118			*pos++ ^= *src++;
119
120		if (!dctx->bytes)
121			internal_polyval_mul(dctx->buffer,
122					    tctx->key_powers[NUM_KEY_POWERS-1]);
123	}
124
125	while (srclen >= POLYVAL_BLOCK_SIZE) {
126		/* allow rescheduling every 4K bytes */
127		nblocks = min(srclen, 4096U) / POLYVAL_BLOCK_SIZE;
128		internal_polyval_update(tctx, src, nblocks, dctx->buffer);
129		srclen -= nblocks * POLYVAL_BLOCK_SIZE;
130		src += nblocks * POLYVAL_BLOCK_SIZE;
131	}
132
133	if (srclen) {
134		dctx->bytes = POLYVAL_BLOCK_SIZE - srclen;
135		pos = dctx->buffer;
136		while (srclen--)
137			*pos++ ^= *src++;
138	}
139
140	return 0;
141}
142
143static int polyval_arm64_final(struct shash_desc *desc, u8 *dst)
144{
145	struct polyval_desc_ctx *dctx = shash_desc_ctx(desc);
146	const struct polyval_tfm_ctx *tctx = crypto_shash_ctx(desc->tfm);
147
148	if (dctx->bytes) {
149		internal_polyval_mul(dctx->buffer,
150				     tctx->key_powers[NUM_KEY_POWERS-1]);
151	}
152
153	memcpy(dst, dctx->buffer, POLYVAL_BLOCK_SIZE);
154
155	return 0;
156}
157
158static struct shash_alg polyval_alg = {
159	.digestsize	= POLYVAL_DIGEST_SIZE,
160	.init		= polyval_arm64_init,
161	.update		= polyval_arm64_update,
162	.final		= polyval_arm64_final,
163	.setkey		= polyval_arm64_setkey,
164	.descsize	= sizeof(struct polyval_desc_ctx),
165	.base		= {
166		.cra_name		= "polyval",
167		.cra_driver_name	= "polyval-ce",
168		.cra_priority		= 200,
169		.cra_blocksize		= POLYVAL_BLOCK_SIZE,
170		.cra_ctxsize		= sizeof(struct polyval_tfm_ctx),
171		.cra_module		= THIS_MODULE,
172	},
173};
174
175static int __init polyval_ce_mod_init(void)
176{
177	return crypto_register_shash(&polyval_alg);
178}
179
180static void __exit polyval_ce_mod_exit(void)
181{
182	crypto_unregister_shash(&polyval_alg);
183}
184
185module_cpu_feature_match(PMULL, polyval_ce_mod_init)
186module_exit(polyval_ce_mod_exit);
187
188MODULE_LICENSE("GPL");
189MODULE_DESCRIPTION("POLYVAL hash function accelerated by ARMv8 Crypto Extensions");
190MODULE_ALIAS_CRYPTO("polyval");
191MODULE_ALIAS_CRYPTO("polyval-ce");
192