1// SPDX-License-Identifier: GPL-2.0
2/*
3 * GHASH routines supporting VMX instructions on the Power 8
4 *
5 * Copyright (C) 2015, 2019 International Business Machines Inc.
6 *
7 * Author: Marcelo Henrique Cerri <mhcerri@br.ibm.com>
8 *
9 * Extended by Daniel Axtens <dja@axtens.net> to replace the fallback
10 * mechanism. The new approach is based on arm64 code, which is:
11 *   Copyright (C) 2014 - 2018 Linaro Ltd. <ard.biesheuvel@linaro.org>
12 */
13
14#include <linux/types.h>
15#include <linux/err.h>
16#include <linux/crypto.h>
17#include <linux/delay.h>
18#include <asm/simd.h>
19#include <asm/switch_to.h>
20#include <crypto/aes.h>
21#include <crypto/ghash.h>
22#include <crypto/scatterwalk.h>
23#include <crypto/internal/hash.h>
24#include <crypto/internal/simd.h>
25#include <crypto/b128ops.h>
26#include "aesp8-ppc.h"
27
28void gcm_init_p8(u128 htable[16], const u64 Xi[2]);
29void gcm_gmult_p8(u64 Xi[2], const u128 htable[16]);
30void gcm_ghash_p8(u64 Xi[2], const u128 htable[16],
31		  const u8 *in, size_t len);
32
33struct p8_ghash_ctx {
34	/* key used by vector asm */
35	u128 htable[16];
36	/* key used by software fallback */
37	be128 key;
38};
39
40struct p8_ghash_desc_ctx {
41	u64 shash[2];
42	u8 buffer[GHASH_DIGEST_SIZE];
43	int bytes;
44};
45
46static int p8_ghash_init(struct shash_desc *desc)
47{
48	struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
49
50	dctx->bytes = 0;
51	memset(dctx->shash, 0, GHASH_DIGEST_SIZE);
52	return 0;
53}
54
55static int p8_ghash_setkey(struct crypto_shash *tfm, const u8 *key,
56			   unsigned int keylen)
57{
58	struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(tfm));
59
60	if (keylen != GHASH_BLOCK_SIZE)
61		return -EINVAL;
62
63	preempt_disable();
64	pagefault_disable();
65	enable_kernel_vsx();
66	gcm_init_p8(ctx->htable, (const u64 *) key);
67	disable_kernel_vsx();
68	pagefault_enable();
69	preempt_enable();
70
71	memcpy(&ctx->key, key, GHASH_BLOCK_SIZE);
72
73	return 0;
74}
75
76static inline void __ghash_block(struct p8_ghash_ctx *ctx,
77				 struct p8_ghash_desc_ctx *dctx)
78{
79	if (crypto_simd_usable()) {
80		preempt_disable();
81		pagefault_disable();
82		enable_kernel_vsx();
83		gcm_ghash_p8(dctx->shash, ctx->htable,
84				dctx->buffer, GHASH_DIGEST_SIZE);
85		disable_kernel_vsx();
86		pagefault_enable();
87		preempt_enable();
88	} else {
89		crypto_xor((u8 *)dctx->shash, dctx->buffer, GHASH_BLOCK_SIZE);
90		gf128mul_lle((be128 *)dctx->shash, &ctx->key);
91	}
92}
93
94static inline void __ghash_blocks(struct p8_ghash_ctx *ctx,
95				  struct p8_ghash_desc_ctx *dctx,
96				  const u8 *src, unsigned int srclen)
97{
98	if (crypto_simd_usable()) {
99		preempt_disable();
100		pagefault_disable();
101		enable_kernel_vsx();
102		gcm_ghash_p8(dctx->shash, ctx->htable,
103				src, srclen);
104		disable_kernel_vsx();
105		pagefault_enable();
106		preempt_enable();
107	} else {
108		while (srclen >= GHASH_BLOCK_SIZE) {
109			crypto_xor((u8 *)dctx->shash, src, GHASH_BLOCK_SIZE);
110			gf128mul_lle((be128 *)dctx->shash, &ctx->key);
111			srclen -= GHASH_BLOCK_SIZE;
112			src += GHASH_BLOCK_SIZE;
113		}
114	}
115}
116
117static int p8_ghash_update(struct shash_desc *desc,
118			   const u8 *src, unsigned int srclen)
119{
120	unsigned int len;
121	struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm));
122	struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
123
124	if (dctx->bytes) {
125		if (dctx->bytes + srclen < GHASH_DIGEST_SIZE) {
126			memcpy(dctx->buffer + dctx->bytes, src,
127				srclen);
128			dctx->bytes += srclen;
129			return 0;
130		}
131		memcpy(dctx->buffer + dctx->bytes, src,
132			GHASH_DIGEST_SIZE - dctx->bytes);
133
134		__ghash_block(ctx, dctx);
135
136		src += GHASH_DIGEST_SIZE - dctx->bytes;
137		srclen -= GHASH_DIGEST_SIZE - dctx->bytes;
138		dctx->bytes = 0;
139	}
140	len = srclen & ~(GHASH_DIGEST_SIZE - 1);
141	if (len) {
142		__ghash_blocks(ctx, dctx, src, len);
143		src += len;
144		srclen -= len;
145	}
146	if (srclen) {
147		memcpy(dctx->buffer, src, srclen);
148		dctx->bytes = srclen;
149	}
150	return 0;
151}
152
153static int p8_ghash_final(struct shash_desc *desc, u8 *out)
154{
155	int i;
156	struct p8_ghash_ctx *ctx = crypto_tfm_ctx(crypto_shash_tfm(desc->tfm));
157	struct p8_ghash_desc_ctx *dctx = shash_desc_ctx(desc);
158
159	if (dctx->bytes) {
160		for (i = dctx->bytes; i < GHASH_DIGEST_SIZE; i++)
161			dctx->buffer[i] = 0;
162		__ghash_block(ctx, dctx);
163		dctx->bytes = 0;
164	}
165	memcpy(out, dctx->shash, GHASH_DIGEST_SIZE);
166	return 0;
167}
168
169struct shash_alg p8_ghash_alg = {
170	.digestsize = GHASH_DIGEST_SIZE,
171	.init = p8_ghash_init,
172	.update = p8_ghash_update,
173	.final = p8_ghash_final,
174	.setkey = p8_ghash_setkey,
175	.descsize = sizeof(struct p8_ghash_desc_ctx)
176		+ sizeof(struct ghash_desc_ctx),
177	.base = {
178		 .cra_name = "ghash",
179		 .cra_driver_name = "p8_ghash",
180		 .cra_priority = 1000,
181		 .cra_blocksize = GHASH_BLOCK_SIZE,
182		 .cra_ctxsize = sizeof(struct p8_ghash_ctx),
183		 .cra_module = THIS_MODULE,
184	},
185};
186