1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Shared glue code for 128bit block ciphers
4 *
5 * Copyright © 2012-2013 Jussi Kivilinna <jussi.kivilinna@iki.fi>
6 *
7 * CBC & ECB parts based on code (crypto/cbc.c,ecb.c) by:
8 *   Copyright (c) 2006 Herbert Xu <herbert@gondor.apana.org.au>
9 * CTR part based on code (crypto/ctr.c) by:
10 *   (C) Copyright IBM Corp. 2007 - Joy Latten <latten@us.ibm.com>
11 */
12
13#include <linux/module.h>
14#include <crypto/b128ops.h>
15#include <crypto/gf128mul.h>
16#include <crypto/internal/skcipher.h>
17#include <crypto/scatterwalk.h>
18#include <crypto/xts.h>
19#include <asm/crypto/glue_helper.h>
20
21int glue_ecb_req_128bit(const struct common_glue_ctx *gctx,
22			struct skcipher_request *req)
23{
24	void *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
25	const unsigned int bsize = 128 / 8;
26	struct skcipher_walk walk;
27	bool fpu_enabled = false;
28	unsigned int nbytes;
29	int err;
30
31	err = skcipher_walk_virt(&walk, req, false);
32
33	while ((nbytes = walk.nbytes)) {
34		const u8 *src = walk.src.virt.addr;
35		u8 *dst = walk.dst.virt.addr;
36		unsigned int func_bytes;
37		unsigned int i;
38
39		fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
40					     &walk, fpu_enabled, nbytes);
41		for (i = 0; i < gctx->num_funcs; i++) {
42			func_bytes = bsize * gctx->funcs[i].num_blocks;
43
44			if (nbytes < func_bytes)
45				continue;
46
47			/* Process multi-block batch */
48			do {
49				gctx->funcs[i].fn_u.ecb(ctx, dst, src);
50				src += func_bytes;
51				dst += func_bytes;
52				nbytes -= func_bytes;
53			} while (nbytes >= func_bytes);
54
55			if (nbytes < bsize)
56				break;
57		}
58		err = skcipher_walk_done(&walk, nbytes);
59	}
60
61	glue_fpu_end(fpu_enabled);
62	return err;
63}
64EXPORT_SYMBOL_GPL(glue_ecb_req_128bit);
65
66int glue_cbc_encrypt_req_128bit(const common_glue_func_t fn,
67				struct skcipher_request *req)
68{
69	void *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
70	const unsigned int bsize = 128 / 8;
71	struct skcipher_walk walk;
72	unsigned int nbytes;
73	int err;
74
75	err = skcipher_walk_virt(&walk, req, false);
76
77	while ((nbytes = walk.nbytes)) {
78		const u128 *src = (u128 *)walk.src.virt.addr;
79		u128 *dst = (u128 *)walk.dst.virt.addr;
80		u128 *iv = (u128 *)walk.iv;
81
82		do {
83			u128_xor(dst, src, iv);
84			fn(ctx, (u8 *)dst, (u8 *)dst);
85			iv = dst;
86			src++;
87			dst++;
88			nbytes -= bsize;
89		} while (nbytes >= bsize);
90
91		*(u128 *)walk.iv = *iv;
92		err = skcipher_walk_done(&walk, nbytes);
93	}
94	return err;
95}
96EXPORT_SYMBOL_GPL(glue_cbc_encrypt_req_128bit);
97
98int glue_cbc_decrypt_req_128bit(const struct common_glue_ctx *gctx,
99				struct skcipher_request *req)
100{
101	void *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
102	const unsigned int bsize = 128 / 8;
103	struct skcipher_walk walk;
104	bool fpu_enabled = false;
105	unsigned int nbytes;
106	int err;
107
108	err = skcipher_walk_virt(&walk, req, false);
109
110	while ((nbytes = walk.nbytes)) {
111		const u128 *src = walk.src.virt.addr;
112		u128 *dst = walk.dst.virt.addr;
113		unsigned int func_bytes, num_blocks;
114		unsigned int i;
115		u128 last_iv;
116
117		fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
118					     &walk, fpu_enabled, nbytes);
119		/* Start of the last block. */
120		src += nbytes / bsize - 1;
121		dst += nbytes / bsize - 1;
122
123		last_iv = *src;
124
125		for (i = 0; i < gctx->num_funcs; i++) {
126			num_blocks = gctx->funcs[i].num_blocks;
127			func_bytes = bsize * num_blocks;
128
129			if (nbytes < func_bytes)
130				continue;
131
132			/* Process multi-block batch */
133			do {
134				src -= num_blocks - 1;
135				dst -= num_blocks - 1;
136
137				gctx->funcs[i].fn_u.cbc(ctx, (u8 *)dst,
138							(const u8 *)src);
139
140				nbytes -= func_bytes;
141				if (nbytes < bsize)
142					goto done;
143
144				u128_xor(dst, dst, --src);
145				dst--;
146			} while (nbytes >= func_bytes);
147		}
148done:
149		u128_xor(dst, dst, (u128 *)walk.iv);
150		*(u128 *)walk.iv = last_iv;
151		err = skcipher_walk_done(&walk, nbytes);
152	}
153
154	glue_fpu_end(fpu_enabled);
155	return err;
156}
157EXPORT_SYMBOL_GPL(glue_cbc_decrypt_req_128bit);
158
159int glue_ctr_req_128bit(const struct common_glue_ctx *gctx,
160			struct skcipher_request *req)
161{
162	void *ctx = crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
163	const unsigned int bsize = 128 / 8;
164	struct skcipher_walk walk;
165	bool fpu_enabled = false;
166	unsigned int nbytes;
167	int err;
168
169	err = skcipher_walk_virt(&walk, req, false);
170
171	while ((nbytes = walk.nbytes) >= bsize) {
172		const u128 *src = walk.src.virt.addr;
173		u128 *dst = walk.dst.virt.addr;
174		unsigned int func_bytes, num_blocks;
175		unsigned int i;
176		le128 ctrblk;
177
178		fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
179					     &walk, fpu_enabled, nbytes);
180
181		be128_to_le128(&ctrblk, (be128 *)walk.iv);
182
183		for (i = 0; i < gctx->num_funcs; i++) {
184			num_blocks = gctx->funcs[i].num_blocks;
185			func_bytes = bsize * num_blocks;
186
187			if (nbytes < func_bytes)
188				continue;
189
190			/* Process multi-block batch */
191			do {
192				gctx->funcs[i].fn_u.ctr(ctx, (u8 *)dst,
193							(const u8 *)src,
194							&ctrblk);
195				src += num_blocks;
196				dst += num_blocks;
197				nbytes -= func_bytes;
198			} while (nbytes >= func_bytes);
199
200			if (nbytes < bsize)
201				break;
202		}
203
204		le128_to_be128((be128 *)walk.iv, &ctrblk);
205		err = skcipher_walk_done(&walk, nbytes);
206	}
207
208	glue_fpu_end(fpu_enabled);
209
210	if (nbytes) {
211		le128 ctrblk;
212		u128 tmp;
213
214		be128_to_le128(&ctrblk, (be128 *)walk.iv);
215		memcpy(&tmp, walk.src.virt.addr, nbytes);
216		gctx->funcs[gctx->num_funcs - 1].fn_u.ctr(ctx, (u8 *)&tmp,
217							  (const u8 *)&tmp,
218							  &ctrblk);
219		memcpy(walk.dst.virt.addr, &tmp, nbytes);
220		le128_to_be128((be128 *)walk.iv, &ctrblk);
221
222		err = skcipher_walk_done(&walk, 0);
223	}
224
225	return err;
226}
227EXPORT_SYMBOL_GPL(glue_ctr_req_128bit);
228
229static unsigned int __glue_xts_req_128bit(const struct common_glue_ctx *gctx,
230					  void *ctx,
231					  struct skcipher_walk *walk)
232{
233	const unsigned int bsize = 128 / 8;
234	unsigned int nbytes = walk->nbytes;
235	u128 *src = walk->src.virt.addr;
236	u128 *dst = walk->dst.virt.addr;
237	unsigned int num_blocks, func_bytes;
238	unsigned int i;
239
240	/* Process multi-block batch */
241	for (i = 0; i < gctx->num_funcs; i++) {
242		num_blocks = gctx->funcs[i].num_blocks;
243		func_bytes = bsize * num_blocks;
244
245		if (nbytes >= func_bytes) {
246			do {
247				gctx->funcs[i].fn_u.xts(ctx, (u8 *)dst,
248							(const u8 *)src,
249							walk->iv);
250
251				src += num_blocks;
252				dst += num_blocks;
253				nbytes -= func_bytes;
254			} while (nbytes >= func_bytes);
255
256			if (nbytes < bsize)
257				goto done;
258		}
259	}
260
261done:
262	return nbytes;
263}
264
265int glue_xts_req_128bit(const struct common_glue_ctx *gctx,
266			struct skcipher_request *req,
267			common_glue_func_t tweak_fn, void *tweak_ctx,
268			void *crypt_ctx, bool decrypt)
269{
270	const bool cts = (req->cryptlen % XTS_BLOCK_SIZE);
271	const unsigned int bsize = 128 / 8;
272	struct skcipher_request subreq;
273	struct skcipher_walk walk;
274	bool fpu_enabled = false;
275	unsigned int nbytes, tail;
276	int err;
277
278	if (req->cryptlen < XTS_BLOCK_SIZE)
279		return -EINVAL;
280
281	if (unlikely(cts)) {
282		struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(req);
283
284		tail = req->cryptlen % XTS_BLOCK_SIZE + XTS_BLOCK_SIZE;
285
286		skcipher_request_set_tfm(&subreq, tfm);
287		skcipher_request_set_callback(&subreq,
288					      crypto_skcipher_get_flags(tfm),
289					      NULL, NULL);
290		skcipher_request_set_crypt(&subreq, req->src, req->dst,
291					   req->cryptlen - tail, req->iv);
292		req = &subreq;
293	}
294
295	err = skcipher_walk_virt(&walk, req, false);
296	nbytes = walk.nbytes;
297	if (err)
298		return err;
299
300	/* set minimum length to bsize, for tweak_fn */
301	fpu_enabled = glue_fpu_begin(bsize, gctx->fpu_blocks_limit,
302				     &walk, fpu_enabled,
303				     nbytes < bsize ? bsize : nbytes);
304
305	/* calculate first value of T */
306	tweak_fn(tweak_ctx, walk.iv, walk.iv);
307
308	while (nbytes) {
309		nbytes = __glue_xts_req_128bit(gctx, crypt_ctx, &walk);
310
311		err = skcipher_walk_done(&walk, nbytes);
312		nbytes = walk.nbytes;
313	}
314
315	if (unlikely(cts)) {
316		u8 *next_tweak, *final_tweak = req->iv;
317		struct scatterlist *src, *dst;
318		struct scatterlist s[2], d[2];
319		le128 b[2];
320
321		dst = src = scatterwalk_ffwd(s, req->src, req->cryptlen);
322		if (req->dst != req->src)
323			dst = scatterwalk_ffwd(d, req->dst, req->cryptlen);
324
325		if (decrypt) {
326			next_tweak = memcpy(b, req->iv, XTS_BLOCK_SIZE);
327			gf128mul_x_ble(b, b);
328		} else {
329			next_tweak = req->iv;
330		}
331
332		skcipher_request_set_crypt(&subreq, src, dst, XTS_BLOCK_SIZE,
333					   next_tweak);
334
335		err = skcipher_walk_virt(&walk, req, false) ?:
336		      skcipher_walk_done(&walk,
337				__glue_xts_req_128bit(gctx, crypt_ctx, &walk));
338		if (err)
339			goto out;
340
341		scatterwalk_map_and_copy(b, dst, 0, XTS_BLOCK_SIZE, 0);
342		memcpy(b + 1, b, tail - XTS_BLOCK_SIZE);
343		scatterwalk_map_and_copy(b, src, XTS_BLOCK_SIZE,
344					 tail - XTS_BLOCK_SIZE, 0);
345		scatterwalk_map_and_copy(b, dst, 0, tail, 1);
346
347		skcipher_request_set_crypt(&subreq, dst, dst, XTS_BLOCK_SIZE,
348					   final_tweak);
349
350		err = skcipher_walk_virt(&walk, req, false) ?:
351		      skcipher_walk_done(&walk,
352				__glue_xts_req_128bit(gctx, crypt_ctx, &walk));
353	}
354
355out:
356	glue_fpu_end(fpu_enabled);
357
358	return err;
359}
360EXPORT_SYMBOL_GPL(glue_xts_req_128bit);
361
362void glue_xts_crypt_128bit_one(const void *ctx, u8 *dst, const u8 *src,
363			       le128 *iv, common_glue_func_t fn)
364{
365	le128 ivblk = *iv;
366
367	/* generate next IV */
368	gf128mul_x_ble(iv, &ivblk);
369
370	/* CC <- T xor C */
371	u128_xor((u128 *)dst, (const u128 *)src, (u128 *)&ivblk);
372
373	/* PP <- D(Key2,CC) */
374	fn(ctx, dst, dst);
375
376	/* P <- T xor PP */
377	u128_xor((u128 *)dst, (u128 *)dst, (u128 *)&ivblk);
378}
379EXPORT_SYMBOL_GPL(glue_xts_crypt_128bit_one);
380
381MODULE_LICENSE("GPL");
382