1 /*
2  * Accelerated CRC32(C) using ARM CRC, NEON and Crypto Extensions instructions
3  *
4  * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License version 2 as
8  * published by the Free Software Foundation.
9  */
10 
11 /* GPL HEADER START
12  *
13  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
14  *
15  * This program is free software; you can redistribute it and/or modify
16  * it under the terms of the GNU General Public License version 2 only,
17  * as published by the Free Software Foundation.
18  *
19  * This program is distributed in the hope that it will be useful, but
20  * WITHOUT ANY WARRANTY; without even the implied warranty of
21  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
22  * General Public License version 2 for more details (a copy is included
23  * in the LICENSE file that accompanied this code).
24  *
25  * You should have received a copy of the GNU General Public License
26  * version 2 along with this program; If not, see http://www.gnu.org/licenses
27  *
28  * Please  visit http://www.xyratex.com/contact if you need additional
29  * information or have any questions.
30  *
31  * GPL HEADER END
32  */
33 
34 /*
35  * Copyright 2012 Xyratex Technology Limited
36  *
37  * Using hardware provided PCLMULQDQ instruction to accelerate the CRC32
38  * calculation.
39  * CRC32 polynomial:0x04c11db7(BE)/0xEDB88320(LE)
40  * PCLMULQDQ is a new instruction in Intel SSE4.2, the reference can be found
41  * at:
42  * https://www.intel.com/products/processor/manuals/
43  * Intel(R) 64 and IA-32 Architectures Software Developer's Manual
44  * Volume 2B: Instruction Set Reference, N-Z
45  *
46  * Authors:   Gregory Prestas <Gregory_Prestas@us.xyratex.com>
47  *	      Alexander Boyko <Alexander_Boyko@xyratex.com>
48  */
49 
50 #include <linux/linkage.h>
51 #include <asm/assembler.h>
52 
53 	.text
54 	.align		6
55 	.arch		armv8-a
56 	.arch_extension	crc
57 	.fpu		crypto-neon-fp-armv8
58 
59 .Lcrc32_constants:
60 	/*
61 	 * [x4*128+32 mod P(x) << 32)]'  << 1   = 0x154442bd4
62 	 * #define CONSTANT_R1  0x154442bd4LL
63 	 *
64 	 * [(x4*128-32 mod P(x) << 32)]' << 1   = 0x1c6e41596
65 	 * #define CONSTANT_R2  0x1c6e41596LL
66 	 */
67 	.quad		0x0000000154442bd4
68 	.quad		0x00000001c6e41596
69 
70 	/*
71 	 * [(x128+32 mod P(x) << 32)]'   << 1   = 0x1751997d0
72 	 * #define CONSTANT_R3  0x1751997d0LL
73 	 *
74 	 * [(x128-32 mod P(x) << 32)]'   << 1   = 0x0ccaa009e
75 	 * #define CONSTANT_R4  0x0ccaa009eLL
76 	 */
77 	.quad		0x00000001751997d0
78 	.quad		0x00000000ccaa009e
79 
80 	/*
81 	 * [(x64 mod P(x) << 32)]'       << 1   = 0x163cd6124
82 	 * #define CONSTANT_R5  0x163cd6124LL
83 	 */
84 	.quad		0x0000000163cd6124
85 	.quad		0x00000000FFFFFFFF
86 
87 	/*
88 	 * #define CRCPOLY_TRUE_LE_FULL 0x1DB710641LL
89 	 *
90 	 * Barrett Reduction constant (u64`) = u` = (x**64 / P(x))`
91 	 *                                                      = 0x1F7011641LL
92 	 * #define CONSTANT_RU  0x1F7011641LL
93 	 */
94 	.quad		0x00000001DB710641
95 	.quad		0x00000001F7011641
96 
97 .Lcrc32c_constants:
98 	.quad		0x00000000740eef02
99 	.quad		0x000000009e4addf8
100 	.quad		0x00000000f20c0dfe
101 	.quad		0x000000014cd00bd6
102 	.quad		0x00000000dd45aab8
103 	.quad		0x00000000FFFFFFFF
104 	.quad		0x0000000105ec76f0
105 	.quad		0x00000000dea713f1
106 
107 	dCONSTANTl	.req	d0
108 	dCONSTANTh	.req	d1
109 	qCONSTANT	.req	q0
110 
111 	BUF		.req	r0
112 	LEN		.req	r1
113 	CRC		.req	r2
114 
115 	qzr		.req	q9
116 
117 	/**
118 	 * Calculate crc32
119 	 * BUF - buffer
120 	 * LEN - sizeof buffer (multiple of 16 bytes), LEN should be > 63
121 	 * CRC - initial crc32
122 	 * return %eax crc32
123 	 * uint crc32_pmull_le(unsigned char const *buffer,
124 	 *                     size_t len, uint crc32)
125 	 */
126 ENTRY(crc32_pmull_le)
127 	adr		r3, .Lcrc32_constants
128 	b		0f
129 
130 ENTRY(crc32c_pmull_le)
131 	adr		r3, .Lcrc32c_constants
132 
133 0:	bic		LEN, LEN, #15
134 	vld1.8		{q1-q2}, [BUF, :128]!
135 	vld1.8		{q3-q4}, [BUF, :128]!
136 	vmov.i8		qzr, #0
137 	vmov.i8		qCONSTANT, #0
138 	vmov.32		dCONSTANTl[0], CRC
139 	veor.8		d2, d2, dCONSTANTl
140 	sub		LEN, LEN, #0x40
141 	cmp		LEN, #0x40
142 	blt		less_64
143 
144 	vld1.64		{qCONSTANT}, [r3]
145 
146 loop_64:		/* 64 bytes Full cache line folding */
147 	sub		LEN, LEN, #0x40
148 
149 	vmull.p64	q5, d3, dCONSTANTh
150 	vmull.p64	q6, d5, dCONSTANTh
151 	vmull.p64	q7, d7, dCONSTANTh
152 	vmull.p64	q8, d9, dCONSTANTh
153 
154 	vmull.p64	q1, d2, dCONSTANTl
155 	vmull.p64	q2, d4, dCONSTANTl
156 	vmull.p64	q3, d6, dCONSTANTl
157 	vmull.p64	q4, d8, dCONSTANTl
158 
159 	veor.8		q1, q1, q5
160 	vld1.8		{q5}, [BUF, :128]!
161 	veor.8		q2, q2, q6
162 	vld1.8		{q6}, [BUF, :128]!
163 	veor.8		q3, q3, q7
164 	vld1.8		{q7}, [BUF, :128]!
165 	veor.8		q4, q4, q8
166 	vld1.8		{q8}, [BUF, :128]!
167 
168 	veor.8		q1, q1, q5
169 	veor.8		q2, q2, q6
170 	veor.8		q3, q3, q7
171 	veor.8		q4, q4, q8
172 
173 	cmp		LEN, #0x40
174 	bge		loop_64
175 
176 less_64:		/* Folding cache line into 128bit */
177 	vldr		dCONSTANTl, [r3, #16]
178 	vldr		dCONSTANTh, [r3, #24]
179 
180 	vmull.p64	q5, d3, dCONSTANTh
181 	vmull.p64	q1, d2, dCONSTANTl
182 	veor.8		q1, q1, q5
183 	veor.8		q1, q1, q2
184 
185 	vmull.p64	q5, d3, dCONSTANTh
186 	vmull.p64	q1, d2, dCONSTANTl
187 	veor.8		q1, q1, q5
188 	veor.8		q1, q1, q3
189 
190 	vmull.p64	q5, d3, dCONSTANTh
191 	vmull.p64	q1, d2, dCONSTANTl
192 	veor.8		q1, q1, q5
193 	veor.8		q1, q1, q4
194 
195 	teq		LEN, #0
196 	beq		fold_64
197 
198 loop_16:		/* Folding rest buffer into 128bit */
199 	subs		LEN, LEN, #0x10
200 
201 	vld1.8		{q2}, [BUF, :128]!
202 	vmull.p64	q5, d3, dCONSTANTh
203 	vmull.p64	q1, d2, dCONSTANTl
204 	veor.8		q1, q1, q5
205 	veor.8		q1, q1, q2
206 
207 	bne		loop_16
208 
209 fold_64:
210 	/* perform the last 64 bit fold, also adds 32 zeroes
211 	 * to the input stream */
212 	vmull.p64	q2, d2, dCONSTANTh
213 	vext.8		q1, q1, qzr, #8
214 	veor.8		q1, q1, q2
215 
216 	/* final 32-bit fold */
217 	vldr		dCONSTANTl, [r3, #32]
218 	vldr		d6, [r3, #40]
219 	vmov.i8		d7, #0
220 
221 	vext.8		q2, q1, qzr, #4
222 	vand.8		d2, d2, d6
223 	vmull.p64	q1, d2, dCONSTANTl
224 	veor.8		q1, q1, q2
225 
226 	/* Finish up with the bit-reversed barrett reduction 64 ==> 32 bits */
227 	vldr		dCONSTANTl, [r3, #48]
228 	vldr		dCONSTANTh, [r3, #56]
229 
230 	vand.8		q2, q1, q3
231 	vext.8		q2, qzr, q2, #8
232 	vmull.p64	q2, d5, dCONSTANTh
233 	vand.8		q2, q2, q3
234 	vmull.p64	q2, d4, dCONSTANTl
235 	veor.8		q1, q1, q2
236 	vmov		r0, s5
237 
238 	bx		lr
239 ENDPROC(crc32_pmull_le)
240 ENDPROC(crc32c_pmull_le)
241 
242 	.macro		__crc32, c
243 	subs		ip, r2, #8
244 	bmi		.Ltail\c
245 
246 	tst		r1, #3
247 	bne		.Lunaligned\c
248 
249 	teq		ip, #0
250 .Laligned8\c:
251 	ldrd		r2, r3, [r1], #8
252 ARM_BE8(rev		r2, r2		)
253 ARM_BE8(rev		r3, r3		)
254 	crc32\c\()w	r0, r0, r2
255 	crc32\c\()w	r0, r0, r3
256 	bxeq		lr
257 	subs		ip, ip, #8
258 	bpl		.Laligned8\c
259 
260 .Ltail\c:
261 	tst		ip, #4
262 	beq		2f
263 	ldr		r3, [r1], #4
264 ARM_BE8(rev		r3, r3		)
265 	crc32\c\()w	r0, r0, r3
266 
267 2:	tst		ip, #2
268 	beq		1f
269 	ldrh		r3, [r1], #2
270 ARM_BE8(rev16		r3, r3		)
271 	crc32\c\()h	r0, r0, r3
272 
273 1:	tst		ip, #1
274 	bxeq		lr
275 	ldrb		r3, [r1]
276 	crc32\c\()b	r0, r0, r3
277 	bx		lr
278 
279 .Lunaligned\c:
280 	tst		r1, #1
281 	beq		2f
282 	ldrb		r3, [r1], #1
283 	subs		r2, r2, #1
284 	crc32\c\()b	r0, r0, r3
285 
286 	tst		r1, #2
287 	beq		0f
288 2:	ldrh		r3, [r1], #2
289 	subs		r2, r2, #2
290 ARM_BE8(rev16		r3, r3		)
291 	crc32\c\()h	r0, r0, r3
292 
293 0:	subs		ip, r2, #8
294 	bpl		.Laligned8\c
295 	b		.Ltail\c
296 	.endm
297 
298 	.align		5
299 ENTRY(crc32_armv8_le)
300 	__crc32
301 ENDPROC(crc32_armv8_le)
302 
303 	.align		5
304 ENTRY(crc32c_armv8_le)
305 	__crc32		c
306 ENDPROC(crc32c_armv8_le)
307