1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3  *  linux/arch/arm/lib/copy_template.s
4  *
5  *  Code template for optimized memory copy functions
6  *
7  *  Author:	Nicolas Pitre
8  *  Created:	Sep 28, 2005
9  *  Copyright:	MontaVista Software, Inc.
10  */
11 
12 /*
13  * Theory of operation
14  * -------------------
15  *
16  * This file provides the core code for a forward memory copy used in
17  * the implementation of memcopy(), copy_to_user() and copy_from_user().
18  *
19  * The including file must define the following accessor macros
20  * according to the need of the given function:
21  *
22  * ldr1w ptr reg abort
23  *
24  *	This loads one word from 'ptr', stores it in 'reg' and increments
25  *	'ptr' to the next word. The 'abort' argument is used for fixup tables.
26  *
27  * ldr4w ptr reg1 reg2 reg3 reg4 abort
28  * ldr8w ptr, reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
29  *
30  *	This loads four or eight words starting from 'ptr', stores them
31  *	in provided registers and increments 'ptr' past those words.
32  *	The'abort' argument is used for fixup tables.
33  *
34  * ldr1b ptr reg cond abort
35  *
36  *	Similar to ldr1w, but it loads a byte and increments 'ptr' one byte.
37  *	It also must apply the condition code if provided, otherwise the
38  *	"al" condition is assumed by default.
39  *
40  * str1w ptr reg abort
41  * str8w ptr reg1 reg2 reg3 reg4 reg5 reg6 reg7 reg8 abort
42  * str1b ptr reg cond abort
43  *
44  *	Same as their ldr* counterparts, but data is stored to 'ptr' location
45  *	rather than being loaded.
46  *
47  * enter reg1 reg2
48  *
49  *	Preserve the provided registers on the stack plus any additional
50  *	data as needed by the implementation including this code. Called
51  *	upon code entry.
52  *
53  * usave reg1 reg2
54  *
55  *	Unwind annotation macro is corresponding for 'enter' macro.
56  *	It tell unwinder that preserved some provided registers on the stack
57  *	and additional data by a prior 'enter' macro.
58  *
59  * exit reg1 reg2
60  *
61  *	Restore registers with the values previously saved with the
62  *	'preserv' macro. Called upon code termination.
63  *
64  * LDR1W_SHIFT
65  * STR1W_SHIFT
66  *
67  *	Correction to be applied to the "ip" register when branching into
68  *	the ldr1w or str1w instructions (some of these macros may expand to
69  *	than one 32bit instruction in Thumb-2)
70  */
71 
72 
73 	UNWIND(	.fnstart			)
74 		enter	r4, lr
75 	UNWIND(	.fnend				)
76 
77 	UNWIND(	.fnstart			)
78 		usave	r4, lr			  @ in first stmdb block
79 
80 		subs	r2, r2, #4
81 		blt	8f
82 		ands	ip, r0, #3
83 	PLD(	pld	[r1, #0]		)
84 		bne	9f
85 		ands	ip, r1, #3
86 		bne	10f
87 
88 1:		subs	r2, r2, #(28)
89 		stmfd	sp!, {r5 - r8}
90 	UNWIND(	.fnend				)
91 
92 	UNWIND(	.fnstart			)
93 		usave	r4, lr
94 	UNWIND(	.save	{r5 - r8}		) @ in second stmfd block
95 		blt	5f
96 
97 	CALGN(	ands	ip, r0, #31		)
98 	CALGN(	rsb	r3, ip, #32		)
99 	CALGN(	sbcsne	r4, r3, r2		)  @ C is always set here
100 	CALGN(	bcs	2f			)
101 	CALGN(	adr	r4, 6f			)
102 	CALGN(	subs	r2, r2, r3		)  @ C gets set
103 	CALGN(	add	pc, r4, ip		)
104 
105 	PLD(	pld	[r1, #0]		)
106 2:	PLD(	subs	r2, r2, #96		)
107 	PLD(	pld	[r1, #28]		)
108 	PLD(	blt	4f			)
109 	PLD(	pld	[r1, #60]		)
110 	PLD(	pld	[r1, #92]		)
111 
112 3:	PLD(	pld	[r1, #124]		)
113 4:		ldr8w	r1, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f
114 		subs	r2, r2, #32
115 		str8w	r0, r3, r4, r5, r6, r7, r8, ip, lr, abort=20f
116 		bge	3b
117 	PLD(	cmn	r2, #96			)
118 	PLD(	bge	4b			)
119 
120 5:		ands	ip, r2, #28
121 		rsb	ip, ip, #32
122 #if LDR1W_SHIFT > 0
123 		lsl	ip, ip, #LDR1W_SHIFT
124 #endif
125 		addne	pc, pc, ip		@ C is always clear here
126 		b	7f
127 6:
128 		.rept	(1 << LDR1W_SHIFT)
129 		W(nop)
130 		.endr
131 		ldr1w	r1, r3, abort=20f
132 		ldr1w	r1, r4, abort=20f
133 		ldr1w	r1, r5, abort=20f
134 		ldr1w	r1, r6, abort=20f
135 		ldr1w	r1, r7, abort=20f
136 		ldr1w	r1, r8, abort=20f
137 		ldr1w	r1, lr, abort=20f
138 
139 #if LDR1W_SHIFT < STR1W_SHIFT
140 		lsl	ip, ip, #STR1W_SHIFT - LDR1W_SHIFT
141 #elif LDR1W_SHIFT > STR1W_SHIFT
142 		lsr	ip, ip, #LDR1W_SHIFT - STR1W_SHIFT
143 #endif
144 		add	pc, pc, ip
145 		nop
146 		.rept	(1 << STR1W_SHIFT)
147 		W(nop)
148 		.endr
149 		str1w	r0, r3, abort=20f
150 		str1w	r0, r4, abort=20f
151 		str1w	r0, r5, abort=20f
152 		str1w	r0, r6, abort=20f
153 		str1w	r0, r7, abort=20f
154 		str1w	r0, r8, abort=20f
155 		str1w	r0, lr, abort=20f
156 
157 	CALGN(	bcs	2b			)
158 
159 7:		ldmfd	sp!, {r5 - r8}
160 	UNWIND(	.fnend				) @ end of second stmfd block
161 
162 	UNWIND(	.fnstart			)
163 		usave	r4, lr			  @ still in first stmdb block
164 8:		movs	r2, r2, lsl #31
165 		ldr1b	r1, r3, ne, abort=21f
166 		ldr1b	r1, r4, cs, abort=21f
167 		ldr1b	r1, ip, cs, abort=21f
168 		str1b	r0, r3, ne, abort=21f
169 		str1b	r0, r4, cs, abort=21f
170 		str1b	r0, ip, cs, abort=21f
171 
172 		exit	r4, pc
173 
174 9:		rsb	ip, ip, #4
175 		cmp	ip, #2
176 		ldr1b	r1, r3, gt, abort=21f
177 		ldr1b	r1, r4, ge, abort=21f
178 		ldr1b	r1, lr, abort=21f
179 		str1b	r0, r3, gt, abort=21f
180 		str1b	r0, r4, ge, abort=21f
181 		subs	r2, r2, ip
182 		str1b	r0, lr, abort=21f
183 		blt	8b
184 		ands	ip, r1, #3
185 		beq	1b
186 
187 10:		bic	r1, r1, #3
188 		cmp	ip, #2
189 		ldr1w	r1, lr, abort=21f
190 		beq	17f
191 		bgt	18f
192 	UNWIND(	.fnend				)
193 
194 
195 		.macro	forward_copy_shift pull push
196 
197 	UNWIND(	.fnstart			)
198 		usave	r4, lr			  @ still in first stmdb block
199 		subs	r2, r2, #28
200 		blt	14f
201 
202 	CALGN(	ands	ip, r0, #31		)
203 	CALGN(	rsb	ip, ip, #32		)
204 	CALGN(	sbcsne	r4, ip, r2		)  @ C is always set here
205 	CALGN(	subcc	r2, r2, ip		)
206 	CALGN(	bcc	15f			)
207 
208 11:		stmfd	sp!, {r5 - r9}
209 	UNWIND(	.fnend				)
210 
211 	UNWIND(	.fnstart			)
212 		usave	r4, lr
213 	UNWIND(	.save	{r5 - r9}		) @ in new second stmfd block
214 	PLD(	pld	[r1, #0]		)
215 	PLD(	subs	r2, r2, #96		)
216 	PLD(	pld	[r1, #28]		)
217 	PLD(	blt	13f			)
218 	PLD(	pld	[r1, #60]		)
219 	PLD(	pld	[r1, #92]		)
220 
221 12:	PLD(	pld	[r1, #124]		)
222 13:		ldr4w	r1, r4, r5, r6, r7, abort=19f
223 		mov	r3, lr, lspull #\pull
224 		subs	r2, r2, #32
225 		ldr4w	r1, r8, r9, ip, lr, abort=19f
226 		orr	r3, r3, r4, lspush #\push
227 		mov	r4, r4, lspull #\pull
228 		orr	r4, r4, r5, lspush #\push
229 		mov	r5, r5, lspull #\pull
230 		orr	r5, r5, r6, lspush #\push
231 		mov	r6, r6, lspull #\pull
232 		orr	r6, r6, r7, lspush #\push
233 		mov	r7, r7, lspull #\pull
234 		orr	r7, r7, r8, lspush #\push
235 		mov	r8, r8, lspull #\pull
236 		orr	r8, r8, r9, lspush #\push
237 		mov	r9, r9, lspull #\pull
238 		orr	r9, r9, ip, lspush #\push
239 		mov	ip, ip, lspull #\pull
240 		orr	ip, ip, lr, lspush #\push
241 		str8w	r0, r3, r4, r5, r6, r7, r8, r9, ip, abort=19f
242 		bge	12b
243 	PLD(	cmn	r2, #96			)
244 	PLD(	bge	13b			)
245 
246 		ldmfd	sp!, {r5 - r9}
247 	UNWIND(	.fnend				) @ end of the second stmfd block
248 
249 	UNWIND(	.fnstart			)
250 		usave	r4, lr			  @ still in first stmdb block
251 14:		ands	ip, r2, #28
252 		beq	16f
253 
254 15:		mov	r3, lr, lspull #\pull
255 		ldr1w	r1, lr, abort=21f
256 		subs	ip, ip, #4
257 		orr	r3, r3, lr, lspush #\push
258 		str1w	r0, r3, abort=21f
259 		bgt	15b
260 	CALGN(	cmp	r2, #0			)
261 	CALGN(	bge	11b			)
262 
263 16:		sub	r1, r1, #(\push / 8)
264 		b	8b
265 	UNWIND(	.fnend				)
266 
267 		.endm
268 
269 
270 		forward_copy_shift	pull=8	push=24
271 
272 17:		forward_copy_shift	pull=16	push=16
273 
274 18:		forward_copy_shift	pull=24	push=8
275 
276 
277 /*
278  * Abort preamble and completion macros.
279  * If a fixup handler is required then those macros must surround it.
280  * It is assumed that the fixup code will handle the private part of
281  * the exit macro.
282  */
283 
284 	.macro	copy_abort_preamble
285 19:	ldmfd	sp!, {r5 - r9}
286 	b	21f
287 20:	ldmfd	sp!, {r5 - r8}
288 21:
289 	.endm
290 
291 	.macro	copy_abort_end
292 	ldmfd	sp!, {r4, pc}
293 	.endm
294 
295