1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * FP/SIMD state saving and restoring macros
4 *
5 * Copyright (C) 2012 ARM Ltd.
6 * Author: Catalin Marinas <catalin.marinas@arm.com>
7 */
8
9.macro fpsimd_save state, tmpnr
10	stp	q0, q1, [\state, #16 * 0]
11	stp	q2, q3, [\state, #16 * 2]
12	stp	q4, q5, [\state, #16 * 4]
13	stp	q6, q7, [\state, #16 * 6]
14	stp	q8, q9, [\state, #16 * 8]
15	stp	q10, q11, [\state, #16 * 10]
16	stp	q12, q13, [\state, #16 * 12]
17	stp	q14, q15, [\state, #16 * 14]
18	stp	q16, q17, [\state, #16 * 16]
19	stp	q18, q19, [\state, #16 * 18]
20	stp	q20, q21, [\state, #16 * 20]
21	stp	q22, q23, [\state, #16 * 22]
22	stp	q24, q25, [\state, #16 * 24]
23	stp	q26, q27, [\state, #16 * 26]
24	stp	q28, q29, [\state, #16 * 28]
25	stp	q30, q31, [\state, #16 * 30]!
26	mrs	x\tmpnr, fpsr
27	str	w\tmpnr, [\state, #16 * 2]
28	mrs	x\tmpnr, fpcr
29	str	w\tmpnr, [\state, #16 * 2 + 4]
30.endm
31
32.macro fpsimd_restore_fpcr state, tmp
33	/*
34	 * Writes to fpcr may be self-synchronising, so avoid restoring
35	 * the register if it hasn't changed.
36	 */
37	mrs	\tmp, fpcr
38	cmp	\tmp, \state
39	b.eq	9999f
40	msr	fpcr, \state
419999:
42.endm
43
44/* Clobbers \state */
45.macro fpsimd_restore state, tmpnr
46	ldp	q0, q1, [\state, #16 * 0]
47	ldp	q2, q3, [\state, #16 * 2]
48	ldp	q4, q5, [\state, #16 * 4]
49	ldp	q6, q7, [\state, #16 * 6]
50	ldp	q8, q9, [\state, #16 * 8]
51	ldp	q10, q11, [\state, #16 * 10]
52	ldp	q12, q13, [\state, #16 * 12]
53	ldp	q14, q15, [\state, #16 * 14]
54	ldp	q16, q17, [\state, #16 * 16]
55	ldp	q18, q19, [\state, #16 * 18]
56	ldp	q20, q21, [\state, #16 * 20]
57	ldp	q22, q23, [\state, #16 * 22]
58	ldp	q24, q25, [\state, #16 * 24]
59	ldp	q26, q27, [\state, #16 * 26]
60	ldp	q28, q29, [\state, #16 * 28]
61	ldp	q30, q31, [\state, #16 * 30]!
62	ldr	w\tmpnr, [\state, #16 * 2]
63	msr	fpsr, x\tmpnr
64	ldr	w\tmpnr, [\state, #16 * 2 + 4]
65	fpsimd_restore_fpcr x\tmpnr, \state
66.endm
67
68/* Sanity-check macros to help avoid encoding garbage instructions */
69
70.macro _check_general_reg nr
71	.if (\nr) < 0 || (\nr) > 30
72		.error "Bad register number \nr."
73	.endif
74.endm
75
76.macro _sve_check_zreg znr
77	.if (\znr) < 0 || (\znr) > 31
78		.error "Bad Scalable Vector Extension vector register number \znr."
79	.endif
80.endm
81
82.macro _sve_check_preg pnr
83	.if (\pnr) < 0 || (\pnr) > 15
84		.error "Bad Scalable Vector Extension predicate register number \pnr."
85	.endif
86.endm
87
88.macro _check_num n, min, max
89	.if (\n) < (\min) || (\n) > (\max)
90		.error "Number \n out of range [\min,\max]"
91	.endif
92.endm
93
94/* SVE instruction encodings for non-SVE-capable assemblers */
95
96/* STR (vector): STR Z\nz, [X\nxbase, #\offset, MUL VL] */
97.macro _sve_str_v nz, nxbase, offset=0
98	_sve_check_zreg \nz
99	_check_general_reg \nxbase
100	_check_num (\offset), -0x100, 0xff
101	.inst	0xe5804000			\
102		| (\nz)				\
103		| ((\nxbase) << 5)		\
104		| (((\offset) & 7) << 10)	\
105		| (((\offset) & 0x1f8) << 13)
106.endm
107
108/* LDR (vector): LDR Z\nz, [X\nxbase, #\offset, MUL VL] */
109.macro _sve_ldr_v nz, nxbase, offset=0
110	_sve_check_zreg \nz
111	_check_general_reg \nxbase
112	_check_num (\offset), -0x100, 0xff
113	.inst	0x85804000			\
114		| (\nz)				\
115		| ((\nxbase) << 5)		\
116		| (((\offset) & 7) << 10)	\
117		| (((\offset) & 0x1f8) << 13)
118.endm
119
120/* STR (predicate): STR P\np, [X\nxbase, #\offset, MUL VL] */
121.macro _sve_str_p np, nxbase, offset=0
122	_sve_check_preg \np
123	_check_general_reg \nxbase
124	_check_num (\offset), -0x100, 0xff
125	.inst	0xe5800000			\
126		| (\np)				\
127		| ((\nxbase) << 5)		\
128		| (((\offset) & 7) << 10)	\
129		| (((\offset) & 0x1f8) << 13)
130.endm
131
132/* LDR (predicate): LDR P\np, [X\nxbase, #\offset, MUL VL] */
133.macro _sve_ldr_p np, nxbase, offset=0
134	_sve_check_preg \np
135	_check_general_reg \nxbase
136	_check_num (\offset), -0x100, 0xff
137	.inst	0x85800000			\
138		| (\np)				\
139		| ((\nxbase) << 5)		\
140		| (((\offset) & 7) << 10)	\
141		| (((\offset) & 0x1f8) << 13)
142.endm
143
144/* RDVL X\nx, #\imm */
145.macro _sve_rdvl nx, imm
146	_check_general_reg \nx
147	_check_num (\imm), -0x20, 0x1f
148	.inst	0x04bf5000			\
149		| (\nx)				\
150		| (((\imm) & 0x3f) << 5)
151.endm
152
153/* RDFFR (unpredicated): RDFFR P\np.B */
154.macro _sve_rdffr np
155	_sve_check_preg \np
156	.inst	0x2519f000			\
157		| (\np)
158.endm
159
160/* WRFFR P\np.B */
161.macro _sve_wrffr np
162	_sve_check_preg \np
163	.inst	0x25289000			\
164		| ((\np) << 5)
165.endm
166
167/* PFALSE P\np.B */
168.macro _sve_pfalse np
169	_sve_check_preg \np
170	.inst	0x2518e400			\
171		| (\np)
172.endm
173
174.macro __for from:req, to:req
175	.if (\from) == (\to)
176		_for__body %\from
177	.else
178		__for %\from, %((\from) + ((\to) - (\from)) / 2)
179		__for %((\from) + ((\to) - (\from)) / 2 + 1), %\to
180	.endif
181.endm
182
183.macro _for var:req, from:req, to:req, insn:vararg
184	.macro _for__body \var:req
185		.noaltmacro
186		\insn
187		.altmacro
188	.endm
189
190	.altmacro
191	__for \from, \to
192	.noaltmacro
193
194	.purgem _for__body
195.endm
196
197/* Update ZCR_EL1.LEN with the new VQ */
198.macro sve_load_vq xvqminus1, xtmp, xtmp2
199		mrs_s		\xtmp, SYS_ZCR_EL1
200		bic		\xtmp2, \xtmp, ZCR_ELx_LEN_MASK
201		orr		\xtmp2, \xtmp2, \xvqminus1
202		cmp		\xtmp2, \xtmp
203		b.eq		921f
204		msr_s		SYS_ZCR_EL1, \xtmp2	//self-synchronising
205921:
206.endm
207
208/* Preserve the first 128-bits of Znz and zero the rest. */
209.macro _sve_flush_z nz
210	_sve_check_zreg \nz
211	mov	v\nz\().16b, v\nz\().16b
212.endm
213
214.macro sve_flush
215 _for n, 0, 31, _sve_flush_z	\n
216 _for n, 0, 15, _sve_pfalse	\n
217		_sve_wrffr	0
218.endm
219
220.macro sve_save nxbase, xpfpsr, nxtmp
221 _for n, 0, 31,	_sve_str_v	\n, \nxbase, \n - 34
222 _for n, 0, 15,	_sve_str_p	\n, \nxbase, \n - 16
223		_sve_rdffr	0
224		_sve_str_p	0, \nxbase
225		_sve_ldr_p	0, \nxbase, -16
226
227		mrs		x\nxtmp, fpsr
228		str		w\nxtmp, [\xpfpsr]
229		mrs		x\nxtmp, fpcr
230		str		w\nxtmp, [\xpfpsr, #4]
231.endm
232
233.macro sve_load nxbase, xpfpsr, xvqminus1, nxtmp, xtmp2
234		sve_load_vq	\xvqminus1, x\nxtmp, \xtmp2
235 _for n, 0, 31,	_sve_ldr_v	\n, \nxbase, \n - 34
236		_sve_ldr_p	0, \nxbase
237		_sve_wrffr	0
238 _for n, 0, 15,	_sve_ldr_p	\n, \nxbase, \n - 16
239
240		ldr		w\nxtmp, [\xpfpsr]
241		msr		fpsr, x\nxtmp
242		ldr		w\nxtmp, [\xpfpsr, #4]
243		msr		fpcr, x\nxtmp
244.endm
245