1/* SPDX-License-Identifier: GPL-2.0 */
2#ifndef __ASM_ALTERNATIVE_MACROS_H
3#define __ASM_ALTERNATIVE_MACROS_H
4
5#include <asm/cpucaps.h>
6
7#define ARM64_CB_PATCH ARM64_NCAPS
8
9/* A64 instructions are always 32 bits. */
10#define	AARCH64_INSN_SIZE		4
11
12#ifndef __ASSEMBLY__
13
14#include <linux/stringify.h>
15
16#define ALTINSTR_ENTRY(feature)					              \
17	" .word 661b - .\n"				/* label           */ \
18	" .word 663f - .\n"				/* new instruction */ \
19	" .hword " __stringify(feature) "\n"		/* feature bit     */ \
20	" .byte 662b-661b\n"				/* source len      */ \
21	" .byte 664f-663f\n"				/* replacement len */
22
23#define ALTINSTR_ENTRY_CB(feature, cb)					      \
24	" .word 661b - .\n"				/* label           */ \
25	" .word " __stringify(cb) "- .\n"		/* callback */	      \
26	" .hword " __stringify(feature) "\n"		/* feature bit     */ \
27	" .byte 662b-661b\n"				/* source len      */ \
28	" .byte 664f-663f\n"				/* replacement len */
29
30/*
31 * alternative assembly primitive:
32 *
33 * If any of these .org directive fail, it means that insn1 and insn2
34 * don't have the same length. This used to be written as
35 *
36 * .if ((664b-663b) != (662b-661b))
37 * 	.error "Alternatives instruction length mismatch"
38 * .endif
39 *
40 * but most assemblers die if insn1 or insn2 have a .inst. This should
41 * be fixed in a binutils release posterior to 2.25.51.0.2 (anything
42 * containing commit 4e4d08cf7399b606 or c1baaddf8861).
43 *
44 * Alternatives with callbacks do not generate replacement instructions.
45 */
46#define __ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg_enabled)	\
47	".if "__stringify(cfg_enabled)" == 1\n"				\
48	"661:\n\t"							\
49	oldinstr "\n"							\
50	"662:\n"							\
51	".pushsection .altinstructions,\"a\"\n"				\
52	ALTINSTR_ENTRY(feature)						\
53	".popsection\n"							\
54	".subsection 1\n"						\
55	"663:\n\t"							\
56	newinstr "\n"							\
57	"664:\n\t"							\
58	".org	. - (664b-663b) + (662b-661b)\n\t"			\
59	".org	. - (662b-661b) + (664b-663b)\n\t"			\
60	".previous\n"							\
61	".endif\n"
62
63#define __ALTERNATIVE_CFG_CB(oldinstr, feature, cfg_enabled, cb)	\
64	".if "__stringify(cfg_enabled)" == 1\n"				\
65	"661:\n\t"							\
66	oldinstr "\n"							\
67	"662:\n"							\
68	".pushsection .altinstructions,\"a\"\n"				\
69	ALTINSTR_ENTRY_CB(feature, cb)					\
70	".popsection\n"							\
71	"663:\n\t"							\
72	"664:\n\t"							\
73	".endif\n"
74
75#define _ALTERNATIVE_CFG(oldinstr, newinstr, feature, cfg, ...)	\
76	__ALTERNATIVE_CFG(oldinstr, newinstr, feature, IS_ENABLED(cfg))
77
78#define ALTERNATIVE_CB(oldinstr, cb) \
79	__ALTERNATIVE_CFG_CB(oldinstr, ARM64_CB_PATCH, 1, cb)
80#else
81
82#include <asm/assembler.h>
83
84.macro altinstruction_entry orig_offset alt_offset feature orig_len alt_len
85	.word \orig_offset - .
86	.word \alt_offset - .
87	.hword \feature
88	.byte \orig_len
89	.byte \alt_len
90.endm
91
92.macro alternative_insn insn1, insn2, cap, enable = 1
93	.if \enable
94661:	\insn1
95662:	.pushsection .altinstructions, "a"
96	altinstruction_entry 661b, 663f, \cap, 662b-661b, 664f-663f
97	.popsection
98	.subsection 1
99663:	\insn2
100664:	.org	. - (664b-663b) + (662b-661b)
101	.org	. - (662b-661b) + (664b-663b)
102	.previous
103	.endif
104.endm
105
106/*
107 * Alternative sequences
108 *
109 * The code for the case where the capability is not present will be
110 * assembled and linked as normal. There are no restrictions on this
111 * code.
112 *
113 * The code for the case where the capability is present will be
114 * assembled into a special section to be used for dynamic patching.
115 * Code for that case must:
116 *
117 * 1. Be exactly the same length (in bytes) as the default code
118 *    sequence.
119 *
120 * 2. Not contain a branch target that is used outside of the
121 *    alternative sequence it is defined in (branches into an
122 *    alternative sequence are not fixed up).
123 */
124
125/*
126 * Begin an alternative code sequence.
127 */
128.macro alternative_if_not cap
129	.set .Lasm_alt_mode, 0
130	.pushsection .altinstructions, "a"
131	altinstruction_entry 661f, 663f, \cap, 662f-661f, 664f-663f
132	.popsection
133661:
134.endm
135
136.macro alternative_if cap
137	.set .Lasm_alt_mode, 1
138	.pushsection .altinstructions, "a"
139	altinstruction_entry 663f, 661f, \cap, 664f-663f, 662f-661f
140	.popsection
141	.subsection 1
142	.align 2	/* So GAS knows label 661 is suitably aligned */
143661:
144.endm
145
146.macro alternative_cb cb
147	.set .Lasm_alt_mode, 0
148	.pushsection .altinstructions, "a"
149	altinstruction_entry 661f, \cb, ARM64_CB_PATCH, 662f-661f, 0
150	.popsection
151661:
152.endm
153
154/*
155 * Provide the other half of the alternative code sequence.
156 */
157.macro alternative_else
158662:
159	.if .Lasm_alt_mode==0
160	.subsection 1
161	.else
162	.previous
163	.endif
164663:
165.endm
166
167/*
168 * Complete an alternative code sequence.
169 */
170.macro alternative_endif
171664:
172	.org	. - (664b-663b) + (662b-661b)
173	.org	. - (662b-661b) + (664b-663b)
174	.if .Lasm_alt_mode==0
175	.previous
176	.endif
177.endm
178
179/*
180 * Callback-based alternative epilogue
181 */
182.macro alternative_cb_end
183662:
184.endm
185
186/*
187 * Provides a trivial alternative or default sequence consisting solely
188 * of NOPs. The number of NOPs is chosen automatically to match the
189 * previous case.
190 */
191.macro alternative_else_nop_endif
192alternative_else
193	nops	(662b-661b) / AARCH64_INSN_SIZE
194alternative_endif
195.endm
196
197#define _ALTERNATIVE_CFG(insn1, insn2, cap, cfg, ...)	\
198	alternative_insn insn1, insn2, cap, IS_ENABLED(cfg)
199
200.macro user_alt, label, oldinstr, newinstr, cond
2019999:	alternative_insn "\oldinstr", "\newinstr", \cond
202	_asm_extable 9999b, \label
203.endm
204
205/*
206 * Generate the assembly for UAO alternatives with exception table entries.
207 * This is complicated as there is no post-increment or pair versions of the
208 * unprivileged instructions, and USER() only works for single instructions.
209 */
210#ifdef CONFIG_ARM64_UAO
211	.macro uao_ldp l, reg1, reg2, addr, post_inc
212		alternative_if_not ARM64_HAS_UAO
2138888:			ldp	\reg1, \reg2, [\addr], \post_inc;
2148889:			nop;
215			nop;
216		alternative_else
217			ldtr	\reg1, [\addr];
218			ldtr	\reg2, [\addr, #8];
219			add	\addr, \addr, \post_inc;
220		alternative_endif
221
222		_asm_extable	8888b,\l;
223		_asm_extable	8889b,\l;
224	.endm
225
226	.macro uao_stp l, reg1, reg2, addr, post_inc
227		alternative_if_not ARM64_HAS_UAO
2288888:			stp	\reg1, \reg2, [\addr], \post_inc;
2298889:			nop;
230			nop;
231		alternative_else
232			sttr	\reg1, [\addr];
233			sttr	\reg2, [\addr, #8];
234			add	\addr, \addr, \post_inc;
235		alternative_endif
236
237		_asm_extable	8888b,\l;
238		_asm_extable	8889b,\l;
239	.endm
240
241	.macro uao_user_alternative l, inst, alt_inst, reg, addr, post_inc
242		alternative_if_not ARM64_HAS_UAO
2438888:			\inst	\reg, [\addr], \post_inc;
244			nop;
245		alternative_else
246			\alt_inst	\reg, [\addr];
247			add		\addr, \addr, \post_inc;
248		alternative_endif
249
250		_asm_extable	8888b,\l;
251	.endm
252#else
253	.macro uao_ldp l, reg1, reg2, addr, post_inc
254		USER(\l, ldp \reg1, \reg2, [\addr], \post_inc)
255	.endm
256	.macro uao_stp l, reg1, reg2, addr, post_inc
257		USER(\l, stp \reg1, \reg2, [\addr], \post_inc)
258	.endm
259	.macro uao_user_alternative l, inst, alt_inst, reg, addr, post_inc
260		USER(\l, \inst \reg, [\addr], \post_inc)
261	.endm
262#endif
263
264#endif  /*  __ASSEMBLY__  */
265
266/*
267 * Usage: asm(ALTERNATIVE(oldinstr, newinstr, feature));
268 *
269 * Usage: asm(ALTERNATIVE(oldinstr, newinstr, feature, CONFIG_FOO));
270 * N.B. If CONFIG_FOO is specified, but not selected, the whole block
271 *      will be omitted, including oldinstr.
272 */
273#define ALTERNATIVE(oldinstr, newinstr, ...)   \
274	_ALTERNATIVE_CFG(oldinstr, newinstr, __VA_ARGS__, 1)
275
276#endif /* __ASM_ALTERNATIVE_MACROS_H */
277