1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3  *  Copyright (C) 1994-2002 Russell King
4  *  Copyright (c) 2003, 2020 ARM Limited
5  *  All Rights Reserved
6  */
7 
8 #include <linux/init.h>
9 #include <linux/linkage.h>
10 #include <asm/assembler.h>
11 #include <asm/page.h>
12 
13 #ifdef __ARMEB__
14 #define LOW_OFFSET	0x4
15 #define HIGH_OFFSET	0x0
16 #else
17 #define LOW_OFFSET	0x0
18 #define HIGH_OFFSET	0x4
19 #endif
20 
21 /*
22  * __fixup_pv_table - patch the stub instructions with the delta between
23  *                    PHYS_OFFSET and PAGE_OFFSET, which is assumed to be
24  *                    2 MiB aligned.
25  *
26  * Called from head.S, which expects the following registers to be preserved:
27  *   r1 = machine no, r2 = atags or dtb,
28  *   r8 = phys_offset, r9 = cpuid, r10 = procinfo
29  */
30 	__HEAD
31 ENTRY(__fixup_pv_table)
32 	mov	r0, r8, lsr #PAGE_SHIFT	@ convert to PFN
33 	str_l	r0, __pv_phys_pfn_offset, r3
34 
35 	adr_l	r0, __pv_offset
36 	subs	r3, r8, #PAGE_OFFSET	@ PHYS_OFFSET - PAGE_OFFSET
37 	mvn	ip, #0
38 	strcc	ip, [r0, #HIGH_OFFSET]	@ save to __pv_offset high bits
39 	str	r3, [r0, #LOW_OFFSET]	@ save to __pv_offset low bits
40 
41 	mov	r0, r3, lsr #21		@ constant for add/sub instructions
42 	teq	r3, r0, lsl #21 	@ must be 2 MiB aligned
43 	bne	0f
44 
45 	adr_l	r4, __pv_table_begin
46 	adr_l	r5, __pv_table_end
47 	b	__fixup_a_pv_table
48 
49 0:	mov	r0, r0			@ deadloop on error
50 	b	0b
51 ENDPROC(__fixup_pv_table)
52 
53 	.text
54 __fixup_a_pv_table:
55 	adr_l	r6, __pv_offset
56 	ldr	r0, [r6, #HIGH_OFFSET]	@ pv_offset high word
57 	ldr	r6, [r6, #LOW_OFFSET]	@ pv_offset low word
58 	cmn	r0, #1
59 #ifdef CONFIG_THUMB2_KERNEL
60 	@
61 	@ The Thumb-2 versions of the patchable sequences are
62 	@
63 	@ phys-to-virt:			movw	<reg>, #offset<31:21>
64 	@				lsl	<reg>, #21
65 	@				sub	<VA>, <PA>, <reg>
66 	@
67 	@ virt-to-phys (non-LPAE):	movw	<reg>, #offset<31:21>
68 	@				lsl	<reg>, #21
69 	@				add	<PA>, <VA>, <reg>
70 	@
71 	@ virt-to-phys (LPAE):		movw	<reg>, #offset<31:21>
72 	@				lsl	<reg>, #21
73 	@				adds	<PAlo>, <VA>, <reg>
74 	@				mov	<PAhi>, #offset<39:32>
75 	@				adc	<PAhi>, <PAhi>, #0
76 	@
77 	@ In the non-LPAE case, all patchable instructions are MOVW
78 	@ instructions, where we need to patch in the offset into the
79 	@ second halfword of the opcode (the 16-bit immediate is encoded
80 	@ as imm4:i:imm3:imm8)
81 	@
82 	@       15       11 10  9           4 3    0  15  14  12 11 8 7    0
83 	@      +-----------+---+-------------+------++---+------+----+------+
84 	@ MOVW | 1 1 1 1 0 | i | 1 0 0 1 0 0 | imm4 || 0 | imm3 | Rd | imm8 |
85 	@      +-----------+---+-------------+------++---+------+----+------+
86 	@
87 	@ In the LPAE case, we also need to patch in the high word of the
88 	@ offset into the immediate field of the MOV instruction, or patch it
89 	@ to a MVN instruction if the offset is negative. In this case, we
90 	@ need to inspect the first halfword of the opcode, to check whether
91 	@ it is MOVW or MOV/MVN, and to perform the MOV to MVN patching if
92 	@ needed. The encoding of the immediate is rather complex for values
93 	@ of i:imm3 != 0b0000, but fortunately, we never need more than 8 lower
94 	@ order bits, which can be patched into imm8 directly (and i:imm3
95 	@ cleared)
96 	@
97 	@      15       11 10  9        5         0  15  14  12 11 8 7    0
98 	@     +-----------+---+---------------------++---+------+----+------+
99 	@ MOV | 1 1 1 1 0 | i | 0 0 0 1 0 0 1 1 1 1 || 0 | imm3 | Rd | imm8 |
100 	@ MVN | 1 1 1 1 0 | i | 0 0 0 1 1 0 1 1 1 1 || 0 | imm3 | Rd | imm8 |
101 	@     +-----------+---+---------------------++---+------+----+------+
102 	@
103 	moveq	r0, #0x200000		@ set bit 21, mov to mvn instruction
104 	lsrs	r3, r6, #29		@ isolate top 3 bits of displacement
105 	ubfx	r6, r6, #21, #8		@ put bits 28:21 into the MOVW imm8 field
106 	bfi	r6, r3, #12, #3		@ put bits 31:29 into the MOVW imm3 field
107 	b	.Lnext
108 .Lloop:	add	r7, r4
109 	adds	r4, #4			@ clears Z flag
110 #ifdef CONFIG_ARM_LPAE
111 	ldrh	ip, [r7]
112 ARM_BE8(rev16	ip, ip)
113 	tst	ip, #0x200		@ MOVW has bit 9 set, MVN has it clear
114 	bne	0f			@ skip to MOVW handling (Z flag is clear)
115 	bic	ip, #0x20		@ clear bit 5 (MVN -> MOV)
116 	orr	ip, ip, r0, lsr #16	@ MOV -> MVN if offset < 0
117 ARM_BE8(rev16	ip, ip)
118 	strh	ip, [r7]
119 	@ Z flag is set
120 0:
121 #endif
122 	ldrh	ip, [r7, #2]
123 ARM_BE8(rev16	ip, ip)
124 	and	ip, #0xf00		@ clear everything except Rd field
125 	orreq	ip, r0			@ Z flag set -> MOV/MVN -> patch in high bits
126 	orrne	ip, r6			@ Z flag clear -> MOVW -> patch in low bits
127 ARM_BE8(rev16	ip, ip)
128 	strh	ip, [r7, #2]
129 #else
130 #ifdef CONFIG_CPU_ENDIAN_BE8
131 @ in BE8, we load data in BE, but instructions still in LE
132 #define PV_BIT24	0x00000001
133 #define PV_IMM8_MASK	0xff000000
134 #define PV_IMMR_MSB	0x00080000
135 #else
136 #define PV_BIT24	0x01000000
137 #define PV_IMM8_MASK	0x000000ff
138 #define PV_IMMR_MSB	0x00000800
139 #endif
140 
141 	@
142 	@ The ARM versions of the patchable sequences are
143 	@
144 	@ phys-to-virt:			sub	<VA>, <PA>, #offset<31:24>, lsl #24
145 	@				sub	<VA>, <PA>, #offset<23:16>, lsl #16
146 	@
147 	@ virt-to-phys (non-LPAE):	add	<PA>, <VA>, #offset<31:24>, lsl #24
148 	@				add	<PA>, <VA>, #offset<23:16>, lsl #16
149 	@
150 	@ virt-to-phys (LPAE):		movw	<reg>, #offset<31:20>
151 	@				adds	<PAlo>, <VA>, <reg>, lsl #20
152 	@				mov	<PAhi>, #offset<39:32>
153 	@				adc	<PAhi>, <PAhi>, #0
154 	@
155 	@ In the non-LPAE case, all patchable instructions are ADD or SUB
156 	@ instructions, where we need to patch in the offset into the
157 	@ immediate field of the opcode, which is emitted with the correct
158 	@ rotation value. (The effective value of the immediate is imm12<7:0>
159 	@ rotated right by [2 * imm12<11:8>] bits)
160 	@
161 	@      31   28 27      23 22  20 19  16 15  12 11    0
162 	@      +------+-----------------+------+------+-------+
163 	@  ADD | cond | 0 0 1 0 1 0 0 0 |  Rn  |  Rd  | imm12 |
164 	@  SUB | cond | 0 0 1 0 0 1 0 0 |  Rn  |  Rd  | imm12 |
165 	@  MOV | cond | 0 0 1 1 1 0 1 0 |  Rn  |  Rd  | imm12 |
166 	@  MVN | cond | 0 0 1 1 1 1 1 0 |  Rn  |  Rd  | imm12 |
167 	@      +------+-----------------+------+------+-------+
168 	@
169 	@ In the LPAE case, we use a MOVW instruction to carry the low offset
170 	@ word, and patch in the high word of the offset into the immediate
171 	@ field of the subsequent MOV instruction, or patch it to a MVN
172 	@ instruction if the offset is negative. We can distinguish MOVW
173 	@ instructions based on bits 23:22 of the opcode, and ADD/SUB can be
174 	@ distinguished from MOV/MVN (all using the encodings above) using
175 	@ bit 24.
176 	@
177 	@      31   28 27      23 22  20 19  16 15  12 11    0
178 	@      +------+-----------------+------+------+-------+
179 	@ MOVW | cond | 0 0 1 1 0 0 0 0 | imm4 |  Rd  | imm12 |
180 	@      +------+-----------------+------+------+-------+
181 	@
182 	moveq	r0, #0x400000		@ set bit 22, mov to mvn instruction
183 	mov	r3, r6, lsr #16		@ put offset bits 31-16 into r3
184 	mov	r6, r6, lsr #24		@ put offset bits 31-24 into r6
185 	and	r3, r3, #0xf0		@ only keep offset bits 23-20 in r3
186 	b	.Lnext
187 .Lloop:	ldr	ip, [r7, r4]
188 #ifdef CONFIG_ARM_LPAE
189 	tst	ip, #PV_BIT24		@ ADD/SUB have bit 24 clear
190 	beq	1f
191 ARM_BE8(rev	ip, ip)
192 	tst	ip, #0xc00000		@ MOVW has bits 23:22 clear
193 	bic	ip, ip, #0x400000	@ clear bit 22
194 	bfc	ip, #0, #12		@ clear imm12 field of MOV[W] instruction
195 	orreq	ip, ip, r6, lsl #4	@ MOVW -> mask in offset bits 31-24
196 	orreq	ip, ip, r3, lsr #4	@ MOVW -> mask in offset bits 23-20
197 	orrne	ip, ip, r0		@ MOV  -> mask in offset bits 7-0 (or bit 22)
198 ARM_BE8(rev	ip, ip)
199 	b	2f
200 1:
201 #endif
202 	tst	ip, #PV_IMMR_MSB		@ rotation value >= 16 ?
203 	bic	ip, ip, #PV_IMM8_MASK
204 	orreq	ip, ip, r6 ARM_BE8(, lsl #24)	@ mask in offset bits 31-24
205 	orrne	ip, ip, r3 ARM_BE8(, lsl #24)	@ mask in offset bits 23-20
206 2:
207 	str	ip, [r7, r4]
208 	add	r4, r4, #4
209 #endif
210 
211 .Lnext:
212 	cmp	r4, r5
213 	ldrcc	r7, [r4]		@ use branch for delay slot
214 	bcc	.Lloop
215 	ret	lr
216 ENDPROC(__fixup_a_pv_table)
217 
218 ENTRY(fixup_pv_table)
219 	stmfd	sp!, {r4 - r7, lr}
220 	mov	r4, r0			@ r0 = table start
221 	add	r5, r0, r1		@ r1 = table size
222 	bl	__fixup_a_pv_table
223 	ldmfd	sp!, {r4 - r7, pc}
224 ENDPROC(fixup_pv_table)
225 
226 	.data
227 	.align	2
228 	.globl	__pv_phys_pfn_offset
229 	.type	__pv_phys_pfn_offset, %object
230 __pv_phys_pfn_offset:
231 	.word	0
232 	.size	__pv_phys_pfn_offset, . -__pv_phys_pfn_offset
233 
234 	.globl	__pv_offset
235 	.type	__pv_offset, %object
236 __pv_offset:
237 	.quad	0
238 	.size	__pv_offset, . -__pv_offset
239