1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3  * ultra.S: Don't expand these all over the place...
4  *
5  * Copyright (C) 1997, 2000, 2008 David S. Miller (davem@davemloft.net)
6  */
7 
8 #include <linux/pgtable.h>
9 #include <asm/asi.h>
10 #include <asm/page.h>
11 #include <asm/spitfire.h>
12 #include <asm/mmu_context.h>
13 #include <asm/mmu.h>
14 #include <asm/pil.h>
15 #include <asm/head.h>
16 #include <asm/thread_info.h>
17 #include <asm/cacheflush.h>
18 #include <asm/hypervisor.h>
19 #include <asm/cpudata.h>
20 
21 	/* Basically, most of the Spitfire vs. Cheetah madness
22 	 * has to do with the fact that Cheetah does not support
23 	 * IMMU flushes out of the secondary context.  Someone needs
24 	 * to throw a south lake birthday party for the folks
25 	 * in Microelectronics who refused to fix this shit.
26 	 */
27 
28 	/* This file is meant to be read efficiently by the CPU, not humans.
29 	 * Staraj sie tego nikomu nie pierdolnac...
30 	 */
31 	.text
32 	.align		32
33 	.globl		__flush_tlb_mm
34 __flush_tlb_mm:		/* 19 insns */
35 	/* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */
36 	ldxa		[%o1] ASI_DMMU, %g2
37 	cmp		%g2, %o0
38 	bne,pn		%icc, __spitfire_flush_tlb_mm_slow
39 	 mov		0x50, %g3
40 	stxa		%g0, [%g3] ASI_DMMU_DEMAP
41 	stxa		%g0, [%g3] ASI_IMMU_DEMAP
42 	sethi		%hi(KERNBASE), %g3
43 	flush		%g3
44 	retl
45 	 nop
46 	nop
47 	nop
48 	nop
49 	nop
50 	nop
51 	nop
52 	nop
53 	nop
54 	nop
55 
56 	.align		32
57 	.globl		__flush_tlb_page
58 __flush_tlb_page:	/* 22 insns */
59 	/* %o0 = context, %o1 = vaddr */
60 	rdpr		%pstate, %g7
61 	andn		%g7, PSTATE_IE, %g2
62 	wrpr		%g2, %pstate
63 	mov		SECONDARY_CONTEXT, %o4
64 	ldxa		[%o4] ASI_DMMU, %g2
65 	stxa		%o0, [%o4] ASI_DMMU
66 	andcc		%o1, 1, %g0
67 	andn		%o1, 1, %o3
68 	be,pn		%icc, 1f
69 	 or		%o3, 0x10, %o3
70 	stxa		%g0, [%o3] ASI_IMMU_DEMAP
71 1:	stxa		%g0, [%o3] ASI_DMMU_DEMAP
72 	membar		#Sync
73 	stxa		%g2, [%o4] ASI_DMMU
74 	sethi		%hi(KERNBASE), %o4
75 	flush		%o4
76 	retl
77 	 wrpr		%g7, 0x0, %pstate
78 	nop
79 	nop
80 	nop
81 	nop
82 
83 	.align		32
84 	.globl		__flush_tlb_pending
85 __flush_tlb_pending:	/* 27 insns */
86 	/* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
87 	rdpr		%pstate, %g7
88 	sllx		%o1, 3, %o1
89 	andn		%g7, PSTATE_IE, %g2
90 	wrpr		%g2, %pstate
91 	mov		SECONDARY_CONTEXT, %o4
92 	ldxa		[%o4] ASI_DMMU, %g2
93 	stxa		%o0, [%o4] ASI_DMMU
94 1:	sub		%o1, (1 << 3), %o1
95 	ldx		[%o2 + %o1], %o3
96 	andcc		%o3, 1, %g0
97 	andn		%o3, 1, %o3
98 	be,pn		%icc, 2f
99 	 or		%o3, 0x10, %o3
100 	stxa		%g0, [%o3] ASI_IMMU_DEMAP
101 2:	stxa		%g0, [%o3] ASI_DMMU_DEMAP
102 	membar		#Sync
103 	brnz,pt		%o1, 1b
104 	 nop
105 	stxa		%g2, [%o4] ASI_DMMU
106 	sethi		%hi(KERNBASE), %o4
107 	flush		%o4
108 	retl
109 	 wrpr		%g7, 0x0, %pstate
110 	nop
111 	nop
112 	nop
113 	nop
114 
115 	.align		32
116 	.globl		__flush_tlb_kernel_range
117 __flush_tlb_kernel_range:	/* 31 insns */
118 	/* %o0=start, %o1=end */
119 	cmp		%o0, %o1
120 	be,pn		%xcc, 2f
121 	 sub		%o1, %o0, %o3
122 	srlx		%o3, 18, %o4
123 	brnz,pn		%o4, __spitfire_flush_tlb_kernel_range_slow
124 	 sethi		%hi(PAGE_SIZE), %o4
125 	sub		%o3, %o4, %o3
126 	or		%o0, 0x20, %o0		! Nucleus
127 1:	stxa		%g0, [%o0 + %o3] ASI_DMMU_DEMAP
128 	stxa		%g0, [%o0 + %o3] ASI_IMMU_DEMAP
129 	membar		#Sync
130 	brnz,pt		%o3, 1b
131 	 sub		%o3, %o4, %o3
132 2:	sethi		%hi(KERNBASE), %o3
133 	flush		%o3
134 	retl
135 	 nop
136 	nop
137 	nop
138 	nop
139 	nop
140 	nop
141 	nop
142 	nop
143 	nop
144 	nop
145 	nop
146 	nop
147 	nop
148 	nop
149 	nop
150 
151 __spitfire_flush_tlb_kernel_range_slow:
152 	mov		63 * 8, %o4
153 1:	ldxa		[%o4] ASI_ITLB_DATA_ACCESS, %o3
154 	andcc		%o3, 0x40, %g0			/* _PAGE_L_4U */
155 	bne,pn		%xcc, 2f
156 	 mov		TLB_TAG_ACCESS, %o3
157 	stxa		%g0, [%o3] ASI_IMMU
158 	stxa		%g0, [%o4] ASI_ITLB_DATA_ACCESS
159 	membar		#Sync
160 2:	ldxa		[%o4] ASI_DTLB_DATA_ACCESS, %o3
161 	andcc		%o3, 0x40, %g0
162 	bne,pn		%xcc, 2f
163 	 mov		TLB_TAG_ACCESS, %o3
164 	stxa		%g0, [%o3] ASI_DMMU
165 	stxa		%g0, [%o4] ASI_DTLB_DATA_ACCESS
166 	membar		#Sync
167 2:	sub		%o4, 8, %o4
168 	brgez,pt	%o4, 1b
169 	 nop
170 	retl
171 	 nop
172 
173 __spitfire_flush_tlb_mm_slow:
174 	rdpr		%pstate, %g1
175 	wrpr		%g1, PSTATE_IE, %pstate
176 	stxa		%o0, [%o1] ASI_DMMU
177 	stxa		%g0, [%g3] ASI_DMMU_DEMAP
178 	stxa		%g0, [%g3] ASI_IMMU_DEMAP
179 	flush		%g6
180 	stxa		%g2, [%o1] ASI_DMMU
181 	sethi		%hi(KERNBASE), %o1
182 	flush		%o1
183 	retl
184 	 wrpr		%g1, 0, %pstate
185 
186 /*
187  * The following code flushes one page_size worth.
188  */
189 	.section .kprobes.text, "ax"
190 	.align		32
191 	.globl		__flush_icache_page
192 __flush_icache_page:	/* %o0 = phys_page */
193 	srlx		%o0, PAGE_SHIFT, %o0
194 	sethi		%hi(PAGE_OFFSET), %g1
195 	sllx		%o0, PAGE_SHIFT, %o0
196 	sethi		%hi(PAGE_SIZE), %g2
197 	ldx		[%g1 + %lo(PAGE_OFFSET)], %g1
198 	add		%o0, %g1, %o0
199 1:	subcc		%g2, 32, %g2
200 	bne,pt		%icc, 1b
201 	 flush		%o0 + %g2
202 	retl
203 	 nop
204 
205 #ifdef DCACHE_ALIASING_POSSIBLE
206 
207 #if (PAGE_SHIFT != 13)
208 #error only page shift of 13 is supported by dcache flush
209 #endif
210 
211 #define DTAG_MASK 0x3
212 
213 	/* This routine is Spitfire specific so the hardcoded
214 	 * D-cache size and line-size are OK.
215 	 */
216 	.align		64
217 	.globl		__flush_dcache_page
218 __flush_dcache_page:	/* %o0=kaddr, %o1=flush_icache */
219 	sethi		%hi(PAGE_OFFSET), %g1
220 	ldx		[%g1 + %lo(PAGE_OFFSET)], %g1
221 	sub		%o0, %g1, %o0			! physical address
222 	srlx		%o0, 11, %o0			! make D-cache TAG
223 	sethi		%hi(1 << 14), %o2		! D-cache size
224 	sub		%o2, (1 << 5), %o2		! D-cache line size
225 1:	ldxa		[%o2] ASI_DCACHE_TAG, %o3	! load D-cache TAG
226 	andcc		%o3, DTAG_MASK, %g0		! Valid?
227 	be,pn		%xcc, 2f			! Nope, branch
228 	 andn		%o3, DTAG_MASK, %o3		! Clear valid bits
229 	cmp		%o3, %o0			! TAG match?
230 	bne,pt		%xcc, 2f			! Nope, branch
231 	 nop
232 	stxa		%g0, [%o2] ASI_DCACHE_TAG	! Invalidate TAG
233 	membar		#Sync
234 2:	brnz,pt		%o2, 1b
235 	 sub		%o2, (1 << 5), %o2		! D-cache line size
236 
237 	/* The I-cache does not snoop local stores so we
238 	 * better flush that too when necessary.
239 	 */
240 	brnz,pt		%o1, __flush_icache_page
241 	 sllx		%o0, 11, %o0
242 	retl
243 	 nop
244 
245 #endif /* DCACHE_ALIASING_POSSIBLE */
246 
247 	.previous
248 
249 	/* Cheetah specific versions, patched at boot time. */
250 __cheetah_flush_tlb_mm: /* 19 insns */
251 	rdpr		%pstate, %g7
252 	andn		%g7, PSTATE_IE, %g2
253 	wrpr		%g2, 0x0, %pstate
254 	wrpr		%g0, 1, %tl
255 	mov		PRIMARY_CONTEXT, %o2
256 	mov		0x40, %g3
257 	ldxa		[%o2] ASI_DMMU, %g2
258 	srlx		%g2, CTX_PGSZ1_NUC_SHIFT, %o1
259 	sllx		%o1, CTX_PGSZ1_NUC_SHIFT, %o1
260 	or		%o0, %o1, %o0	/* Preserve nucleus page size fields */
261 	stxa		%o0, [%o2] ASI_DMMU
262 	stxa		%g0, [%g3] ASI_DMMU_DEMAP
263 	stxa		%g0, [%g3] ASI_IMMU_DEMAP
264 	stxa		%g2, [%o2] ASI_DMMU
265 	sethi		%hi(KERNBASE), %o2
266 	flush		%o2
267 	wrpr		%g0, 0, %tl
268 	retl
269 	 wrpr		%g7, 0x0, %pstate
270 
271 __cheetah_flush_tlb_page:	/* 22 insns */
272 	/* %o0 = context, %o1 = vaddr */
273 	rdpr		%pstate, %g7
274 	andn		%g7, PSTATE_IE, %g2
275 	wrpr		%g2, 0x0, %pstate
276 	wrpr		%g0, 1, %tl
277 	mov		PRIMARY_CONTEXT, %o4
278 	ldxa		[%o4] ASI_DMMU, %g2
279 	srlx		%g2, CTX_PGSZ1_NUC_SHIFT, %o3
280 	sllx		%o3, CTX_PGSZ1_NUC_SHIFT, %o3
281 	or		%o0, %o3, %o0	/* Preserve nucleus page size fields */
282 	stxa		%o0, [%o4] ASI_DMMU
283 	andcc		%o1, 1, %g0
284 	be,pn		%icc, 1f
285 	 andn		%o1, 1, %o3
286 	stxa		%g0, [%o3] ASI_IMMU_DEMAP
287 1:	stxa		%g0, [%o3] ASI_DMMU_DEMAP
288 	membar		#Sync
289 	stxa		%g2, [%o4] ASI_DMMU
290 	sethi		%hi(KERNBASE), %o4
291 	flush		%o4
292 	wrpr		%g0, 0, %tl
293 	retl
294 	 wrpr		%g7, 0x0, %pstate
295 
296 __cheetah_flush_tlb_pending:	/* 27 insns */
297 	/* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
298 	rdpr		%pstate, %g7
299 	sllx		%o1, 3, %o1
300 	andn		%g7, PSTATE_IE, %g2
301 	wrpr		%g2, 0x0, %pstate
302 	wrpr		%g0, 1, %tl
303 	mov		PRIMARY_CONTEXT, %o4
304 	ldxa		[%o4] ASI_DMMU, %g2
305 	srlx		%g2, CTX_PGSZ1_NUC_SHIFT, %o3
306 	sllx		%o3, CTX_PGSZ1_NUC_SHIFT, %o3
307 	or		%o0, %o3, %o0	/* Preserve nucleus page size fields */
308 	stxa		%o0, [%o4] ASI_DMMU
309 1:	sub		%o1, (1 << 3), %o1
310 	ldx		[%o2 + %o1], %o3
311 	andcc		%o3, 1, %g0
312 	be,pn		%icc, 2f
313 	 andn		%o3, 1, %o3
314 	stxa		%g0, [%o3] ASI_IMMU_DEMAP
315 2:	stxa		%g0, [%o3] ASI_DMMU_DEMAP
316 	membar		#Sync
317 	brnz,pt		%o1, 1b
318 	 nop
319 	stxa		%g2, [%o4] ASI_DMMU
320 	sethi		%hi(KERNBASE), %o4
321 	flush		%o4
322 	wrpr		%g0, 0, %tl
323 	retl
324 	 wrpr		%g7, 0x0, %pstate
325 
326 __cheetah_flush_tlb_kernel_range:	/* 31 insns */
327 	/* %o0=start, %o1=end */
328 	cmp		%o0, %o1
329 	be,pn		%xcc, 2f
330 	 sub		%o1, %o0, %o3
331 	srlx		%o3, 18, %o4
332 	brnz,pn		%o4, 3f
333 	 sethi		%hi(PAGE_SIZE), %o4
334 	sub		%o3, %o4, %o3
335 	or		%o0, 0x20, %o0		! Nucleus
336 1:	stxa		%g0, [%o0 + %o3] ASI_DMMU_DEMAP
337 	stxa		%g0, [%o0 + %o3] ASI_IMMU_DEMAP
338 	membar		#Sync
339 	brnz,pt		%o3, 1b
340 	 sub		%o3, %o4, %o3
341 2:	sethi		%hi(KERNBASE), %o3
342 	flush		%o3
343 	retl
344 	 nop
345 3:	mov		0x80, %o4
346 	stxa		%g0, [%o4] ASI_DMMU_DEMAP
347 	membar		#Sync
348 	stxa		%g0, [%o4] ASI_IMMU_DEMAP
349 	membar		#Sync
350 	retl
351 	 nop
352 	nop
353 	nop
354 	nop
355 	nop
356 	nop
357 	nop
358 	nop
359 
360 #ifdef DCACHE_ALIASING_POSSIBLE
361 __cheetah_flush_dcache_page: /* 11 insns */
362 	sethi		%hi(PAGE_OFFSET), %g1
363 	ldx		[%g1 + %lo(PAGE_OFFSET)], %g1
364 	sub		%o0, %g1, %o0
365 	sethi		%hi(PAGE_SIZE), %o4
366 1:	subcc		%o4, (1 << 5), %o4
367 	stxa		%g0, [%o0 + %o4] ASI_DCACHE_INVALIDATE
368 	membar		#Sync
369 	bne,pt		%icc, 1b
370 	 nop
371 	retl		/* I-cache flush never needed on Cheetah, see callers. */
372 	 nop
373 #endif /* DCACHE_ALIASING_POSSIBLE */
374 
375 	/* Hypervisor specific versions, patched at boot time.  */
376 __hypervisor_tlb_tl0_error:
377 	save		%sp, -192, %sp
378 	mov		%i0, %o0
379 	call		hypervisor_tlbop_error
380 	 mov		%i1, %o1
381 	ret
382 	 restore
383 
384 __hypervisor_flush_tlb_mm: /* 19 insns */
385 	mov		%o0, %o2	/* ARG2: mmu context */
386 	mov		0, %o0		/* ARG0: CPU lists unimplemented */
387 	mov		0, %o1		/* ARG1: CPU lists unimplemented */
388 	mov		HV_MMU_ALL, %o3	/* ARG3: flags */
389 	mov		HV_FAST_MMU_DEMAP_CTX, %o5
390 	ta		HV_FAST_TRAP
391 	brnz,pn		%o0, 1f
392 	 mov		HV_FAST_MMU_DEMAP_CTX, %o1
393 	retl
394 	 nop
395 1:	sethi		%hi(__hypervisor_tlb_tl0_error), %o5
396 	jmpl		%o5 + %lo(__hypervisor_tlb_tl0_error), %g0
397 	 nop
398 	nop
399 	nop
400 	nop
401 	nop
402 	nop
403 	nop
404 
405 __hypervisor_flush_tlb_page: /* 22 insns */
406 	/* %o0 = context, %o1 = vaddr */
407 	mov		%o0, %g2
408 	mov		%o1, %o0              /* ARG0: vaddr + IMMU-bit */
409 	mov		%g2, %o1	      /* ARG1: mmu context */
410 	mov		HV_MMU_ALL, %o2	      /* ARG2: flags */
411 	srlx		%o0, PAGE_SHIFT, %o0
412 	sllx		%o0, PAGE_SHIFT, %o0
413 	ta		HV_MMU_UNMAP_ADDR_TRAP
414 	brnz,pn		%o0, 1f
415 	 mov		HV_MMU_UNMAP_ADDR_TRAP, %o1
416 	retl
417 	 nop
418 1:	sethi		%hi(__hypervisor_tlb_tl0_error), %o2
419 	jmpl		%o2 + %lo(__hypervisor_tlb_tl0_error), %g0
420 	 nop
421 	nop
422 	nop
423 	nop
424 	nop
425 	nop
426 	nop
427 	nop
428 	nop
429 
430 __hypervisor_flush_tlb_pending: /* 27 insns */
431 	/* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
432 	sllx		%o1, 3, %g1
433 	mov		%o2, %g2
434 	mov		%o0, %g3
435 1:	sub		%g1, (1 << 3), %g1
436 	ldx		[%g2 + %g1], %o0      /* ARG0: vaddr + IMMU-bit */
437 	mov		%g3, %o1	      /* ARG1: mmu context */
438 	mov		HV_MMU_ALL, %o2	      /* ARG2: flags */
439 	srlx		%o0, PAGE_SHIFT, %o0
440 	sllx		%o0, PAGE_SHIFT, %o0
441 	ta		HV_MMU_UNMAP_ADDR_TRAP
442 	brnz,pn		%o0, 1f
443 	 mov		HV_MMU_UNMAP_ADDR_TRAP, %o1
444 	brnz,pt		%g1, 1b
445 	 nop
446 	retl
447 	 nop
448 1:	sethi		%hi(__hypervisor_tlb_tl0_error), %o2
449 	jmpl		%o2 + %lo(__hypervisor_tlb_tl0_error), %g0
450 	 nop
451 	nop
452 	nop
453 	nop
454 	nop
455 	nop
456 	nop
457 	nop
458 	nop
459 
460 __hypervisor_flush_tlb_kernel_range: /* 31 insns */
461 	/* %o0=start, %o1=end */
462 	cmp		%o0, %o1
463 	be,pn		%xcc, 2f
464 	 sub		%o1, %o0, %g2
465 	srlx		%g2, 18, %g3
466 	brnz,pn		%g3, 4f
467 	 mov		%o0, %g1
468 	sethi		%hi(PAGE_SIZE), %g3
469 	sub		%g2, %g3, %g2
470 1:	add		%g1, %g2, %o0	/* ARG0: virtual address */
471 	mov		0, %o1		/* ARG1: mmu context */
472 	mov		HV_MMU_ALL, %o2	/* ARG2: flags */
473 	ta		HV_MMU_UNMAP_ADDR_TRAP
474 	brnz,pn		%o0, 3f
475 	 mov		HV_MMU_UNMAP_ADDR_TRAP, %o1
476 	brnz,pt		%g2, 1b
477 	 sub		%g2, %g3, %g2
478 2:	retl
479 	 nop
480 3:	sethi		%hi(__hypervisor_tlb_tl0_error), %o2
481 	jmpl		%o2 + %lo(__hypervisor_tlb_tl0_error), %g0
482 	 nop
483 4:	mov		0, %o0		/* ARG0: CPU lists unimplemented */
484 	mov		0, %o1		/* ARG1: CPU lists unimplemented */
485 	mov		0, %o2		/* ARG2: mmu context == nucleus */
486 	mov		HV_MMU_ALL, %o3	/* ARG3: flags */
487 	mov		HV_FAST_MMU_DEMAP_CTX, %o5
488 	ta		HV_FAST_TRAP
489 	brnz,pn		%o0, 3b
490 	 mov		HV_FAST_MMU_DEMAP_CTX, %o1
491 	retl
492 	 nop
493 
494 #ifdef DCACHE_ALIASING_POSSIBLE
495 	/* XXX Niagara and friends have an 8K cache, so no aliasing is
496 	 * XXX possible, but nothing explicit in the Hypervisor API
497 	 * XXX guarantees this.
498 	 */
499 __hypervisor_flush_dcache_page:	/* 2 insns */
500 	retl
501 	 nop
502 #endif
503 
504 tlb_patch_one:
505 1:	lduw		[%o1], %g1
506 	stw		%g1, [%o0]
507 	flush		%o0
508 	subcc		%o2, 1, %o2
509 	add		%o1, 4, %o1
510 	bne,pt		%icc, 1b
511 	 add		%o0, 4, %o0
512 	retl
513 	 nop
514 
515 #ifdef CONFIG_SMP
516 	/* These are all called by the slaves of a cross call, at
517 	 * trap level 1, with interrupts fully disabled.
518 	 *
519 	 * Register usage:
520 	 *   %g5	mm->context	(all tlb flushes)
521 	 *   %g1	address arg 1	(tlb page and range flushes)
522 	 *   %g7	address arg 2	(tlb range flush only)
523 	 *
524 	 *   %g6	scratch 1
525 	 *   %g2	scratch 2
526 	 *   %g3	scratch 3
527 	 *   %g4	scratch 4
528 	 */
529 	.align		32
530 	.globl		xcall_flush_tlb_mm
531 xcall_flush_tlb_mm:	/* 24 insns */
532 	mov		PRIMARY_CONTEXT, %g2
533 	ldxa		[%g2] ASI_DMMU, %g3
534 	srlx		%g3, CTX_PGSZ1_NUC_SHIFT, %g4
535 	sllx		%g4, CTX_PGSZ1_NUC_SHIFT, %g4
536 	or		%g5, %g4, %g5	/* Preserve nucleus page size fields */
537 	stxa		%g5, [%g2] ASI_DMMU
538 	mov		0x40, %g4
539 	stxa		%g0, [%g4] ASI_DMMU_DEMAP
540 	stxa		%g0, [%g4] ASI_IMMU_DEMAP
541 	stxa		%g3, [%g2] ASI_DMMU
542 	retry
543 	nop
544 	nop
545 	nop
546 	nop
547 	nop
548 	nop
549 	nop
550 	nop
551 	nop
552 	nop
553 	nop
554 	nop
555 	nop
556 
557 	.globl		xcall_flush_tlb_page
558 xcall_flush_tlb_page:	/* 20 insns */
559 	/* %g5=context, %g1=vaddr */
560 	mov		PRIMARY_CONTEXT, %g4
561 	ldxa		[%g4] ASI_DMMU, %g2
562 	srlx		%g2, CTX_PGSZ1_NUC_SHIFT, %g4
563 	sllx		%g4, CTX_PGSZ1_NUC_SHIFT, %g4
564 	or		%g5, %g4, %g5
565 	mov		PRIMARY_CONTEXT, %g4
566 	stxa		%g5, [%g4] ASI_DMMU
567 	andcc		%g1, 0x1, %g0
568 	be,pn		%icc, 2f
569 	 andn		%g1, 0x1, %g5
570 	stxa		%g0, [%g5] ASI_IMMU_DEMAP
571 2:	stxa		%g0, [%g5] ASI_DMMU_DEMAP
572 	membar		#Sync
573 	stxa		%g2, [%g4] ASI_DMMU
574 	retry
575 	nop
576 	nop
577 	nop
578 	nop
579 	nop
580 
581 	.globl		xcall_flush_tlb_kernel_range
582 xcall_flush_tlb_kernel_range:	/* 44 insns */
583 	sethi		%hi(PAGE_SIZE - 1), %g2
584 	or		%g2, %lo(PAGE_SIZE - 1), %g2
585 	andn		%g1, %g2, %g1
586 	andn		%g7, %g2, %g7
587 	sub		%g7, %g1, %g3
588 	srlx		%g3, 18, %g2
589 	brnz,pn		%g2, 2f
590 	 sethi		%hi(PAGE_SIZE), %g2
591 	sub		%g3, %g2, %g3
592 	or		%g1, 0x20, %g1		! Nucleus
593 1:	stxa		%g0, [%g1 + %g3] ASI_DMMU_DEMAP
594 	stxa		%g0, [%g1 + %g3] ASI_IMMU_DEMAP
595 	membar		#Sync
596 	brnz,pt		%g3, 1b
597 	 sub		%g3, %g2, %g3
598 	retry
599 2:	mov		63 * 8, %g1
600 1:	ldxa		[%g1] ASI_ITLB_DATA_ACCESS, %g2
601 	andcc		%g2, 0x40, %g0			/* _PAGE_L_4U */
602 	bne,pn		%xcc, 2f
603 	 mov		TLB_TAG_ACCESS, %g2
604 	stxa		%g0, [%g2] ASI_IMMU
605 	stxa		%g0, [%g1] ASI_ITLB_DATA_ACCESS
606 	membar		#Sync
607 2:	ldxa		[%g1] ASI_DTLB_DATA_ACCESS, %g2
608 	andcc		%g2, 0x40, %g0
609 	bne,pn		%xcc, 2f
610 	 mov		TLB_TAG_ACCESS, %g2
611 	stxa		%g0, [%g2] ASI_DMMU
612 	stxa		%g0, [%g1] ASI_DTLB_DATA_ACCESS
613 	membar		#Sync
614 2:	sub		%g1, 8, %g1
615 	brgez,pt	%g1, 1b
616 	 nop
617 	retry
618 	nop
619 	nop
620 	nop
621 	nop
622 	nop
623 	nop
624 	nop
625 	nop
626 	nop
627 
628 	/* This runs in a very controlled environment, so we do
629 	 * not need to worry about BH races etc.
630 	 */
631 	.globl		xcall_sync_tick
632 xcall_sync_tick:
633 
634 661:	rdpr		%pstate, %g2
635 	wrpr		%g2, PSTATE_IG | PSTATE_AG, %pstate
636 	.section	.sun4v_2insn_patch, "ax"
637 	.word		661b
638 	nop
639 	nop
640 	.previous
641 
642 	rdpr		%pil, %g2
643 	wrpr		%g0, PIL_NORMAL_MAX, %pil
644 	sethi		%hi(109f), %g7
645 	b,pt		%xcc, etrap_irq
646 109:	 or		%g7, %lo(109b), %g7
647 #ifdef CONFIG_TRACE_IRQFLAGS
648 	call		trace_hardirqs_off
649 	 nop
650 #endif
651 	call		smp_synchronize_tick_client
652 	 nop
653 	b		rtrap_xcall
654 	 ldx		[%sp + PTREGS_OFF + PT_V9_TSTATE], %l1
655 
656 	.globl		xcall_fetch_glob_regs
657 xcall_fetch_glob_regs:
658 	sethi		%hi(global_cpu_snapshot), %g1
659 	or		%g1, %lo(global_cpu_snapshot), %g1
660 	__GET_CPUID(%g2)
661 	sllx		%g2, 6, %g3
662 	add		%g1, %g3, %g1
663 	rdpr		%tstate, %g7
664 	stx		%g7, [%g1 + GR_SNAP_TSTATE]
665 	rdpr		%tpc, %g7
666 	stx		%g7, [%g1 + GR_SNAP_TPC]
667 	rdpr		%tnpc, %g7
668 	stx		%g7, [%g1 + GR_SNAP_TNPC]
669 	stx		%o7, [%g1 + GR_SNAP_O7]
670 	stx		%i7, [%g1 + GR_SNAP_I7]
671 	/* Don't try this at home kids... */
672 	rdpr		%cwp, %g3
673 	sub		%g3, 1, %g7
674 	wrpr		%g7, %cwp
675 	mov		%i7, %g7
676 	wrpr		%g3, %cwp
677 	stx		%g7, [%g1 + GR_SNAP_RPC]
678 	sethi		%hi(trap_block), %g7
679 	or		%g7, %lo(trap_block), %g7
680 	sllx		%g2, TRAP_BLOCK_SZ_SHIFT, %g2
681 	add		%g7, %g2, %g7
682 	ldx		[%g7 + TRAP_PER_CPU_THREAD], %g3
683 	stx		%g3, [%g1 + GR_SNAP_THREAD]
684 	retry
685 
686 	.globl		xcall_fetch_glob_pmu
687 xcall_fetch_glob_pmu:
688 	sethi		%hi(global_cpu_snapshot), %g1
689 	or		%g1, %lo(global_cpu_snapshot), %g1
690 	__GET_CPUID(%g2)
691 	sllx		%g2, 6, %g3
692 	add		%g1, %g3, %g1
693 	rd		%pic, %g7
694 	stx		%g7, [%g1 + (4 * 8)]
695 	rd		%pcr, %g7
696 	stx		%g7, [%g1 + (0 * 8)]
697 	retry
698 
699 	.globl		xcall_fetch_glob_pmu_n4
700 xcall_fetch_glob_pmu_n4:
701 	sethi		%hi(global_cpu_snapshot), %g1
702 	or		%g1, %lo(global_cpu_snapshot), %g1
703 	__GET_CPUID(%g2)
704 	sllx		%g2, 6, %g3
705 	add		%g1, %g3, %g1
706 
707 	ldxa		[%g0] ASI_PIC, %g7
708 	stx		%g7, [%g1 + (4 * 8)]
709 	mov		0x08, %g3
710 	ldxa		[%g3] ASI_PIC, %g7
711 	stx		%g7, [%g1 + (5 * 8)]
712 	mov		0x10, %g3
713 	ldxa		[%g3] ASI_PIC, %g7
714 	stx		%g7, [%g1 + (6 * 8)]
715 	mov		0x18, %g3
716 	ldxa		[%g3] ASI_PIC, %g7
717 	stx		%g7, [%g1 + (7 * 8)]
718 
719 	mov		%o0, %g2
720 	mov		%o1, %g3
721 	mov		%o5, %g7
722 
723 	mov		HV_FAST_VT_GET_PERFREG, %o5
724 	mov		3, %o0
725 	ta		HV_FAST_TRAP
726 	stx		%o1, [%g1 + (3 * 8)]
727 	mov		HV_FAST_VT_GET_PERFREG, %o5
728 	mov		2, %o0
729 	ta		HV_FAST_TRAP
730 	stx		%o1, [%g1 + (2 * 8)]
731 	mov		HV_FAST_VT_GET_PERFREG, %o5
732 	mov		1, %o0
733 	ta		HV_FAST_TRAP
734 	stx		%o1, [%g1 + (1 * 8)]
735 	mov		HV_FAST_VT_GET_PERFREG, %o5
736 	mov		0, %o0
737 	ta		HV_FAST_TRAP
738 	stx		%o1, [%g1 + (0 * 8)]
739 
740 	mov		%g2, %o0
741 	mov		%g3, %o1
742 	mov		%g7, %o5
743 
744 	retry
745 
746 __cheetah_xcall_flush_tlb_kernel_range:	/* 44 insns */
747 	sethi		%hi(PAGE_SIZE - 1), %g2
748 	or		%g2, %lo(PAGE_SIZE - 1), %g2
749 	andn		%g1, %g2, %g1
750 	andn		%g7, %g2, %g7
751 	sub		%g7, %g1, %g3
752 	srlx		%g3, 18, %g2
753 	brnz,pn		%g2, 2f
754 	 sethi		%hi(PAGE_SIZE), %g2
755 	sub		%g3, %g2, %g3
756 	or		%g1, 0x20, %g1		! Nucleus
757 1:	stxa		%g0, [%g1 + %g3] ASI_DMMU_DEMAP
758 	stxa		%g0, [%g1 + %g3] ASI_IMMU_DEMAP
759 	membar		#Sync
760 	brnz,pt		%g3, 1b
761 	 sub		%g3, %g2, %g3
762 	retry
763 2:	mov		0x80, %g2
764 	stxa		%g0, [%g2] ASI_DMMU_DEMAP
765 	membar		#Sync
766 	stxa		%g0, [%g2] ASI_IMMU_DEMAP
767 	membar		#Sync
768 	retry
769 	nop
770 	nop
771 	nop
772 	nop
773 	nop
774 	nop
775 	nop
776 	nop
777 	nop
778 	nop
779 	nop
780 	nop
781 	nop
782 	nop
783 	nop
784 	nop
785 	nop
786 	nop
787 	nop
788 	nop
789 	nop
790 	nop
791 
792 #ifdef DCACHE_ALIASING_POSSIBLE
793 	.align		32
794 	.globl		xcall_flush_dcache_page_cheetah
795 xcall_flush_dcache_page_cheetah: /* %g1 == physical page address */
796 	sethi		%hi(PAGE_SIZE), %g3
797 1:	subcc		%g3, (1 << 5), %g3
798 	stxa		%g0, [%g1 + %g3] ASI_DCACHE_INVALIDATE
799 	membar		#Sync
800 	bne,pt		%icc, 1b
801 	 nop
802 	retry
803 	nop
804 #endif /* DCACHE_ALIASING_POSSIBLE */
805 
806 	.globl		xcall_flush_dcache_page_spitfire
807 xcall_flush_dcache_page_spitfire: /* %g1 == physical page address
808 				     %g7 == kernel page virtual address
809 				     %g5 == (page->mapping != NULL)  */
810 #ifdef DCACHE_ALIASING_POSSIBLE
811 	srlx		%g1, (13 - 2), %g1	! Form tag comparitor
812 	sethi		%hi(L1DCACHE_SIZE), %g3	! D$ size == 16K
813 	sub		%g3, (1 << 5), %g3	! D$ linesize == 32
814 1:	ldxa		[%g3] ASI_DCACHE_TAG, %g2
815 	andcc		%g2, 0x3, %g0
816 	be,pn		%xcc, 2f
817 	 andn		%g2, 0x3, %g2
818 	cmp		%g2, %g1
819 
820 	bne,pt		%xcc, 2f
821 	 nop
822 	stxa		%g0, [%g3] ASI_DCACHE_TAG
823 	membar		#Sync
824 2:	cmp		%g3, 0
825 	bne,pt		%xcc, 1b
826 	 sub		%g3, (1 << 5), %g3
827 
828 	brz,pn		%g5, 2f
829 #endif /* DCACHE_ALIASING_POSSIBLE */
830 	 sethi		%hi(PAGE_SIZE), %g3
831 
832 1:	flush		%g7
833 	subcc		%g3, (1 << 5), %g3
834 	bne,pt		%icc, 1b
835 	 add		%g7, (1 << 5), %g7
836 
837 2:	retry
838 	nop
839 	nop
840 
841 	/* %g5:	error
842 	 * %g6:	tlb op
843 	 */
844 __hypervisor_tlb_xcall_error:
845 	mov	%g5, %g4
846 	mov	%g6, %g5
847 	ba,pt	%xcc, etrap
848 	 rd	%pc, %g7
849 	mov	%l4, %o0
850 	call	hypervisor_tlbop_error_xcall
851 	 mov	%l5, %o1
852 	ba,a,pt	%xcc, rtrap
853 
854 	.globl		__hypervisor_xcall_flush_tlb_mm
855 __hypervisor_xcall_flush_tlb_mm: /* 24 insns */
856 	/* %g5=ctx, g1,g2,g3,g4,g7=scratch, %g6=unusable */
857 	mov		%o0, %g2
858 	mov		%o1, %g3
859 	mov		%o2, %g4
860 	mov		%o3, %g1
861 	mov		%o5, %g7
862 	clr		%o0		/* ARG0: CPU lists unimplemented */
863 	clr		%o1		/* ARG1: CPU lists unimplemented */
864 	mov		%g5, %o2	/* ARG2: mmu context */
865 	mov		HV_MMU_ALL, %o3	/* ARG3: flags */
866 	mov		HV_FAST_MMU_DEMAP_CTX, %o5
867 	ta		HV_FAST_TRAP
868 	mov		HV_FAST_MMU_DEMAP_CTX, %g6
869 	brnz,pn		%o0, 1f
870 	 mov		%o0, %g5
871 	mov		%g2, %o0
872 	mov		%g3, %o1
873 	mov		%g4, %o2
874 	mov		%g1, %o3
875 	mov		%g7, %o5
876 	membar		#Sync
877 	retry
878 1:	sethi		%hi(__hypervisor_tlb_xcall_error), %g4
879 	jmpl		%g4 + %lo(__hypervisor_tlb_xcall_error), %g0
880 	 nop
881 
882 	.globl		__hypervisor_xcall_flush_tlb_page
883 __hypervisor_xcall_flush_tlb_page: /* 20 insns */
884 	/* %g5=ctx, %g1=vaddr */
885 	mov		%o0, %g2
886 	mov		%o1, %g3
887 	mov		%o2, %g4
888 	mov		%g1, %o0	        /* ARG0: virtual address */
889 	mov		%g5, %o1		/* ARG1: mmu context */
890 	mov		HV_MMU_ALL, %o2		/* ARG2: flags */
891 	srlx		%o0, PAGE_SHIFT, %o0
892 	sllx		%o0, PAGE_SHIFT, %o0
893 	ta		HV_MMU_UNMAP_ADDR_TRAP
894 	mov		HV_MMU_UNMAP_ADDR_TRAP, %g6
895 	brnz,a,pn	%o0, 1f
896 	 mov		%o0, %g5
897 	mov		%g2, %o0
898 	mov		%g3, %o1
899 	mov		%g4, %o2
900 	membar		#Sync
901 	retry
902 1:	sethi		%hi(__hypervisor_tlb_xcall_error), %g4
903 	jmpl		%g4 + %lo(__hypervisor_tlb_xcall_error), %g0
904 	 nop
905 
906 	.globl		__hypervisor_xcall_flush_tlb_kernel_range
907 __hypervisor_xcall_flush_tlb_kernel_range: /* 44 insns */
908 	/* %g1=start, %g7=end, g2,g3,g4,g5,g6=scratch */
909 	sethi		%hi(PAGE_SIZE - 1), %g2
910 	or		%g2, %lo(PAGE_SIZE - 1), %g2
911 	andn		%g1, %g2, %g1
912 	andn		%g7, %g2, %g7
913 	sub		%g7, %g1, %g3
914 	srlx		%g3, 18, %g7
915 	add		%g2, 1, %g2
916 	sub		%g3, %g2, %g3
917 	mov		%o0, %g2
918 	mov		%o1, %g4
919 	brnz,pn		%g7, 2f
920 	 mov		%o2, %g7
921 1:	add		%g1, %g3, %o0	/* ARG0: virtual address */
922 	mov		0, %o1		/* ARG1: mmu context */
923 	mov		HV_MMU_ALL, %o2	/* ARG2: flags */
924 	ta		HV_MMU_UNMAP_ADDR_TRAP
925 	mov		HV_MMU_UNMAP_ADDR_TRAP, %g6
926 	brnz,pn		%o0, 1f
927 	 mov		%o0, %g5
928 	sethi		%hi(PAGE_SIZE), %o2
929 	brnz,pt		%g3, 1b
930 	 sub		%g3, %o2, %g3
931 5:	mov		%g2, %o0
932 	mov		%g4, %o1
933 	mov		%g7, %o2
934 	membar		#Sync
935 	retry
936 1:	sethi		%hi(__hypervisor_tlb_xcall_error), %g4
937 	jmpl		%g4 + %lo(__hypervisor_tlb_xcall_error), %g0
938 	 nop
939 2:	mov		%o3, %g1
940 	mov		%o5, %g3
941 	mov		0, %o0		/* ARG0: CPU lists unimplemented */
942 	mov		0, %o1		/* ARG1: CPU lists unimplemented */
943 	mov		0, %o2		/* ARG2: mmu context == nucleus */
944 	mov		HV_MMU_ALL, %o3	/* ARG3: flags */
945 	mov		HV_FAST_MMU_DEMAP_CTX, %o5
946 	ta		HV_FAST_TRAP
947 	mov		%g1, %o3
948 	brz,pt		%o0, 5b
949 	 mov		%g3, %o5
950 	mov		HV_FAST_MMU_DEMAP_CTX, %g6
951 	ba,pt		%xcc, 1b
952 	 clr		%g5
953 
954 	/* These just get rescheduled to PIL vectors. */
955 	.globl		xcall_call_function
956 xcall_call_function:
957 	wr		%g0, (1 << PIL_SMP_CALL_FUNC), %set_softint
958 	retry
959 
960 	.globl		xcall_call_function_single
961 xcall_call_function_single:
962 	wr		%g0, (1 << PIL_SMP_CALL_FUNC_SNGL), %set_softint
963 	retry
964 
965 	.globl		xcall_receive_signal
966 xcall_receive_signal:
967 	wr		%g0, (1 << PIL_SMP_RECEIVE_SIGNAL), %set_softint
968 	retry
969 
970 	.globl		xcall_capture
971 xcall_capture:
972 	wr		%g0, (1 << PIL_SMP_CAPTURE), %set_softint
973 	retry
974 
975 #ifdef CONFIG_KGDB
976 	.globl		xcall_kgdb_capture
977 xcall_kgdb_capture:
978 	wr		%g0, (1 << PIL_KGDB_CAPTURE), %set_softint
979 	retry
980 #endif
981 
982 #endif /* CONFIG_SMP */
983 
984 	.globl		cheetah_patch_cachetlbops
985 cheetah_patch_cachetlbops:
986 	save		%sp, -128, %sp
987 
988 	sethi		%hi(__flush_tlb_mm), %o0
989 	or		%o0, %lo(__flush_tlb_mm), %o0
990 	sethi		%hi(__cheetah_flush_tlb_mm), %o1
991 	or		%o1, %lo(__cheetah_flush_tlb_mm), %o1
992 	call		tlb_patch_one
993 	 mov		19, %o2
994 
995 	sethi		%hi(__flush_tlb_page), %o0
996 	or		%o0, %lo(__flush_tlb_page), %o0
997 	sethi		%hi(__cheetah_flush_tlb_page), %o1
998 	or		%o1, %lo(__cheetah_flush_tlb_page), %o1
999 	call		tlb_patch_one
1000 	 mov		22, %o2
1001 
1002 	sethi		%hi(__flush_tlb_pending), %o0
1003 	or		%o0, %lo(__flush_tlb_pending), %o0
1004 	sethi		%hi(__cheetah_flush_tlb_pending), %o1
1005 	or		%o1, %lo(__cheetah_flush_tlb_pending), %o1
1006 	call		tlb_patch_one
1007 	 mov		27, %o2
1008 
1009 	sethi		%hi(__flush_tlb_kernel_range), %o0
1010 	or		%o0, %lo(__flush_tlb_kernel_range), %o0
1011 	sethi		%hi(__cheetah_flush_tlb_kernel_range), %o1
1012 	or		%o1, %lo(__cheetah_flush_tlb_kernel_range), %o1
1013 	call		tlb_patch_one
1014 	 mov		31, %o2
1015 
1016 #ifdef DCACHE_ALIASING_POSSIBLE
1017 	sethi		%hi(__flush_dcache_page), %o0
1018 	or		%o0, %lo(__flush_dcache_page), %o0
1019 	sethi		%hi(__cheetah_flush_dcache_page), %o1
1020 	or		%o1, %lo(__cheetah_flush_dcache_page), %o1
1021 	call		tlb_patch_one
1022 	 mov		11, %o2
1023 #endif /* DCACHE_ALIASING_POSSIBLE */
1024 
1025 #ifdef CONFIG_SMP
1026 	sethi		%hi(xcall_flush_tlb_kernel_range), %o0
1027 	or		%o0, %lo(xcall_flush_tlb_kernel_range), %o0
1028 	sethi		%hi(__cheetah_xcall_flush_tlb_kernel_range), %o1
1029 	or		%o1, %lo(__cheetah_xcall_flush_tlb_kernel_range), %o1
1030 	call		tlb_patch_one
1031 	 mov		44, %o2
1032 #endif /* CONFIG_SMP */
1033 
1034 	ret
1035 	 restore
1036 
1037 	.globl		hypervisor_patch_cachetlbops
1038 hypervisor_patch_cachetlbops:
1039 	save		%sp, -128, %sp
1040 
1041 	sethi		%hi(__flush_tlb_mm), %o0
1042 	or		%o0, %lo(__flush_tlb_mm), %o0
1043 	sethi		%hi(__hypervisor_flush_tlb_mm), %o1
1044 	or		%o1, %lo(__hypervisor_flush_tlb_mm), %o1
1045 	call		tlb_patch_one
1046 	 mov		19, %o2
1047 
1048 	sethi		%hi(__flush_tlb_page), %o0
1049 	or		%o0, %lo(__flush_tlb_page), %o0
1050 	sethi		%hi(__hypervisor_flush_tlb_page), %o1
1051 	or		%o1, %lo(__hypervisor_flush_tlb_page), %o1
1052 	call		tlb_patch_one
1053 	 mov		22, %o2
1054 
1055 	sethi		%hi(__flush_tlb_pending), %o0
1056 	or		%o0, %lo(__flush_tlb_pending), %o0
1057 	sethi		%hi(__hypervisor_flush_tlb_pending), %o1
1058 	or		%o1, %lo(__hypervisor_flush_tlb_pending), %o1
1059 	call		tlb_patch_one
1060 	 mov		27, %o2
1061 
1062 	sethi		%hi(__flush_tlb_kernel_range), %o0
1063 	or		%o0, %lo(__flush_tlb_kernel_range), %o0
1064 	sethi		%hi(__hypervisor_flush_tlb_kernel_range), %o1
1065 	or		%o1, %lo(__hypervisor_flush_tlb_kernel_range), %o1
1066 	call		tlb_patch_one
1067 	 mov		31, %o2
1068 
1069 #ifdef DCACHE_ALIASING_POSSIBLE
1070 	sethi		%hi(__flush_dcache_page), %o0
1071 	or		%o0, %lo(__flush_dcache_page), %o0
1072 	sethi		%hi(__hypervisor_flush_dcache_page), %o1
1073 	or		%o1, %lo(__hypervisor_flush_dcache_page), %o1
1074 	call		tlb_patch_one
1075 	 mov		2, %o2
1076 #endif /* DCACHE_ALIASING_POSSIBLE */
1077 
1078 #ifdef CONFIG_SMP
1079 	sethi		%hi(xcall_flush_tlb_mm), %o0
1080 	or		%o0, %lo(xcall_flush_tlb_mm), %o0
1081 	sethi		%hi(__hypervisor_xcall_flush_tlb_mm), %o1
1082 	or		%o1, %lo(__hypervisor_xcall_flush_tlb_mm), %o1
1083 	call		tlb_patch_one
1084 	 mov		24, %o2
1085 
1086 	sethi		%hi(xcall_flush_tlb_page), %o0
1087 	or		%o0, %lo(xcall_flush_tlb_page), %o0
1088 	sethi		%hi(__hypervisor_xcall_flush_tlb_page), %o1
1089 	or		%o1, %lo(__hypervisor_xcall_flush_tlb_page), %o1
1090 	call		tlb_patch_one
1091 	 mov		20, %o2
1092 
1093 	sethi		%hi(xcall_flush_tlb_kernel_range), %o0
1094 	or		%o0, %lo(xcall_flush_tlb_kernel_range), %o0
1095 	sethi		%hi(__hypervisor_xcall_flush_tlb_kernel_range), %o1
1096 	or		%o1, %lo(__hypervisor_xcall_flush_tlb_kernel_range), %o1
1097 	call		tlb_patch_one
1098 	 mov		44, %o2
1099 #endif /* CONFIG_SMP */
1100 
1101 	ret
1102 	 restore
1103