1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * Copyright (C) 2020 Loongson Technology Corporation Limited
4 */
5#include <asm/asm.h>
6#include <asm/export.h>
7#include <asm/loongarchregs.h>
8#include <asm/page.h>
9#include <asm/pgtable-bits.h>
10#include <asm/regdef.h>
11#include <asm/stackframe.h>
12
13#ifdef CONFIG_64BIT
14#include <asm/pgtable-64.h>
15#endif
16
17#define INVTLB_ADDR_GFALSE_AND_ASID	5
18
19#define PTRS_PER_PGD_BITS	(PAGE_SHIFT - 3)
20#define PTRS_PER_PUD_BITS	(PAGE_SHIFT - 3)
21#define PTRS_PER_PMD_BITS	(PAGE_SHIFT - 3)
22#define PTRS_PER_PTE_BITS	(PAGE_SHIFT - 3)
23
24	.macro tlb_do_page_fault, write
25	SYM_CODE_START(tlb_do_page_fault_\write)
26	SAVE_ALL
27	csrrd		a2, LOONGARCH_CSR_BADV
28	move		a0, sp
29	REG_S		a2, sp, PT_BVADDR
30	li.w		a1, \write
31	la.abs		t0, do_page_fault
32	jirl		ra, t0, 0
33	UNWIND_HINT_REGS
34	RESTORE_ALL_AND_RET
35	SYM_CODE_END(tlb_do_page_fault_\write)
36	.endm
37
38	tlb_do_page_fault 0
39	tlb_do_page_fault 1
40
41SYM_CODE_START(handle_tlb_protect)
42	BACKUP_T0T1
43	SAVE_ALL
44	move		a0, sp
45	move		a1, zero
46	csrrd		a2, LOONGARCH_CSR_BADV
47	REG_S		a2, sp, PT_BVADDR
48	la.abs		t0, do_page_fault
49	jirl		ra, t0, 0
50	UNWIND_HINT_REGS
51	RESTORE_ALL_AND_RET
52SYM_CODE_END(handle_tlb_protect)
53
54SYM_CODE_START(handle_tlb_load)
55	csrwr		t0, EXCEPTION_KS0
56	csrwr		t1, EXCEPTION_KS1
57	csrwr		ra, EXCEPTION_KS2
58
59	/*
60	 * The vmalloc handling is not in the hotpath.
61	 */
62	csrrd		t0, LOONGARCH_CSR_BADV
63	bltz		t0, vmalloc_load
64	csrrd		t1, LOONGARCH_CSR_PGDL
65
66vmalloc_done_load:
67	/* Get PGD offset in bytes */
68	bstrpick.d	ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT
69	alsl.d		t1, ra, t1, 3
70#if CONFIG_PGTABLE_LEVELS > 3
71	ld.d		t1, t1, 0
72	bstrpick.d	ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT
73	alsl.d		t1, ra, t1, 3
74#endif
75#if CONFIG_PGTABLE_LEVELS > 2
76	ld.d		t1, t1, 0
77	bstrpick.d	ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT
78	alsl.d		t1, ra, t1, 3
79#endif
80	ld.d		ra, t1, 0
81
82	/*
83	 * For huge tlb entries, pmde doesn't contain an address but
84	 * instead contains the tlb pte. Check the PAGE_HUGE bit and
85	 * see if we need to jump to huge tlb processing.
86	 */
87	rotri.d		ra, ra, _PAGE_HUGE_SHIFT + 1
88	bltz		ra, tlb_huge_update_load
89
90	rotri.d		ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1)
91	bstrpick.d	t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT
92	alsl.d		t1, t0, ra, _PTE_T_LOG2
93
94#ifdef CONFIG_SMP
95smp_pgtable_change_load:
96	ll.d		t0, t1, 0
97#else
98	ld.d		t0, t1, 0
99#endif
100	andi		ra, t0, _PAGE_PRESENT
101	beqz		ra, nopage_tlb_load
102
103	ori		t0, t0, _PAGE_VALID
104#ifdef CONFIG_SMP
105	sc.d		t0, t1, 0
106	beqz		t0, smp_pgtable_change_load
107#else
108	st.d		t0, t1, 0
109#endif
110	tlbsrch
111	bstrins.d	t1, zero, 3, 3
112	ld.d		t0, t1, 0
113	ld.d		t1, t1, 8
114	csrwr		t0, LOONGARCH_CSR_TLBELO0
115	csrwr		t1, LOONGARCH_CSR_TLBELO1
116	tlbwr
117
118	csrrd		t0, EXCEPTION_KS0
119	csrrd		t1, EXCEPTION_KS1
120	csrrd		ra, EXCEPTION_KS2
121	ertn
122
123#ifdef CONFIG_64BIT
124vmalloc_load:
125	la.abs		t1, swapper_pg_dir
126	b		vmalloc_done_load
127#endif
128
129	/* This is the entry point of a huge page. */
130tlb_huge_update_load:
131#ifdef CONFIG_SMP
132	ll.d		ra, t1, 0
133#endif
134	andi		t0, ra, _PAGE_PRESENT
135	beqz		t0, nopage_tlb_load
136
137#ifdef CONFIG_SMP
138	ori		t0, ra, _PAGE_VALID
139	sc.d		t0, t1, 0
140	beqz		t0, tlb_huge_update_load
141	ori		t0, ra, _PAGE_VALID
142#else
143	rotri.d		ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1)
144	ori		t0, ra, _PAGE_VALID
145	st.d		t0, t1, 0
146#endif
147	csrrd		ra, LOONGARCH_CSR_ASID
148	csrrd		t1, LOONGARCH_CSR_BADV
149	andi		ra, ra, CSR_ASID_ASID
150	invtlb		INVTLB_ADDR_GFALSE_AND_ASID, ra, t1
151
152	/*
153	 * A huge PTE describes an area the size of the
154	 * configured huge page size. This is twice the
155	 * of the large TLB entry size we intend to use.
156	 * A TLB entry half the size of the configured
157	 * huge page size is configured into entrylo0
158	 * and entrylo1 to cover the contiguous huge PTE
159	 * address space.
160	 */
161	/* Huge page: Move Global bit */
162	xori		t0, t0, _PAGE_HUGE
163	lu12i.w		t1, _PAGE_HGLOBAL >> 12
164	and		t1, t0, t1
165	srli.d		t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT)
166	or		t0, t0, t1
167
168	move		ra, t0
169	csrwr		ra, LOONGARCH_CSR_TLBELO0
170
171	/* Convert to entrylo1 */
172	addi.d		t1, zero, 1
173	slli.d		t1, t1, (HPAGE_SHIFT - 1)
174	add.d		t0, t0, t1
175	csrwr		t0, LOONGARCH_CSR_TLBELO1
176
177	/* Set huge page tlb entry size */
178	addu16i.d	t0, zero, (CSR_TLBIDX_PS >> 16)
179	addu16i.d	t1, zero, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16))
180	csrxchg		t1, t0, LOONGARCH_CSR_TLBIDX
181
182	tlbfill
183
184	addu16i.d	t0, zero, (CSR_TLBIDX_PS >> 16)
185	addu16i.d	t1, zero, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16))
186	csrxchg		t1, t0, LOONGARCH_CSR_TLBIDX
187
188	csrrd		t0, EXCEPTION_KS0
189	csrrd		t1, EXCEPTION_KS1
190	csrrd		ra, EXCEPTION_KS2
191	ertn
192
193nopage_tlb_load:
194	dbar		0x700
195	csrrd		ra, EXCEPTION_KS2
196	la.abs		t0, tlb_do_page_fault_0
197	jr		t0
198SYM_CODE_END(handle_tlb_load)
199
200SYM_CODE_START(handle_tlb_load_ptw)
201	csrwr		t0, LOONGARCH_CSR_KS0
202	csrwr		t1, LOONGARCH_CSR_KS1
203	la.abs		t0, tlb_do_page_fault_0
204	jirl		zero, t0, 0
205SYM_CODE_END(handle_tlb_load_ptw)
206
207SYM_CODE_START(handle_tlb_store)
208	csrwr		t0, EXCEPTION_KS0
209	csrwr		t1, EXCEPTION_KS1
210	csrwr		ra, EXCEPTION_KS2
211
212	/*
213	 * The vmalloc handling is not in the hotpath.
214	 */
215	csrrd		t0, LOONGARCH_CSR_BADV
216	bltz		t0, vmalloc_store
217	csrrd		t1, LOONGARCH_CSR_PGDL
218
219vmalloc_done_store:
220	/* Get PGD offset in bytes */
221	bstrpick.d	ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT
222	alsl.d		t1, ra, t1, 3
223#if CONFIG_PGTABLE_LEVELS > 3
224	ld.d		t1, t1, 0
225	bstrpick.d	ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT
226	alsl.d		t1, ra, t1, 3
227#endif
228#if CONFIG_PGTABLE_LEVELS > 2
229	ld.d		t1, t1, 0
230	bstrpick.d	ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT
231	alsl.d		t1, ra, t1, 3
232#endif
233	ld.d		ra, t1, 0
234
235	/*
236	 * For huge tlb entries, pmde doesn't contain an address but
237	 * instead contains the tlb pte. Check the PAGE_HUGE bit and
238	 * see if we need to jump to huge tlb processing.
239	 */
240	rotri.d		ra, ra, _PAGE_HUGE_SHIFT + 1
241	bltz		ra, tlb_huge_update_store
242
243	rotri.d		ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1)
244	bstrpick.d	t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT
245	alsl.d		t1, t0, ra, _PTE_T_LOG2
246
247#ifdef CONFIG_SMP
248smp_pgtable_change_store:
249	ll.d		t0, t1, 0
250#else
251	ld.d		t0, t1, 0
252#endif
253	andi		ra, t0, _PAGE_PRESENT | _PAGE_WRITE
254	xori		ra, ra, _PAGE_PRESENT | _PAGE_WRITE
255	bnez		ra, nopage_tlb_store
256
257	ori		t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
258#ifdef CONFIG_SMP
259	sc.d		t0, t1, 0
260	beqz		t0, smp_pgtable_change_store
261#else
262	st.d		t0, t1, 0
263#endif
264	tlbsrch
265	bstrins.d	t1, zero, 3, 3
266	ld.d		t0, t1, 0
267	ld.d		t1, t1, 8
268	csrwr		t0, LOONGARCH_CSR_TLBELO0
269	csrwr		t1, LOONGARCH_CSR_TLBELO1
270	tlbwr
271
272	csrrd		t0, EXCEPTION_KS0
273	csrrd		t1, EXCEPTION_KS1
274	csrrd		ra, EXCEPTION_KS2
275	ertn
276
277#ifdef CONFIG_64BIT
278vmalloc_store:
279	la.abs		t1, swapper_pg_dir
280	b		vmalloc_done_store
281#endif
282
283	/* This is the entry point of a huge page. */
284tlb_huge_update_store:
285#ifdef CONFIG_SMP
286	ll.d		ra, t1, 0
287#endif
288	andi		t0, ra, _PAGE_PRESENT | _PAGE_WRITE
289	xori		t0, t0, _PAGE_PRESENT | _PAGE_WRITE
290	bnez		t0, nopage_tlb_store
291
292#ifdef CONFIG_SMP
293	ori		t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
294	sc.d		t0, t1, 0
295	beqz		t0, tlb_huge_update_store
296	ori		t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
297#else
298	rotri.d		ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1)
299	ori		t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
300	st.d		t0, t1, 0
301#endif
302	csrrd		ra, LOONGARCH_CSR_ASID
303	csrrd		t1, LOONGARCH_CSR_BADV
304	andi		ra, ra, CSR_ASID_ASID
305	invtlb		INVTLB_ADDR_GFALSE_AND_ASID, ra, t1
306
307	/*
308	 * A huge PTE describes an area the size of the
309	 * configured huge page size. This is twice the
310	 * of the large TLB entry size we intend to use.
311	 * A TLB entry half the size of the configured
312	 * huge page size is configured into entrylo0
313	 * and entrylo1 to cover the contiguous huge PTE
314	 * address space.
315	 */
316	/* Huge page: Move Global bit */
317	xori		t0, t0, _PAGE_HUGE
318	lu12i.w		t1, _PAGE_HGLOBAL >> 12
319	and		t1, t0, t1
320	srli.d		t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT)
321	or		t0, t0, t1
322
323	move		ra, t0
324	csrwr		ra, LOONGARCH_CSR_TLBELO0
325
326	/* Convert to entrylo1 */
327	addi.d		t1, zero, 1
328	slli.d		t1, t1, (HPAGE_SHIFT - 1)
329	add.d		t0, t0, t1
330	csrwr		t0, LOONGARCH_CSR_TLBELO1
331
332	/* Set huge page tlb entry size */
333	addu16i.d	t0, zero, (CSR_TLBIDX_PS >> 16)
334	addu16i.d	t1, zero, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16))
335	csrxchg		t1, t0, LOONGARCH_CSR_TLBIDX
336
337	tlbfill
338
339	/* Reset default page size */
340	addu16i.d	t0, zero, (CSR_TLBIDX_PS >> 16)
341	addu16i.d	t1, zero, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16))
342	csrxchg		t1, t0, LOONGARCH_CSR_TLBIDX
343
344	csrrd		t0, EXCEPTION_KS0
345	csrrd		t1, EXCEPTION_KS1
346	csrrd		ra, EXCEPTION_KS2
347	ertn
348
349nopage_tlb_store:
350	dbar		0x700
351	csrrd		ra, EXCEPTION_KS2
352	la.abs		t0, tlb_do_page_fault_1
353	jr		t0
354SYM_CODE_END(handle_tlb_store)
355
356SYM_CODE_START(handle_tlb_store_ptw)
357	csrwr		t0, LOONGARCH_CSR_KS0
358	csrwr		t1, LOONGARCH_CSR_KS1
359	la.abs		t0, tlb_do_page_fault_1
360	jirl		zero, t0, 0
361SYM_CODE_END(handle_tlb_store_ptw)
362
363SYM_CODE_START(handle_tlb_modify)
364	csrwr		t0, EXCEPTION_KS0
365	csrwr		t1, EXCEPTION_KS1
366	csrwr		ra, EXCEPTION_KS2
367
368	/*
369	 * The vmalloc handling is not in the hotpath.
370	 */
371	csrrd		t0, LOONGARCH_CSR_BADV
372	bltz		t0, vmalloc_modify
373	csrrd		t1, LOONGARCH_CSR_PGDL
374
375vmalloc_done_modify:
376	/* Get PGD offset in bytes */
377	bstrpick.d	ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT
378	alsl.d		t1, ra, t1, 3
379#if CONFIG_PGTABLE_LEVELS > 3
380	ld.d		t1, t1, 0
381	bstrpick.d	ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT
382	alsl.d		t1, ra, t1, 3
383#endif
384#if CONFIG_PGTABLE_LEVELS > 2
385	ld.d		t1, t1, 0
386	bstrpick.d	ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT
387	alsl.d		t1, ra, t1, 3
388#endif
389	ld.d		ra, t1, 0
390
391	/*
392	 * For huge tlb entries, pmde doesn't contain an address but
393	 * instead contains the tlb pte. Check the PAGE_HUGE bit and
394	 * see if we need to jump to huge tlb processing.
395	 */
396	rotri.d		ra, ra, _PAGE_HUGE_SHIFT + 1
397	bltz		ra, tlb_huge_update_modify
398
399	rotri.d		ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1)
400	bstrpick.d	t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT
401	alsl.d		t1, t0, ra, _PTE_T_LOG2
402
403#ifdef CONFIG_SMP
404smp_pgtable_change_modify:
405	ll.d		t0, t1, 0
406#else
407	ld.d		t0, t1, 0
408#endif
409	andi		ra, t0, _PAGE_WRITE
410	beqz		ra, nopage_tlb_modify
411
412	ori		t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
413#ifdef CONFIG_SMP
414	sc.d		t0, t1, 0
415	beqz		t0, smp_pgtable_change_modify
416#else
417	st.d		t0, t1, 0
418#endif
419	tlbsrch
420	bstrins.d	t1, zero, 3, 3
421	ld.d		t0, t1, 0
422	ld.d		t1, t1, 8
423	csrwr		t0, LOONGARCH_CSR_TLBELO0
424	csrwr		t1, LOONGARCH_CSR_TLBELO1
425	tlbwr
426
427	csrrd		t0, EXCEPTION_KS0
428	csrrd		t1, EXCEPTION_KS1
429	csrrd		ra, EXCEPTION_KS2
430	ertn
431
432#ifdef CONFIG_64BIT
433vmalloc_modify:
434	la.abs		t1, swapper_pg_dir
435	b		vmalloc_done_modify
436#endif
437
438	/* This is the entry point of a huge page. */
439tlb_huge_update_modify:
440#ifdef CONFIG_SMP
441	ll.d		ra, t1, 0
442#endif
443	andi		t0, ra, _PAGE_WRITE
444	beqz		t0, nopage_tlb_modify
445
446#ifdef CONFIG_SMP
447	ori		t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
448	sc.d		t0, t1, 0
449	beqz		t0, tlb_huge_update_modify
450	ori		t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
451#else
452	rotri.d		ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1)
453	ori		t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
454	st.d		t0, t1, 0
455#endif
456	csrrd		ra, LOONGARCH_CSR_ASID
457	csrrd		t1, LOONGARCH_CSR_BADV
458	andi		ra, ra, CSR_ASID_ASID
459	invtlb		INVTLB_ADDR_GFALSE_AND_ASID, ra, t1
460
461	/*
462	 * A huge PTE describes an area the size of the
463	 * configured huge page size. This is twice the
464	 * of the large TLB entry size we intend to use.
465	 * A TLB entry half the size of the configured
466	 * huge page size is configured into entrylo0
467	 * and entrylo1 to cover the contiguous huge PTE
468	 * address space.
469	 */
470	/* Huge page: Move Global bit */
471	xori		t0, t0, _PAGE_HUGE
472	lu12i.w		t1, _PAGE_HGLOBAL >> 12
473	and		t1, t0, t1
474	srli.d		t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT)
475	or		t0, t0, t1
476
477	move		ra, t0
478	csrwr		ra, LOONGARCH_CSR_TLBELO0
479
480	/* Convert to entrylo1 */
481	addi.d		t1, zero, 1
482	slli.d		t1, t1, (HPAGE_SHIFT - 1)
483	add.d		t0, t0, t1
484	csrwr		t0, LOONGARCH_CSR_TLBELO1
485
486	/* Set huge page tlb entry size */
487	addu16i.d	t0, zero, (CSR_TLBIDX_PS >> 16)
488	addu16i.d	t1, zero, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16))
489	csrxchg		t1, t0, LOONGARCH_CSR_TLBIDX
490
491	tlbfill
492
493	/* Reset default page size */
494	addu16i.d	t0, zero, (CSR_TLBIDX_PS >> 16)
495	addu16i.d	t1, zero, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16))
496	csrxchg		t1, t0, LOONGARCH_CSR_TLBIDX
497
498	csrrd		t0, EXCEPTION_KS0
499	csrrd		t1, EXCEPTION_KS1
500	csrrd		ra, EXCEPTION_KS2
501	ertn
502
503nopage_tlb_modify:
504	dbar		0x700
505	csrrd		ra, EXCEPTION_KS2
506	la.abs		t0, tlb_do_page_fault_1
507	jr		t0
508SYM_CODE_END(handle_tlb_modify)
509
510SYM_CODE_START(handle_tlb_modify_ptw)
511	csrwr		t0, LOONGARCH_CSR_KS0
512	csrwr		t1, LOONGARCH_CSR_KS1
513	la.abs		t0, tlb_do_page_fault_1
514	jirl		zero, t0, 0
515SYM_CODE_END(handle_tlb_modify_ptw)
516
517SYM_CODE_START(handle_tlb_refill)
518	csrwr		t0, LOONGARCH_CSR_TLBRSAVE
519	csrrd		t0, LOONGARCH_CSR_PGD
520	lddir		t0, t0, 3
521#if CONFIG_PGTABLE_LEVELS > 3
522	lddir		t0, t0, 2
523#endif
524#if CONFIG_PGTABLE_LEVELS > 2
525	lddir		t0, t0, 1
526#endif
527	ldpte		t0, 0
528	ldpte		t0, 1
529	tlbfill
530	csrrd		t0, LOONGARCH_CSR_TLBRSAVE
531	ertn
532SYM_CODE_END(handle_tlb_refill)
533