18c2ecf20Sopenharmony_ci/* SPDX-License-Identifier: GPL-2.0 */ 28c2ecf20Sopenharmony_ci/* 38c2ecf20Sopenharmony_ci * Copyright (C) 2020 Loongson Technology Corporation Limited 48c2ecf20Sopenharmony_ci */ 58c2ecf20Sopenharmony_ci#include <asm/asm.h> 68c2ecf20Sopenharmony_ci#include <asm/export.h> 78c2ecf20Sopenharmony_ci#include <asm/loongarchregs.h> 88c2ecf20Sopenharmony_ci#include <asm/page.h> 98c2ecf20Sopenharmony_ci#include <asm/pgtable-bits.h> 108c2ecf20Sopenharmony_ci#include <asm/regdef.h> 118c2ecf20Sopenharmony_ci#include <asm/stackframe.h> 128c2ecf20Sopenharmony_ci 138c2ecf20Sopenharmony_ci#ifdef CONFIG_64BIT 148c2ecf20Sopenharmony_ci#include <asm/pgtable-64.h> 158c2ecf20Sopenharmony_ci#endif 168c2ecf20Sopenharmony_ci 178c2ecf20Sopenharmony_ci#define INVTLB_ADDR_GFALSE_AND_ASID 5 188c2ecf20Sopenharmony_ci 198c2ecf20Sopenharmony_ci#define PTRS_PER_PGD_BITS (PAGE_SHIFT - 3) 208c2ecf20Sopenharmony_ci#define PTRS_PER_PUD_BITS (PAGE_SHIFT - 3) 218c2ecf20Sopenharmony_ci#define PTRS_PER_PMD_BITS (PAGE_SHIFT - 3) 228c2ecf20Sopenharmony_ci#define PTRS_PER_PTE_BITS (PAGE_SHIFT - 3) 238c2ecf20Sopenharmony_ci 248c2ecf20Sopenharmony_ci .macro tlb_do_page_fault, write 258c2ecf20Sopenharmony_ci SYM_CODE_START(tlb_do_page_fault_\write) 268c2ecf20Sopenharmony_ci SAVE_ALL 278c2ecf20Sopenharmony_ci csrrd a2, LOONGARCH_CSR_BADV 288c2ecf20Sopenharmony_ci move a0, sp 298c2ecf20Sopenharmony_ci REG_S a2, sp, PT_BVADDR 308c2ecf20Sopenharmony_ci li.w a1, \write 318c2ecf20Sopenharmony_ci la.abs t0, do_page_fault 328c2ecf20Sopenharmony_ci jirl ra, t0, 0 338c2ecf20Sopenharmony_ci UNWIND_HINT_REGS 348c2ecf20Sopenharmony_ci RESTORE_ALL_AND_RET 358c2ecf20Sopenharmony_ci SYM_CODE_END(tlb_do_page_fault_\write) 368c2ecf20Sopenharmony_ci .endm 378c2ecf20Sopenharmony_ci 388c2ecf20Sopenharmony_ci tlb_do_page_fault 0 398c2ecf20Sopenharmony_ci tlb_do_page_fault 1 408c2ecf20Sopenharmony_ci 418c2ecf20Sopenharmony_ciSYM_CODE_START(handle_tlb_protect) 428c2ecf20Sopenharmony_ci BACKUP_T0T1 438c2ecf20Sopenharmony_ci SAVE_ALL 448c2ecf20Sopenharmony_ci move a0, sp 458c2ecf20Sopenharmony_ci move a1, zero 468c2ecf20Sopenharmony_ci csrrd a2, LOONGARCH_CSR_BADV 478c2ecf20Sopenharmony_ci REG_S a2, sp, PT_BVADDR 488c2ecf20Sopenharmony_ci la.abs t0, do_page_fault 498c2ecf20Sopenharmony_ci jirl ra, t0, 0 508c2ecf20Sopenharmony_ci UNWIND_HINT_REGS 518c2ecf20Sopenharmony_ci RESTORE_ALL_AND_RET 528c2ecf20Sopenharmony_ciSYM_CODE_END(handle_tlb_protect) 538c2ecf20Sopenharmony_ci 548c2ecf20Sopenharmony_ciSYM_CODE_START(handle_tlb_load) 558c2ecf20Sopenharmony_ci csrwr t0, EXCEPTION_KS0 568c2ecf20Sopenharmony_ci csrwr t1, EXCEPTION_KS1 578c2ecf20Sopenharmony_ci csrwr ra, EXCEPTION_KS2 588c2ecf20Sopenharmony_ci 598c2ecf20Sopenharmony_ci /* 608c2ecf20Sopenharmony_ci * The vmalloc handling is not in the hotpath. 618c2ecf20Sopenharmony_ci */ 628c2ecf20Sopenharmony_ci csrrd t0, LOONGARCH_CSR_BADV 638c2ecf20Sopenharmony_ci bltz t0, vmalloc_load 648c2ecf20Sopenharmony_ci csrrd t1, LOONGARCH_CSR_PGDL 658c2ecf20Sopenharmony_ci 668c2ecf20Sopenharmony_civmalloc_done_load: 678c2ecf20Sopenharmony_ci /* Get PGD offset in bytes */ 688c2ecf20Sopenharmony_ci bstrpick.d ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT 698c2ecf20Sopenharmony_ci alsl.d t1, ra, t1, 3 708c2ecf20Sopenharmony_ci#if CONFIG_PGTABLE_LEVELS > 3 718c2ecf20Sopenharmony_ci ld.d t1, t1, 0 728c2ecf20Sopenharmony_ci bstrpick.d ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT 738c2ecf20Sopenharmony_ci alsl.d t1, ra, t1, 3 748c2ecf20Sopenharmony_ci#endif 758c2ecf20Sopenharmony_ci#if CONFIG_PGTABLE_LEVELS > 2 768c2ecf20Sopenharmony_ci ld.d t1, t1, 0 778c2ecf20Sopenharmony_ci bstrpick.d ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT 788c2ecf20Sopenharmony_ci alsl.d t1, ra, t1, 3 798c2ecf20Sopenharmony_ci#endif 808c2ecf20Sopenharmony_ci ld.d ra, t1, 0 818c2ecf20Sopenharmony_ci 828c2ecf20Sopenharmony_ci /* 838c2ecf20Sopenharmony_ci * For huge tlb entries, pmde doesn't contain an address but 848c2ecf20Sopenharmony_ci * instead contains the tlb pte. Check the PAGE_HUGE bit and 858c2ecf20Sopenharmony_ci * see if we need to jump to huge tlb processing. 868c2ecf20Sopenharmony_ci */ 878c2ecf20Sopenharmony_ci rotri.d ra, ra, _PAGE_HUGE_SHIFT + 1 888c2ecf20Sopenharmony_ci bltz ra, tlb_huge_update_load 898c2ecf20Sopenharmony_ci 908c2ecf20Sopenharmony_ci rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) 918c2ecf20Sopenharmony_ci bstrpick.d t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT 928c2ecf20Sopenharmony_ci alsl.d t1, t0, ra, _PTE_T_LOG2 938c2ecf20Sopenharmony_ci 948c2ecf20Sopenharmony_ci#ifdef CONFIG_SMP 958c2ecf20Sopenharmony_cismp_pgtable_change_load: 968c2ecf20Sopenharmony_ci ll.d t0, t1, 0 978c2ecf20Sopenharmony_ci#else 988c2ecf20Sopenharmony_ci ld.d t0, t1, 0 998c2ecf20Sopenharmony_ci#endif 1008c2ecf20Sopenharmony_ci andi ra, t0, _PAGE_PRESENT 1018c2ecf20Sopenharmony_ci beqz ra, nopage_tlb_load 1028c2ecf20Sopenharmony_ci 1038c2ecf20Sopenharmony_ci ori t0, t0, _PAGE_VALID 1048c2ecf20Sopenharmony_ci#ifdef CONFIG_SMP 1058c2ecf20Sopenharmony_ci sc.d t0, t1, 0 1068c2ecf20Sopenharmony_ci beqz t0, smp_pgtable_change_load 1078c2ecf20Sopenharmony_ci#else 1088c2ecf20Sopenharmony_ci st.d t0, t1, 0 1098c2ecf20Sopenharmony_ci#endif 1108c2ecf20Sopenharmony_ci tlbsrch 1118c2ecf20Sopenharmony_ci bstrins.d t1, zero, 3, 3 1128c2ecf20Sopenharmony_ci ld.d t0, t1, 0 1138c2ecf20Sopenharmony_ci ld.d t1, t1, 8 1148c2ecf20Sopenharmony_ci csrwr t0, LOONGARCH_CSR_TLBELO0 1158c2ecf20Sopenharmony_ci csrwr t1, LOONGARCH_CSR_TLBELO1 1168c2ecf20Sopenharmony_ci tlbwr 1178c2ecf20Sopenharmony_ci 1188c2ecf20Sopenharmony_ci csrrd t0, EXCEPTION_KS0 1198c2ecf20Sopenharmony_ci csrrd t1, EXCEPTION_KS1 1208c2ecf20Sopenharmony_ci csrrd ra, EXCEPTION_KS2 1218c2ecf20Sopenharmony_ci ertn 1228c2ecf20Sopenharmony_ci 1238c2ecf20Sopenharmony_ci#ifdef CONFIG_64BIT 1248c2ecf20Sopenharmony_civmalloc_load: 1258c2ecf20Sopenharmony_ci la.abs t1, swapper_pg_dir 1268c2ecf20Sopenharmony_ci b vmalloc_done_load 1278c2ecf20Sopenharmony_ci#endif 1288c2ecf20Sopenharmony_ci 1298c2ecf20Sopenharmony_ci /* This is the entry point of a huge page. */ 1308c2ecf20Sopenharmony_citlb_huge_update_load: 1318c2ecf20Sopenharmony_ci#ifdef CONFIG_SMP 1328c2ecf20Sopenharmony_ci ll.d ra, t1, 0 1338c2ecf20Sopenharmony_ci#endif 1348c2ecf20Sopenharmony_ci andi t0, ra, _PAGE_PRESENT 1358c2ecf20Sopenharmony_ci beqz t0, nopage_tlb_load 1368c2ecf20Sopenharmony_ci 1378c2ecf20Sopenharmony_ci#ifdef CONFIG_SMP 1388c2ecf20Sopenharmony_ci ori t0, ra, _PAGE_VALID 1398c2ecf20Sopenharmony_ci sc.d t0, t1, 0 1408c2ecf20Sopenharmony_ci beqz t0, tlb_huge_update_load 1418c2ecf20Sopenharmony_ci ori t0, ra, _PAGE_VALID 1428c2ecf20Sopenharmony_ci#else 1438c2ecf20Sopenharmony_ci rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) 1448c2ecf20Sopenharmony_ci ori t0, ra, _PAGE_VALID 1458c2ecf20Sopenharmony_ci st.d t0, t1, 0 1468c2ecf20Sopenharmony_ci#endif 1478c2ecf20Sopenharmony_ci csrrd ra, LOONGARCH_CSR_ASID 1488c2ecf20Sopenharmony_ci csrrd t1, LOONGARCH_CSR_BADV 1498c2ecf20Sopenharmony_ci andi ra, ra, CSR_ASID_ASID 1508c2ecf20Sopenharmony_ci invtlb INVTLB_ADDR_GFALSE_AND_ASID, ra, t1 1518c2ecf20Sopenharmony_ci 1528c2ecf20Sopenharmony_ci /* 1538c2ecf20Sopenharmony_ci * A huge PTE describes an area the size of the 1548c2ecf20Sopenharmony_ci * configured huge page size. This is twice the 1558c2ecf20Sopenharmony_ci * of the large TLB entry size we intend to use. 1568c2ecf20Sopenharmony_ci * A TLB entry half the size of the configured 1578c2ecf20Sopenharmony_ci * huge page size is configured into entrylo0 1588c2ecf20Sopenharmony_ci * and entrylo1 to cover the contiguous huge PTE 1598c2ecf20Sopenharmony_ci * address space. 1608c2ecf20Sopenharmony_ci */ 1618c2ecf20Sopenharmony_ci /* Huge page: Move Global bit */ 1628c2ecf20Sopenharmony_ci xori t0, t0, _PAGE_HUGE 1638c2ecf20Sopenharmony_ci lu12i.w t1, _PAGE_HGLOBAL >> 12 1648c2ecf20Sopenharmony_ci and t1, t0, t1 1658c2ecf20Sopenharmony_ci srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) 1668c2ecf20Sopenharmony_ci or t0, t0, t1 1678c2ecf20Sopenharmony_ci 1688c2ecf20Sopenharmony_ci move ra, t0 1698c2ecf20Sopenharmony_ci csrwr ra, LOONGARCH_CSR_TLBELO0 1708c2ecf20Sopenharmony_ci 1718c2ecf20Sopenharmony_ci /* Convert to entrylo1 */ 1728c2ecf20Sopenharmony_ci addi.d t1, zero, 1 1738c2ecf20Sopenharmony_ci slli.d t1, t1, (HPAGE_SHIFT - 1) 1748c2ecf20Sopenharmony_ci add.d t0, t0, t1 1758c2ecf20Sopenharmony_ci csrwr t0, LOONGARCH_CSR_TLBELO1 1768c2ecf20Sopenharmony_ci 1778c2ecf20Sopenharmony_ci /* Set huge page tlb entry size */ 1788c2ecf20Sopenharmony_ci addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) 1798c2ecf20Sopenharmony_ci addu16i.d t1, zero, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) 1808c2ecf20Sopenharmony_ci csrxchg t1, t0, LOONGARCH_CSR_TLBIDX 1818c2ecf20Sopenharmony_ci 1828c2ecf20Sopenharmony_ci tlbfill 1838c2ecf20Sopenharmony_ci 1848c2ecf20Sopenharmony_ci addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) 1858c2ecf20Sopenharmony_ci addu16i.d t1, zero, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) 1868c2ecf20Sopenharmony_ci csrxchg t1, t0, LOONGARCH_CSR_TLBIDX 1878c2ecf20Sopenharmony_ci 1888c2ecf20Sopenharmony_ci csrrd t0, EXCEPTION_KS0 1898c2ecf20Sopenharmony_ci csrrd t1, EXCEPTION_KS1 1908c2ecf20Sopenharmony_ci csrrd ra, EXCEPTION_KS2 1918c2ecf20Sopenharmony_ci ertn 1928c2ecf20Sopenharmony_ci 1938c2ecf20Sopenharmony_cinopage_tlb_load: 1948c2ecf20Sopenharmony_ci dbar 0x700 1958c2ecf20Sopenharmony_ci csrrd ra, EXCEPTION_KS2 1968c2ecf20Sopenharmony_ci la.abs t0, tlb_do_page_fault_0 1978c2ecf20Sopenharmony_ci jr t0 1988c2ecf20Sopenharmony_ciSYM_CODE_END(handle_tlb_load) 1998c2ecf20Sopenharmony_ci 2008c2ecf20Sopenharmony_ciSYM_CODE_START(handle_tlb_load_ptw) 2018c2ecf20Sopenharmony_ci csrwr t0, LOONGARCH_CSR_KS0 2028c2ecf20Sopenharmony_ci csrwr t1, LOONGARCH_CSR_KS1 2038c2ecf20Sopenharmony_ci la.abs t0, tlb_do_page_fault_0 2048c2ecf20Sopenharmony_ci jirl zero, t0, 0 2058c2ecf20Sopenharmony_ciSYM_CODE_END(handle_tlb_load_ptw) 2068c2ecf20Sopenharmony_ci 2078c2ecf20Sopenharmony_ciSYM_CODE_START(handle_tlb_store) 2088c2ecf20Sopenharmony_ci csrwr t0, EXCEPTION_KS0 2098c2ecf20Sopenharmony_ci csrwr t1, EXCEPTION_KS1 2108c2ecf20Sopenharmony_ci csrwr ra, EXCEPTION_KS2 2118c2ecf20Sopenharmony_ci 2128c2ecf20Sopenharmony_ci /* 2138c2ecf20Sopenharmony_ci * The vmalloc handling is not in the hotpath. 2148c2ecf20Sopenharmony_ci */ 2158c2ecf20Sopenharmony_ci csrrd t0, LOONGARCH_CSR_BADV 2168c2ecf20Sopenharmony_ci bltz t0, vmalloc_store 2178c2ecf20Sopenharmony_ci csrrd t1, LOONGARCH_CSR_PGDL 2188c2ecf20Sopenharmony_ci 2198c2ecf20Sopenharmony_civmalloc_done_store: 2208c2ecf20Sopenharmony_ci /* Get PGD offset in bytes */ 2218c2ecf20Sopenharmony_ci bstrpick.d ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT 2228c2ecf20Sopenharmony_ci alsl.d t1, ra, t1, 3 2238c2ecf20Sopenharmony_ci#if CONFIG_PGTABLE_LEVELS > 3 2248c2ecf20Sopenharmony_ci ld.d t1, t1, 0 2258c2ecf20Sopenharmony_ci bstrpick.d ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT 2268c2ecf20Sopenharmony_ci alsl.d t1, ra, t1, 3 2278c2ecf20Sopenharmony_ci#endif 2288c2ecf20Sopenharmony_ci#if CONFIG_PGTABLE_LEVELS > 2 2298c2ecf20Sopenharmony_ci ld.d t1, t1, 0 2308c2ecf20Sopenharmony_ci bstrpick.d ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT 2318c2ecf20Sopenharmony_ci alsl.d t1, ra, t1, 3 2328c2ecf20Sopenharmony_ci#endif 2338c2ecf20Sopenharmony_ci ld.d ra, t1, 0 2348c2ecf20Sopenharmony_ci 2358c2ecf20Sopenharmony_ci /* 2368c2ecf20Sopenharmony_ci * For huge tlb entries, pmde doesn't contain an address but 2378c2ecf20Sopenharmony_ci * instead contains the tlb pte. Check the PAGE_HUGE bit and 2388c2ecf20Sopenharmony_ci * see if we need to jump to huge tlb processing. 2398c2ecf20Sopenharmony_ci */ 2408c2ecf20Sopenharmony_ci rotri.d ra, ra, _PAGE_HUGE_SHIFT + 1 2418c2ecf20Sopenharmony_ci bltz ra, tlb_huge_update_store 2428c2ecf20Sopenharmony_ci 2438c2ecf20Sopenharmony_ci rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) 2448c2ecf20Sopenharmony_ci bstrpick.d t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT 2458c2ecf20Sopenharmony_ci alsl.d t1, t0, ra, _PTE_T_LOG2 2468c2ecf20Sopenharmony_ci 2478c2ecf20Sopenharmony_ci#ifdef CONFIG_SMP 2488c2ecf20Sopenharmony_cismp_pgtable_change_store: 2498c2ecf20Sopenharmony_ci ll.d t0, t1, 0 2508c2ecf20Sopenharmony_ci#else 2518c2ecf20Sopenharmony_ci ld.d t0, t1, 0 2528c2ecf20Sopenharmony_ci#endif 2538c2ecf20Sopenharmony_ci andi ra, t0, _PAGE_PRESENT | _PAGE_WRITE 2548c2ecf20Sopenharmony_ci xori ra, ra, _PAGE_PRESENT | _PAGE_WRITE 2558c2ecf20Sopenharmony_ci bnez ra, nopage_tlb_store 2568c2ecf20Sopenharmony_ci 2578c2ecf20Sopenharmony_ci ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 2588c2ecf20Sopenharmony_ci#ifdef CONFIG_SMP 2598c2ecf20Sopenharmony_ci sc.d t0, t1, 0 2608c2ecf20Sopenharmony_ci beqz t0, smp_pgtable_change_store 2618c2ecf20Sopenharmony_ci#else 2628c2ecf20Sopenharmony_ci st.d t0, t1, 0 2638c2ecf20Sopenharmony_ci#endif 2648c2ecf20Sopenharmony_ci tlbsrch 2658c2ecf20Sopenharmony_ci bstrins.d t1, zero, 3, 3 2668c2ecf20Sopenharmony_ci ld.d t0, t1, 0 2678c2ecf20Sopenharmony_ci ld.d t1, t1, 8 2688c2ecf20Sopenharmony_ci csrwr t0, LOONGARCH_CSR_TLBELO0 2698c2ecf20Sopenharmony_ci csrwr t1, LOONGARCH_CSR_TLBELO1 2708c2ecf20Sopenharmony_ci tlbwr 2718c2ecf20Sopenharmony_ci 2728c2ecf20Sopenharmony_ci csrrd t0, EXCEPTION_KS0 2738c2ecf20Sopenharmony_ci csrrd t1, EXCEPTION_KS1 2748c2ecf20Sopenharmony_ci csrrd ra, EXCEPTION_KS2 2758c2ecf20Sopenharmony_ci ertn 2768c2ecf20Sopenharmony_ci 2778c2ecf20Sopenharmony_ci#ifdef CONFIG_64BIT 2788c2ecf20Sopenharmony_civmalloc_store: 2798c2ecf20Sopenharmony_ci la.abs t1, swapper_pg_dir 2808c2ecf20Sopenharmony_ci b vmalloc_done_store 2818c2ecf20Sopenharmony_ci#endif 2828c2ecf20Sopenharmony_ci 2838c2ecf20Sopenharmony_ci /* This is the entry point of a huge page. */ 2848c2ecf20Sopenharmony_citlb_huge_update_store: 2858c2ecf20Sopenharmony_ci#ifdef CONFIG_SMP 2868c2ecf20Sopenharmony_ci ll.d ra, t1, 0 2878c2ecf20Sopenharmony_ci#endif 2888c2ecf20Sopenharmony_ci andi t0, ra, _PAGE_PRESENT | _PAGE_WRITE 2898c2ecf20Sopenharmony_ci xori t0, t0, _PAGE_PRESENT | _PAGE_WRITE 2908c2ecf20Sopenharmony_ci bnez t0, nopage_tlb_store 2918c2ecf20Sopenharmony_ci 2928c2ecf20Sopenharmony_ci#ifdef CONFIG_SMP 2938c2ecf20Sopenharmony_ci ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 2948c2ecf20Sopenharmony_ci sc.d t0, t1, 0 2958c2ecf20Sopenharmony_ci beqz t0, tlb_huge_update_store 2968c2ecf20Sopenharmony_ci ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 2978c2ecf20Sopenharmony_ci#else 2988c2ecf20Sopenharmony_ci rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) 2998c2ecf20Sopenharmony_ci ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 3008c2ecf20Sopenharmony_ci st.d t0, t1, 0 3018c2ecf20Sopenharmony_ci#endif 3028c2ecf20Sopenharmony_ci csrrd ra, LOONGARCH_CSR_ASID 3038c2ecf20Sopenharmony_ci csrrd t1, LOONGARCH_CSR_BADV 3048c2ecf20Sopenharmony_ci andi ra, ra, CSR_ASID_ASID 3058c2ecf20Sopenharmony_ci invtlb INVTLB_ADDR_GFALSE_AND_ASID, ra, t1 3068c2ecf20Sopenharmony_ci 3078c2ecf20Sopenharmony_ci /* 3088c2ecf20Sopenharmony_ci * A huge PTE describes an area the size of the 3098c2ecf20Sopenharmony_ci * configured huge page size. This is twice the 3108c2ecf20Sopenharmony_ci * of the large TLB entry size we intend to use. 3118c2ecf20Sopenharmony_ci * A TLB entry half the size of the configured 3128c2ecf20Sopenharmony_ci * huge page size is configured into entrylo0 3138c2ecf20Sopenharmony_ci * and entrylo1 to cover the contiguous huge PTE 3148c2ecf20Sopenharmony_ci * address space. 3158c2ecf20Sopenharmony_ci */ 3168c2ecf20Sopenharmony_ci /* Huge page: Move Global bit */ 3178c2ecf20Sopenharmony_ci xori t0, t0, _PAGE_HUGE 3188c2ecf20Sopenharmony_ci lu12i.w t1, _PAGE_HGLOBAL >> 12 3198c2ecf20Sopenharmony_ci and t1, t0, t1 3208c2ecf20Sopenharmony_ci srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) 3218c2ecf20Sopenharmony_ci or t0, t0, t1 3228c2ecf20Sopenharmony_ci 3238c2ecf20Sopenharmony_ci move ra, t0 3248c2ecf20Sopenharmony_ci csrwr ra, LOONGARCH_CSR_TLBELO0 3258c2ecf20Sopenharmony_ci 3268c2ecf20Sopenharmony_ci /* Convert to entrylo1 */ 3278c2ecf20Sopenharmony_ci addi.d t1, zero, 1 3288c2ecf20Sopenharmony_ci slli.d t1, t1, (HPAGE_SHIFT - 1) 3298c2ecf20Sopenharmony_ci add.d t0, t0, t1 3308c2ecf20Sopenharmony_ci csrwr t0, LOONGARCH_CSR_TLBELO1 3318c2ecf20Sopenharmony_ci 3328c2ecf20Sopenharmony_ci /* Set huge page tlb entry size */ 3338c2ecf20Sopenharmony_ci addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) 3348c2ecf20Sopenharmony_ci addu16i.d t1, zero, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) 3358c2ecf20Sopenharmony_ci csrxchg t1, t0, LOONGARCH_CSR_TLBIDX 3368c2ecf20Sopenharmony_ci 3378c2ecf20Sopenharmony_ci tlbfill 3388c2ecf20Sopenharmony_ci 3398c2ecf20Sopenharmony_ci /* Reset default page size */ 3408c2ecf20Sopenharmony_ci addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) 3418c2ecf20Sopenharmony_ci addu16i.d t1, zero, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) 3428c2ecf20Sopenharmony_ci csrxchg t1, t0, LOONGARCH_CSR_TLBIDX 3438c2ecf20Sopenharmony_ci 3448c2ecf20Sopenharmony_ci csrrd t0, EXCEPTION_KS0 3458c2ecf20Sopenharmony_ci csrrd t1, EXCEPTION_KS1 3468c2ecf20Sopenharmony_ci csrrd ra, EXCEPTION_KS2 3478c2ecf20Sopenharmony_ci ertn 3488c2ecf20Sopenharmony_ci 3498c2ecf20Sopenharmony_cinopage_tlb_store: 3508c2ecf20Sopenharmony_ci dbar 0x700 3518c2ecf20Sopenharmony_ci csrrd ra, EXCEPTION_KS2 3528c2ecf20Sopenharmony_ci la.abs t0, tlb_do_page_fault_1 3538c2ecf20Sopenharmony_ci jr t0 3548c2ecf20Sopenharmony_ciSYM_CODE_END(handle_tlb_store) 3558c2ecf20Sopenharmony_ci 3568c2ecf20Sopenharmony_ciSYM_CODE_START(handle_tlb_store_ptw) 3578c2ecf20Sopenharmony_ci csrwr t0, LOONGARCH_CSR_KS0 3588c2ecf20Sopenharmony_ci csrwr t1, LOONGARCH_CSR_KS1 3598c2ecf20Sopenharmony_ci la.abs t0, tlb_do_page_fault_1 3608c2ecf20Sopenharmony_ci jirl zero, t0, 0 3618c2ecf20Sopenharmony_ciSYM_CODE_END(handle_tlb_store_ptw) 3628c2ecf20Sopenharmony_ci 3638c2ecf20Sopenharmony_ciSYM_CODE_START(handle_tlb_modify) 3648c2ecf20Sopenharmony_ci csrwr t0, EXCEPTION_KS0 3658c2ecf20Sopenharmony_ci csrwr t1, EXCEPTION_KS1 3668c2ecf20Sopenharmony_ci csrwr ra, EXCEPTION_KS2 3678c2ecf20Sopenharmony_ci 3688c2ecf20Sopenharmony_ci /* 3698c2ecf20Sopenharmony_ci * The vmalloc handling is not in the hotpath. 3708c2ecf20Sopenharmony_ci */ 3718c2ecf20Sopenharmony_ci csrrd t0, LOONGARCH_CSR_BADV 3728c2ecf20Sopenharmony_ci bltz t0, vmalloc_modify 3738c2ecf20Sopenharmony_ci csrrd t1, LOONGARCH_CSR_PGDL 3748c2ecf20Sopenharmony_ci 3758c2ecf20Sopenharmony_civmalloc_done_modify: 3768c2ecf20Sopenharmony_ci /* Get PGD offset in bytes */ 3778c2ecf20Sopenharmony_ci bstrpick.d ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT 3788c2ecf20Sopenharmony_ci alsl.d t1, ra, t1, 3 3798c2ecf20Sopenharmony_ci#if CONFIG_PGTABLE_LEVELS > 3 3808c2ecf20Sopenharmony_ci ld.d t1, t1, 0 3818c2ecf20Sopenharmony_ci bstrpick.d ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT 3828c2ecf20Sopenharmony_ci alsl.d t1, ra, t1, 3 3838c2ecf20Sopenharmony_ci#endif 3848c2ecf20Sopenharmony_ci#if CONFIG_PGTABLE_LEVELS > 2 3858c2ecf20Sopenharmony_ci ld.d t1, t1, 0 3868c2ecf20Sopenharmony_ci bstrpick.d ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT 3878c2ecf20Sopenharmony_ci alsl.d t1, ra, t1, 3 3888c2ecf20Sopenharmony_ci#endif 3898c2ecf20Sopenharmony_ci ld.d ra, t1, 0 3908c2ecf20Sopenharmony_ci 3918c2ecf20Sopenharmony_ci /* 3928c2ecf20Sopenharmony_ci * For huge tlb entries, pmde doesn't contain an address but 3938c2ecf20Sopenharmony_ci * instead contains the tlb pte. Check the PAGE_HUGE bit and 3948c2ecf20Sopenharmony_ci * see if we need to jump to huge tlb processing. 3958c2ecf20Sopenharmony_ci */ 3968c2ecf20Sopenharmony_ci rotri.d ra, ra, _PAGE_HUGE_SHIFT + 1 3978c2ecf20Sopenharmony_ci bltz ra, tlb_huge_update_modify 3988c2ecf20Sopenharmony_ci 3998c2ecf20Sopenharmony_ci rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) 4008c2ecf20Sopenharmony_ci bstrpick.d t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT 4018c2ecf20Sopenharmony_ci alsl.d t1, t0, ra, _PTE_T_LOG2 4028c2ecf20Sopenharmony_ci 4038c2ecf20Sopenharmony_ci#ifdef CONFIG_SMP 4048c2ecf20Sopenharmony_cismp_pgtable_change_modify: 4058c2ecf20Sopenharmony_ci ll.d t0, t1, 0 4068c2ecf20Sopenharmony_ci#else 4078c2ecf20Sopenharmony_ci ld.d t0, t1, 0 4088c2ecf20Sopenharmony_ci#endif 4098c2ecf20Sopenharmony_ci andi ra, t0, _PAGE_WRITE 4108c2ecf20Sopenharmony_ci beqz ra, nopage_tlb_modify 4118c2ecf20Sopenharmony_ci 4128c2ecf20Sopenharmony_ci ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 4138c2ecf20Sopenharmony_ci#ifdef CONFIG_SMP 4148c2ecf20Sopenharmony_ci sc.d t0, t1, 0 4158c2ecf20Sopenharmony_ci beqz t0, smp_pgtable_change_modify 4168c2ecf20Sopenharmony_ci#else 4178c2ecf20Sopenharmony_ci st.d t0, t1, 0 4188c2ecf20Sopenharmony_ci#endif 4198c2ecf20Sopenharmony_ci tlbsrch 4208c2ecf20Sopenharmony_ci bstrins.d t1, zero, 3, 3 4218c2ecf20Sopenharmony_ci ld.d t0, t1, 0 4228c2ecf20Sopenharmony_ci ld.d t1, t1, 8 4238c2ecf20Sopenharmony_ci csrwr t0, LOONGARCH_CSR_TLBELO0 4248c2ecf20Sopenharmony_ci csrwr t1, LOONGARCH_CSR_TLBELO1 4258c2ecf20Sopenharmony_ci tlbwr 4268c2ecf20Sopenharmony_ci 4278c2ecf20Sopenharmony_ci csrrd t0, EXCEPTION_KS0 4288c2ecf20Sopenharmony_ci csrrd t1, EXCEPTION_KS1 4298c2ecf20Sopenharmony_ci csrrd ra, EXCEPTION_KS2 4308c2ecf20Sopenharmony_ci ertn 4318c2ecf20Sopenharmony_ci 4328c2ecf20Sopenharmony_ci#ifdef CONFIG_64BIT 4338c2ecf20Sopenharmony_civmalloc_modify: 4348c2ecf20Sopenharmony_ci la.abs t1, swapper_pg_dir 4358c2ecf20Sopenharmony_ci b vmalloc_done_modify 4368c2ecf20Sopenharmony_ci#endif 4378c2ecf20Sopenharmony_ci 4388c2ecf20Sopenharmony_ci /* This is the entry point of a huge page. */ 4398c2ecf20Sopenharmony_citlb_huge_update_modify: 4408c2ecf20Sopenharmony_ci#ifdef CONFIG_SMP 4418c2ecf20Sopenharmony_ci ll.d ra, t1, 0 4428c2ecf20Sopenharmony_ci#endif 4438c2ecf20Sopenharmony_ci andi t0, ra, _PAGE_WRITE 4448c2ecf20Sopenharmony_ci beqz t0, nopage_tlb_modify 4458c2ecf20Sopenharmony_ci 4468c2ecf20Sopenharmony_ci#ifdef CONFIG_SMP 4478c2ecf20Sopenharmony_ci ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 4488c2ecf20Sopenharmony_ci sc.d t0, t1, 0 4498c2ecf20Sopenharmony_ci beqz t0, tlb_huge_update_modify 4508c2ecf20Sopenharmony_ci ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 4518c2ecf20Sopenharmony_ci#else 4528c2ecf20Sopenharmony_ci rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) 4538c2ecf20Sopenharmony_ci ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 4548c2ecf20Sopenharmony_ci st.d t0, t1, 0 4558c2ecf20Sopenharmony_ci#endif 4568c2ecf20Sopenharmony_ci csrrd ra, LOONGARCH_CSR_ASID 4578c2ecf20Sopenharmony_ci csrrd t1, LOONGARCH_CSR_BADV 4588c2ecf20Sopenharmony_ci andi ra, ra, CSR_ASID_ASID 4598c2ecf20Sopenharmony_ci invtlb INVTLB_ADDR_GFALSE_AND_ASID, ra, t1 4608c2ecf20Sopenharmony_ci 4618c2ecf20Sopenharmony_ci /* 4628c2ecf20Sopenharmony_ci * A huge PTE describes an area the size of the 4638c2ecf20Sopenharmony_ci * configured huge page size. This is twice the 4648c2ecf20Sopenharmony_ci * of the large TLB entry size we intend to use. 4658c2ecf20Sopenharmony_ci * A TLB entry half the size of the configured 4668c2ecf20Sopenharmony_ci * huge page size is configured into entrylo0 4678c2ecf20Sopenharmony_ci * and entrylo1 to cover the contiguous huge PTE 4688c2ecf20Sopenharmony_ci * address space. 4698c2ecf20Sopenharmony_ci */ 4708c2ecf20Sopenharmony_ci /* Huge page: Move Global bit */ 4718c2ecf20Sopenharmony_ci xori t0, t0, _PAGE_HUGE 4728c2ecf20Sopenharmony_ci lu12i.w t1, _PAGE_HGLOBAL >> 12 4738c2ecf20Sopenharmony_ci and t1, t0, t1 4748c2ecf20Sopenharmony_ci srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) 4758c2ecf20Sopenharmony_ci or t0, t0, t1 4768c2ecf20Sopenharmony_ci 4778c2ecf20Sopenharmony_ci move ra, t0 4788c2ecf20Sopenharmony_ci csrwr ra, LOONGARCH_CSR_TLBELO0 4798c2ecf20Sopenharmony_ci 4808c2ecf20Sopenharmony_ci /* Convert to entrylo1 */ 4818c2ecf20Sopenharmony_ci addi.d t1, zero, 1 4828c2ecf20Sopenharmony_ci slli.d t1, t1, (HPAGE_SHIFT - 1) 4838c2ecf20Sopenharmony_ci add.d t0, t0, t1 4848c2ecf20Sopenharmony_ci csrwr t0, LOONGARCH_CSR_TLBELO1 4858c2ecf20Sopenharmony_ci 4868c2ecf20Sopenharmony_ci /* Set huge page tlb entry size */ 4878c2ecf20Sopenharmony_ci addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) 4888c2ecf20Sopenharmony_ci addu16i.d t1, zero, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) 4898c2ecf20Sopenharmony_ci csrxchg t1, t0, LOONGARCH_CSR_TLBIDX 4908c2ecf20Sopenharmony_ci 4918c2ecf20Sopenharmony_ci tlbfill 4928c2ecf20Sopenharmony_ci 4938c2ecf20Sopenharmony_ci /* Reset default page size */ 4948c2ecf20Sopenharmony_ci addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) 4958c2ecf20Sopenharmony_ci addu16i.d t1, zero, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) 4968c2ecf20Sopenharmony_ci csrxchg t1, t0, LOONGARCH_CSR_TLBIDX 4978c2ecf20Sopenharmony_ci 4988c2ecf20Sopenharmony_ci csrrd t0, EXCEPTION_KS0 4998c2ecf20Sopenharmony_ci csrrd t1, EXCEPTION_KS1 5008c2ecf20Sopenharmony_ci csrrd ra, EXCEPTION_KS2 5018c2ecf20Sopenharmony_ci ertn 5028c2ecf20Sopenharmony_ci 5038c2ecf20Sopenharmony_cinopage_tlb_modify: 5048c2ecf20Sopenharmony_ci dbar 0x700 5058c2ecf20Sopenharmony_ci csrrd ra, EXCEPTION_KS2 5068c2ecf20Sopenharmony_ci la.abs t0, tlb_do_page_fault_1 5078c2ecf20Sopenharmony_ci jr t0 5088c2ecf20Sopenharmony_ciSYM_CODE_END(handle_tlb_modify) 5098c2ecf20Sopenharmony_ci 5108c2ecf20Sopenharmony_ciSYM_CODE_START(handle_tlb_modify_ptw) 5118c2ecf20Sopenharmony_ci csrwr t0, LOONGARCH_CSR_KS0 5128c2ecf20Sopenharmony_ci csrwr t1, LOONGARCH_CSR_KS1 5138c2ecf20Sopenharmony_ci la.abs t0, tlb_do_page_fault_1 5148c2ecf20Sopenharmony_ci jirl zero, t0, 0 5158c2ecf20Sopenharmony_ciSYM_CODE_END(handle_tlb_modify_ptw) 5168c2ecf20Sopenharmony_ci 5178c2ecf20Sopenharmony_ciSYM_CODE_START(handle_tlb_refill) 5188c2ecf20Sopenharmony_ci csrwr t0, LOONGARCH_CSR_TLBRSAVE 5198c2ecf20Sopenharmony_ci csrrd t0, LOONGARCH_CSR_PGD 5208c2ecf20Sopenharmony_ci lddir t0, t0, 3 5218c2ecf20Sopenharmony_ci#if CONFIG_PGTABLE_LEVELS > 3 5228c2ecf20Sopenharmony_ci lddir t0, t0, 2 5238c2ecf20Sopenharmony_ci#endif 5248c2ecf20Sopenharmony_ci#if CONFIG_PGTABLE_LEVELS > 2 5258c2ecf20Sopenharmony_ci lddir t0, t0, 1 5268c2ecf20Sopenharmony_ci#endif 5278c2ecf20Sopenharmony_ci ldpte t0, 0 5288c2ecf20Sopenharmony_ci ldpte t0, 1 5298c2ecf20Sopenharmony_ci tlbfill 5308c2ecf20Sopenharmony_ci csrrd t0, LOONGARCH_CSR_TLBRSAVE 5318c2ecf20Sopenharmony_ci ertn 5328c2ecf20Sopenharmony_ciSYM_CODE_END(handle_tlb_refill) 533