162306a36Sopenharmony_ci/* SPDX-License-Identifier: GPL-2.0 */ 262306a36Sopenharmony_ci/* 362306a36Sopenharmony_ci * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 462306a36Sopenharmony_ci */ 562306a36Sopenharmony_ci#include <asm/asm.h> 662306a36Sopenharmony_ci#include <asm/loongarch.h> 762306a36Sopenharmony_ci#include <asm/page.h> 862306a36Sopenharmony_ci#include <asm/pgtable.h> 962306a36Sopenharmony_ci#include <asm/regdef.h> 1062306a36Sopenharmony_ci#include <asm/stackframe.h> 1162306a36Sopenharmony_ci 1262306a36Sopenharmony_ci#define INVTLB_ADDR_GFALSE_AND_ASID 5 1362306a36Sopenharmony_ci 1462306a36Sopenharmony_ci#define PTRS_PER_PGD_BITS (PAGE_SHIFT - 3) 1562306a36Sopenharmony_ci#define PTRS_PER_PUD_BITS (PAGE_SHIFT - 3) 1662306a36Sopenharmony_ci#define PTRS_PER_PMD_BITS (PAGE_SHIFT - 3) 1762306a36Sopenharmony_ci#define PTRS_PER_PTE_BITS (PAGE_SHIFT - 3) 1862306a36Sopenharmony_ci 1962306a36Sopenharmony_ci .macro tlb_do_page_fault, write 2062306a36Sopenharmony_ci SYM_CODE_START(tlb_do_page_fault_\write) 2162306a36Sopenharmony_ci SAVE_ALL 2262306a36Sopenharmony_ci csrrd a2, LOONGARCH_CSR_BADV 2362306a36Sopenharmony_ci move a0, sp 2462306a36Sopenharmony_ci REG_S a2, sp, PT_BVADDR 2562306a36Sopenharmony_ci li.w a1, \write 2662306a36Sopenharmony_ci bl do_page_fault 2762306a36Sopenharmony_ci RESTORE_ALL_AND_RET 2862306a36Sopenharmony_ci SYM_CODE_END(tlb_do_page_fault_\write) 2962306a36Sopenharmony_ci .endm 3062306a36Sopenharmony_ci 3162306a36Sopenharmony_ci tlb_do_page_fault 0 3262306a36Sopenharmony_ci tlb_do_page_fault 1 3362306a36Sopenharmony_ci 3462306a36Sopenharmony_ciSYM_CODE_START(handle_tlb_protect) 3562306a36Sopenharmony_ci BACKUP_T0T1 3662306a36Sopenharmony_ci SAVE_ALL 3762306a36Sopenharmony_ci move a0, sp 3862306a36Sopenharmony_ci move a1, zero 3962306a36Sopenharmony_ci csrrd a2, LOONGARCH_CSR_BADV 4062306a36Sopenharmony_ci REG_S a2, sp, PT_BVADDR 4162306a36Sopenharmony_ci la_abs t0, do_page_fault 4262306a36Sopenharmony_ci jirl ra, t0, 0 4362306a36Sopenharmony_ci RESTORE_ALL_AND_RET 4462306a36Sopenharmony_ciSYM_CODE_END(handle_tlb_protect) 4562306a36Sopenharmony_ci 4662306a36Sopenharmony_ciSYM_CODE_START(handle_tlb_load) 4762306a36Sopenharmony_ci csrwr t0, EXCEPTION_KS0 4862306a36Sopenharmony_ci csrwr t1, EXCEPTION_KS1 4962306a36Sopenharmony_ci csrwr ra, EXCEPTION_KS2 5062306a36Sopenharmony_ci 5162306a36Sopenharmony_ci /* 5262306a36Sopenharmony_ci * The vmalloc handling is not in the hotpath. 5362306a36Sopenharmony_ci */ 5462306a36Sopenharmony_ci csrrd t0, LOONGARCH_CSR_BADV 5562306a36Sopenharmony_ci bltz t0, vmalloc_load 5662306a36Sopenharmony_ci csrrd t1, LOONGARCH_CSR_PGDL 5762306a36Sopenharmony_ci 5862306a36Sopenharmony_civmalloc_done_load: 5962306a36Sopenharmony_ci /* Get PGD offset in bytes */ 6062306a36Sopenharmony_ci bstrpick.d ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT 6162306a36Sopenharmony_ci alsl.d t1, ra, t1, 3 6262306a36Sopenharmony_ci#if CONFIG_PGTABLE_LEVELS > 3 6362306a36Sopenharmony_ci ld.d t1, t1, 0 6462306a36Sopenharmony_ci bstrpick.d ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT 6562306a36Sopenharmony_ci alsl.d t1, ra, t1, 3 6662306a36Sopenharmony_ci#endif 6762306a36Sopenharmony_ci#if CONFIG_PGTABLE_LEVELS > 2 6862306a36Sopenharmony_ci ld.d t1, t1, 0 6962306a36Sopenharmony_ci bstrpick.d ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT 7062306a36Sopenharmony_ci alsl.d t1, ra, t1, 3 7162306a36Sopenharmony_ci#endif 7262306a36Sopenharmony_ci ld.d ra, t1, 0 7362306a36Sopenharmony_ci 7462306a36Sopenharmony_ci /* 7562306a36Sopenharmony_ci * For huge tlb entries, pmde doesn't contain an address but 7662306a36Sopenharmony_ci * instead contains the tlb pte. Check the PAGE_HUGE bit and 7762306a36Sopenharmony_ci * see if we need to jump to huge tlb processing. 7862306a36Sopenharmony_ci */ 7962306a36Sopenharmony_ci rotri.d ra, ra, _PAGE_HUGE_SHIFT + 1 8062306a36Sopenharmony_ci bltz ra, tlb_huge_update_load 8162306a36Sopenharmony_ci 8262306a36Sopenharmony_ci rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) 8362306a36Sopenharmony_ci bstrpick.d t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT 8462306a36Sopenharmony_ci alsl.d t1, t0, ra, _PTE_T_LOG2 8562306a36Sopenharmony_ci 8662306a36Sopenharmony_ci#ifdef CONFIG_SMP 8762306a36Sopenharmony_cismp_pgtable_change_load: 8862306a36Sopenharmony_ci ll.d t0, t1, 0 8962306a36Sopenharmony_ci#else 9062306a36Sopenharmony_ci ld.d t0, t1, 0 9162306a36Sopenharmony_ci#endif 9262306a36Sopenharmony_ci andi ra, t0, _PAGE_PRESENT 9362306a36Sopenharmony_ci beqz ra, nopage_tlb_load 9462306a36Sopenharmony_ci 9562306a36Sopenharmony_ci ori t0, t0, _PAGE_VALID 9662306a36Sopenharmony_ci#ifdef CONFIG_SMP 9762306a36Sopenharmony_ci sc.d t0, t1, 0 9862306a36Sopenharmony_ci beqz t0, smp_pgtable_change_load 9962306a36Sopenharmony_ci#else 10062306a36Sopenharmony_ci st.d t0, t1, 0 10162306a36Sopenharmony_ci#endif 10262306a36Sopenharmony_ci tlbsrch 10362306a36Sopenharmony_ci bstrins.d t1, zero, 3, 3 10462306a36Sopenharmony_ci ld.d t0, t1, 0 10562306a36Sopenharmony_ci ld.d t1, t1, 8 10662306a36Sopenharmony_ci csrwr t0, LOONGARCH_CSR_TLBELO0 10762306a36Sopenharmony_ci csrwr t1, LOONGARCH_CSR_TLBELO1 10862306a36Sopenharmony_ci tlbwr 10962306a36Sopenharmony_ci 11062306a36Sopenharmony_ci csrrd t0, EXCEPTION_KS0 11162306a36Sopenharmony_ci csrrd t1, EXCEPTION_KS1 11262306a36Sopenharmony_ci csrrd ra, EXCEPTION_KS2 11362306a36Sopenharmony_ci ertn 11462306a36Sopenharmony_ci 11562306a36Sopenharmony_ci#ifdef CONFIG_64BIT 11662306a36Sopenharmony_civmalloc_load: 11762306a36Sopenharmony_ci la_abs t1, swapper_pg_dir 11862306a36Sopenharmony_ci b vmalloc_done_load 11962306a36Sopenharmony_ci#endif 12062306a36Sopenharmony_ci 12162306a36Sopenharmony_ci /* This is the entry point of a huge page. */ 12262306a36Sopenharmony_citlb_huge_update_load: 12362306a36Sopenharmony_ci#ifdef CONFIG_SMP 12462306a36Sopenharmony_ci ll.d ra, t1, 0 12562306a36Sopenharmony_ci#endif 12662306a36Sopenharmony_ci andi t0, ra, _PAGE_PRESENT 12762306a36Sopenharmony_ci beqz t0, nopage_tlb_load 12862306a36Sopenharmony_ci 12962306a36Sopenharmony_ci#ifdef CONFIG_SMP 13062306a36Sopenharmony_ci ori t0, ra, _PAGE_VALID 13162306a36Sopenharmony_ci sc.d t0, t1, 0 13262306a36Sopenharmony_ci beqz t0, tlb_huge_update_load 13362306a36Sopenharmony_ci ori t0, ra, _PAGE_VALID 13462306a36Sopenharmony_ci#else 13562306a36Sopenharmony_ci rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) 13662306a36Sopenharmony_ci ori t0, ra, _PAGE_VALID 13762306a36Sopenharmony_ci st.d t0, t1, 0 13862306a36Sopenharmony_ci#endif 13962306a36Sopenharmony_ci csrrd ra, LOONGARCH_CSR_ASID 14062306a36Sopenharmony_ci csrrd t1, LOONGARCH_CSR_BADV 14162306a36Sopenharmony_ci andi ra, ra, CSR_ASID_ASID 14262306a36Sopenharmony_ci invtlb INVTLB_ADDR_GFALSE_AND_ASID, ra, t1 14362306a36Sopenharmony_ci 14462306a36Sopenharmony_ci /* 14562306a36Sopenharmony_ci * A huge PTE describes an area the size of the 14662306a36Sopenharmony_ci * configured huge page size. This is twice the 14762306a36Sopenharmony_ci * of the large TLB entry size we intend to use. 14862306a36Sopenharmony_ci * A TLB entry half the size of the configured 14962306a36Sopenharmony_ci * huge page size is configured into entrylo0 15062306a36Sopenharmony_ci * and entrylo1 to cover the contiguous huge PTE 15162306a36Sopenharmony_ci * address space. 15262306a36Sopenharmony_ci */ 15362306a36Sopenharmony_ci /* Huge page: Move Global bit */ 15462306a36Sopenharmony_ci xori t0, t0, _PAGE_HUGE 15562306a36Sopenharmony_ci lu12i.w t1, _PAGE_HGLOBAL >> 12 15662306a36Sopenharmony_ci and t1, t0, t1 15762306a36Sopenharmony_ci srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) 15862306a36Sopenharmony_ci or t0, t0, t1 15962306a36Sopenharmony_ci 16062306a36Sopenharmony_ci move ra, t0 16162306a36Sopenharmony_ci csrwr ra, LOONGARCH_CSR_TLBELO0 16262306a36Sopenharmony_ci 16362306a36Sopenharmony_ci /* Convert to entrylo1 */ 16462306a36Sopenharmony_ci addi.d t1, zero, 1 16562306a36Sopenharmony_ci slli.d t1, t1, (HPAGE_SHIFT - 1) 16662306a36Sopenharmony_ci add.d t0, t0, t1 16762306a36Sopenharmony_ci csrwr t0, LOONGARCH_CSR_TLBELO1 16862306a36Sopenharmony_ci 16962306a36Sopenharmony_ci /* Set huge page tlb entry size */ 17062306a36Sopenharmony_ci addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) 17162306a36Sopenharmony_ci addu16i.d t1, zero, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) 17262306a36Sopenharmony_ci csrxchg t1, t0, LOONGARCH_CSR_TLBIDX 17362306a36Sopenharmony_ci 17462306a36Sopenharmony_ci tlbfill 17562306a36Sopenharmony_ci 17662306a36Sopenharmony_ci addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) 17762306a36Sopenharmony_ci addu16i.d t1, zero, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) 17862306a36Sopenharmony_ci csrxchg t1, t0, LOONGARCH_CSR_TLBIDX 17962306a36Sopenharmony_ci 18062306a36Sopenharmony_ci csrrd t0, EXCEPTION_KS0 18162306a36Sopenharmony_ci csrrd t1, EXCEPTION_KS1 18262306a36Sopenharmony_ci csrrd ra, EXCEPTION_KS2 18362306a36Sopenharmony_ci ertn 18462306a36Sopenharmony_ci 18562306a36Sopenharmony_cinopage_tlb_load: 18662306a36Sopenharmony_ci dbar 0x700 18762306a36Sopenharmony_ci csrrd ra, EXCEPTION_KS2 18862306a36Sopenharmony_ci la_abs t0, tlb_do_page_fault_0 18962306a36Sopenharmony_ci jr t0 19062306a36Sopenharmony_ciSYM_CODE_END(handle_tlb_load) 19162306a36Sopenharmony_ci 19262306a36Sopenharmony_ciSYM_CODE_START(handle_tlb_load_ptw) 19362306a36Sopenharmony_ci csrwr t0, LOONGARCH_CSR_KS0 19462306a36Sopenharmony_ci csrwr t1, LOONGARCH_CSR_KS1 19562306a36Sopenharmony_ci la_abs t0, tlb_do_page_fault_0 19662306a36Sopenharmony_ci jr t0 19762306a36Sopenharmony_ciSYM_CODE_END(handle_tlb_load_ptw) 19862306a36Sopenharmony_ci 19962306a36Sopenharmony_ciSYM_CODE_START(handle_tlb_store) 20062306a36Sopenharmony_ci csrwr t0, EXCEPTION_KS0 20162306a36Sopenharmony_ci csrwr t1, EXCEPTION_KS1 20262306a36Sopenharmony_ci csrwr ra, EXCEPTION_KS2 20362306a36Sopenharmony_ci 20462306a36Sopenharmony_ci /* 20562306a36Sopenharmony_ci * The vmalloc handling is not in the hotpath. 20662306a36Sopenharmony_ci */ 20762306a36Sopenharmony_ci csrrd t0, LOONGARCH_CSR_BADV 20862306a36Sopenharmony_ci bltz t0, vmalloc_store 20962306a36Sopenharmony_ci csrrd t1, LOONGARCH_CSR_PGDL 21062306a36Sopenharmony_ci 21162306a36Sopenharmony_civmalloc_done_store: 21262306a36Sopenharmony_ci /* Get PGD offset in bytes */ 21362306a36Sopenharmony_ci bstrpick.d ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT 21462306a36Sopenharmony_ci alsl.d t1, ra, t1, 3 21562306a36Sopenharmony_ci#if CONFIG_PGTABLE_LEVELS > 3 21662306a36Sopenharmony_ci ld.d t1, t1, 0 21762306a36Sopenharmony_ci bstrpick.d ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT 21862306a36Sopenharmony_ci alsl.d t1, ra, t1, 3 21962306a36Sopenharmony_ci#endif 22062306a36Sopenharmony_ci#if CONFIG_PGTABLE_LEVELS > 2 22162306a36Sopenharmony_ci ld.d t1, t1, 0 22262306a36Sopenharmony_ci bstrpick.d ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT 22362306a36Sopenharmony_ci alsl.d t1, ra, t1, 3 22462306a36Sopenharmony_ci#endif 22562306a36Sopenharmony_ci ld.d ra, t1, 0 22662306a36Sopenharmony_ci 22762306a36Sopenharmony_ci /* 22862306a36Sopenharmony_ci * For huge tlb entries, pmde doesn't contain an address but 22962306a36Sopenharmony_ci * instead contains the tlb pte. Check the PAGE_HUGE bit and 23062306a36Sopenharmony_ci * see if we need to jump to huge tlb processing. 23162306a36Sopenharmony_ci */ 23262306a36Sopenharmony_ci rotri.d ra, ra, _PAGE_HUGE_SHIFT + 1 23362306a36Sopenharmony_ci bltz ra, tlb_huge_update_store 23462306a36Sopenharmony_ci 23562306a36Sopenharmony_ci rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) 23662306a36Sopenharmony_ci bstrpick.d t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT 23762306a36Sopenharmony_ci alsl.d t1, t0, ra, _PTE_T_LOG2 23862306a36Sopenharmony_ci 23962306a36Sopenharmony_ci#ifdef CONFIG_SMP 24062306a36Sopenharmony_cismp_pgtable_change_store: 24162306a36Sopenharmony_ci ll.d t0, t1, 0 24262306a36Sopenharmony_ci#else 24362306a36Sopenharmony_ci ld.d t0, t1, 0 24462306a36Sopenharmony_ci#endif 24562306a36Sopenharmony_ci andi ra, t0, _PAGE_PRESENT | _PAGE_WRITE 24662306a36Sopenharmony_ci xori ra, ra, _PAGE_PRESENT | _PAGE_WRITE 24762306a36Sopenharmony_ci bnez ra, nopage_tlb_store 24862306a36Sopenharmony_ci 24962306a36Sopenharmony_ci ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 25062306a36Sopenharmony_ci#ifdef CONFIG_SMP 25162306a36Sopenharmony_ci sc.d t0, t1, 0 25262306a36Sopenharmony_ci beqz t0, smp_pgtable_change_store 25362306a36Sopenharmony_ci#else 25462306a36Sopenharmony_ci st.d t0, t1, 0 25562306a36Sopenharmony_ci#endif 25662306a36Sopenharmony_ci tlbsrch 25762306a36Sopenharmony_ci bstrins.d t1, zero, 3, 3 25862306a36Sopenharmony_ci ld.d t0, t1, 0 25962306a36Sopenharmony_ci ld.d t1, t1, 8 26062306a36Sopenharmony_ci csrwr t0, LOONGARCH_CSR_TLBELO0 26162306a36Sopenharmony_ci csrwr t1, LOONGARCH_CSR_TLBELO1 26262306a36Sopenharmony_ci tlbwr 26362306a36Sopenharmony_ci 26462306a36Sopenharmony_ci csrrd t0, EXCEPTION_KS0 26562306a36Sopenharmony_ci csrrd t1, EXCEPTION_KS1 26662306a36Sopenharmony_ci csrrd ra, EXCEPTION_KS2 26762306a36Sopenharmony_ci ertn 26862306a36Sopenharmony_ci 26962306a36Sopenharmony_ci#ifdef CONFIG_64BIT 27062306a36Sopenharmony_civmalloc_store: 27162306a36Sopenharmony_ci la_abs t1, swapper_pg_dir 27262306a36Sopenharmony_ci b vmalloc_done_store 27362306a36Sopenharmony_ci#endif 27462306a36Sopenharmony_ci 27562306a36Sopenharmony_ci /* This is the entry point of a huge page. */ 27662306a36Sopenharmony_citlb_huge_update_store: 27762306a36Sopenharmony_ci#ifdef CONFIG_SMP 27862306a36Sopenharmony_ci ll.d ra, t1, 0 27962306a36Sopenharmony_ci#endif 28062306a36Sopenharmony_ci andi t0, ra, _PAGE_PRESENT | _PAGE_WRITE 28162306a36Sopenharmony_ci xori t0, t0, _PAGE_PRESENT | _PAGE_WRITE 28262306a36Sopenharmony_ci bnez t0, nopage_tlb_store 28362306a36Sopenharmony_ci 28462306a36Sopenharmony_ci#ifdef CONFIG_SMP 28562306a36Sopenharmony_ci ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 28662306a36Sopenharmony_ci sc.d t0, t1, 0 28762306a36Sopenharmony_ci beqz t0, tlb_huge_update_store 28862306a36Sopenharmony_ci ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 28962306a36Sopenharmony_ci#else 29062306a36Sopenharmony_ci rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) 29162306a36Sopenharmony_ci ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 29262306a36Sopenharmony_ci st.d t0, t1, 0 29362306a36Sopenharmony_ci#endif 29462306a36Sopenharmony_ci csrrd ra, LOONGARCH_CSR_ASID 29562306a36Sopenharmony_ci csrrd t1, LOONGARCH_CSR_BADV 29662306a36Sopenharmony_ci andi ra, ra, CSR_ASID_ASID 29762306a36Sopenharmony_ci invtlb INVTLB_ADDR_GFALSE_AND_ASID, ra, t1 29862306a36Sopenharmony_ci 29962306a36Sopenharmony_ci /* 30062306a36Sopenharmony_ci * A huge PTE describes an area the size of the 30162306a36Sopenharmony_ci * configured huge page size. This is twice the 30262306a36Sopenharmony_ci * of the large TLB entry size we intend to use. 30362306a36Sopenharmony_ci * A TLB entry half the size of the configured 30462306a36Sopenharmony_ci * huge page size is configured into entrylo0 30562306a36Sopenharmony_ci * and entrylo1 to cover the contiguous huge PTE 30662306a36Sopenharmony_ci * address space. 30762306a36Sopenharmony_ci */ 30862306a36Sopenharmony_ci /* Huge page: Move Global bit */ 30962306a36Sopenharmony_ci xori t0, t0, _PAGE_HUGE 31062306a36Sopenharmony_ci lu12i.w t1, _PAGE_HGLOBAL >> 12 31162306a36Sopenharmony_ci and t1, t0, t1 31262306a36Sopenharmony_ci srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) 31362306a36Sopenharmony_ci or t0, t0, t1 31462306a36Sopenharmony_ci 31562306a36Sopenharmony_ci move ra, t0 31662306a36Sopenharmony_ci csrwr ra, LOONGARCH_CSR_TLBELO0 31762306a36Sopenharmony_ci 31862306a36Sopenharmony_ci /* Convert to entrylo1 */ 31962306a36Sopenharmony_ci addi.d t1, zero, 1 32062306a36Sopenharmony_ci slli.d t1, t1, (HPAGE_SHIFT - 1) 32162306a36Sopenharmony_ci add.d t0, t0, t1 32262306a36Sopenharmony_ci csrwr t0, LOONGARCH_CSR_TLBELO1 32362306a36Sopenharmony_ci 32462306a36Sopenharmony_ci /* Set huge page tlb entry size */ 32562306a36Sopenharmony_ci addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) 32662306a36Sopenharmony_ci addu16i.d t1, zero, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) 32762306a36Sopenharmony_ci csrxchg t1, t0, LOONGARCH_CSR_TLBIDX 32862306a36Sopenharmony_ci 32962306a36Sopenharmony_ci tlbfill 33062306a36Sopenharmony_ci 33162306a36Sopenharmony_ci /* Reset default page size */ 33262306a36Sopenharmony_ci addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) 33362306a36Sopenharmony_ci addu16i.d t1, zero, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) 33462306a36Sopenharmony_ci csrxchg t1, t0, LOONGARCH_CSR_TLBIDX 33562306a36Sopenharmony_ci 33662306a36Sopenharmony_ci csrrd t0, EXCEPTION_KS0 33762306a36Sopenharmony_ci csrrd t1, EXCEPTION_KS1 33862306a36Sopenharmony_ci csrrd ra, EXCEPTION_KS2 33962306a36Sopenharmony_ci ertn 34062306a36Sopenharmony_ci 34162306a36Sopenharmony_cinopage_tlb_store: 34262306a36Sopenharmony_ci dbar 0x700 34362306a36Sopenharmony_ci csrrd ra, EXCEPTION_KS2 34462306a36Sopenharmony_ci la_abs t0, tlb_do_page_fault_1 34562306a36Sopenharmony_ci jr t0 34662306a36Sopenharmony_ciSYM_CODE_END(handle_tlb_store) 34762306a36Sopenharmony_ci 34862306a36Sopenharmony_ciSYM_CODE_START(handle_tlb_store_ptw) 34962306a36Sopenharmony_ci csrwr t0, LOONGARCH_CSR_KS0 35062306a36Sopenharmony_ci csrwr t1, LOONGARCH_CSR_KS1 35162306a36Sopenharmony_ci la_abs t0, tlb_do_page_fault_1 35262306a36Sopenharmony_ci jr t0 35362306a36Sopenharmony_ciSYM_CODE_END(handle_tlb_store_ptw) 35462306a36Sopenharmony_ci 35562306a36Sopenharmony_ciSYM_CODE_START(handle_tlb_modify) 35662306a36Sopenharmony_ci csrwr t0, EXCEPTION_KS0 35762306a36Sopenharmony_ci csrwr t1, EXCEPTION_KS1 35862306a36Sopenharmony_ci csrwr ra, EXCEPTION_KS2 35962306a36Sopenharmony_ci 36062306a36Sopenharmony_ci /* 36162306a36Sopenharmony_ci * The vmalloc handling is not in the hotpath. 36262306a36Sopenharmony_ci */ 36362306a36Sopenharmony_ci csrrd t0, LOONGARCH_CSR_BADV 36462306a36Sopenharmony_ci bltz t0, vmalloc_modify 36562306a36Sopenharmony_ci csrrd t1, LOONGARCH_CSR_PGDL 36662306a36Sopenharmony_ci 36762306a36Sopenharmony_civmalloc_done_modify: 36862306a36Sopenharmony_ci /* Get PGD offset in bytes */ 36962306a36Sopenharmony_ci bstrpick.d ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT 37062306a36Sopenharmony_ci alsl.d t1, ra, t1, 3 37162306a36Sopenharmony_ci#if CONFIG_PGTABLE_LEVELS > 3 37262306a36Sopenharmony_ci ld.d t1, t1, 0 37362306a36Sopenharmony_ci bstrpick.d ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT 37462306a36Sopenharmony_ci alsl.d t1, ra, t1, 3 37562306a36Sopenharmony_ci#endif 37662306a36Sopenharmony_ci#if CONFIG_PGTABLE_LEVELS > 2 37762306a36Sopenharmony_ci ld.d t1, t1, 0 37862306a36Sopenharmony_ci bstrpick.d ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT 37962306a36Sopenharmony_ci alsl.d t1, ra, t1, 3 38062306a36Sopenharmony_ci#endif 38162306a36Sopenharmony_ci ld.d ra, t1, 0 38262306a36Sopenharmony_ci 38362306a36Sopenharmony_ci /* 38462306a36Sopenharmony_ci * For huge tlb entries, pmde doesn't contain an address but 38562306a36Sopenharmony_ci * instead contains the tlb pte. Check the PAGE_HUGE bit and 38662306a36Sopenharmony_ci * see if we need to jump to huge tlb processing. 38762306a36Sopenharmony_ci */ 38862306a36Sopenharmony_ci rotri.d ra, ra, _PAGE_HUGE_SHIFT + 1 38962306a36Sopenharmony_ci bltz ra, tlb_huge_update_modify 39062306a36Sopenharmony_ci 39162306a36Sopenharmony_ci rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) 39262306a36Sopenharmony_ci bstrpick.d t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT 39362306a36Sopenharmony_ci alsl.d t1, t0, ra, _PTE_T_LOG2 39462306a36Sopenharmony_ci 39562306a36Sopenharmony_ci#ifdef CONFIG_SMP 39662306a36Sopenharmony_cismp_pgtable_change_modify: 39762306a36Sopenharmony_ci ll.d t0, t1, 0 39862306a36Sopenharmony_ci#else 39962306a36Sopenharmony_ci ld.d t0, t1, 0 40062306a36Sopenharmony_ci#endif 40162306a36Sopenharmony_ci andi ra, t0, _PAGE_WRITE 40262306a36Sopenharmony_ci beqz ra, nopage_tlb_modify 40362306a36Sopenharmony_ci 40462306a36Sopenharmony_ci ori t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 40562306a36Sopenharmony_ci#ifdef CONFIG_SMP 40662306a36Sopenharmony_ci sc.d t0, t1, 0 40762306a36Sopenharmony_ci beqz t0, smp_pgtable_change_modify 40862306a36Sopenharmony_ci#else 40962306a36Sopenharmony_ci st.d t0, t1, 0 41062306a36Sopenharmony_ci#endif 41162306a36Sopenharmony_ci tlbsrch 41262306a36Sopenharmony_ci bstrins.d t1, zero, 3, 3 41362306a36Sopenharmony_ci ld.d t0, t1, 0 41462306a36Sopenharmony_ci ld.d t1, t1, 8 41562306a36Sopenharmony_ci csrwr t0, LOONGARCH_CSR_TLBELO0 41662306a36Sopenharmony_ci csrwr t1, LOONGARCH_CSR_TLBELO1 41762306a36Sopenharmony_ci tlbwr 41862306a36Sopenharmony_ci 41962306a36Sopenharmony_ci csrrd t0, EXCEPTION_KS0 42062306a36Sopenharmony_ci csrrd t1, EXCEPTION_KS1 42162306a36Sopenharmony_ci csrrd ra, EXCEPTION_KS2 42262306a36Sopenharmony_ci ertn 42362306a36Sopenharmony_ci 42462306a36Sopenharmony_ci#ifdef CONFIG_64BIT 42562306a36Sopenharmony_civmalloc_modify: 42662306a36Sopenharmony_ci la_abs t1, swapper_pg_dir 42762306a36Sopenharmony_ci b vmalloc_done_modify 42862306a36Sopenharmony_ci#endif 42962306a36Sopenharmony_ci 43062306a36Sopenharmony_ci /* This is the entry point of a huge page. */ 43162306a36Sopenharmony_citlb_huge_update_modify: 43262306a36Sopenharmony_ci#ifdef CONFIG_SMP 43362306a36Sopenharmony_ci ll.d ra, t1, 0 43462306a36Sopenharmony_ci#endif 43562306a36Sopenharmony_ci andi t0, ra, _PAGE_WRITE 43662306a36Sopenharmony_ci beqz t0, nopage_tlb_modify 43762306a36Sopenharmony_ci 43862306a36Sopenharmony_ci#ifdef CONFIG_SMP 43962306a36Sopenharmony_ci ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 44062306a36Sopenharmony_ci sc.d t0, t1, 0 44162306a36Sopenharmony_ci beqz t0, tlb_huge_update_modify 44262306a36Sopenharmony_ci ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 44362306a36Sopenharmony_ci#else 44462306a36Sopenharmony_ci rotri.d ra, ra, 64 - (_PAGE_HUGE_SHIFT + 1) 44562306a36Sopenharmony_ci ori t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED) 44662306a36Sopenharmony_ci st.d t0, t1, 0 44762306a36Sopenharmony_ci#endif 44862306a36Sopenharmony_ci csrrd ra, LOONGARCH_CSR_ASID 44962306a36Sopenharmony_ci csrrd t1, LOONGARCH_CSR_BADV 45062306a36Sopenharmony_ci andi ra, ra, CSR_ASID_ASID 45162306a36Sopenharmony_ci invtlb INVTLB_ADDR_GFALSE_AND_ASID, ra, t1 45262306a36Sopenharmony_ci 45362306a36Sopenharmony_ci /* 45462306a36Sopenharmony_ci * A huge PTE describes an area the size of the 45562306a36Sopenharmony_ci * configured huge page size. This is twice the 45662306a36Sopenharmony_ci * of the large TLB entry size we intend to use. 45762306a36Sopenharmony_ci * A TLB entry half the size of the configured 45862306a36Sopenharmony_ci * huge page size is configured into entrylo0 45962306a36Sopenharmony_ci * and entrylo1 to cover the contiguous huge PTE 46062306a36Sopenharmony_ci * address space. 46162306a36Sopenharmony_ci */ 46262306a36Sopenharmony_ci /* Huge page: Move Global bit */ 46362306a36Sopenharmony_ci xori t0, t0, _PAGE_HUGE 46462306a36Sopenharmony_ci lu12i.w t1, _PAGE_HGLOBAL >> 12 46562306a36Sopenharmony_ci and t1, t0, t1 46662306a36Sopenharmony_ci srli.d t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT) 46762306a36Sopenharmony_ci or t0, t0, t1 46862306a36Sopenharmony_ci 46962306a36Sopenharmony_ci move ra, t0 47062306a36Sopenharmony_ci csrwr ra, LOONGARCH_CSR_TLBELO0 47162306a36Sopenharmony_ci 47262306a36Sopenharmony_ci /* Convert to entrylo1 */ 47362306a36Sopenharmony_ci addi.d t1, zero, 1 47462306a36Sopenharmony_ci slli.d t1, t1, (HPAGE_SHIFT - 1) 47562306a36Sopenharmony_ci add.d t0, t0, t1 47662306a36Sopenharmony_ci csrwr t0, LOONGARCH_CSR_TLBELO1 47762306a36Sopenharmony_ci 47862306a36Sopenharmony_ci /* Set huge page tlb entry size */ 47962306a36Sopenharmony_ci addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) 48062306a36Sopenharmony_ci addu16i.d t1, zero, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) 48162306a36Sopenharmony_ci csrxchg t1, t0, LOONGARCH_CSR_TLBIDX 48262306a36Sopenharmony_ci 48362306a36Sopenharmony_ci tlbfill 48462306a36Sopenharmony_ci 48562306a36Sopenharmony_ci /* Reset default page size */ 48662306a36Sopenharmony_ci addu16i.d t0, zero, (CSR_TLBIDX_PS >> 16) 48762306a36Sopenharmony_ci addu16i.d t1, zero, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT - 16)) 48862306a36Sopenharmony_ci csrxchg t1, t0, LOONGARCH_CSR_TLBIDX 48962306a36Sopenharmony_ci 49062306a36Sopenharmony_ci csrrd t0, EXCEPTION_KS0 49162306a36Sopenharmony_ci csrrd t1, EXCEPTION_KS1 49262306a36Sopenharmony_ci csrrd ra, EXCEPTION_KS2 49362306a36Sopenharmony_ci ertn 49462306a36Sopenharmony_ci 49562306a36Sopenharmony_cinopage_tlb_modify: 49662306a36Sopenharmony_ci dbar 0x700 49762306a36Sopenharmony_ci csrrd ra, EXCEPTION_KS2 49862306a36Sopenharmony_ci la_abs t0, tlb_do_page_fault_1 49962306a36Sopenharmony_ci jr t0 50062306a36Sopenharmony_ciSYM_CODE_END(handle_tlb_modify) 50162306a36Sopenharmony_ci 50262306a36Sopenharmony_ciSYM_CODE_START(handle_tlb_modify_ptw) 50362306a36Sopenharmony_ci csrwr t0, LOONGARCH_CSR_KS0 50462306a36Sopenharmony_ci csrwr t1, LOONGARCH_CSR_KS1 50562306a36Sopenharmony_ci la_abs t0, tlb_do_page_fault_1 50662306a36Sopenharmony_ci jr t0 50762306a36Sopenharmony_ciSYM_CODE_END(handle_tlb_modify_ptw) 50862306a36Sopenharmony_ci 50962306a36Sopenharmony_ciSYM_CODE_START(handle_tlb_refill) 51062306a36Sopenharmony_ci csrwr t0, LOONGARCH_CSR_TLBRSAVE 51162306a36Sopenharmony_ci csrrd t0, LOONGARCH_CSR_PGD 51262306a36Sopenharmony_ci lddir t0, t0, 3 51362306a36Sopenharmony_ci#if CONFIG_PGTABLE_LEVELS > 3 51462306a36Sopenharmony_ci lddir t0, t0, 2 51562306a36Sopenharmony_ci#endif 51662306a36Sopenharmony_ci#if CONFIG_PGTABLE_LEVELS > 2 51762306a36Sopenharmony_ci lddir t0, t0, 1 51862306a36Sopenharmony_ci#endif 51962306a36Sopenharmony_ci ldpte t0, 0 52062306a36Sopenharmony_ci ldpte t0, 1 52162306a36Sopenharmony_ci tlbfill 52262306a36Sopenharmony_ci csrrd t0, LOONGARCH_CSR_TLBRSAVE 52362306a36Sopenharmony_ci ertn 52462306a36Sopenharmony_ciSYM_CODE_END(handle_tlb_refill) 525