1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (C) 2019 Andes Technology Corporation
3
4 #include <linux/pfn.h>
5 #include <linux/init_task.h>
6 #include <linux/kasan.h>
7 #include <linux/kernel.h>
8 #include <linux/memblock.h>
9 #include <linux/pgtable.h>
10 #include <asm/tlbflush.h>
11 #include <asm/fixmap.h>
12
13 extern pgd_t early_pg_dir[PTRS_PER_PGD];
kasan_early_init(void)14 asmlinkage void __init kasan_early_init(void)
15 {
16 uintptr_t i;
17 pgd_t *pgd = early_pg_dir + pgd_index(KASAN_SHADOW_START);
18
19 BUILD_BUG_ON(KASAN_SHADOW_OFFSET !=
20 KASAN_SHADOW_END - (1UL << (64 - KASAN_SHADOW_SCALE_SHIFT)));
21
22 for (i = 0; i < PTRS_PER_PTE; ++i)
23 set_pte(kasan_early_shadow_pte + i,
24 pfn_pte(virt_to_pfn(kasan_early_shadow_page), PAGE_KERNEL));
25
26 for (i = 0; i < PTRS_PER_PMD; ++i)
27 set_pmd(kasan_early_shadow_pmd + i,
28 pfn_pmd(PFN_DOWN
29 (__pa((uintptr_t) kasan_early_shadow_pte)),
30 __pgprot(_PAGE_TABLE)));
31
32 for (i = KASAN_SHADOW_START; i < KASAN_SHADOW_END;
33 i += PGDIR_SIZE, ++pgd)
34 set_pgd(pgd,
35 pfn_pgd(PFN_DOWN
36 (__pa(((uintptr_t) kasan_early_shadow_pmd))),
37 __pgprot(_PAGE_TABLE)));
38
39 /* init for swapper_pg_dir */
40 pgd = pgd_offset_k(KASAN_SHADOW_START);
41
42 for (i = KASAN_SHADOW_START; i < KASAN_SHADOW_END;
43 i += PGDIR_SIZE, ++pgd)
44 set_pgd(pgd,
45 pfn_pgd(PFN_DOWN
46 (__pa(((uintptr_t) kasan_early_shadow_pmd))),
47 __pgprot(_PAGE_TABLE)));
48
49 local_flush_tlb_all();
50 }
51
populate(void *start, void *end)52 static void __init populate(void *start, void *end)
53 {
54 unsigned long i, offset;
55 unsigned long vaddr = (unsigned long)start & PAGE_MASK;
56 unsigned long vend = PAGE_ALIGN((unsigned long)end);
57 unsigned long n_pages = (vend - vaddr) / PAGE_SIZE;
58 unsigned long n_ptes =
59 ((n_pages + PTRS_PER_PTE) & -PTRS_PER_PTE) / PTRS_PER_PTE;
60 unsigned long n_pmds =
61 ((n_ptes + PTRS_PER_PMD) & -PTRS_PER_PMD) / PTRS_PER_PMD;
62
63 pte_t *pte =
64 memblock_alloc(n_ptes * PTRS_PER_PTE * sizeof(pte_t), PAGE_SIZE);
65 pmd_t *pmd =
66 memblock_alloc(n_pmds * PTRS_PER_PMD * sizeof(pmd_t), PAGE_SIZE);
67 pgd_t *pgd = pgd_offset_k(vaddr);
68
69 for (i = 0; i < n_pages; i++) {
70 phys_addr_t phys = memblock_phys_alloc(PAGE_SIZE, PAGE_SIZE);
71 set_pte(&pte[i], pfn_pte(PHYS_PFN(phys), PAGE_KERNEL));
72 }
73
74 for (i = 0, offset = 0; i < n_ptes; i++, offset += PTRS_PER_PTE)
75 set_pmd(&pmd[i],
76 pfn_pmd(PFN_DOWN(__pa(&pte[offset])),
77 __pgprot(_PAGE_TABLE)));
78
79 for (i = 0, offset = 0; i < n_pmds; i++, offset += PTRS_PER_PMD)
80 set_pgd(&pgd[i],
81 pfn_pgd(PFN_DOWN(__pa(&pmd[offset])),
82 __pgprot(_PAGE_TABLE)));
83
84 local_flush_tlb_all();
85 memset(start, 0, end - start);
86 }
87
kasan_init(void)88 void __init kasan_init(void)
89 {
90 phys_addr_t p_start, p_end;
91 u64 i;
92
93 /*
94 * Populate all kernel virtual address space with kasan_early_shadow_page
95 * except for the linear mapping and the modules/kernel/BPF mapping.
96 */
97 kasan_populate_early_shadow((void *)KASAN_SHADOW_START,
98 (void *)kasan_mem_to_shadow((void *)
99 VMALLOC_END));
100
101 /* Populate the linear mapping */
102 for_each_mem_range(i, &p_start, &p_end) {
103 void *start = (void *)__va(p_start);
104 void *end = (void *)__va(p_end);
105
106 if (start >= end)
107 break;
108
109 populate(kasan_mem_to_shadow(start), kasan_mem_to_shadow(end));
110 };
111
112 /* Populate kernel, BPF, modules mapping */
113 kasan_populate(kasan_mem_to_shadow((const void *)MODULES_VADDR),
114 kasan_mem_to_shadow((const void *)BPF_JIT_REGION_END));
115
116 for (i = 0; i < PTRS_PER_PTE; i++)
117 set_pte(&kasan_early_shadow_pte[i],
118 mk_pte(virt_to_page(kasan_early_shadow_page),
119 __pgprot(_PAGE_PRESENT | _PAGE_READ |
120 _PAGE_ACCESSED)));
121
122 memset(kasan_early_shadow_page, 0, PAGE_SIZE);
123 init_task.kasan_depth = 0;
124 }
125