1 // SPDX-License-Identifier: GPL-2.0
2
3 #include <linux/mm.h>
4 #include <linux/smp.h>
5 #include <linux/sched.h>
6 #include <asm/sbi.h>
7 #include <asm/mmu_context.h>
8
local_flush_tlb_all_asid(unsigned long asid)9 static inline void local_flush_tlb_all_asid(unsigned long asid)
10 {
11 if (asid != FLUSH_TLB_NO_ASID)
12 __asm__ __volatile__ ("sfence.vma x0, %0"
13 :
14 : "r" (asid)
15 : "memory");
16 else
17 local_flush_tlb_all();
18 }
19
local_flush_tlb_page_asid(unsigned long addr, unsigned long asid)20 static inline void local_flush_tlb_page_asid(unsigned long addr,
21 unsigned long asid)
22 {
23 if (asid != FLUSH_TLB_NO_ASID)
24 __asm__ __volatile__ ("sfence.vma %0, %1"
25 :
26 : "r" (addr), "r" (asid)
27 : "memory");
28 else
29 local_flush_tlb_page(addr);
30 }
31
32 /*
33 * Flush entire TLB if number of entries to be flushed is greater
34 * than the threshold below.
35 */
36 static unsigned long tlb_flush_all_threshold __read_mostly = 64;
37
local_flush_tlb_range_threshold_asid(unsigned long start, unsigned long size, unsigned long stride, unsigned long asid)38 static void local_flush_tlb_range_threshold_asid(unsigned long start,
39 unsigned long size,
40 unsigned long stride,
41 unsigned long asid)
42 {
43 unsigned long nr_ptes_in_range = DIV_ROUND_UP(size, stride);
44 int i;
45
46 if (nr_ptes_in_range > tlb_flush_all_threshold) {
47 local_flush_tlb_all_asid(asid);
48 return;
49 }
50
51 for (i = 0; i < nr_ptes_in_range; ++i) {
52 local_flush_tlb_page_asid(start, asid);
53 start += stride;
54 }
55 }
56
local_flush_tlb_range_asid(unsigned long start, unsigned long size, unsigned long stride, unsigned long asid)57 static inline void local_flush_tlb_range_asid(unsigned long start,
58 unsigned long size, unsigned long stride, unsigned long asid)
59 {
60 if (size <= stride)
61 local_flush_tlb_page_asid(start, asid);
62 else if (size == FLUSH_TLB_MAX_SIZE)
63 local_flush_tlb_all_asid(asid);
64 else
65 local_flush_tlb_range_threshold_asid(start, size, stride, asid);
66 }
67
68 /* Flush a range of kernel pages without broadcasting */
local_flush_tlb_kernel_range(unsigned long start, unsigned long end)69 void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
70 {
71 local_flush_tlb_range_asid(start, end - start, PAGE_SIZE, FLUSH_TLB_NO_ASID);
72 }
73
__ipi_flush_tlb_all(void *info)74 static void __ipi_flush_tlb_all(void *info)
75 {
76 local_flush_tlb_all();
77 }
78
flush_tlb_all(void)79 void flush_tlb_all(void)
80 {
81 if (riscv_use_ipi_for_rfence())
82 on_each_cpu(__ipi_flush_tlb_all, NULL, 1);
83 else
84 sbi_remote_sfence_vma_asid(NULL, 0, FLUSH_TLB_MAX_SIZE, FLUSH_TLB_NO_ASID);
85 }
86
87 struct flush_tlb_range_data {
88 unsigned long asid;
89 unsigned long start;
90 unsigned long size;
91 unsigned long stride;
92 };
93
__ipi_flush_tlb_range_asid(void *info)94 static void __ipi_flush_tlb_range_asid(void *info)
95 {
96 struct flush_tlb_range_data *d = info;
97
98 local_flush_tlb_range_asid(d->start, d->size, d->stride, d->asid);
99 }
100
__flush_tlb_range(struct mm_struct *mm, unsigned long start, unsigned long size, unsigned long stride)101 static void __flush_tlb_range(struct mm_struct *mm, unsigned long start,
102 unsigned long size, unsigned long stride)
103 {
104 struct flush_tlb_range_data ftd;
105 const struct cpumask *cmask;
106 unsigned long asid = FLUSH_TLB_NO_ASID;
107 bool broadcast;
108
109 if (mm) {
110 unsigned int cpuid;
111
112 cmask = mm_cpumask(mm);
113 if (cpumask_empty(cmask))
114 return;
115
116 cpuid = get_cpu();
117 /* check if the tlbflush needs to be sent to other CPUs */
118 broadcast = cpumask_any_but(cmask, cpuid) < nr_cpu_ids;
119
120 if (static_branch_unlikely(&use_asid_allocator))
121 asid = atomic_long_read(&mm->context.id) & asid_mask;
122 } else {
123 cmask = cpu_online_mask;
124 broadcast = true;
125 }
126
127 if (broadcast) {
128 if (riscv_use_ipi_for_rfence()) {
129 ftd.asid = asid;
130 ftd.start = start;
131 ftd.size = size;
132 ftd.stride = stride;
133 on_each_cpu_mask(cmask,
134 __ipi_flush_tlb_range_asid,
135 &ftd, 1);
136 } else
137 sbi_remote_sfence_vma_asid(cmask,
138 start, size, asid);
139 } else {
140 local_flush_tlb_range_asid(start, size, stride, asid);
141 }
142
143 if (mm)
144 put_cpu();
145 }
146
flush_tlb_mm(struct mm_struct *mm)147 void flush_tlb_mm(struct mm_struct *mm)
148 {
149 __flush_tlb_range(mm, 0, FLUSH_TLB_MAX_SIZE, PAGE_SIZE);
150 }
151
flush_tlb_mm_range(struct mm_struct *mm, unsigned long start, unsigned long end, unsigned int page_size)152 void flush_tlb_mm_range(struct mm_struct *mm,
153 unsigned long start, unsigned long end,
154 unsigned int page_size)
155 {
156 __flush_tlb_range(mm, start, end - start, page_size);
157 }
158
flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)159 void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
160 {
161 __flush_tlb_range(vma->vm_mm, addr, PAGE_SIZE, PAGE_SIZE);
162 }
163
flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)164 void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
165 unsigned long end)
166 {
167 __flush_tlb_range(vma->vm_mm, start, end - start, PAGE_SIZE);
168 }
169
flush_tlb_kernel_range(unsigned long start, unsigned long end)170 void flush_tlb_kernel_range(unsigned long start, unsigned long end)
171 {
172 __flush_tlb_range(NULL, start, end - start, PAGE_SIZE);
173 }
174
175 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end)176 void flush_pmd_tlb_range(struct vm_area_struct *vma, unsigned long start,
177 unsigned long end)
178 {
179 __flush_tlb_range(vma->vm_mm, start, end - start, PMD_SIZE);
180 }
181 #endif
182