Lines Matching refs:addr
154 static void init_pte(pmd_t *pmdp, unsigned long addr, unsigned long end,
159 ptep = pte_set_fixmap_offset(pmdp, addr);
173 } while (ptep++, addr += PAGE_SIZE, addr != end);
178 static void alloc_init_cont_pte(pmd_t *pmdp, unsigned long addr,
200 next = pte_cont_addr_end(addr, end);
203 if ((((addr | next | phys) & ~CONT_PTE_MASK) == 0) &&
207 init_pte(pmdp, addr, next, phys, __prot);
209 phys += next - addr;
210 } while (addr = next, addr != end);
213 static void init_pmd(pud_t *pudp, unsigned long addr, unsigned long end,
220 pmdp = pmd_set_fixmap_offset(pudp, addr);
224 next = pmd_addr_end(addr, end);
227 if (((addr | next | phys) & ~SECTION_MASK) == 0 &&
238 alloc_init_cont_pte(pmdp, addr, next, phys, prot,
244 phys += next - addr;
245 } while (pmdp++, addr = next, addr != end);
250 static void alloc_init_cont_pmd(pud_t *pudp, unsigned long addr,
274 next = pmd_cont_addr_end(addr, end);
277 if ((((addr | next | phys) & ~CONT_PMD_MASK) == 0) &&
281 init_pmd(pudp, addr, next, phys, __prot, pgtable_alloc, flags);
283 phys += next - addr;
284 } while (addr = next, addr != end);
287 static inline bool use_1G_block(unsigned long addr, unsigned long next,
293 if (((addr | next | phys) & ~PUD_MASK) != 0)
299 static void alloc_init_pud(pgd_t *pgdp, unsigned long addr, unsigned long end,
306 p4d_t *p4dp = p4d_offset(pgdp, addr);
324 pudp = pud_set_fixmap_offset(p4dp, addr);
328 next = pud_addr_end(addr, end);
333 if (use_1G_block(addr, next, phys) &&
344 alloc_init_cont_pmd(pudp, addr, next, phys, prot,
350 phys += next - addr;
351 } while (pudp++, addr = next, addr != end);
364 unsigned long addr, end, next;
375 addr = virt & PAGE_MASK;
379 next = pgd_addr_end(addr, end);
380 alloc_init_pud(pgdp, addr, next, phys, prot, pgtable_alloc,
382 phys += next - addr;
383 } while (pgdp++, addr = next, addr != end);
604 vma->addr = va_start;
771 int kern_addr_valid(unsigned long addr)
779 addr = arch_kasan_reset_tag(addr);
780 if ((((long)addr) >> VA_BITS) != -1UL)
783 pgdp = pgd_offset_k(addr);
787 p4dp = p4d_offset(pgdp, addr);
791 pudp = pud_offset(p4dp, addr);
799 pmdp = pmd_offset(pudp, addr);
807 ptep = pte_offset_kernel(pmdp, addr);
851 static void unmap_hotplug_pte_range(pmd_t *pmdp, unsigned long addr,
858 ptep = pte_offset_kernel(pmdp, addr);
864 pte_clear(&init_mm, addr, ptep);
865 flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
869 } while (addr += PAGE_SIZE, addr < end);
872 static void unmap_hotplug_pmd_range(pud_t *pudp, unsigned long addr,
880 next = pmd_addr_end(addr, end);
881 pmdp = pmd_offset(pudp, addr);
894 flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
901 unmap_hotplug_pte_range(pmdp, addr, next, free_mapped, altmap);
902 } while (addr = next, addr < end);
905 static void unmap_hotplug_pud_range(p4d_t *p4dp, unsigned long addr,
913 next = pud_addr_end(addr, end);
914 pudp = pud_offset(p4dp, addr);
927 flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
934 unmap_hotplug_pmd_range(pudp, addr, next, free_mapped, altmap);
935 } while (addr = next, addr < end);
938 static void unmap_hotplug_p4d_range(pgd_t *pgdp, unsigned long addr,
946 next = p4d_addr_end(addr, end);
947 p4dp = p4d_offset(pgdp, addr);
953 unmap_hotplug_pud_range(p4dp, addr, next, free_mapped, altmap);
954 } while (addr = next, addr < end);
957 static void unmap_hotplug_range(unsigned long addr, unsigned long end,
972 next = pgd_addr_end(addr, end);
973 pgdp = pgd_offset_k(addr);
979 unmap_hotplug_p4d_range(pgdp, addr, next, free_mapped, altmap);
980 } while (addr = next, addr < end);
983 static void free_empty_pte_table(pmd_t *pmdp, unsigned long addr,
988 unsigned long i, start = addr;
991 ptep = pte_offset_kernel(pmdp, addr);
999 } while (addr += PAGE_SIZE, addr < end);
1020 static void free_empty_pmd_table(pud_t *pudp, unsigned long addr,
1025 unsigned long i, next, start = addr;
1028 next = pmd_addr_end(addr, end);
1029 pmdp = pmd_offset(pudp, addr);
1035 free_empty_pte_table(pmdp, addr, next, floor, ceiling);
1036 } while (addr = next, addr < end);
1060 static void free_empty_pud_table(p4d_t *p4dp, unsigned long addr,
1065 unsigned long i, next, start = addr;
1068 next = pud_addr_end(addr, end);
1069 pudp = pud_offset(p4dp, addr);
1075 free_empty_pmd_table(pudp, addr, next, floor, ceiling);
1076 } while (addr = next, addr < end);
1100 static void free_empty_p4d_table(pgd_t *pgdp, unsigned long addr,
1108 next = p4d_addr_end(addr, end);
1109 p4dp = p4d_offset(pgdp, addr);
1115 free_empty_pud_table(p4dp, addr, next, floor, ceiling);
1116 } while (addr = next, addr < end);
1119 static void free_empty_tables(unsigned long addr, unsigned long end,
1126 next = pgd_addr_end(addr, end);
1127 pgdp = pgd_offset_k(addr);
1133 free_empty_p4d_table(pgdp, addr, next, floor, ceiling);
1134 } while (addr = next, addr < end);
1149 unsigned long addr = start;
1157 next = pmd_addr_end(addr, end);
1159 pgdp = vmemmap_pgd_populate(addr, node);
1163 p4dp = vmemmap_p4d_populate(pgdp, addr, node);
1167 pudp = vmemmap_pud_populate(p4dp, addr, node);
1171 pmdp = pmd_offset(pudp, addr);
1181 vmemmap_verify((pte_t *)pmdp, node, addr, next);
1182 } while (addr = next, addr != end);
1199 static inline pud_t * fixmap_pud(unsigned long addr)
1201 pgd_t *pgdp = pgd_offset_k(addr);
1202 p4d_t *p4dp = p4d_offset(pgdp, addr);
1207 return pud_offset_kimg(p4dp, addr);
1210 static inline pmd_t * fixmap_pmd(unsigned long addr)
1212 pud_t *pudp = fixmap_pud(addr);
1217 return pmd_offset_kimg(pudp, addr);
1220 static inline pte_t * fixmap_pte(unsigned long addr)
1222 return &bm_pte[pte_index(addr)];
1237 unsigned long addr = FIXADDR_START;
1239 pgdp = pgd_offset_k(addr);
1240 p4dp = p4d_offset(pgdp, addr);
1250 pudp = pud_offset_kimg(p4dp, addr);
1254 pudp = fixmap_pud(addr);
1258 pmdp = fixmap_pmd(addr);
1291 unsigned long addr = __fix_to_virt(idx);
1296 ptep = fixmap_pte(addr);
1301 pte_clear(&init_mm, addr, ptep);
1302 flush_tlb_kernel_range(addr, addr+PAGE_SIZE);
1424 int pmd_free_pte_page(pmd_t *pmdp, unsigned long addr)
1436 table = pte_offset_kernel(pmdp, addr);
1438 __flush_tlb_kernel_pgtable(addr);
1443 int pud_free_pmd_page(pud_t *pudp, unsigned long addr)
1457 table = pmd_offset(pudp, addr);
1459 next = addr;
1460 end = addr + PUD_SIZE;
1466 __flush_tlb_kernel_pgtable(addr);
1471 int p4d_free_pud_page(p4d_t *p4d, unsigned long addr)