1// SPDX-License-Identifier: GPL-2.0 2#include <linux/mm.h> 3#include <linux/gfp.h> 4#include <linux/hugetlb.h> 5#include <asm/pgalloc.h> 6#include <asm/tlb.h> 7#include <asm/fixmap.h> 8#include <asm/mtrr.h> 9 10#ifdef CONFIG_DYNAMIC_PHYSICAL_MASK 11phys_addr_t physical_mask __ro_after_init = (1ULL << __PHYSICAL_MASK_SHIFT) - 1; 12EXPORT_SYMBOL(physical_mask); 13#endif 14 15#ifdef CONFIG_HIGHPTE 16#define PGTABLE_HIGHMEM __GFP_HIGHMEM 17#else 18#define PGTABLE_HIGHMEM 0 19#endif 20 21#ifndef CONFIG_PARAVIRT 22static inline 23void paravirt_tlb_remove_table(struct mmu_gather *tlb, void *table) 24{ 25 tlb_remove_page(tlb, table); 26} 27#endif 28 29gfp_t __userpte_alloc_gfp = GFP_PGTABLE_USER | PGTABLE_HIGHMEM; 30 31pgtable_t pte_alloc_one(struct mm_struct *mm) 32{ 33 return __pte_alloc_one(mm, __userpte_alloc_gfp); 34} 35 36static int __init setup_userpte(char *arg) 37{ 38 if (!arg) 39 return -EINVAL; 40 41 /* 42 * "userpte=nohigh" disables allocation of user pagetables in 43 * high memory. 44 */ 45 if (strcmp(arg, "nohigh") == 0) 46 __userpte_alloc_gfp &= ~__GFP_HIGHMEM; 47 else 48 return -EINVAL; 49 return 0; 50} 51early_param("userpte", setup_userpte); 52 53void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte) 54{ 55 pagetable_pte_dtor(page_ptdesc(pte)); 56 paravirt_release_pte(page_to_pfn(pte)); 57 paravirt_tlb_remove_table(tlb, pte); 58} 59 60#if CONFIG_PGTABLE_LEVELS > 2 61void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd) 62{ 63 struct ptdesc *ptdesc = virt_to_ptdesc(pmd); 64 paravirt_release_pmd(__pa(pmd) >> PAGE_SHIFT); 65 /* 66 * NOTE! For PAE, any changes to the top page-directory-pointer-table 67 * entries need a full cr3 reload to flush. 68 */ 69#ifdef CONFIG_X86_PAE 70 tlb->need_flush_all = 1; 71#endif 72 pagetable_pmd_dtor(ptdesc); 73 paravirt_tlb_remove_table(tlb, ptdesc_page(ptdesc)); 74} 75 76#if CONFIG_PGTABLE_LEVELS > 3 77void ___pud_free_tlb(struct mmu_gather *tlb, pud_t *pud) 78{ 79 paravirt_release_pud(__pa(pud) >> PAGE_SHIFT); 80 paravirt_tlb_remove_table(tlb, virt_to_page(pud)); 81} 82 83#if CONFIG_PGTABLE_LEVELS > 4 84void ___p4d_free_tlb(struct mmu_gather *tlb, p4d_t *p4d) 85{ 86 paravirt_release_p4d(__pa(p4d) >> PAGE_SHIFT); 87 paravirt_tlb_remove_table(tlb, virt_to_page(p4d)); 88} 89#endif /* CONFIG_PGTABLE_LEVELS > 4 */ 90#endif /* CONFIG_PGTABLE_LEVELS > 3 */ 91#endif /* CONFIG_PGTABLE_LEVELS > 2 */ 92 93static inline void pgd_list_add(pgd_t *pgd) 94{ 95 struct ptdesc *ptdesc = virt_to_ptdesc(pgd); 96 97 list_add(&ptdesc->pt_list, &pgd_list); 98} 99 100static inline void pgd_list_del(pgd_t *pgd) 101{ 102 struct ptdesc *ptdesc = virt_to_ptdesc(pgd); 103 104 list_del(&ptdesc->pt_list); 105} 106 107#define UNSHARED_PTRS_PER_PGD \ 108 (SHARED_KERNEL_PMD ? KERNEL_PGD_BOUNDARY : PTRS_PER_PGD) 109#define MAX_UNSHARED_PTRS_PER_PGD \ 110 max_t(size_t, KERNEL_PGD_BOUNDARY, PTRS_PER_PGD) 111 112 113static void pgd_set_mm(pgd_t *pgd, struct mm_struct *mm) 114{ 115 virt_to_ptdesc(pgd)->pt_mm = mm; 116} 117 118struct mm_struct *pgd_page_get_mm(struct page *page) 119{ 120 return page_ptdesc(page)->pt_mm; 121} 122 123static void pgd_ctor(struct mm_struct *mm, pgd_t *pgd) 124{ 125 /* If the pgd points to a shared pagetable level (either the 126 ptes in non-PAE, or shared PMD in PAE), then just copy the 127 references from swapper_pg_dir. */ 128 if (CONFIG_PGTABLE_LEVELS == 2 || 129 (CONFIG_PGTABLE_LEVELS == 3 && SHARED_KERNEL_PMD) || 130 CONFIG_PGTABLE_LEVELS >= 4) { 131 clone_pgd_range(pgd + KERNEL_PGD_BOUNDARY, 132 swapper_pg_dir + KERNEL_PGD_BOUNDARY, 133 KERNEL_PGD_PTRS); 134 } 135 136 /* list required to sync kernel mapping updates */ 137 if (!SHARED_KERNEL_PMD) { 138 pgd_set_mm(pgd, mm); 139 pgd_list_add(pgd); 140 } 141} 142 143static void pgd_dtor(pgd_t *pgd) 144{ 145 if (SHARED_KERNEL_PMD) 146 return; 147 148 spin_lock(&pgd_lock); 149 pgd_list_del(pgd); 150 spin_unlock(&pgd_lock); 151} 152 153/* 154 * List of all pgd's needed for non-PAE so it can invalidate entries 155 * in both cached and uncached pgd's; not needed for PAE since the 156 * kernel pmd is shared. If PAE were not to share the pmd a similar 157 * tactic would be needed. This is essentially codepath-based locking 158 * against pageattr.c; it is the unique case in which a valid change 159 * of kernel pagetables can't be lazily synchronized by vmalloc faults. 160 * vmalloc faults work because attached pagetables are never freed. 161 * -- nyc 162 */ 163 164#ifdef CONFIG_X86_PAE 165/* 166 * In PAE mode, we need to do a cr3 reload (=tlb flush) when 167 * updating the top-level pagetable entries to guarantee the 168 * processor notices the update. Since this is expensive, and 169 * all 4 top-level entries are used almost immediately in a 170 * new process's life, we just pre-populate them here. 171 * 172 * Also, if we're in a paravirt environment where the kernel pmd is 173 * not shared between pagetables (!SHARED_KERNEL_PMDS), we allocate 174 * and initialize the kernel pmds here. 175 */ 176#define PREALLOCATED_PMDS UNSHARED_PTRS_PER_PGD 177#define MAX_PREALLOCATED_PMDS MAX_UNSHARED_PTRS_PER_PGD 178 179/* 180 * We allocate separate PMDs for the kernel part of the user page-table 181 * when PTI is enabled. We need them to map the per-process LDT into the 182 * user-space page-table. 183 */ 184#define PREALLOCATED_USER_PMDS (boot_cpu_has(X86_FEATURE_PTI) ? \ 185 KERNEL_PGD_PTRS : 0) 186#define MAX_PREALLOCATED_USER_PMDS KERNEL_PGD_PTRS 187 188void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd) 189{ 190 paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT); 191 192 /* Note: almost everything apart from _PAGE_PRESENT is 193 reserved at the pmd (PDPT) level. */ 194 set_pud(pudp, __pud(__pa(pmd) | _PAGE_PRESENT)); 195 196 /* 197 * According to Intel App note "TLBs, Paging-Structure Caches, 198 * and Their Invalidation", April 2007, document 317080-001, 199 * section 8.1: in PAE mode we explicitly have to flush the 200 * TLB via cr3 if the top-level pgd is changed... 201 */ 202 flush_tlb_mm(mm); 203} 204#else /* !CONFIG_X86_PAE */ 205 206/* No need to prepopulate any pagetable entries in non-PAE modes. */ 207#define PREALLOCATED_PMDS 0 208#define MAX_PREALLOCATED_PMDS 0 209#define PREALLOCATED_USER_PMDS 0 210#define MAX_PREALLOCATED_USER_PMDS 0 211#endif /* CONFIG_X86_PAE */ 212 213static void free_pmds(struct mm_struct *mm, pmd_t *pmds[], int count) 214{ 215 int i; 216 struct ptdesc *ptdesc; 217 218 for (i = 0; i < count; i++) 219 if (pmds[i]) { 220 ptdesc = virt_to_ptdesc(pmds[i]); 221 222 pagetable_pmd_dtor(ptdesc); 223 pagetable_free(ptdesc); 224 mm_dec_nr_pmds(mm); 225 } 226} 227 228static int preallocate_pmds(struct mm_struct *mm, pmd_t *pmds[], int count) 229{ 230 int i; 231 bool failed = false; 232 gfp_t gfp = GFP_PGTABLE_USER; 233 234 if (mm == &init_mm) 235 gfp &= ~__GFP_ACCOUNT; 236 gfp &= ~__GFP_HIGHMEM; 237 238 for (i = 0; i < count; i++) { 239 pmd_t *pmd = NULL; 240 struct ptdesc *ptdesc = pagetable_alloc(gfp, 0); 241 242 if (!ptdesc) 243 failed = true; 244 if (ptdesc && !pagetable_pmd_ctor(ptdesc)) { 245 pagetable_free(ptdesc); 246 ptdesc = NULL; 247 failed = true; 248 } 249 if (ptdesc) { 250 mm_inc_nr_pmds(mm); 251 pmd = ptdesc_address(ptdesc); 252 } 253 254 pmds[i] = pmd; 255 } 256 257 if (failed) { 258 free_pmds(mm, pmds, count); 259 return -ENOMEM; 260 } 261 262 return 0; 263} 264 265/* 266 * Mop up any pmd pages which may still be attached to the pgd. 267 * Normally they will be freed by munmap/exit_mmap, but any pmd we 268 * preallocate which never got a corresponding vma will need to be 269 * freed manually. 270 */ 271static void mop_up_one_pmd(struct mm_struct *mm, pgd_t *pgdp) 272{ 273 pgd_t pgd = *pgdp; 274 275 if (pgd_val(pgd) != 0) { 276 pmd_t *pmd = (pmd_t *)pgd_page_vaddr(pgd); 277 278 pgd_clear(pgdp); 279 280 paravirt_release_pmd(pgd_val(pgd) >> PAGE_SHIFT); 281 pmd_free(mm, pmd); 282 mm_dec_nr_pmds(mm); 283 } 284} 285 286static void pgd_mop_up_pmds(struct mm_struct *mm, pgd_t *pgdp) 287{ 288 int i; 289 290 for (i = 0; i < PREALLOCATED_PMDS; i++) 291 mop_up_one_pmd(mm, &pgdp[i]); 292 293#ifdef CONFIG_PAGE_TABLE_ISOLATION 294 295 if (!boot_cpu_has(X86_FEATURE_PTI)) 296 return; 297 298 pgdp = kernel_to_user_pgdp(pgdp); 299 300 for (i = 0; i < PREALLOCATED_USER_PMDS; i++) 301 mop_up_one_pmd(mm, &pgdp[i + KERNEL_PGD_BOUNDARY]); 302#endif 303} 304 305static void pgd_prepopulate_pmd(struct mm_struct *mm, pgd_t *pgd, pmd_t *pmds[]) 306{ 307 p4d_t *p4d; 308 pud_t *pud; 309 int i; 310 311 p4d = p4d_offset(pgd, 0); 312 pud = pud_offset(p4d, 0); 313 314 for (i = 0; i < PREALLOCATED_PMDS; i++, pud++) { 315 pmd_t *pmd = pmds[i]; 316 317 if (i >= KERNEL_PGD_BOUNDARY) 318 memcpy(pmd, (pmd_t *)pgd_page_vaddr(swapper_pg_dir[i]), 319 sizeof(pmd_t) * PTRS_PER_PMD); 320 321 pud_populate(mm, pud, pmd); 322 } 323} 324 325#ifdef CONFIG_PAGE_TABLE_ISOLATION 326static void pgd_prepopulate_user_pmd(struct mm_struct *mm, 327 pgd_t *k_pgd, pmd_t *pmds[]) 328{ 329 pgd_t *s_pgd = kernel_to_user_pgdp(swapper_pg_dir); 330 pgd_t *u_pgd = kernel_to_user_pgdp(k_pgd); 331 p4d_t *u_p4d; 332 pud_t *u_pud; 333 int i; 334 335 u_p4d = p4d_offset(u_pgd, 0); 336 u_pud = pud_offset(u_p4d, 0); 337 338 s_pgd += KERNEL_PGD_BOUNDARY; 339 u_pud += KERNEL_PGD_BOUNDARY; 340 341 for (i = 0; i < PREALLOCATED_USER_PMDS; i++, u_pud++, s_pgd++) { 342 pmd_t *pmd = pmds[i]; 343 344 memcpy(pmd, (pmd_t *)pgd_page_vaddr(*s_pgd), 345 sizeof(pmd_t) * PTRS_PER_PMD); 346 347 pud_populate(mm, u_pud, pmd); 348 } 349 350} 351#else 352static void pgd_prepopulate_user_pmd(struct mm_struct *mm, 353 pgd_t *k_pgd, pmd_t *pmds[]) 354{ 355} 356#endif 357/* 358 * Xen paravirt assumes pgd table should be in one page. 64 bit kernel also 359 * assumes that pgd should be in one page. 360 * 361 * But kernel with PAE paging that is not running as a Xen domain 362 * only needs to allocate 32 bytes for pgd instead of one page. 363 */ 364#ifdef CONFIG_X86_PAE 365 366#include <linux/slab.h> 367 368#define PGD_SIZE (PTRS_PER_PGD * sizeof(pgd_t)) 369#define PGD_ALIGN 32 370 371static struct kmem_cache *pgd_cache; 372 373void __init pgtable_cache_init(void) 374{ 375 /* 376 * When PAE kernel is running as a Xen domain, it does not use 377 * shared kernel pmd. And this requires a whole page for pgd. 378 */ 379 if (!SHARED_KERNEL_PMD) 380 return; 381 382 /* 383 * when PAE kernel is not running as a Xen domain, it uses 384 * shared kernel pmd. Shared kernel pmd does not require a whole 385 * page for pgd. We are able to just allocate a 32-byte for pgd. 386 * During boot time, we create a 32-byte slab for pgd table allocation. 387 */ 388 pgd_cache = kmem_cache_create("pgd_cache", PGD_SIZE, PGD_ALIGN, 389 SLAB_PANIC, NULL); 390} 391 392static inline pgd_t *_pgd_alloc(void) 393{ 394 /* 395 * If no SHARED_KERNEL_PMD, PAE kernel is running as a Xen domain. 396 * We allocate one page for pgd. 397 */ 398 if (!SHARED_KERNEL_PMD) 399 return (pgd_t *)__get_free_pages(GFP_PGTABLE_USER, 400 PGD_ALLOCATION_ORDER); 401 402 /* 403 * Now PAE kernel is not running as a Xen domain. We can allocate 404 * a 32-byte slab for pgd to save memory space. 405 */ 406 return kmem_cache_alloc(pgd_cache, GFP_PGTABLE_USER); 407} 408 409static inline void _pgd_free(pgd_t *pgd) 410{ 411 if (!SHARED_KERNEL_PMD) 412 free_pages((unsigned long)pgd, PGD_ALLOCATION_ORDER); 413 else 414 kmem_cache_free(pgd_cache, pgd); 415} 416#else 417 418static inline pgd_t *_pgd_alloc(void) 419{ 420 return (pgd_t *)__get_free_pages(GFP_PGTABLE_USER, 421 PGD_ALLOCATION_ORDER); 422} 423 424static inline void _pgd_free(pgd_t *pgd) 425{ 426 free_pages((unsigned long)pgd, PGD_ALLOCATION_ORDER); 427} 428#endif /* CONFIG_X86_PAE */ 429 430pgd_t *pgd_alloc(struct mm_struct *mm) 431{ 432 pgd_t *pgd; 433 pmd_t *u_pmds[MAX_PREALLOCATED_USER_PMDS]; 434 pmd_t *pmds[MAX_PREALLOCATED_PMDS]; 435 436 pgd = _pgd_alloc(); 437 438 if (pgd == NULL) 439 goto out; 440 441 mm->pgd = pgd; 442 443 if (sizeof(pmds) != 0 && 444 preallocate_pmds(mm, pmds, PREALLOCATED_PMDS) != 0) 445 goto out_free_pgd; 446 447 if (sizeof(u_pmds) != 0 && 448 preallocate_pmds(mm, u_pmds, PREALLOCATED_USER_PMDS) != 0) 449 goto out_free_pmds; 450 451 if (paravirt_pgd_alloc(mm) != 0) 452 goto out_free_user_pmds; 453 454 /* 455 * Make sure that pre-populating the pmds is atomic with 456 * respect to anything walking the pgd_list, so that they 457 * never see a partially populated pgd. 458 */ 459 spin_lock(&pgd_lock); 460 461 pgd_ctor(mm, pgd); 462 if (sizeof(pmds) != 0) 463 pgd_prepopulate_pmd(mm, pgd, pmds); 464 465 if (sizeof(u_pmds) != 0) 466 pgd_prepopulate_user_pmd(mm, pgd, u_pmds); 467 468 spin_unlock(&pgd_lock); 469 470 return pgd; 471 472out_free_user_pmds: 473 if (sizeof(u_pmds) != 0) 474 free_pmds(mm, u_pmds, PREALLOCATED_USER_PMDS); 475out_free_pmds: 476 if (sizeof(pmds) != 0) 477 free_pmds(mm, pmds, PREALLOCATED_PMDS); 478out_free_pgd: 479 _pgd_free(pgd); 480out: 481 return NULL; 482} 483 484void pgd_free(struct mm_struct *mm, pgd_t *pgd) 485{ 486 pgd_mop_up_pmds(mm, pgd); 487 pgd_dtor(pgd); 488 paravirt_pgd_free(mm, pgd); 489 _pgd_free(pgd); 490} 491 492/* 493 * Used to set accessed or dirty bits in the page table entries 494 * on other architectures. On x86, the accessed and dirty bits 495 * are tracked by hardware. However, do_wp_page calls this function 496 * to also make the pte writeable at the same time the dirty bit is 497 * set. In that case we do actually need to write the PTE. 498 */ 499int ptep_set_access_flags(struct vm_area_struct *vma, 500 unsigned long address, pte_t *ptep, 501 pte_t entry, int dirty) 502{ 503 int changed = !pte_same(*ptep, entry); 504 505 if (changed && dirty) 506 set_pte(ptep, entry); 507 508 return changed; 509} 510 511#ifdef CONFIG_TRANSPARENT_HUGEPAGE 512int pmdp_set_access_flags(struct vm_area_struct *vma, 513 unsigned long address, pmd_t *pmdp, 514 pmd_t entry, int dirty) 515{ 516 int changed = !pmd_same(*pmdp, entry); 517 518 VM_BUG_ON(address & ~HPAGE_PMD_MASK); 519 520 if (changed && dirty) { 521 set_pmd(pmdp, entry); 522 /* 523 * We had a write-protection fault here and changed the pmd 524 * to to more permissive. No need to flush the TLB for that, 525 * #PF is architecturally guaranteed to do that and in the 526 * worst-case we'll generate a spurious fault. 527 */ 528 } 529 530 return changed; 531} 532 533int pudp_set_access_flags(struct vm_area_struct *vma, unsigned long address, 534 pud_t *pudp, pud_t entry, int dirty) 535{ 536 int changed = !pud_same(*pudp, entry); 537 538 VM_BUG_ON(address & ~HPAGE_PUD_MASK); 539 540 if (changed && dirty) { 541 set_pud(pudp, entry); 542 /* 543 * We had a write-protection fault here and changed the pud 544 * to to more permissive. No need to flush the TLB for that, 545 * #PF is architecturally guaranteed to do that and in the 546 * worst-case we'll generate a spurious fault. 547 */ 548 } 549 550 return changed; 551} 552#endif 553 554int ptep_test_and_clear_young(struct vm_area_struct *vma, 555 unsigned long addr, pte_t *ptep) 556{ 557 int ret = 0; 558 559 if (pte_young(*ptep)) 560 ret = test_and_clear_bit(_PAGE_BIT_ACCESSED, 561 (unsigned long *) &ptep->pte); 562 563 return ret; 564} 565 566#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_ARCH_HAS_NONLEAF_PMD_YOUNG) 567int pmdp_test_and_clear_young(struct vm_area_struct *vma, 568 unsigned long addr, pmd_t *pmdp) 569{ 570 int ret = 0; 571 572 if (pmd_young(*pmdp)) 573 ret = test_and_clear_bit(_PAGE_BIT_ACCESSED, 574 (unsigned long *)pmdp); 575 576 return ret; 577} 578#endif 579 580#ifdef CONFIG_TRANSPARENT_HUGEPAGE 581int pudp_test_and_clear_young(struct vm_area_struct *vma, 582 unsigned long addr, pud_t *pudp) 583{ 584 int ret = 0; 585 586 if (pud_young(*pudp)) 587 ret = test_and_clear_bit(_PAGE_BIT_ACCESSED, 588 (unsigned long *)pudp); 589 590 return ret; 591} 592#endif 593 594int ptep_clear_flush_young(struct vm_area_struct *vma, 595 unsigned long address, pte_t *ptep) 596{ 597 /* 598 * On x86 CPUs, clearing the accessed bit without a TLB flush 599 * doesn't cause data corruption. [ It could cause incorrect 600 * page aging and the (mistaken) reclaim of hot pages, but the 601 * chance of that should be relatively low. ] 602 * 603 * So as a performance optimization don't flush the TLB when 604 * clearing the accessed bit, it will eventually be flushed by 605 * a context switch or a VM operation anyway. [ In the rare 606 * event of it not getting flushed for a long time the delay 607 * shouldn't really matter because there's no real memory 608 * pressure for swapout to react to. ] 609 */ 610 return ptep_test_and_clear_young(vma, address, ptep); 611} 612 613#ifdef CONFIG_TRANSPARENT_HUGEPAGE 614int pmdp_clear_flush_young(struct vm_area_struct *vma, 615 unsigned long address, pmd_t *pmdp) 616{ 617 int young; 618 619 VM_BUG_ON(address & ~HPAGE_PMD_MASK); 620 621 young = pmdp_test_and_clear_young(vma, address, pmdp); 622 if (young) 623 flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE); 624 625 return young; 626} 627 628pmd_t pmdp_invalidate_ad(struct vm_area_struct *vma, unsigned long address, 629 pmd_t *pmdp) 630{ 631 /* 632 * No flush is necessary. Once an invalid PTE is established, the PTE's 633 * access and dirty bits cannot be updated. 634 */ 635 return pmdp_establish(vma, address, pmdp, pmd_mkinvalid(*pmdp)); 636} 637#endif 638 639/** 640 * reserve_top_address - reserves a hole in the top of kernel address space 641 * @reserve - size of hole to reserve 642 * 643 * Can be used to relocate the fixmap area and poke a hole in the top 644 * of kernel address space to make room for a hypervisor. 645 */ 646void __init reserve_top_address(unsigned long reserve) 647{ 648#ifdef CONFIG_X86_32 649 BUG_ON(fixmaps_set > 0); 650 __FIXADDR_TOP = round_down(-reserve, 1 << PMD_SHIFT) - PAGE_SIZE; 651 printk(KERN_INFO "Reserving virtual address space above 0x%08lx (rounded to 0x%08lx)\n", 652 -reserve, __FIXADDR_TOP + PAGE_SIZE); 653#endif 654} 655 656int fixmaps_set; 657 658void __native_set_fixmap(enum fixed_addresses idx, pte_t pte) 659{ 660 unsigned long address = __fix_to_virt(idx); 661 662#ifdef CONFIG_X86_64 663 /* 664 * Ensure that the static initial page tables are covering the 665 * fixmap completely. 666 */ 667 BUILD_BUG_ON(__end_of_permanent_fixed_addresses > 668 (FIXMAP_PMD_NUM * PTRS_PER_PTE)); 669#endif 670 671 if (idx >= __end_of_fixed_addresses) { 672 BUG(); 673 return; 674 } 675 set_pte_vaddr(address, pte); 676 fixmaps_set++; 677} 678 679void native_set_fixmap(unsigned /* enum fixed_addresses */ idx, 680 phys_addr_t phys, pgprot_t flags) 681{ 682 /* Sanitize 'prot' against any unsupported bits: */ 683 pgprot_val(flags) &= __default_kernel_pte_mask; 684 685 __native_set_fixmap(idx, pfn_pte(phys >> PAGE_SHIFT, flags)); 686} 687 688#ifdef CONFIG_HAVE_ARCH_HUGE_VMAP 689#ifdef CONFIG_X86_5LEVEL 690/** 691 * p4d_set_huge - setup kernel P4D mapping 692 * 693 * No 512GB pages yet -- always return 0 694 */ 695int p4d_set_huge(p4d_t *p4d, phys_addr_t addr, pgprot_t prot) 696{ 697 return 0; 698} 699 700/** 701 * p4d_clear_huge - clear kernel P4D mapping when it is set 702 * 703 * No 512GB pages yet -- always return 0 704 */ 705void p4d_clear_huge(p4d_t *p4d) 706{ 707} 708#endif 709 710/** 711 * pud_set_huge - setup kernel PUD mapping 712 * 713 * MTRRs can override PAT memory types with 4KiB granularity. Therefore, this 714 * function sets up a huge page only if the complete range has the same MTRR 715 * caching mode. 716 * 717 * Callers should try to decrease page size (1GB -> 2MB -> 4K) if the bigger 718 * page mapping attempt fails. 719 * 720 * Returns 1 on success and 0 on failure. 721 */ 722int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot) 723{ 724 u8 uniform; 725 726 mtrr_type_lookup(addr, addr + PUD_SIZE, &uniform); 727 if (!uniform) 728 return 0; 729 730 /* Bail out if we are we on a populated non-leaf entry: */ 731 if (pud_present(*pud) && !pud_huge(*pud)) 732 return 0; 733 734 set_pte((pte_t *)pud, pfn_pte( 735 (u64)addr >> PAGE_SHIFT, 736 __pgprot(protval_4k_2_large(pgprot_val(prot)) | _PAGE_PSE))); 737 738 return 1; 739} 740 741/** 742 * pmd_set_huge - setup kernel PMD mapping 743 * 744 * See text over pud_set_huge() above. 745 * 746 * Returns 1 on success and 0 on failure. 747 */ 748int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot) 749{ 750 u8 uniform; 751 752 mtrr_type_lookup(addr, addr + PMD_SIZE, &uniform); 753 if (!uniform) { 754 pr_warn_once("%s: Cannot satisfy [mem %#010llx-%#010llx] with a huge-page mapping due to MTRR override.\n", 755 __func__, addr, addr + PMD_SIZE); 756 return 0; 757 } 758 759 /* Bail out if we are we on a populated non-leaf entry: */ 760 if (pmd_present(*pmd) && !pmd_huge(*pmd)) 761 return 0; 762 763 set_pte((pte_t *)pmd, pfn_pte( 764 (u64)addr >> PAGE_SHIFT, 765 __pgprot(protval_4k_2_large(pgprot_val(prot)) | _PAGE_PSE))); 766 767 return 1; 768} 769 770/** 771 * pud_clear_huge - clear kernel PUD mapping when it is set 772 * 773 * Returns 1 on success and 0 on failure (no PUD map is found). 774 */ 775int pud_clear_huge(pud_t *pud) 776{ 777 if (pud_large(*pud)) { 778 pud_clear(pud); 779 return 1; 780 } 781 782 return 0; 783} 784 785/** 786 * pmd_clear_huge - clear kernel PMD mapping when it is set 787 * 788 * Returns 1 on success and 0 on failure (no PMD map is found). 789 */ 790int pmd_clear_huge(pmd_t *pmd) 791{ 792 if (pmd_large(*pmd)) { 793 pmd_clear(pmd); 794 return 1; 795 } 796 797 return 0; 798} 799 800#ifdef CONFIG_X86_64 801/** 802 * pud_free_pmd_page - Clear pud entry and free pmd page. 803 * @pud: Pointer to a PUD. 804 * @addr: Virtual address associated with pud. 805 * 806 * Context: The pud range has been unmapped and TLB purged. 807 * Return: 1 if clearing the entry succeeded. 0 otherwise. 808 * 809 * NOTE: Callers must allow a single page allocation. 810 */ 811int pud_free_pmd_page(pud_t *pud, unsigned long addr) 812{ 813 pmd_t *pmd, *pmd_sv; 814 pte_t *pte; 815 int i; 816 817 pmd = pud_pgtable(*pud); 818 pmd_sv = (pmd_t *)__get_free_page(GFP_KERNEL); 819 if (!pmd_sv) 820 return 0; 821 822 for (i = 0; i < PTRS_PER_PMD; i++) { 823 pmd_sv[i] = pmd[i]; 824 if (!pmd_none(pmd[i])) 825 pmd_clear(&pmd[i]); 826 } 827 828 pud_clear(pud); 829 830 /* INVLPG to clear all paging-structure caches */ 831 flush_tlb_kernel_range(addr, addr + PAGE_SIZE-1); 832 833 for (i = 0; i < PTRS_PER_PMD; i++) { 834 if (!pmd_none(pmd_sv[i])) { 835 pte = (pte_t *)pmd_page_vaddr(pmd_sv[i]); 836 free_page((unsigned long)pte); 837 } 838 } 839 840 free_page((unsigned long)pmd_sv); 841 842 pagetable_pmd_dtor(virt_to_ptdesc(pmd)); 843 free_page((unsigned long)pmd); 844 845 return 1; 846} 847 848/** 849 * pmd_free_pte_page - Clear pmd entry and free pte page. 850 * @pmd: Pointer to a PMD. 851 * @addr: Virtual address associated with pmd. 852 * 853 * Context: The pmd range has been unmapped and TLB purged. 854 * Return: 1 if clearing the entry succeeded. 0 otherwise. 855 */ 856int pmd_free_pte_page(pmd_t *pmd, unsigned long addr) 857{ 858 pte_t *pte; 859 860 pte = (pte_t *)pmd_page_vaddr(*pmd); 861 pmd_clear(pmd); 862 863 /* INVLPG to clear all paging-structure caches */ 864 flush_tlb_kernel_range(addr, addr + PAGE_SIZE-1); 865 866 free_page((unsigned long)pte); 867 868 return 1; 869} 870 871#else /* !CONFIG_X86_64 */ 872 873/* 874 * Disable free page handling on x86-PAE. This assures that ioremap() 875 * does not update sync'd pmd entries. See vmalloc_sync_one(). 876 */ 877int pmd_free_pte_page(pmd_t *pmd, unsigned long addr) 878{ 879 return pmd_none(*pmd); 880} 881 882#endif /* CONFIG_X86_64 */ 883#endif /* CONFIG_HAVE_ARCH_HUGE_VMAP */ 884 885pte_t pte_mkwrite(pte_t pte, struct vm_area_struct *vma) 886{ 887 if (vma->vm_flags & VM_SHADOW_STACK) 888 return pte_mkwrite_shstk(pte); 889 890 pte = pte_mkwrite_novma(pte); 891 892 return pte_clear_saveddirty(pte); 893} 894 895pmd_t pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma) 896{ 897 if (vma->vm_flags & VM_SHADOW_STACK) 898 return pmd_mkwrite_shstk(pmd); 899 900 pmd = pmd_mkwrite_novma(pmd); 901 902 return pmd_clear_saveddirty(pmd); 903} 904 905void arch_check_zapped_pte(struct vm_area_struct *vma, pte_t pte) 906{ 907 /* 908 * Hardware before shadow stack can (rarely) set Dirty=1 909 * on a Write=0 PTE. So the below condition 910 * only indicates a software bug when shadow stack is 911 * supported by the HW. This checking is covered in 912 * pte_shstk(). 913 */ 914 VM_WARN_ON_ONCE(!(vma->vm_flags & VM_SHADOW_STACK) && 915 pte_shstk(pte)); 916} 917 918void arch_check_zapped_pmd(struct vm_area_struct *vma, pmd_t pmd) 919{ 920 /* See note in arch_check_zapped_pte() */ 921 VM_WARN_ON_ONCE(!(vma->vm_flags & VM_SHADOW_STACK) && 922 pmd_shstk(pmd)); 923} 924