1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Copyright 2002 Andi Kleen, SuSE Labs. 4 * Thanks to Ben LaHaise for precious feedback. 5 */ 6#include <linux/highmem.h> 7#include <linux/memblock.h> 8#include <linux/sched.h> 9#include <linux/mm.h> 10#include <linux/interrupt.h> 11#include <linux/seq_file.h> 12#include <linux/proc_fs.h> 13#include <linux/debugfs.h> 14#include <linux/pfn.h> 15#include <linux/percpu.h> 16#include <linux/gfp.h> 17#include <linux/pci.h> 18#include <linux/vmalloc.h> 19#include <linux/libnvdimm.h> 20#include <linux/vmstat.h> 21#include <linux/kernel.h> 22#include <linux/cc_platform.h> 23#include <linux/set_memory.h> 24#include <linux/memregion.h> 25 26#include <asm/e820/api.h> 27#include <asm/processor.h> 28#include <asm/tlbflush.h> 29#include <asm/sections.h> 30#include <asm/setup.h> 31#include <linux/uaccess.h> 32#include <asm/pgalloc.h> 33#include <asm/proto.h> 34#include <asm/memtype.h> 35#include <asm/hyperv-tlfs.h> 36#include <asm/mshyperv.h> 37 38#include "../mm_internal.h" 39 40/* 41 * The current flushing context - we pass it instead of 5 arguments: 42 */ 43struct cpa_data { 44 unsigned long *vaddr; 45 pgd_t *pgd; 46 pgprot_t mask_set; 47 pgprot_t mask_clr; 48 unsigned long numpages; 49 unsigned long curpage; 50 unsigned long pfn; 51 unsigned int flags; 52 unsigned int force_split : 1, 53 force_static_prot : 1, 54 force_flush_all : 1; 55 struct page **pages; 56}; 57 58enum cpa_warn { 59 CPA_CONFLICT, 60 CPA_PROTECT, 61 CPA_DETECT, 62}; 63 64static const int cpa_warn_level = CPA_PROTECT; 65 66/* 67 * Serialize cpa() (for !DEBUG_PAGEALLOC which uses large identity mappings) 68 * using cpa_lock. So that we don't allow any other cpu, with stale large tlb 69 * entries change the page attribute in parallel to some other cpu 70 * splitting a large page entry along with changing the attribute. 71 */ 72static DEFINE_SPINLOCK(cpa_lock); 73 74#define CPA_FLUSHTLB 1 75#define CPA_ARRAY 2 76#define CPA_PAGES_ARRAY 4 77#define CPA_NO_CHECK_ALIAS 8 /* Do not search for aliases */ 78 79static inline pgprot_t cachemode2pgprot(enum page_cache_mode pcm) 80{ 81 return __pgprot(cachemode2protval(pcm)); 82} 83 84#ifdef CONFIG_PROC_FS 85static unsigned long direct_pages_count[PG_LEVEL_NUM]; 86 87void update_page_count(int level, unsigned long pages) 88{ 89 /* Protect against CPA */ 90 spin_lock(&pgd_lock); 91 direct_pages_count[level] += pages; 92 spin_unlock(&pgd_lock); 93} 94 95static void split_page_count(int level) 96{ 97 if (direct_pages_count[level] == 0) 98 return; 99 100 direct_pages_count[level]--; 101 if (system_state == SYSTEM_RUNNING) { 102 if (level == PG_LEVEL_2M) 103 count_vm_event(DIRECT_MAP_LEVEL2_SPLIT); 104 else if (level == PG_LEVEL_1G) 105 count_vm_event(DIRECT_MAP_LEVEL3_SPLIT); 106 } 107 direct_pages_count[level - 1] += PTRS_PER_PTE; 108} 109 110void arch_report_meminfo(struct seq_file *m) 111{ 112 seq_printf(m, "DirectMap4k: %8lu kB\n", 113 direct_pages_count[PG_LEVEL_4K] << 2); 114#if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE) 115 seq_printf(m, "DirectMap2M: %8lu kB\n", 116 direct_pages_count[PG_LEVEL_2M] << 11); 117#else 118 seq_printf(m, "DirectMap4M: %8lu kB\n", 119 direct_pages_count[PG_LEVEL_2M] << 12); 120#endif 121 if (direct_gbpages) 122 seq_printf(m, "DirectMap1G: %8lu kB\n", 123 direct_pages_count[PG_LEVEL_1G] << 20); 124} 125#else 126static inline void split_page_count(int level) { } 127#endif 128 129#ifdef CONFIG_X86_CPA_STATISTICS 130 131static unsigned long cpa_1g_checked; 132static unsigned long cpa_1g_sameprot; 133static unsigned long cpa_1g_preserved; 134static unsigned long cpa_2m_checked; 135static unsigned long cpa_2m_sameprot; 136static unsigned long cpa_2m_preserved; 137static unsigned long cpa_4k_install; 138 139static inline void cpa_inc_1g_checked(void) 140{ 141 cpa_1g_checked++; 142} 143 144static inline void cpa_inc_2m_checked(void) 145{ 146 cpa_2m_checked++; 147} 148 149static inline void cpa_inc_4k_install(void) 150{ 151 data_race(cpa_4k_install++); 152} 153 154static inline void cpa_inc_lp_sameprot(int level) 155{ 156 if (level == PG_LEVEL_1G) 157 cpa_1g_sameprot++; 158 else 159 cpa_2m_sameprot++; 160} 161 162static inline void cpa_inc_lp_preserved(int level) 163{ 164 if (level == PG_LEVEL_1G) 165 cpa_1g_preserved++; 166 else 167 cpa_2m_preserved++; 168} 169 170static int cpastats_show(struct seq_file *m, void *p) 171{ 172 seq_printf(m, "1G pages checked: %16lu\n", cpa_1g_checked); 173 seq_printf(m, "1G pages sameprot: %16lu\n", cpa_1g_sameprot); 174 seq_printf(m, "1G pages preserved: %16lu\n", cpa_1g_preserved); 175 seq_printf(m, "2M pages checked: %16lu\n", cpa_2m_checked); 176 seq_printf(m, "2M pages sameprot: %16lu\n", cpa_2m_sameprot); 177 seq_printf(m, "2M pages preserved: %16lu\n", cpa_2m_preserved); 178 seq_printf(m, "4K pages set-checked: %16lu\n", cpa_4k_install); 179 return 0; 180} 181 182static int cpastats_open(struct inode *inode, struct file *file) 183{ 184 return single_open(file, cpastats_show, NULL); 185} 186 187static const struct file_operations cpastats_fops = { 188 .open = cpastats_open, 189 .read = seq_read, 190 .llseek = seq_lseek, 191 .release = single_release, 192}; 193 194static int __init cpa_stats_init(void) 195{ 196 debugfs_create_file("cpa_stats", S_IRUSR, arch_debugfs_dir, NULL, 197 &cpastats_fops); 198 return 0; 199} 200late_initcall(cpa_stats_init); 201#else 202static inline void cpa_inc_1g_checked(void) { } 203static inline void cpa_inc_2m_checked(void) { } 204static inline void cpa_inc_4k_install(void) { } 205static inline void cpa_inc_lp_sameprot(int level) { } 206static inline void cpa_inc_lp_preserved(int level) { } 207#endif 208 209 210static inline int 211within(unsigned long addr, unsigned long start, unsigned long end) 212{ 213 return addr >= start && addr < end; 214} 215 216static inline int 217within_inclusive(unsigned long addr, unsigned long start, unsigned long end) 218{ 219 return addr >= start && addr <= end; 220} 221 222#ifdef CONFIG_X86_64 223 224/* 225 * The kernel image is mapped into two places in the virtual address space 226 * (addresses without KASLR, of course): 227 * 228 * 1. The kernel direct map (0xffff880000000000) 229 * 2. The "high kernel map" (0xffffffff81000000) 230 * 231 * We actually execute out of #2. If we get the address of a kernel symbol, it 232 * points to #2, but almost all physical-to-virtual translations point to #1. 233 * 234 * This is so that we can have both a directmap of all physical memory *and* 235 * take full advantage of the limited (s32) immediate addressing range (2G) 236 * of x86_64. 237 * 238 * See Documentation/arch/x86/x86_64/mm.rst for more detail. 239 */ 240 241static inline unsigned long highmap_start_pfn(void) 242{ 243 return __pa_symbol(_text) >> PAGE_SHIFT; 244} 245 246static inline unsigned long highmap_end_pfn(void) 247{ 248 /* Do not reference physical address outside the kernel. */ 249 return __pa_symbol(roundup(_brk_end, PMD_SIZE) - 1) >> PAGE_SHIFT; 250} 251 252static bool __cpa_pfn_in_highmap(unsigned long pfn) 253{ 254 /* 255 * Kernel text has an alias mapping at a high address, known 256 * here as "highmap". 257 */ 258 return within_inclusive(pfn, highmap_start_pfn(), highmap_end_pfn()); 259} 260 261#else 262 263static bool __cpa_pfn_in_highmap(unsigned long pfn) 264{ 265 /* There is no highmap on 32-bit */ 266 return false; 267} 268 269#endif 270 271/* 272 * See set_mce_nospec(). 273 * 274 * Machine check recovery code needs to change cache mode of poisoned pages to 275 * UC to avoid speculative access logging another error. But passing the 276 * address of the 1:1 mapping to set_memory_uc() is a fine way to encourage a 277 * speculative access. So we cheat and flip the top bit of the address. This 278 * works fine for the code that updates the page tables. But at the end of the 279 * process we need to flush the TLB and cache and the non-canonical address 280 * causes a #GP fault when used by the INVLPG and CLFLUSH instructions. 281 * 282 * But in the common case we already have a canonical address. This code 283 * will fix the top bit if needed and is a no-op otherwise. 284 */ 285static inline unsigned long fix_addr(unsigned long addr) 286{ 287#ifdef CONFIG_X86_64 288 return (long)(addr << 1) >> 1; 289#else 290 return addr; 291#endif 292} 293 294static unsigned long __cpa_addr(struct cpa_data *cpa, unsigned long idx) 295{ 296 if (cpa->flags & CPA_PAGES_ARRAY) { 297 struct page *page = cpa->pages[idx]; 298 299 if (unlikely(PageHighMem(page))) 300 return 0; 301 302 return (unsigned long)page_address(page); 303 } 304 305 if (cpa->flags & CPA_ARRAY) 306 return cpa->vaddr[idx]; 307 308 return *cpa->vaddr + idx * PAGE_SIZE; 309} 310 311/* 312 * Flushing functions 313 */ 314 315static void clflush_cache_range_opt(void *vaddr, unsigned int size) 316{ 317 const unsigned long clflush_size = boot_cpu_data.x86_clflush_size; 318 void *p = (void *)((unsigned long)vaddr & ~(clflush_size - 1)); 319 void *vend = vaddr + size; 320 321 if (p >= vend) 322 return; 323 324 for (; p < vend; p += clflush_size) 325 clflushopt(p); 326} 327 328/** 329 * clflush_cache_range - flush a cache range with clflush 330 * @vaddr: virtual start address 331 * @size: number of bytes to flush 332 * 333 * CLFLUSHOPT is an unordered instruction which needs fencing with MFENCE or 334 * SFENCE to avoid ordering issues. 335 */ 336void clflush_cache_range(void *vaddr, unsigned int size) 337{ 338 mb(); 339 clflush_cache_range_opt(vaddr, size); 340 mb(); 341} 342EXPORT_SYMBOL_GPL(clflush_cache_range); 343 344#ifdef CONFIG_ARCH_HAS_PMEM_API 345void arch_invalidate_pmem(void *addr, size_t size) 346{ 347 clflush_cache_range(addr, size); 348} 349EXPORT_SYMBOL_GPL(arch_invalidate_pmem); 350#endif 351 352#ifdef CONFIG_ARCH_HAS_CPU_CACHE_INVALIDATE_MEMREGION 353bool cpu_cache_has_invalidate_memregion(void) 354{ 355 return !cpu_feature_enabled(X86_FEATURE_HYPERVISOR); 356} 357EXPORT_SYMBOL_NS_GPL(cpu_cache_has_invalidate_memregion, DEVMEM); 358 359int cpu_cache_invalidate_memregion(int res_desc) 360{ 361 if (WARN_ON_ONCE(!cpu_cache_has_invalidate_memregion())) 362 return -ENXIO; 363 wbinvd_on_all_cpus(); 364 return 0; 365} 366EXPORT_SYMBOL_NS_GPL(cpu_cache_invalidate_memregion, DEVMEM); 367#endif 368 369static void __cpa_flush_all(void *arg) 370{ 371 unsigned long cache = (unsigned long)arg; 372 373 /* 374 * Flush all to work around Errata in early athlons regarding 375 * large page flushing. 376 */ 377 __flush_tlb_all(); 378 379 if (cache && boot_cpu_data.x86 >= 4) 380 wbinvd(); 381} 382 383static void cpa_flush_all(unsigned long cache) 384{ 385 BUG_ON(irqs_disabled() && !early_boot_irqs_disabled); 386 387 on_each_cpu(__cpa_flush_all, (void *) cache, 1); 388} 389 390static void __cpa_flush_tlb(void *data) 391{ 392 struct cpa_data *cpa = data; 393 unsigned int i; 394 395 for (i = 0; i < cpa->numpages; i++) 396 flush_tlb_one_kernel(fix_addr(__cpa_addr(cpa, i))); 397} 398 399static void cpa_flush(struct cpa_data *data, int cache) 400{ 401 struct cpa_data *cpa = data; 402 unsigned int i; 403 404 BUG_ON(irqs_disabled() && !early_boot_irqs_disabled); 405 406 if (cache && !static_cpu_has(X86_FEATURE_CLFLUSH)) { 407 cpa_flush_all(cache); 408 return; 409 } 410 411 if (cpa->force_flush_all || cpa->numpages > tlb_single_page_flush_ceiling) 412 flush_tlb_all(); 413 else 414 on_each_cpu(__cpa_flush_tlb, cpa, 1); 415 416 if (!cache) 417 return; 418 419 mb(); 420 for (i = 0; i < cpa->numpages; i++) { 421 unsigned long addr = __cpa_addr(cpa, i); 422 unsigned int level; 423 424 pte_t *pte = lookup_address(addr, &level); 425 426 /* 427 * Only flush present addresses: 428 */ 429 if (pte && (pte_val(*pte) & _PAGE_PRESENT)) 430 clflush_cache_range_opt((void *)fix_addr(addr), PAGE_SIZE); 431 } 432 mb(); 433} 434 435static bool overlaps(unsigned long r1_start, unsigned long r1_end, 436 unsigned long r2_start, unsigned long r2_end) 437{ 438 return (r1_start <= r2_end && r1_end >= r2_start) || 439 (r2_start <= r1_end && r2_end >= r1_start); 440} 441 442#ifdef CONFIG_PCI_BIOS 443/* 444 * The BIOS area between 640k and 1Mb needs to be executable for PCI BIOS 445 * based config access (CONFIG_PCI_GOBIOS) support. 446 */ 447#define BIOS_PFN PFN_DOWN(BIOS_BEGIN) 448#define BIOS_PFN_END PFN_DOWN(BIOS_END - 1) 449 450static pgprotval_t protect_pci_bios(unsigned long spfn, unsigned long epfn) 451{ 452 if (pcibios_enabled && overlaps(spfn, epfn, BIOS_PFN, BIOS_PFN_END)) 453 return _PAGE_NX; 454 return 0; 455} 456#else 457static pgprotval_t protect_pci_bios(unsigned long spfn, unsigned long epfn) 458{ 459 return 0; 460} 461#endif 462 463/* 464 * The .rodata section needs to be read-only. Using the pfn catches all 465 * aliases. This also includes __ro_after_init, so do not enforce until 466 * kernel_set_to_readonly is true. 467 */ 468static pgprotval_t protect_rodata(unsigned long spfn, unsigned long epfn) 469{ 470 unsigned long epfn_ro, spfn_ro = PFN_DOWN(__pa_symbol(__start_rodata)); 471 472 /* 473 * Note: __end_rodata is at page aligned and not inclusive, so 474 * subtract 1 to get the last enforced PFN in the rodata area. 475 */ 476 epfn_ro = PFN_DOWN(__pa_symbol(__end_rodata)) - 1; 477 478 if (kernel_set_to_readonly && overlaps(spfn, epfn, spfn_ro, epfn_ro)) 479 return _PAGE_RW; 480 return 0; 481} 482 483/* 484 * Protect kernel text against becoming non executable by forbidding 485 * _PAGE_NX. This protects only the high kernel mapping (_text -> _etext) 486 * out of which the kernel actually executes. Do not protect the low 487 * mapping. 488 * 489 * This does not cover __inittext since that is gone after boot. 490 */ 491static pgprotval_t protect_kernel_text(unsigned long start, unsigned long end) 492{ 493 unsigned long t_end = (unsigned long)_etext - 1; 494 unsigned long t_start = (unsigned long)_text; 495 496 if (overlaps(start, end, t_start, t_end)) 497 return _PAGE_NX; 498 return 0; 499} 500 501#if defined(CONFIG_X86_64) 502/* 503 * Once the kernel maps the text as RO (kernel_set_to_readonly is set), 504 * kernel text mappings for the large page aligned text, rodata sections 505 * will be always read-only. For the kernel identity mappings covering the 506 * holes caused by this alignment can be anything that user asks. 507 * 508 * This will preserve the large page mappings for kernel text/data at no 509 * extra cost. 510 */ 511static pgprotval_t protect_kernel_text_ro(unsigned long start, 512 unsigned long end) 513{ 514 unsigned long t_end = (unsigned long)__end_rodata_hpage_align - 1; 515 unsigned long t_start = (unsigned long)_text; 516 unsigned int level; 517 518 if (!kernel_set_to_readonly || !overlaps(start, end, t_start, t_end)) 519 return 0; 520 /* 521 * Don't enforce the !RW mapping for the kernel text mapping, if 522 * the current mapping is already using small page mapping. No 523 * need to work hard to preserve large page mappings in this case. 524 * 525 * This also fixes the Linux Xen paravirt guest boot failure caused 526 * by unexpected read-only mappings for kernel identity 527 * mappings. In this paravirt guest case, the kernel text mapping 528 * and the kernel identity mapping share the same page-table pages, 529 * so the protections for kernel text and identity mappings have to 530 * be the same. 531 */ 532 if (lookup_address(start, &level) && (level != PG_LEVEL_4K)) 533 return _PAGE_RW; 534 return 0; 535} 536#else 537static pgprotval_t protect_kernel_text_ro(unsigned long start, 538 unsigned long end) 539{ 540 return 0; 541} 542#endif 543 544static inline bool conflicts(pgprot_t prot, pgprotval_t val) 545{ 546 return (pgprot_val(prot) & ~val) != pgprot_val(prot); 547} 548 549static inline void check_conflict(int warnlvl, pgprot_t prot, pgprotval_t val, 550 unsigned long start, unsigned long end, 551 unsigned long pfn, const char *txt) 552{ 553 static const char *lvltxt[] = { 554 [CPA_CONFLICT] = "conflict", 555 [CPA_PROTECT] = "protect", 556 [CPA_DETECT] = "detect", 557 }; 558 559 if (warnlvl > cpa_warn_level || !conflicts(prot, val)) 560 return; 561 562 pr_warn("CPA %8s %10s: 0x%016lx - 0x%016lx PFN %lx req %016llx prevent %016llx\n", 563 lvltxt[warnlvl], txt, start, end, pfn, (unsigned long long)pgprot_val(prot), 564 (unsigned long long)val); 565} 566 567/* 568 * Certain areas of memory on x86 require very specific protection flags, 569 * for example the BIOS area or kernel text. Callers don't always get this 570 * right (again, ioremap() on BIOS memory is not uncommon) so this function 571 * checks and fixes these known static required protection bits. 572 */ 573static inline pgprot_t static_protections(pgprot_t prot, unsigned long start, 574 unsigned long pfn, unsigned long npg, 575 unsigned long lpsize, int warnlvl) 576{ 577 pgprotval_t forbidden, res; 578 unsigned long end; 579 580 /* 581 * There is no point in checking RW/NX conflicts when the requested 582 * mapping is setting the page !PRESENT. 583 */ 584 if (!(pgprot_val(prot) & _PAGE_PRESENT)) 585 return prot; 586 587 /* Operate on the virtual address */ 588 end = start + npg * PAGE_SIZE - 1; 589 590 res = protect_kernel_text(start, end); 591 check_conflict(warnlvl, prot, res, start, end, pfn, "Text NX"); 592 forbidden = res; 593 594 /* 595 * Special case to preserve a large page. If the change spawns the 596 * full large page mapping then there is no point to split it 597 * up. Happens with ftrace and is going to be removed once ftrace 598 * switched to text_poke(). 599 */ 600 if (lpsize != (npg * PAGE_SIZE) || (start & (lpsize - 1))) { 601 res = protect_kernel_text_ro(start, end); 602 check_conflict(warnlvl, prot, res, start, end, pfn, "Text RO"); 603 forbidden |= res; 604 } 605 606 /* Check the PFN directly */ 607 res = protect_pci_bios(pfn, pfn + npg - 1); 608 check_conflict(warnlvl, prot, res, start, end, pfn, "PCIBIOS NX"); 609 forbidden |= res; 610 611 res = protect_rodata(pfn, pfn + npg - 1); 612 check_conflict(warnlvl, prot, res, start, end, pfn, "Rodata RO"); 613 forbidden |= res; 614 615 return __pgprot(pgprot_val(prot) & ~forbidden); 616} 617 618/* 619 * Validate strict W^X semantics. 620 */ 621static inline pgprot_t verify_rwx(pgprot_t old, pgprot_t new, unsigned long start, 622 unsigned long pfn, unsigned long npg) 623{ 624 unsigned long end; 625 626 /* 627 * 32-bit has some unfixable W+X issues, like EFI code 628 * and writeable data being in the same page. Disable 629 * detection and enforcement there. 630 */ 631 if (IS_ENABLED(CONFIG_X86_32)) 632 return new; 633 634 /* Only verify when NX is supported: */ 635 if (!(__supported_pte_mask & _PAGE_NX)) 636 return new; 637 638 if (!((pgprot_val(old) ^ pgprot_val(new)) & (_PAGE_RW | _PAGE_NX))) 639 return new; 640 641 if ((pgprot_val(new) & (_PAGE_RW | _PAGE_NX)) != _PAGE_RW) 642 return new; 643 644 end = start + npg * PAGE_SIZE - 1; 645 WARN_ONCE(1, "CPA detected W^X violation: %016llx -> %016llx range: 0x%016lx - 0x%016lx PFN %lx\n", 646 (unsigned long long)pgprot_val(old), 647 (unsigned long long)pgprot_val(new), 648 start, end, pfn); 649 650 /* 651 * For now, allow all permission change attempts by returning the 652 * attempted permissions. This can 'return old' to actively 653 * refuse the permission change at a later time. 654 */ 655 return new; 656} 657 658/* 659 * Lookup the page table entry for a virtual address in a specific pgd. 660 * Return a pointer to the entry and the level of the mapping. 661 */ 662pte_t *lookup_address_in_pgd(pgd_t *pgd, unsigned long address, 663 unsigned int *level) 664{ 665 p4d_t *p4d; 666 pud_t *pud; 667 pmd_t *pmd; 668 669 *level = PG_LEVEL_NONE; 670 671 if (pgd_none(*pgd)) 672 return NULL; 673 674 p4d = p4d_offset(pgd, address); 675 if (p4d_none(*p4d)) 676 return NULL; 677 678 *level = PG_LEVEL_512G; 679 if (p4d_large(*p4d) || !p4d_present(*p4d)) 680 return (pte_t *)p4d; 681 682 pud = pud_offset(p4d, address); 683 if (pud_none(*pud)) 684 return NULL; 685 686 *level = PG_LEVEL_1G; 687 if (pud_large(*pud) || !pud_present(*pud)) 688 return (pte_t *)pud; 689 690 pmd = pmd_offset(pud, address); 691 if (pmd_none(*pmd)) 692 return NULL; 693 694 *level = PG_LEVEL_2M; 695 if (pmd_large(*pmd) || !pmd_present(*pmd)) 696 return (pte_t *)pmd; 697 698 *level = PG_LEVEL_4K; 699 700 return pte_offset_kernel(pmd, address); 701} 702 703/* 704 * Lookup the page table entry for a virtual address. Return a pointer 705 * to the entry and the level of the mapping. 706 * 707 * Note: We return pud and pmd either when the entry is marked large 708 * or when the present bit is not set. Otherwise we would return a 709 * pointer to a nonexisting mapping. 710 */ 711pte_t *lookup_address(unsigned long address, unsigned int *level) 712{ 713 return lookup_address_in_pgd(pgd_offset_k(address), address, level); 714} 715EXPORT_SYMBOL_GPL(lookup_address); 716 717static pte_t *_lookup_address_cpa(struct cpa_data *cpa, unsigned long address, 718 unsigned int *level) 719{ 720 if (cpa->pgd) 721 return lookup_address_in_pgd(cpa->pgd + pgd_index(address), 722 address, level); 723 724 return lookup_address(address, level); 725} 726 727/* 728 * Lookup the PMD entry for a virtual address. Return a pointer to the entry 729 * or NULL if not present. 730 */ 731pmd_t *lookup_pmd_address(unsigned long address) 732{ 733 pgd_t *pgd; 734 p4d_t *p4d; 735 pud_t *pud; 736 737 pgd = pgd_offset_k(address); 738 if (pgd_none(*pgd)) 739 return NULL; 740 741 p4d = p4d_offset(pgd, address); 742 if (p4d_none(*p4d) || p4d_large(*p4d) || !p4d_present(*p4d)) 743 return NULL; 744 745 pud = pud_offset(p4d, address); 746 if (pud_none(*pud) || pud_large(*pud) || !pud_present(*pud)) 747 return NULL; 748 749 return pmd_offset(pud, address); 750} 751 752/* 753 * This is necessary because __pa() does not work on some 754 * kinds of memory, like vmalloc() or the alloc_remap() 755 * areas on 32-bit NUMA systems. The percpu areas can 756 * end up in this kind of memory, for instance. 757 * 758 * This could be optimized, but it is only intended to be 759 * used at initialization time, and keeping it 760 * unoptimized should increase the testing coverage for 761 * the more obscure platforms. 762 */ 763phys_addr_t slow_virt_to_phys(void *__virt_addr) 764{ 765 unsigned long virt_addr = (unsigned long)__virt_addr; 766 phys_addr_t phys_addr; 767 unsigned long offset; 768 enum pg_level level; 769 pte_t *pte; 770 771 pte = lookup_address(virt_addr, &level); 772 BUG_ON(!pte); 773 774 /* 775 * pXX_pfn() returns unsigned long, which must be cast to phys_addr_t 776 * before being left-shifted PAGE_SHIFT bits -- this trick is to 777 * make 32-PAE kernel work correctly. 778 */ 779 switch (level) { 780 case PG_LEVEL_1G: 781 phys_addr = (phys_addr_t)pud_pfn(*(pud_t *)pte) << PAGE_SHIFT; 782 offset = virt_addr & ~PUD_MASK; 783 break; 784 case PG_LEVEL_2M: 785 phys_addr = (phys_addr_t)pmd_pfn(*(pmd_t *)pte) << PAGE_SHIFT; 786 offset = virt_addr & ~PMD_MASK; 787 break; 788 default: 789 phys_addr = (phys_addr_t)pte_pfn(*pte) << PAGE_SHIFT; 790 offset = virt_addr & ~PAGE_MASK; 791 } 792 793 return (phys_addr_t)(phys_addr | offset); 794} 795EXPORT_SYMBOL_GPL(slow_virt_to_phys); 796 797/* 798 * Set the new pmd in all the pgds we know about: 799 */ 800static void __set_pmd_pte(pte_t *kpte, unsigned long address, pte_t pte) 801{ 802 /* change init_mm */ 803 set_pte_atomic(kpte, pte); 804#ifdef CONFIG_X86_32 805 if (!SHARED_KERNEL_PMD) { 806 struct page *page; 807 808 list_for_each_entry(page, &pgd_list, lru) { 809 pgd_t *pgd; 810 p4d_t *p4d; 811 pud_t *pud; 812 pmd_t *pmd; 813 814 pgd = (pgd_t *)page_address(page) + pgd_index(address); 815 p4d = p4d_offset(pgd, address); 816 pud = pud_offset(p4d, address); 817 pmd = pmd_offset(pud, address); 818 set_pte_atomic((pte_t *)pmd, pte); 819 } 820 } 821#endif 822} 823 824static pgprot_t pgprot_clear_protnone_bits(pgprot_t prot) 825{ 826 /* 827 * _PAGE_GLOBAL means "global page" for present PTEs. 828 * But, it is also used to indicate _PAGE_PROTNONE 829 * for non-present PTEs. 830 * 831 * This ensures that a _PAGE_GLOBAL PTE going from 832 * present to non-present is not confused as 833 * _PAGE_PROTNONE. 834 */ 835 if (!(pgprot_val(prot) & _PAGE_PRESENT)) 836 pgprot_val(prot) &= ~_PAGE_GLOBAL; 837 838 return prot; 839} 840 841static int __should_split_large_page(pte_t *kpte, unsigned long address, 842 struct cpa_data *cpa) 843{ 844 unsigned long numpages, pmask, psize, lpaddr, pfn, old_pfn; 845 pgprot_t old_prot, new_prot, req_prot, chk_prot; 846 pte_t new_pte, *tmp; 847 enum pg_level level; 848 849 /* 850 * Check for races, another CPU might have split this page 851 * up already: 852 */ 853 tmp = _lookup_address_cpa(cpa, address, &level); 854 if (tmp != kpte) 855 return 1; 856 857 switch (level) { 858 case PG_LEVEL_2M: 859 old_prot = pmd_pgprot(*(pmd_t *)kpte); 860 old_pfn = pmd_pfn(*(pmd_t *)kpte); 861 cpa_inc_2m_checked(); 862 break; 863 case PG_LEVEL_1G: 864 old_prot = pud_pgprot(*(pud_t *)kpte); 865 old_pfn = pud_pfn(*(pud_t *)kpte); 866 cpa_inc_1g_checked(); 867 break; 868 default: 869 return -EINVAL; 870 } 871 872 psize = page_level_size(level); 873 pmask = page_level_mask(level); 874 875 /* 876 * Calculate the number of pages, which fit into this large 877 * page starting at address: 878 */ 879 lpaddr = (address + psize) & pmask; 880 numpages = (lpaddr - address) >> PAGE_SHIFT; 881 if (numpages < cpa->numpages) 882 cpa->numpages = numpages; 883 884 /* 885 * We are safe now. Check whether the new pgprot is the same: 886 * Convert protection attributes to 4k-format, as cpa->mask* are set 887 * up accordingly. 888 */ 889 890 /* Clear PSE (aka _PAGE_PAT) and move PAT bit to correct position */ 891 req_prot = pgprot_large_2_4k(old_prot); 892 893 pgprot_val(req_prot) &= ~pgprot_val(cpa->mask_clr); 894 pgprot_val(req_prot) |= pgprot_val(cpa->mask_set); 895 896 /* 897 * req_prot is in format of 4k pages. It must be converted to large 898 * page format: the caching mode includes the PAT bit located at 899 * different bit positions in the two formats. 900 */ 901 req_prot = pgprot_4k_2_large(req_prot); 902 req_prot = pgprot_clear_protnone_bits(req_prot); 903 if (pgprot_val(req_prot) & _PAGE_PRESENT) 904 pgprot_val(req_prot) |= _PAGE_PSE; 905 906 /* 907 * old_pfn points to the large page base pfn. So we need to add the 908 * offset of the virtual address: 909 */ 910 pfn = old_pfn + ((address & (psize - 1)) >> PAGE_SHIFT); 911 cpa->pfn = pfn; 912 913 /* 914 * Calculate the large page base address and the number of 4K pages 915 * in the large page 916 */ 917 lpaddr = address & pmask; 918 numpages = psize >> PAGE_SHIFT; 919 920 /* 921 * Sanity check that the existing mapping is correct versus the static 922 * protections. static_protections() guards against !PRESENT, so no 923 * extra conditional required here. 924 */ 925 chk_prot = static_protections(old_prot, lpaddr, old_pfn, numpages, 926 psize, CPA_CONFLICT); 927 928 if (WARN_ON_ONCE(pgprot_val(chk_prot) != pgprot_val(old_prot))) { 929 /* 930 * Split the large page and tell the split code to 931 * enforce static protections. 932 */ 933 cpa->force_static_prot = 1; 934 return 1; 935 } 936 937 /* 938 * Optimization: If the requested pgprot is the same as the current 939 * pgprot, then the large page can be preserved and no updates are 940 * required independent of alignment and length of the requested 941 * range. The above already established that the current pgprot is 942 * correct, which in consequence makes the requested pgprot correct 943 * as well if it is the same. The static protection scan below will 944 * not come to a different conclusion. 945 */ 946 if (pgprot_val(req_prot) == pgprot_val(old_prot)) { 947 cpa_inc_lp_sameprot(level); 948 return 0; 949 } 950 951 /* 952 * If the requested range does not cover the full page, split it up 953 */ 954 if (address != lpaddr || cpa->numpages != numpages) 955 return 1; 956 957 /* 958 * Check whether the requested pgprot is conflicting with a static 959 * protection requirement in the large page. 960 */ 961 new_prot = static_protections(req_prot, lpaddr, old_pfn, numpages, 962 psize, CPA_DETECT); 963 964 new_prot = verify_rwx(old_prot, new_prot, lpaddr, old_pfn, numpages); 965 966 /* 967 * If there is a conflict, split the large page. 968 * 969 * There used to be a 4k wise evaluation trying really hard to 970 * preserve the large pages, but experimentation has shown, that this 971 * does not help at all. There might be corner cases which would 972 * preserve one large page occasionally, but it's really not worth the 973 * extra code and cycles for the common case. 974 */ 975 if (pgprot_val(req_prot) != pgprot_val(new_prot)) 976 return 1; 977 978 /* All checks passed. Update the large page mapping. */ 979 new_pte = pfn_pte(old_pfn, new_prot); 980 __set_pmd_pte(kpte, address, new_pte); 981 cpa->flags |= CPA_FLUSHTLB; 982 cpa_inc_lp_preserved(level); 983 return 0; 984} 985 986static int should_split_large_page(pte_t *kpte, unsigned long address, 987 struct cpa_data *cpa) 988{ 989 int do_split; 990 991 if (cpa->force_split) 992 return 1; 993 994 spin_lock(&pgd_lock); 995 do_split = __should_split_large_page(kpte, address, cpa); 996 spin_unlock(&pgd_lock); 997 998 return do_split; 999} 1000 1001static void split_set_pte(struct cpa_data *cpa, pte_t *pte, unsigned long pfn, 1002 pgprot_t ref_prot, unsigned long address, 1003 unsigned long size) 1004{ 1005 unsigned int npg = PFN_DOWN(size); 1006 pgprot_t prot; 1007 1008 /* 1009 * If should_split_large_page() discovered an inconsistent mapping, 1010 * remove the invalid protection in the split mapping. 1011 */ 1012 if (!cpa->force_static_prot) 1013 goto set; 1014 1015 /* Hand in lpsize = 0 to enforce the protection mechanism */ 1016 prot = static_protections(ref_prot, address, pfn, npg, 0, CPA_PROTECT); 1017 1018 if (pgprot_val(prot) == pgprot_val(ref_prot)) 1019 goto set; 1020 1021 /* 1022 * If this is splitting a PMD, fix it up. PUD splits cannot be 1023 * fixed trivially as that would require to rescan the newly 1024 * installed PMD mappings after returning from split_large_page() 1025 * so an eventual further split can allocate the necessary PTE 1026 * pages. Warn for now and revisit it in case this actually 1027 * happens. 1028 */ 1029 if (size == PAGE_SIZE) 1030 ref_prot = prot; 1031 else 1032 pr_warn_once("CPA: Cannot fixup static protections for PUD split\n"); 1033set: 1034 set_pte(pte, pfn_pte(pfn, ref_prot)); 1035} 1036 1037static int 1038__split_large_page(struct cpa_data *cpa, pte_t *kpte, unsigned long address, 1039 struct page *base) 1040{ 1041 unsigned long lpaddr, lpinc, ref_pfn, pfn, pfninc = 1; 1042 pte_t *pbase = (pte_t *)page_address(base); 1043 unsigned int i, level; 1044 pgprot_t ref_prot; 1045 pte_t *tmp; 1046 1047 spin_lock(&pgd_lock); 1048 /* 1049 * Check for races, another CPU might have split this page 1050 * up for us already: 1051 */ 1052 tmp = _lookup_address_cpa(cpa, address, &level); 1053 if (tmp != kpte) { 1054 spin_unlock(&pgd_lock); 1055 return 1; 1056 } 1057 1058 paravirt_alloc_pte(&init_mm, page_to_pfn(base)); 1059 1060 switch (level) { 1061 case PG_LEVEL_2M: 1062 ref_prot = pmd_pgprot(*(pmd_t *)kpte); 1063 /* 1064 * Clear PSE (aka _PAGE_PAT) and move 1065 * PAT bit to correct position. 1066 */ 1067 ref_prot = pgprot_large_2_4k(ref_prot); 1068 ref_pfn = pmd_pfn(*(pmd_t *)kpte); 1069 lpaddr = address & PMD_MASK; 1070 lpinc = PAGE_SIZE; 1071 break; 1072 1073 case PG_LEVEL_1G: 1074 ref_prot = pud_pgprot(*(pud_t *)kpte); 1075 ref_pfn = pud_pfn(*(pud_t *)kpte); 1076 pfninc = PMD_SIZE >> PAGE_SHIFT; 1077 lpaddr = address & PUD_MASK; 1078 lpinc = PMD_SIZE; 1079 /* 1080 * Clear the PSE flags if the PRESENT flag is not set 1081 * otherwise pmd_present/pmd_huge will return true 1082 * even on a non present pmd. 1083 */ 1084 if (!(pgprot_val(ref_prot) & _PAGE_PRESENT)) 1085 pgprot_val(ref_prot) &= ~_PAGE_PSE; 1086 break; 1087 1088 default: 1089 spin_unlock(&pgd_lock); 1090 return 1; 1091 } 1092 1093 ref_prot = pgprot_clear_protnone_bits(ref_prot); 1094 1095 /* 1096 * Get the target pfn from the original entry: 1097 */ 1098 pfn = ref_pfn; 1099 for (i = 0; i < PTRS_PER_PTE; i++, pfn += pfninc, lpaddr += lpinc) 1100 split_set_pte(cpa, pbase + i, pfn, ref_prot, lpaddr, lpinc); 1101 1102 if (virt_addr_valid(address)) { 1103 unsigned long pfn = PFN_DOWN(__pa(address)); 1104 1105 if (pfn_range_is_mapped(pfn, pfn + 1)) 1106 split_page_count(level); 1107 } 1108 1109 /* 1110 * Install the new, split up pagetable. 1111 * 1112 * We use the standard kernel pagetable protections for the new 1113 * pagetable protections, the actual ptes set above control the 1114 * primary protection behavior: 1115 */ 1116 __set_pmd_pte(kpte, address, mk_pte(base, __pgprot(_KERNPG_TABLE))); 1117 1118 /* 1119 * Do a global flush tlb after splitting the large page 1120 * and before we do the actual change page attribute in the PTE. 1121 * 1122 * Without this, we violate the TLB application note, that says: 1123 * "The TLBs may contain both ordinary and large-page 1124 * translations for a 4-KByte range of linear addresses. This 1125 * may occur if software modifies the paging structures so that 1126 * the page size used for the address range changes. If the two 1127 * translations differ with respect to page frame or attributes 1128 * (e.g., permissions), processor behavior is undefined and may 1129 * be implementation-specific." 1130 * 1131 * We do this global tlb flush inside the cpa_lock, so that we 1132 * don't allow any other cpu, with stale tlb entries change the 1133 * page attribute in parallel, that also falls into the 1134 * just split large page entry. 1135 */ 1136 flush_tlb_all(); 1137 spin_unlock(&pgd_lock); 1138 1139 return 0; 1140} 1141 1142static int split_large_page(struct cpa_data *cpa, pte_t *kpte, 1143 unsigned long address) 1144{ 1145 struct page *base; 1146 1147 if (!debug_pagealloc_enabled()) 1148 spin_unlock(&cpa_lock); 1149 base = alloc_pages(GFP_KERNEL, 0); 1150 if (!debug_pagealloc_enabled()) 1151 spin_lock(&cpa_lock); 1152 if (!base) 1153 return -ENOMEM; 1154 1155 if (__split_large_page(cpa, kpte, address, base)) 1156 __free_page(base); 1157 1158 return 0; 1159} 1160 1161static bool try_to_free_pte_page(pte_t *pte) 1162{ 1163 int i; 1164 1165 for (i = 0; i < PTRS_PER_PTE; i++) 1166 if (!pte_none(pte[i])) 1167 return false; 1168 1169 free_page((unsigned long)pte); 1170 return true; 1171} 1172 1173static bool try_to_free_pmd_page(pmd_t *pmd) 1174{ 1175 int i; 1176 1177 for (i = 0; i < PTRS_PER_PMD; i++) 1178 if (!pmd_none(pmd[i])) 1179 return false; 1180 1181 free_page((unsigned long)pmd); 1182 return true; 1183} 1184 1185static bool unmap_pte_range(pmd_t *pmd, unsigned long start, unsigned long end) 1186{ 1187 pte_t *pte = pte_offset_kernel(pmd, start); 1188 1189 while (start < end) { 1190 set_pte(pte, __pte(0)); 1191 1192 start += PAGE_SIZE; 1193 pte++; 1194 } 1195 1196 if (try_to_free_pte_page((pte_t *)pmd_page_vaddr(*pmd))) { 1197 pmd_clear(pmd); 1198 return true; 1199 } 1200 return false; 1201} 1202 1203static void __unmap_pmd_range(pud_t *pud, pmd_t *pmd, 1204 unsigned long start, unsigned long end) 1205{ 1206 if (unmap_pte_range(pmd, start, end)) 1207 if (try_to_free_pmd_page(pud_pgtable(*pud))) 1208 pud_clear(pud); 1209} 1210 1211static void unmap_pmd_range(pud_t *pud, unsigned long start, unsigned long end) 1212{ 1213 pmd_t *pmd = pmd_offset(pud, start); 1214 1215 /* 1216 * Not on a 2MB page boundary? 1217 */ 1218 if (start & (PMD_SIZE - 1)) { 1219 unsigned long next_page = (start + PMD_SIZE) & PMD_MASK; 1220 unsigned long pre_end = min_t(unsigned long, end, next_page); 1221 1222 __unmap_pmd_range(pud, pmd, start, pre_end); 1223 1224 start = pre_end; 1225 pmd++; 1226 } 1227 1228 /* 1229 * Try to unmap in 2M chunks. 1230 */ 1231 while (end - start >= PMD_SIZE) { 1232 if (pmd_large(*pmd)) 1233 pmd_clear(pmd); 1234 else 1235 __unmap_pmd_range(pud, pmd, start, start + PMD_SIZE); 1236 1237 start += PMD_SIZE; 1238 pmd++; 1239 } 1240 1241 /* 1242 * 4K leftovers? 1243 */ 1244 if (start < end) 1245 return __unmap_pmd_range(pud, pmd, start, end); 1246 1247 /* 1248 * Try again to free the PMD page if haven't succeeded above. 1249 */ 1250 if (!pud_none(*pud)) 1251 if (try_to_free_pmd_page(pud_pgtable(*pud))) 1252 pud_clear(pud); 1253} 1254 1255static void unmap_pud_range(p4d_t *p4d, unsigned long start, unsigned long end) 1256{ 1257 pud_t *pud = pud_offset(p4d, start); 1258 1259 /* 1260 * Not on a GB page boundary? 1261 */ 1262 if (start & (PUD_SIZE - 1)) { 1263 unsigned long next_page = (start + PUD_SIZE) & PUD_MASK; 1264 unsigned long pre_end = min_t(unsigned long, end, next_page); 1265 1266 unmap_pmd_range(pud, start, pre_end); 1267 1268 start = pre_end; 1269 pud++; 1270 } 1271 1272 /* 1273 * Try to unmap in 1G chunks? 1274 */ 1275 while (end - start >= PUD_SIZE) { 1276 1277 if (pud_large(*pud)) 1278 pud_clear(pud); 1279 else 1280 unmap_pmd_range(pud, start, start + PUD_SIZE); 1281 1282 start += PUD_SIZE; 1283 pud++; 1284 } 1285 1286 /* 1287 * 2M leftovers? 1288 */ 1289 if (start < end) 1290 unmap_pmd_range(pud, start, end); 1291 1292 /* 1293 * No need to try to free the PUD page because we'll free it in 1294 * populate_pgd's error path 1295 */ 1296} 1297 1298static int alloc_pte_page(pmd_t *pmd) 1299{ 1300 pte_t *pte = (pte_t *)get_zeroed_page(GFP_KERNEL); 1301 if (!pte) 1302 return -1; 1303 1304 set_pmd(pmd, __pmd(__pa(pte) | _KERNPG_TABLE)); 1305 return 0; 1306} 1307 1308static int alloc_pmd_page(pud_t *pud) 1309{ 1310 pmd_t *pmd = (pmd_t *)get_zeroed_page(GFP_KERNEL); 1311 if (!pmd) 1312 return -1; 1313 1314 set_pud(pud, __pud(__pa(pmd) | _KERNPG_TABLE)); 1315 return 0; 1316} 1317 1318static void populate_pte(struct cpa_data *cpa, 1319 unsigned long start, unsigned long end, 1320 unsigned num_pages, pmd_t *pmd, pgprot_t pgprot) 1321{ 1322 pte_t *pte; 1323 1324 pte = pte_offset_kernel(pmd, start); 1325 1326 pgprot = pgprot_clear_protnone_bits(pgprot); 1327 1328 while (num_pages-- && start < end) { 1329 set_pte(pte, pfn_pte(cpa->pfn, pgprot)); 1330 1331 start += PAGE_SIZE; 1332 cpa->pfn++; 1333 pte++; 1334 } 1335} 1336 1337static long populate_pmd(struct cpa_data *cpa, 1338 unsigned long start, unsigned long end, 1339 unsigned num_pages, pud_t *pud, pgprot_t pgprot) 1340{ 1341 long cur_pages = 0; 1342 pmd_t *pmd; 1343 pgprot_t pmd_pgprot; 1344 1345 /* 1346 * Not on a 2M boundary? 1347 */ 1348 if (start & (PMD_SIZE - 1)) { 1349 unsigned long pre_end = start + (num_pages << PAGE_SHIFT); 1350 unsigned long next_page = (start + PMD_SIZE) & PMD_MASK; 1351 1352 pre_end = min_t(unsigned long, pre_end, next_page); 1353 cur_pages = (pre_end - start) >> PAGE_SHIFT; 1354 cur_pages = min_t(unsigned int, num_pages, cur_pages); 1355 1356 /* 1357 * Need a PTE page? 1358 */ 1359 pmd = pmd_offset(pud, start); 1360 if (pmd_none(*pmd)) 1361 if (alloc_pte_page(pmd)) 1362 return -1; 1363 1364 populate_pte(cpa, start, pre_end, cur_pages, pmd, pgprot); 1365 1366 start = pre_end; 1367 } 1368 1369 /* 1370 * We mapped them all? 1371 */ 1372 if (num_pages == cur_pages) 1373 return cur_pages; 1374 1375 pmd_pgprot = pgprot_4k_2_large(pgprot); 1376 1377 while (end - start >= PMD_SIZE) { 1378 1379 /* 1380 * We cannot use a 1G page so allocate a PMD page if needed. 1381 */ 1382 if (pud_none(*pud)) 1383 if (alloc_pmd_page(pud)) 1384 return -1; 1385 1386 pmd = pmd_offset(pud, start); 1387 1388 set_pmd(pmd, pmd_mkhuge(pfn_pmd(cpa->pfn, 1389 canon_pgprot(pmd_pgprot)))); 1390 1391 start += PMD_SIZE; 1392 cpa->pfn += PMD_SIZE >> PAGE_SHIFT; 1393 cur_pages += PMD_SIZE >> PAGE_SHIFT; 1394 } 1395 1396 /* 1397 * Map trailing 4K pages. 1398 */ 1399 if (start < end) { 1400 pmd = pmd_offset(pud, start); 1401 if (pmd_none(*pmd)) 1402 if (alloc_pte_page(pmd)) 1403 return -1; 1404 1405 populate_pte(cpa, start, end, num_pages - cur_pages, 1406 pmd, pgprot); 1407 } 1408 return num_pages; 1409} 1410 1411static int populate_pud(struct cpa_data *cpa, unsigned long start, p4d_t *p4d, 1412 pgprot_t pgprot) 1413{ 1414 pud_t *pud; 1415 unsigned long end; 1416 long cur_pages = 0; 1417 pgprot_t pud_pgprot; 1418 1419 end = start + (cpa->numpages << PAGE_SHIFT); 1420 1421 /* 1422 * Not on a Gb page boundary? => map everything up to it with 1423 * smaller pages. 1424 */ 1425 if (start & (PUD_SIZE - 1)) { 1426 unsigned long pre_end; 1427 unsigned long next_page = (start + PUD_SIZE) & PUD_MASK; 1428 1429 pre_end = min_t(unsigned long, end, next_page); 1430 cur_pages = (pre_end - start) >> PAGE_SHIFT; 1431 cur_pages = min_t(int, (int)cpa->numpages, cur_pages); 1432 1433 pud = pud_offset(p4d, start); 1434 1435 /* 1436 * Need a PMD page? 1437 */ 1438 if (pud_none(*pud)) 1439 if (alloc_pmd_page(pud)) 1440 return -1; 1441 1442 cur_pages = populate_pmd(cpa, start, pre_end, cur_pages, 1443 pud, pgprot); 1444 if (cur_pages < 0) 1445 return cur_pages; 1446 1447 start = pre_end; 1448 } 1449 1450 /* We mapped them all? */ 1451 if (cpa->numpages == cur_pages) 1452 return cur_pages; 1453 1454 pud = pud_offset(p4d, start); 1455 pud_pgprot = pgprot_4k_2_large(pgprot); 1456 1457 /* 1458 * Map everything starting from the Gb boundary, possibly with 1G pages 1459 */ 1460 while (boot_cpu_has(X86_FEATURE_GBPAGES) && end - start >= PUD_SIZE) { 1461 set_pud(pud, pud_mkhuge(pfn_pud(cpa->pfn, 1462 canon_pgprot(pud_pgprot)))); 1463 1464 start += PUD_SIZE; 1465 cpa->pfn += PUD_SIZE >> PAGE_SHIFT; 1466 cur_pages += PUD_SIZE >> PAGE_SHIFT; 1467 pud++; 1468 } 1469 1470 /* Map trailing leftover */ 1471 if (start < end) { 1472 long tmp; 1473 1474 pud = pud_offset(p4d, start); 1475 if (pud_none(*pud)) 1476 if (alloc_pmd_page(pud)) 1477 return -1; 1478 1479 tmp = populate_pmd(cpa, start, end, cpa->numpages - cur_pages, 1480 pud, pgprot); 1481 if (tmp < 0) 1482 return cur_pages; 1483 1484 cur_pages += tmp; 1485 } 1486 return cur_pages; 1487} 1488 1489/* 1490 * Restrictions for kernel page table do not necessarily apply when mapping in 1491 * an alternate PGD. 1492 */ 1493static int populate_pgd(struct cpa_data *cpa, unsigned long addr) 1494{ 1495 pgprot_t pgprot = __pgprot(_KERNPG_TABLE); 1496 pud_t *pud = NULL; /* shut up gcc */ 1497 p4d_t *p4d; 1498 pgd_t *pgd_entry; 1499 long ret; 1500 1501 pgd_entry = cpa->pgd + pgd_index(addr); 1502 1503 if (pgd_none(*pgd_entry)) { 1504 p4d = (p4d_t *)get_zeroed_page(GFP_KERNEL); 1505 if (!p4d) 1506 return -1; 1507 1508 set_pgd(pgd_entry, __pgd(__pa(p4d) | _KERNPG_TABLE)); 1509 } 1510 1511 /* 1512 * Allocate a PUD page and hand it down for mapping. 1513 */ 1514 p4d = p4d_offset(pgd_entry, addr); 1515 if (p4d_none(*p4d)) { 1516 pud = (pud_t *)get_zeroed_page(GFP_KERNEL); 1517 if (!pud) 1518 return -1; 1519 1520 set_p4d(p4d, __p4d(__pa(pud) | _KERNPG_TABLE)); 1521 } 1522 1523 pgprot_val(pgprot) &= ~pgprot_val(cpa->mask_clr); 1524 pgprot_val(pgprot) |= pgprot_val(cpa->mask_set); 1525 1526 ret = populate_pud(cpa, addr, p4d, pgprot); 1527 if (ret < 0) { 1528 /* 1529 * Leave the PUD page in place in case some other CPU or thread 1530 * already found it, but remove any useless entries we just 1531 * added to it. 1532 */ 1533 unmap_pud_range(p4d, addr, 1534 addr + (cpa->numpages << PAGE_SHIFT)); 1535 return ret; 1536 } 1537 1538 cpa->numpages = ret; 1539 return 0; 1540} 1541 1542static int __cpa_process_fault(struct cpa_data *cpa, unsigned long vaddr, 1543 int primary) 1544{ 1545 if (cpa->pgd) { 1546 /* 1547 * Right now, we only execute this code path when mapping 1548 * the EFI virtual memory map regions, no other users 1549 * provide a ->pgd value. This may change in the future. 1550 */ 1551 return populate_pgd(cpa, vaddr); 1552 } 1553 1554 /* 1555 * Ignore all non primary paths. 1556 */ 1557 if (!primary) { 1558 cpa->numpages = 1; 1559 return 0; 1560 } 1561 1562 /* 1563 * Ignore the NULL PTE for kernel identity mapping, as it is expected 1564 * to have holes. 1565 * Also set numpages to '1' indicating that we processed cpa req for 1566 * one virtual address page and its pfn. TBD: numpages can be set based 1567 * on the initial value and the level returned by lookup_address(). 1568 */ 1569 if (within(vaddr, PAGE_OFFSET, 1570 PAGE_OFFSET + (max_pfn_mapped << PAGE_SHIFT))) { 1571 cpa->numpages = 1; 1572 cpa->pfn = __pa(vaddr) >> PAGE_SHIFT; 1573 return 0; 1574 1575 } else if (__cpa_pfn_in_highmap(cpa->pfn)) { 1576 /* Faults in the highmap are OK, so do not warn: */ 1577 return -EFAULT; 1578 } else { 1579 WARN(1, KERN_WARNING "CPA: called for zero pte. " 1580 "vaddr = %lx cpa->vaddr = %lx\n", vaddr, 1581 *cpa->vaddr); 1582 1583 return -EFAULT; 1584 } 1585} 1586 1587static int __change_page_attr(struct cpa_data *cpa, int primary) 1588{ 1589 unsigned long address; 1590 int do_split, err; 1591 unsigned int level; 1592 pte_t *kpte, old_pte; 1593 1594 address = __cpa_addr(cpa, cpa->curpage); 1595repeat: 1596 kpte = _lookup_address_cpa(cpa, address, &level); 1597 if (!kpte) 1598 return __cpa_process_fault(cpa, address, primary); 1599 1600 old_pte = *kpte; 1601 if (pte_none(old_pte)) 1602 return __cpa_process_fault(cpa, address, primary); 1603 1604 if (level == PG_LEVEL_4K) { 1605 pte_t new_pte; 1606 pgprot_t old_prot = pte_pgprot(old_pte); 1607 pgprot_t new_prot = pte_pgprot(old_pte); 1608 unsigned long pfn = pte_pfn(old_pte); 1609 1610 pgprot_val(new_prot) &= ~pgprot_val(cpa->mask_clr); 1611 pgprot_val(new_prot) |= pgprot_val(cpa->mask_set); 1612 1613 cpa_inc_4k_install(); 1614 /* Hand in lpsize = 0 to enforce the protection mechanism */ 1615 new_prot = static_protections(new_prot, address, pfn, 1, 0, 1616 CPA_PROTECT); 1617 1618 new_prot = verify_rwx(old_prot, new_prot, address, pfn, 1); 1619 1620 new_prot = pgprot_clear_protnone_bits(new_prot); 1621 1622 /* 1623 * We need to keep the pfn from the existing PTE, 1624 * after all we're only going to change it's attributes 1625 * not the memory it points to 1626 */ 1627 new_pte = pfn_pte(pfn, new_prot); 1628 cpa->pfn = pfn; 1629 /* 1630 * Do we really change anything ? 1631 */ 1632 if (pte_val(old_pte) != pte_val(new_pte)) { 1633 set_pte_atomic(kpte, new_pte); 1634 cpa->flags |= CPA_FLUSHTLB; 1635 } 1636 cpa->numpages = 1; 1637 return 0; 1638 } 1639 1640 /* 1641 * Check, whether we can keep the large page intact 1642 * and just change the pte: 1643 */ 1644 do_split = should_split_large_page(kpte, address, cpa); 1645 /* 1646 * When the range fits into the existing large page, 1647 * return. cp->numpages and cpa->tlbflush have been updated in 1648 * try_large_page: 1649 */ 1650 if (do_split <= 0) 1651 return do_split; 1652 1653 /* 1654 * We have to split the large page: 1655 */ 1656 err = split_large_page(cpa, kpte, address); 1657 if (!err) 1658 goto repeat; 1659 1660 return err; 1661} 1662 1663static int __change_page_attr_set_clr(struct cpa_data *cpa, int primary); 1664 1665/* 1666 * Check the directmap and "high kernel map" 'aliases'. 1667 */ 1668static int cpa_process_alias(struct cpa_data *cpa) 1669{ 1670 struct cpa_data alias_cpa; 1671 unsigned long laddr = (unsigned long)__va(cpa->pfn << PAGE_SHIFT); 1672 unsigned long vaddr; 1673 int ret; 1674 1675 if (!pfn_range_is_mapped(cpa->pfn, cpa->pfn + 1)) 1676 return 0; 1677 1678 /* 1679 * No need to redo, when the primary call touched the direct 1680 * mapping already: 1681 */ 1682 vaddr = __cpa_addr(cpa, cpa->curpage); 1683 if (!(within(vaddr, PAGE_OFFSET, 1684 PAGE_OFFSET + (max_pfn_mapped << PAGE_SHIFT)))) { 1685 1686 alias_cpa = *cpa; 1687 alias_cpa.vaddr = &laddr; 1688 alias_cpa.flags &= ~(CPA_PAGES_ARRAY | CPA_ARRAY); 1689 alias_cpa.curpage = 0; 1690 1691 /* Directmap always has NX set, do not modify. */ 1692 if (__supported_pte_mask & _PAGE_NX) { 1693 alias_cpa.mask_clr.pgprot &= ~_PAGE_NX; 1694 alias_cpa.mask_set.pgprot &= ~_PAGE_NX; 1695 } 1696 1697 cpa->force_flush_all = 1; 1698 1699 ret = __change_page_attr_set_clr(&alias_cpa, 0); 1700 if (ret) 1701 return ret; 1702 } 1703 1704#ifdef CONFIG_X86_64 1705 /* 1706 * If the primary call didn't touch the high mapping already 1707 * and the physical address is inside the kernel map, we need 1708 * to touch the high mapped kernel as well: 1709 */ 1710 if (!within(vaddr, (unsigned long)_text, _brk_end) && 1711 __cpa_pfn_in_highmap(cpa->pfn)) { 1712 unsigned long temp_cpa_vaddr = (cpa->pfn << PAGE_SHIFT) + 1713 __START_KERNEL_map - phys_base; 1714 alias_cpa = *cpa; 1715 alias_cpa.vaddr = &temp_cpa_vaddr; 1716 alias_cpa.flags &= ~(CPA_PAGES_ARRAY | CPA_ARRAY); 1717 alias_cpa.curpage = 0; 1718 1719 /* 1720 * [_text, _brk_end) also covers data, do not modify NX except 1721 * in cases where the highmap is the primary target. 1722 */ 1723 if (__supported_pte_mask & _PAGE_NX) { 1724 alias_cpa.mask_clr.pgprot &= ~_PAGE_NX; 1725 alias_cpa.mask_set.pgprot &= ~_PAGE_NX; 1726 } 1727 1728 cpa->force_flush_all = 1; 1729 /* 1730 * The high mapping range is imprecise, so ignore the 1731 * return value. 1732 */ 1733 __change_page_attr_set_clr(&alias_cpa, 0); 1734 } 1735#endif 1736 1737 return 0; 1738} 1739 1740static int __change_page_attr_set_clr(struct cpa_data *cpa, int primary) 1741{ 1742 unsigned long numpages = cpa->numpages; 1743 unsigned long rempages = numpages; 1744 int ret = 0; 1745 1746 /* 1747 * No changes, easy! 1748 */ 1749 if (!(pgprot_val(cpa->mask_set) | pgprot_val(cpa->mask_clr)) && 1750 !cpa->force_split) 1751 return ret; 1752 1753 while (rempages) { 1754 /* 1755 * Store the remaining nr of pages for the large page 1756 * preservation check. 1757 */ 1758 cpa->numpages = rempages; 1759 /* for array changes, we can't use large page */ 1760 if (cpa->flags & (CPA_ARRAY | CPA_PAGES_ARRAY)) 1761 cpa->numpages = 1; 1762 1763 if (!debug_pagealloc_enabled()) 1764 spin_lock(&cpa_lock); 1765 ret = __change_page_attr(cpa, primary); 1766 if (!debug_pagealloc_enabled()) 1767 spin_unlock(&cpa_lock); 1768 if (ret) 1769 goto out; 1770 1771 if (primary && !(cpa->flags & CPA_NO_CHECK_ALIAS)) { 1772 ret = cpa_process_alias(cpa); 1773 if (ret) 1774 goto out; 1775 } 1776 1777 /* 1778 * Adjust the number of pages with the result of the 1779 * CPA operation. Either a large page has been 1780 * preserved or a single page update happened. 1781 */ 1782 BUG_ON(cpa->numpages > rempages || !cpa->numpages); 1783 rempages -= cpa->numpages; 1784 cpa->curpage += cpa->numpages; 1785 } 1786 1787out: 1788 /* Restore the original numpages */ 1789 cpa->numpages = numpages; 1790 return ret; 1791} 1792 1793static int change_page_attr_set_clr(unsigned long *addr, int numpages, 1794 pgprot_t mask_set, pgprot_t mask_clr, 1795 int force_split, int in_flag, 1796 struct page **pages) 1797{ 1798 struct cpa_data cpa; 1799 int ret, cache; 1800 1801 memset(&cpa, 0, sizeof(cpa)); 1802 1803 /* 1804 * Check, if we are requested to set a not supported 1805 * feature. Clearing non-supported features is OK. 1806 */ 1807 mask_set = canon_pgprot(mask_set); 1808 1809 if (!pgprot_val(mask_set) && !pgprot_val(mask_clr) && !force_split) 1810 return 0; 1811 1812 /* Ensure we are PAGE_SIZE aligned */ 1813 if (in_flag & CPA_ARRAY) { 1814 int i; 1815 for (i = 0; i < numpages; i++) { 1816 if (addr[i] & ~PAGE_MASK) { 1817 addr[i] &= PAGE_MASK; 1818 WARN_ON_ONCE(1); 1819 } 1820 } 1821 } else if (!(in_flag & CPA_PAGES_ARRAY)) { 1822 /* 1823 * in_flag of CPA_PAGES_ARRAY implies it is aligned. 1824 * No need to check in that case 1825 */ 1826 if (*addr & ~PAGE_MASK) { 1827 *addr &= PAGE_MASK; 1828 /* 1829 * People should not be passing in unaligned addresses: 1830 */ 1831 WARN_ON_ONCE(1); 1832 } 1833 } 1834 1835 /* Must avoid aliasing mappings in the highmem code */ 1836 kmap_flush_unused(); 1837 1838 vm_unmap_aliases(); 1839 1840 cpa.vaddr = addr; 1841 cpa.pages = pages; 1842 cpa.numpages = numpages; 1843 cpa.mask_set = mask_set; 1844 cpa.mask_clr = mask_clr; 1845 cpa.flags = in_flag; 1846 cpa.curpage = 0; 1847 cpa.force_split = force_split; 1848 1849 ret = __change_page_attr_set_clr(&cpa, 1); 1850 1851 /* 1852 * Check whether we really changed something: 1853 */ 1854 if (!(cpa.flags & CPA_FLUSHTLB)) 1855 goto out; 1856 1857 /* 1858 * No need to flush, when we did not set any of the caching 1859 * attributes: 1860 */ 1861 cache = !!pgprot2cachemode(mask_set); 1862 1863 /* 1864 * On error; flush everything to be sure. 1865 */ 1866 if (ret) { 1867 cpa_flush_all(cache); 1868 goto out; 1869 } 1870 1871 cpa_flush(&cpa, cache); 1872out: 1873 return ret; 1874} 1875 1876static inline int change_page_attr_set(unsigned long *addr, int numpages, 1877 pgprot_t mask, int array) 1878{ 1879 return change_page_attr_set_clr(addr, numpages, mask, __pgprot(0), 0, 1880 (array ? CPA_ARRAY : 0), NULL); 1881} 1882 1883static inline int change_page_attr_clear(unsigned long *addr, int numpages, 1884 pgprot_t mask, int array) 1885{ 1886 return change_page_attr_set_clr(addr, numpages, __pgprot(0), mask, 0, 1887 (array ? CPA_ARRAY : 0), NULL); 1888} 1889 1890static inline int cpa_set_pages_array(struct page **pages, int numpages, 1891 pgprot_t mask) 1892{ 1893 return change_page_attr_set_clr(NULL, numpages, mask, __pgprot(0), 0, 1894 CPA_PAGES_ARRAY, pages); 1895} 1896 1897static inline int cpa_clear_pages_array(struct page **pages, int numpages, 1898 pgprot_t mask) 1899{ 1900 return change_page_attr_set_clr(NULL, numpages, __pgprot(0), mask, 0, 1901 CPA_PAGES_ARRAY, pages); 1902} 1903 1904/* 1905 * __set_memory_prot is an internal helper for callers that have been passed 1906 * a pgprot_t value from upper layers and a reservation has already been taken. 1907 * If you want to set the pgprot to a specific page protocol, use the 1908 * set_memory_xx() functions. 1909 */ 1910int __set_memory_prot(unsigned long addr, int numpages, pgprot_t prot) 1911{ 1912 return change_page_attr_set_clr(&addr, numpages, prot, 1913 __pgprot(~pgprot_val(prot)), 0, 0, 1914 NULL); 1915} 1916 1917int _set_memory_uc(unsigned long addr, int numpages) 1918{ 1919 /* 1920 * for now UC MINUS. see comments in ioremap() 1921 * If you really need strong UC use ioremap_uc(), but note 1922 * that you cannot override IO areas with set_memory_*() as 1923 * these helpers cannot work with IO memory. 1924 */ 1925 return change_page_attr_set(&addr, numpages, 1926 cachemode2pgprot(_PAGE_CACHE_MODE_UC_MINUS), 1927 0); 1928} 1929 1930int set_memory_uc(unsigned long addr, int numpages) 1931{ 1932 int ret; 1933 1934 /* 1935 * for now UC MINUS. see comments in ioremap() 1936 */ 1937 ret = memtype_reserve(__pa(addr), __pa(addr) + numpages * PAGE_SIZE, 1938 _PAGE_CACHE_MODE_UC_MINUS, NULL); 1939 if (ret) 1940 goto out_err; 1941 1942 ret = _set_memory_uc(addr, numpages); 1943 if (ret) 1944 goto out_free; 1945 1946 return 0; 1947 1948out_free: 1949 memtype_free(__pa(addr), __pa(addr) + numpages * PAGE_SIZE); 1950out_err: 1951 return ret; 1952} 1953EXPORT_SYMBOL(set_memory_uc); 1954 1955int _set_memory_wc(unsigned long addr, int numpages) 1956{ 1957 int ret; 1958 1959 ret = change_page_attr_set(&addr, numpages, 1960 cachemode2pgprot(_PAGE_CACHE_MODE_UC_MINUS), 1961 0); 1962 if (!ret) { 1963 ret = change_page_attr_set_clr(&addr, numpages, 1964 cachemode2pgprot(_PAGE_CACHE_MODE_WC), 1965 __pgprot(_PAGE_CACHE_MASK), 1966 0, 0, NULL); 1967 } 1968 return ret; 1969} 1970 1971int set_memory_wc(unsigned long addr, int numpages) 1972{ 1973 int ret; 1974 1975 ret = memtype_reserve(__pa(addr), __pa(addr) + numpages * PAGE_SIZE, 1976 _PAGE_CACHE_MODE_WC, NULL); 1977 if (ret) 1978 return ret; 1979 1980 ret = _set_memory_wc(addr, numpages); 1981 if (ret) 1982 memtype_free(__pa(addr), __pa(addr) + numpages * PAGE_SIZE); 1983 1984 return ret; 1985} 1986EXPORT_SYMBOL(set_memory_wc); 1987 1988int _set_memory_wt(unsigned long addr, int numpages) 1989{ 1990 return change_page_attr_set(&addr, numpages, 1991 cachemode2pgprot(_PAGE_CACHE_MODE_WT), 0); 1992} 1993 1994int _set_memory_wb(unsigned long addr, int numpages) 1995{ 1996 /* WB cache mode is hard wired to all cache attribute bits being 0 */ 1997 return change_page_attr_clear(&addr, numpages, 1998 __pgprot(_PAGE_CACHE_MASK), 0); 1999} 2000 2001int set_memory_wb(unsigned long addr, int numpages) 2002{ 2003 int ret; 2004 2005 ret = _set_memory_wb(addr, numpages); 2006 if (ret) 2007 return ret; 2008 2009 memtype_free(__pa(addr), __pa(addr) + numpages * PAGE_SIZE); 2010 return 0; 2011} 2012EXPORT_SYMBOL(set_memory_wb); 2013 2014/* Prevent speculative access to a page by marking it not-present */ 2015#ifdef CONFIG_X86_64 2016int set_mce_nospec(unsigned long pfn) 2017{ 2018 unsigned long decoy_addr; 2019 int rc; 2020 2021 /* SGX pages are not in the 1:1 map */ 2022 if (arch_is_platform_page(pfn << PAGE_SHIFT)) 2023 return 0; 2024 /* 2025 * We would like to just call: 2026 * set_memory_XX((unsigned long)pfn_to_kaddr(pfn), 1); 2027 * but doing that would radically increase the odds of a 2028 * speculative access to the poison page because we'd have 2029 * the virtual address of the kernel 1:1 mapping sitting 2030 * around in registers. 2031 * Instead we get tricky. We create a non-canonical address 2032 * that looks just like the one we want, but has bit 63 flipped. 2033 * This relies on set_memory_XX() properly sanitizing any __pa() 2034 * results with __PHYSICAL_MASK or PTE_PFN_MASK. 2035 */ 2036 decoy_addr = (pfn << PAGE_SHIFT) + (PAGE_OFFSET ^ BIT(63)); 2037 2038 rc = set_memory_np(decoy_addr, 1); 2039 if (rc) 2040 pr_warn("Could not invalidate pfn=0x%lx from 1:1 map\n", pfn); 2041 return rc; 2042} 2043 2044static int set_memory_p(unsigned long *addr, int numpages) 2045{ 2046 return change_page_attr_set(addr, numpages, __pgprot(_PAGE_PRESENT), 0); 2047} 2048 2049/* Restore full speculative operation to the pfn. */ 2050int clear_mce_nospec(unsigned long pfn) 2051{ 2052 unsigned long addr = (unsigned long) pfn_to_kaddr(pfn); 2053 2054 return set_memory_p(&addr, 1); 2055} 2056EXPORT_SYMBOL_GPL(clear_mce_nospec); 2057#endif /* CONFIG_X86_64 */ 2058 2059int set_memory_x(unsigned long addr, int numpages) 2060{ 2061 if (!(__supported_pte_mask & _PAGE_NX)) 2062 return 0; 2063 2064 return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_NX), 0); 2065} 2066 2067int set_memory_nx(unsigned long addr, int numpages) 2068{ 2069 if (!(__supported_pte_mask & _PAGE_NX)) 2070 return 0; 2071 2072 return change_page_attr_set(&addr, numpages, __pgprot(_PAGE_NX), 0); 2073} 2074 2075int set_memory_ro(unsigned long addr, int numpages) 2076{ 2077 return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_RW | _PAGE_DIRTY), 0); 2078} 2079 2080int set_memory_rox(unsigned long addr, int numpages) 2081{ 2082 pgprot_t clr = __pgprot(_PAGE_RW | _PAGE_DIRTY); 2083 2084 if (__supported_pte_mask & _PAGE_NX) 2085 clr.pgprot |= _PAGE_NX; 2086 2087 return change_page_attr_clear(&addr, numpages, clr, 0); 2088} 2089 2090int set_memory_rw(unsigned long addr, int numpages) 2091{ 2092 return change_page_attr_set(&addr, numpages, __pgprot(_PAGE_RW), 0); 2093} 2094 2095int set_memory_np(unsigned long addr, int numpages) 2096{ 2097 return change_page_attr_clear(&addr, numpages, __pgprot(_PAGE_PRESENT), 0); 2098} 2099 2100int set_memory_np_noalias(unsigned long addr, int numpages) 2101{ 2102 return change_page_attr_set_clr(&addr, numpages, __pgprot(0), 2103 __pgprot(_PAGE_PRESENT), 0, 2104 CPA_NO_CHECK_ALIAS, NULL); 2105} 2106 2107int set_memory_4k(unsigned long addr, int numpages) 2108{ 2109 return change_page_attr_set_clr(&addr, numpages, __pgprot(0), 2110 __pgprot(0), 1, 0, NULL); 2111} 2112 2113int set_memory_nonglobal(unsigned long addr, int numpages) 2114{ 2115 return change_page_attr_clear(&addr, numpages, 2116 __pgprot(_PAGE_GLOBAL), 0); 2117} 2118 2119int set_memory_global(unsigned long addr, int numpages) 2120{ 2121 return change_page_attr_set(&addr, numpages, 2122 __pgprot(_PAGE_GLOBAL), 0); 2123} 2124 2125/* 2126 * __set_memory_enc_pgtable() is used for the hypervisors that get 2127 * informed about "encryption" status via page tables. 2128 */ 2129static int __set_memory_enc_pgtable(unsigned long addr, int numpages, bool enc) 2130{ 2131 pgprot_t empty = __pgprot(0); 2132 struct cpa_data cpa; 2133 int ret; 2134 2135 /* Should not be working on unaligned addresses */ 2136 if (WARN_ONCE(addr & ~PAGE_MASK, "misaligned address: %#lx\n", addr)) 2137 addr &= PAGE_MASK; 2138 2139 memset(&cpa, 0, sizeof(cpa)); 2140 cpa.vaddr = &addr; 2141 cpa.numpages = numpages; 2142 cpa.mask_set = enc ? pgprot_encrypted(empty) : pgprot_decrypted(empty); 2143 cpa.mask_clr = enc ? pgprot_decrypted(empty) : pgprot_encrypted(empty); 2144 cpa.pgd = init_mm.pgd; 2145 2146 /* Must avoid aliasing mappings in the highmem code */ 2147 kmap_flush_unused(); 2148 vm_unmap_aliases(); 2149 2150 /* Flush the caches as needed before changing the encryption attribute. */ 2151 if (x86_platform.guest.enc_tlb_flush_required(enc)) 2152 cpa_flush(&cpa, x86_platform.guest.enc_cache_flush_required()); 2153 2154 /* Notify hypervisor that we are about to set/clr encryption attribute. */ 2155 if (!x86_platform.guest.enc_status_change_prepare(addr, numpages, enc)) 2156 return -EIO; 2157 2158 ret = __change_page_attr_set_clr(&cpa, 1); 2159 2160 /* 2161 * After changing the encryption attribute, we need to flush TLBs again 2162 * in case any speculative TLB caching occurred (but no need to flush 2163 * caches again). We could just use cpa_flush_all(), but in case TLB 2164 * flushing gets optimized in the cpa_flush() path use the same logic 2165 * as above. 2166 */ 2167 cpa_flush(&cpa, 0); 2168 2169 /* Notify hypervisor that we have successfully set/clr encryption attribute. */ 2170 if (!ret) { 2171 if (!x86_platform.guest.enc_status_change_finish(addr, numpages, enc)) 2172 ret = -EIO; 2173 } 2174 2175 return ret; 2176} 2177 2178static int __set_memory_enc_dec(unsigned long addr, int numpages, bool enc) 2179{ 2180 if (cc_platform_has(CC_ATTR_MEM_ENCRYPT)) 2181 return __set_memory_enc_pgtable(addr, numpages, enc); 2182 2183 return 0; 2184} 2185 2186int set_memory_encrypted(unsigned long addr, int numpages) 2187{ 2188 return __set_memory_enc_dec(addr, numpages, true); 2189} 2190EXPORT_SYMBOL_GPL(set_memory_encrypted); 2191 2192int set_memory_decrypted(unsigned long addr, int numpages) 2193{ 2194 return __set_memory_enc_dec(addr, numpages, false); 2195} 2196EXPORT_SYMBOL_GPL(set_memory_decrypted); 2197 2198int set_pages_uc(struct page *page, int numpages) 2199{ 2200 unsigned long addr = (unsigned long)page_address(page); 2201 2202 return set_memory_uc(addr, numpages); 2203} 2204EXPORT_SYMBOL(set_pages_uc); 2205 2206static int _set_pages_array(struct page **pages, int numpages, 2207 enum page_cache_mode new_type) 2208{ 2209 unsigned long start; 2210 unsigned long end; 2211 enum page_cache_mode set_type; 2212 int i; 2213 int free_idx; 2214 int ret; 2215 2216 for (i = 0; i < numpages; i++) { 2217 if (PageHighMem(pages[i])) 2218 continue; 2219 start = page_to_pfn(pages[i]) << PAGE_SHIFT; 2220 end = start + PAGE_SIZE; 2221 if (memtype_reserve(start, end, new_type, NULL)) 2222 goto err_out; 2223 } 2224 2225 /* If WC, set to UC- first and then WC */ 2226 set_type = (new_type == _PAGE_CACHE_MODE_WC) ? 2227 _PAGE_CACHE_MODE_UC_MINUS : new_type; 2228 2229 ret = cpa_set_pages_array(pages, numpages, 2230 cachemode2pgprot(set_type)); 2231 if (!ret && new_type == _PAGE_CACHE_MODE_WC) 2232 ret = change_page_attr_set_clr(NULL, numpages, 2233 cachemode2pgprot( 2234 _PAGE_CACHE_MODE_WC), 2235 __pgprot(_PAGE_CACHE_MASK), 2236 0, CPA_PAGES_ARRAY, pages); 2237 if (ret) 2238 goto err_out; 2239 return 0; /* Success */ 2240err_out: 2241 free_idx = i; 2242 for (i = 0; i < free_idx; i++) { 2243 if (PageHighMem(pages[i])) 2244 continue; 2245 start = page_to_pfn(pages[i]) << PAGE_SHIFT; 2246 end = start + PAGE_SIZE; 2247 memtype_free(start, end); 2248 } 2249 return -EINVAL; 2250} 2251 2252int set_pages_array_uc(struct page **pages, int numpages) 2253{ 2254 return _set_pages_array(pages, numpages, _PAGE_CACHE_MODE_UC_MINUS); 2255} 2256EXPORT_SYMBOL(set_pages_array_uc); 2257 2258int set_pages_array_wc(struct page **pages, int numpages) 2259{ 2260 return _set_pages_array(pages, numpages, _PAGE_CACHE_MODE_WC); 2261} 2262EXPORT_SYMBOL(set_pages_array_wc); 2263 2264int set_pages_wb(struct page *page, int numpages) 2265{ 2266 unsigned long addr = (unsigned long)page_address(page); 2267 2268 return set_memory_wb(addr, numpages); 2269} 2270EXPORT_SYMBOL(set_pages_wb); 2271 2272int set_pages_array_wb(struct page **pages, int numpages) 2273{ 2274 int retval; 2275 unsigned long start; 2276 unsigned long end; 2277 int i; 2278 2279 /* WB cache mode is hard wired to all cache attribute bits being 0 */ 2280 retval = cpa_clear_pages_array(pages, numpages, 2281 __pgprot(_PAGE_CACHE_MASK)); 2282 if (retval) 2283 return retval; 2284 2285 for (i = 0; i < numpages; i++) { 2286 if (PageHighMem(pages[i])) 2287 continue; 2288 start = page_to_pfn(pages[i]) << PAGE_SHIFT; 2289 end = start + PAGE_SIZE; 2290 memtype_free(start, end); 2291 } 2292 2293 return 0; 2294} 2295EXPORT_SYMBOL(set_pages_array_wb); 2296 2297int set_pages_ro(struct page *page, int numpages) 2298{ 2299 unsigned long addr = (unsigned long)page_address(page); 2300 2301 return set_memory_ro(addr, numpages); 2302} 2303 2304int set_pages_rw(struct page *page, int numpages) 2305{ 2306 unsigned long addr = (unsigned long)page_address(page); 2307 2308 return set_memory_rw(addr, numpages); 2309} 2310 2311static int __set_pages_p(struct page *page, int numpages) 2312{ 2313 unsigned long tempaddr = (unsigned long) page_address(page); 2314 struct cpa_data cpa = { .vaddr = &tempaddr, 2315 .pgd = NULL, 2316 .numpages = numpages, 2317 .mask_set = __pgprot(_PAGE_PRESENT | _PAGE_RW), 2318 .mask_clr = __pgprot(0), 2319 .flags = CPA_NO_CHECK_ALIAS }; 2320 2321 /* 2322 * No alias checking needed for setting present flag. otherwise, 2323 * we may need to break large pages for 64-bit kernel text 2324 * mappings (this adds to complexity if we want to do this from 2325 * atomic context especially). Let's keep it simple! 2326 */ 2327 return __change_page_attr_set_clr(&cpa, 1); 2328} 2329 2330static int __set_pages_np(struct page *page, int numpages) 2331{ 2332 unsigned long tempaddr = (unsigned long) page_address(page); 2333 struct cpa_data cpa = { .vaddr = &tempaddr, 2334 .pgd = NULL, 2335 .numpages = numpages, 2336 .mask_set = __pgprot(0), 2337 .mask_clr = __pgprot(_PAGE_PRESENT | _PAGE_RW), 2338 .flags = CPA_NO_CHECK_ALIAS }; 2339 2340 /* 2341 * No alias checking needed for setting not present flag. otherwise, 2342 * we may need to break large pages for 64-bit kernel text 2343 * mappings (this adds to complexity if we want to do this from 2344 * atomic context especially). Let's keep it simple! 2345 */ 2346 return __change_page_attr_set_clr(&cpa, 1); 2347} 2348 2349int set_direct_map_invalid_noflush(struct page *page) 2350{ 2351 return __set_pages_np(page, 1); 2352} 2353 2354int set_direct_map_default_noflush(struct page *page) 2355{ 2356 return __set_pages_p(page, 1); 2357} 2358 2359#ifdef CONFIG_DEBUG_PAGEALLOC 2360void __kernel_map_pages(struct page *page, int numpages, int enable) 2361{ 2362 if (PageHighMem(page)) 2363 return; 2364 if (!enable) { 2365 debug_check_no_locks_freed(page_address(page), 2366 numpages * PAGE_SIZE); 2367 } 2368 2369 /* 2370 * The return value is ignored as the calls cannot fail. 2371 * Large pages for identity mappings are not used at boot time 2372 * and hence no memory allocations during large page split. 2373 */ 2374 if (enable) 2375 __set_pages_p(page, numpages); 2376 else 2377 __set_pages_np(page, numpages); 2378 2379 /* 2380 * We should perform an IPI and flush all tlbs, 2381 * but that can deadlock->flush only current cpu. 2382 * Preemption needs to be disabled around __flush_tlb_all() due to 2383 * CR3 reload in __native_flush_tlb(). 2384 */ 2385 preempt_disable(); 2386 __flush_tlb_all(); 2387 preempt_enable(); 2388 2389 arch_flush_lazy_mmu_mode(); 2390} 2391#endif /* CONFIG_DEBUG_PAGEALLOC */ 2392 2393bool kernel_page_present(struct page *page) 2394{ 2395 unsigned int level; 2396 pte_t *pte; 2397 2398 if (PageHighMem(page)) 2399 return false; 2400 2401 pte = lookup_address((unsigned long)page_address(page), &level); 2402 return (pte_val(*pte) & _PAGE_PRESENT); 2403} 2404 2405int __init kernel_map_pages_in_pgd(pgd_t *pgd, u64 pfn, unsigned long address, 2406 unsigned numpages, unsigned long page_flags) 2407{ 2408 int retval = -EINVAL; 2409 2410 struct cpa_data cpa = { 2411 .vaddr = &address, 2412 .pfn = pfn, 2413 .pgd = pgd, 2414 .numpages = numpages, 2415 .mask_set = __pgprot(0), 2416 .mask_clr = __pgprot(~page_flags & (_PAGE_NX|_PAGE_RW)), 2417 .flags = CPA_NO_CHECK_ALIAS, 2418 }; 2419 2420 WARN_ONCE(num_online_cpus() > 1, "Don't call after initializing SMP"); 2421 2422 if (!(__supported_pte_mask & _PAGE_NX)) 2423 goto out; 2424 2425 if (!(page_flags & _PAGE_ENC)) 2426 cpa.mask_clr = pgprot_encrypted(cpa.mask_clr); 2427 2428 cpa.mask_set = __pgprot(_PAGE_PRESENT | page_flags); 2429 2430 retval = __change_page_attr_set_clr(&cpa, 1); 2431 __flush_tlb_all(); 2432 2433out: 2434 return retval; 2435} 2436 2437/* 2438 * __flush_tlb_all() flushes mappings only on current CPU and hence this 2439 * function shouldn't be used in an SMP environment. Presently, it's used only 2440 * during boot (way before smp_init()) by EFI subsystem and hence is ok. 2441 */ 2442int __init kernel_unmap_pages_in_pgd(pgd_t *pgd, unsigned long address, 2443 unsigned long numpages) 2444{ 2445 int retval; 2446 2447 /* 2448 * The typical sequence for unmapping is to find a pte through 2449 * lookup_address_in_pgd() (ideally, it should never return NULL because 2450 * the address is already mapped) and change it's protections. As pfn is 2451 * the *target* of a mapping, it's not useful while unmapping. 2452 */ 2453 struct cpa_data cpa = { 2454 .vaddr = &address, 2455 .pfn = 0, 2456 .pgd = pgd, 2457 .numpages = numpages, 2458 .mask_set = __pgprot(0), 2459 .mask_clr = __pgprot(_PAGE_PRESENT | _PAGE_RW), 2460 .flags = CPA_NO_CHECK_ALIAS, 2461 }; 2462 2463 WARN_ONCE(num_online_cpus() > 1, "Don't call after initializing SMP"); 2464 2465 retval = __change_page_attr_set_clr(&cpa, 1); 2466 __flush_tlb_all(); 2467 2468 return retval; 2469} 2470 2471/* 2472 * The testcases use internal knowledge of the implementation that shouldn't 2473 * be exposed to the rest of the kernel. Include these directly here. 2474 */ 2475#ifdef CONFIG_CPA_DEBUG 2476#include "cpa-test.c" 2477#endif 2478