1// SPDX-License-Identifier: GPL-2.0-or-later 2/* 3 * This file contains the routines for TLB flushing. 4 * On machines where the MMU does not use a hash table to store virtual to 5 * physical translations (ie, SW loaded TLBs or Book3E compilant processors, 6 * this does -not- include 603 however which shares the implementation with 7 * hash based processors) 8 * 9 * -- BenH 10 * 11 * Copyright 2008,2009 Ben Herrenschmidt <benh@kernel.crashing.org> 12 * IBM Corp. 13 * 14 * Derived from arch/ppc/mm/init.c: 15 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org) 16 * 17 * Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au) 18 * and Cort Dougan (PReP) (cort@cs.nmt.edu) 19 * Copyright (C) 1996 Paul Mackerras 20 * 21 * Derived from "arch/i386/mm/init.c" 22 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds 23 */ 24 25#include <linux/kernel.h> 26#include <linux/export.h> 27#include <linux/mm.h> 28#include <linux/init.h> 29#include <linux/highmem.h> 30#include <linux/pagemap.h> 31#include <linux/preempt.h> 32#include <linux/spinlock.h> 33#include <linux/memblock.h> 34#include <linux/of_fdt.h> 35#include <linux/hugetlb.h> 36 37#include <asm/pgalloc.h> 38#include <asm/tlbflush.h> 39#include <asm/tlb.h> 40#include <asm/code-patching.h> 41#include <asm/cputhreads.h> 42#include <asm/hugetlb.h> 43#include <asm/paca.h> 44 45#include <mm/mmu_decl.h> 46 47/* 48 * This struct lists the sw-supported page sizes. The hardawre MMU may support 49 * other sizes not listed here. The .ind field is only used on MMUs that have 50 * indirect page table entries. 51 */ 52#if defined(CONFIG_PPC_BOOK3E_MMU) || defined(CONFIG_PPC_8xx) 53#ifdef CONFIG_PPC_FSL_BOOK3E 54struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT] = { 55 [MMU_PAGE_4K] = { 56 .shift = 12, 57 .enc = BOOK3E_PAGESZ_4K, 58 }, 59 [MMU_PAGE_2M] = { 60 .shift = 21, 61 .enc = BOOK3E_PAGESZ_2M, 62 }, 63 [MMU_PAGE_4M] = { 64 .shift = 22, 65 .enc = BOOK3E_PAGESZ_4M, 66 }, 67 [MMU_PAGE_16M] = { 68 .shift = 24, 69 .enc = BOOK3E_PAGESZ_16M, 70 }, 71 [MMU_PAGE_64M] = { 72 .shift = 26, 73 .enc = BOOK3E_PAGESZ_64M, 74 }, 75 [MMU_PAGE_256M] = { 76 .shift = 28, 77 .enc = BOOK3E_PAGESZ_256M, 78 }, 79 [MMU_PAGE_1G] = { 80 .shift = 30, 81 .enc = BOOK3E_PAGESZ_1GB, 82 }, 83}; 84#elif defined(CONFIG_PPC_8xx) 85struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT] = { 86 [MMU_PAGE_4K] = { 87 .shift = 12, 88 }, 89 [MMU_PAGE_16K] = { 90 .shift = 14, 91 }, 92 [MMU_PAGE_512K] = { 93 .shift = 19, 94 }, 95 [MMU_PAGE_8M] = { 96 .shift = 23, 97 }, 98}; 99#else 100struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT] = { 101 [MMU_PAGE_4K] = { 102 .shift = 12, 103 .ind = 20, 104 .enc = BOOK3E_PAGESZ_4K, 105 }, 106 [MMU_PAGE_16K] = { 107 .shift = 14, 108 .enc = BOOK3E_PAGESZ_16K, 109 }, 110 [MMU_PAGE_64K] = { 111 .shift = 16, 112 .ind = 28, 113 .enc = BOOK3E_PAGESZ_64K, 114 }, 115 [MMU_PAGE_1M] = { 116 .shift = 20, 117 .enc = BOOK3E_PAGESZ_1M, 118 }, 119 [MMU_PAGE_16M] = { 120 .shift = 24, 121 .ind = 36, 122 .enc = BOOK3E_PAGESZ_16M, 123 }, 124 [MMU_PAGE_256M] = { 125 .shift = 28, 126 .enc = BOOK3E_PAGESZ_256M, 127 }, 128 [MMU_PAGE_1G] = { 129 .shift = 30, 130 .enc = BOOK3E_PAGESZ_1GB, 131 }, 132}; 133#endif /* CONFIG_FSL_BOOKE */ 134 135static inline int mmu_get_tsize(int psize) 136{ 137 return mmu_psize_defs[psize].enc; 138} 139#else 140static inline int mmu_get_tsize(int psize) 141{ 142 /* This isn't used on !Book3E for now */ 143 return 0; 144} 145#endif /* CONFIG_PPC_BOOK3E_MMU */ 146 147/* The variables below are currently only used on 64-bit Book3E 148 * though this will probably be made common with other nohash 149 * implementations at some point 150 */ 151#ifdef CONFIG_PPC64 152 153int mmu_linear_psize; /* Page size used for the linear mapping */ 154int mmu_pte_psize; /* Page size used for PTE pages */ 155int mmu_vmemmap_psize; /* Page size used for the virtual mem map */ 156int book3e_htw_mode; /* HW tablewalk? Value is PPC_HTW_* */ 157unsigned long linear_map_top; /* Top of linear mapping */ 158 159 160/* 161 * Number of bytes to add to SPRN_SPRG_TLB_EXFRAME on crit/mcheck/debug 162 * exceptions. This is used for bolted and e6500 TLB miss handlers which 163 * do not modify this SPRG in the TLB miss code; for other TLB miss handlers, 164 * this is set to zero. 165 */ 166int extlb_level_exc; 167 168#endif /* CONFIG_PPC64 */ 169 170#ifdef CONFIG_PPC_FSL_BOOK3E 171/* next_tlbcam_idx is used to round-robin tlbcam entry assignment */ 172DEFINE_PER_CPU(int, next_tlbcam_idx); 173EXPORT_PER_CPU_SYMBOL(next_tlbcam_idx); 174#endif 175 176/* 177 * Base TLB flushing operations: 178 * 179 * - flush_tlb_mm(mm) flushes the specified mm context TLB's 180 * - flush_tlb_page(vma, vmaddr) flushes one page 181 * - flush_tlb_range(vma, start, end) flushes a range of pages 182 * - flush_tlb_kernel_range(start, end) flushes kernel pages 183 * 184 * - local_* variants of page and mm only apply to the current 185 * processor 186 */ 187 188/* 189 * These are the base non-SMP variants of page and mm flushing 190 */ 191void local_flush_tlb_mm(struct mm_struct *mm) 192{ 193 unsigned int pid; 194 195 preempt_disable(); 196 pid = mm->context.id; 197 if (pid != MMU_NO_CONTEXT) 198 _tlbil_pid(pid); 199 preempt_enable(); 200} 201EXPORT_SYMBOL(local_flush_tlb_mm); 202 203void __local_flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr, 204 int tsize, int ind) 205{ 206 unsigned int pid; 207 208 preempt_disable(); 209 pid = mm ? mm->context.id : 0; 210 if (pid != MMU_NO_CONTEXT) 211 _tlbil_va(vmaddr, pid, tsize, ind); 212 preempt_enable(); 213} 214 215void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr) 216{ 217 __local_flush_tlb_page(vma ? vma->vm_mm : NULL, vmaddr, 218 mmu_get_tsize(mmu_virtual_psize), 0); 219} 220EXPORT_SYMBOL(local_flush_tlb_page); 221 222/* 223 * And here are the SMP non-local implementations 224 */ 225#ifdef CONFIG_SMP 226 227static DEFINE_RAW_SPINLOCK(tlbivax_lock); 228 229struct tlb_flush_param { 230 unsigned long addr; 231 unsigned int pid; 232 unsigned int tsize; 233 unsigned int ind; 234}; 235 236static void do_flush_tlb_mm_ipi(void *param) 237{ 238 struct tlb_flush_param *p = param; 239 240 _tlbil_pid(p ? p->pid : 0); 241} 242 243static void do_flush_tlb_page_ipi(void *param) 244{ 245 struct tlb_flush_param *p = param; 246 247 _tlbil_va(p->addr, p->pid, p->tsize, p->ind); 248} 249 250 251/* Note on invalidations and PID: 252 * 253 * We snapshot the PID with preempt disabled. At this point, it can still 254 * change either because: 255 * - our context is being stolen (PID -> NO_CONTEXT) on another CPU 256 * - we are invaliating some target that isn't currently running here 257 * and is concurrently acquiring a new PID on another CPU 258 * - some other CPU is re-acquiring a lost PID for this mm 259 * etc... 260 * 261 * However, this shouldn't be a problem as we only guarantee 262 * invalidation of TLB entries present prior to this call, so we 263 * don't care about the PID changing, and invalidating a stale PID 264 * is generally harmless. 265 */ 266 267void flush_tlb_mm(struct mm_struct *mm) 268{ 269 unsigned int pid; 270 271 preempt_disable(); 272 pid = mm->context.id; 273 if (unlikely(pid == MMU_NO_CONTEXT)) 274 goto no_context; 275 if (!mm_is_core_local(mm)) { 276 struct tlb_flush_param p = { .pid = pid }; 277 /* Ignores smp_processor_id() even if set. */ 278 smp_call_function_many(mm_cpumask(mm), 279 do_flush_tlb_mm_ipi, &p, 1); 280 } 281 _tlbil_pid(pid); 282 no_context: 283 preempt_enable(); 284} 285EXPORT_SYMBOL(flush_tlb_mm); 286 287void __flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr, 288 int tsize, int ind) 289{ 290 struct cpumask *cpu_mask; 291 unsigned int pid; 292 293 /* 294 * This function as well as __local_flush_tlb_page() must only be called 295 * for user contexts. 296 */ 297 if (WARN_ON(!mm)) 298 return; 299 300 preempt_disable(); 301 pid = mm->context.id; 302 if (unlikely(pid == MMU_NO_CONTEXT)) 303 goto bail; 304 cpu_mask = mm_cpumask(mm); 305 if (!mm_is_core_local(mm)) { 306 /* If broadcast tlbivax is supported, use it */ 307 if (mmu_has_feature(MMU_FTR_USE_TLBIVAX_BCAST)) { 308 int lock = mmu_has_feature(MMU_FTR_LOCK_BCAST_INVAL); 309 if (lock) 310 raw_spin_lock(&tlbivax_lock); 311 _tlbivax_bcast(vmaddr, pid, tsize, ind); 312 if (lock) 313 raw_spin_unlock(&tlbivax_lock); 314 goto bail; 315 } else { 316 struct tlb_flush_param p = { 317 .pid = pid, 318 .addr = vmaddr, 319 .tsize = tsize, 320 .ind = ind, 321 }; 322 /* Ignores smp_processor_id() even if set in cpu_mask */ 323 smp_call_function_many(cpu_mask, 324 do_flush_tlb_page_ipi, &p, 1); 325 } 326 } 327 _tlbil_va(vmaddr, pid, tsize, ind); 328 bail: 329 preempt_enable(); 330} 331 332void flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr) 333{ 334#ifdef CONFIG_HUGETLB_PAGE 335 if (vma && is_vm_hugetlb_page(vma)) 336 flush_hugetlb_page(vma, vmaddr); 337#endif 338 339 __flush_tlb_page(vma ? vma->vm_mm : NULL, vmaddr, 340 mmu_get_tsize(mmu_virtual_psize), 0); 341} 342EXPORT_SYMBOL(flush_tlb_page); 343 344#endif /* CONFIG_SMP */ 345 346#ifdef CONFIG_PPC_47x 347void __init early_init_mmu_47x(void) 348{ 349#ifdef CONFIG_SMP 350 unsigned long root = of_get_flat_dt_root(); 351 if (of_get_flat_dt_prop(root, "cooperative-partition", NULL)) 352 mmu_clear_feature(MMU_FTR_USE_TLBIVAX_BCAST); 353#endif /* CONFIG_SMP */ 354} 355#endif /* CONFIG_PPC_47x */ 356 357/* 358 * Flush kernel TLB entries in the given range 359 */ 360void flush_tlb_kernel_range(unsigned long start, unsigned long end) 361{ 362#ifdef CONFIG_SMP 363 preempt_disable(); 364 smp_call_function(do_flush_tlb_mm_ipi, NULL, 1); 365 _tlbil_pid(0); 366 preempt_enable(); 367#else 368 _tlbil_pid(0); 369#endif 370} 371EXPORT_SYMBOL(flush_tlb_kernel_range); 372 373/* 374 * Currently, for range flushing, we just do a full mm flush. This should 375 * be optimized based on a threshold on the size of the range, since 376 * some implementation can stack multiple tlbivax before a tlbsync but 377 * for now, we keep it that way 378 */ 379void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, 380 unsigned long end) 381 382{ 383 if (end - start == PAGE_SIZE && !(start & ~PAGE_MASK)) 384 flush_tlb_page(vma, start); 385 else 386 flush_tlb_mm(vma->vm_mm); 387} 388EXPORT_SYMBOL(flush_tlb_range); 389 390void tlb_flush(struct mmu_gather *tlb) 391{ 392 flush_tlb_mm(tlb->mm); 393} 394 395/* 396 * Below are functions specific to the 64-bit variant of Book3E though that 397 * may change in the future 398 */ 399 400#ifdef CONFIG_PPC64 401 402/* 403 * Handling of virtual linear page tables or indirect TLB entries 404 * flushing when PTE pages are freed 405 */ 406void tlb_flush_pgtable(struct mmu_gather *tlb, unsigned long address) 407{ 408 int tsize = mmu_psize_defs[mmu_pte_psize].enc; 409 410 if (book3e_htw_mode != PPC_HTW_NONE) { 411 unsigned long start = address & PMD_MASK; 412 unsigned long end = address + PMD_SIZE; 413 unsigned long size = 1UL << mmu_psize_defs[mmu_pte_psize].shift; 414 415 /* This isn't the most optimal, ideally we would factor out the 416 * while preempt & CPU mask mucking around, or even the IPI but 417 * it will do for now 418 */ 419 while (start < end) { 420 __flush_tlb_page(tlb->mm, start, tsize, 1); 421 start += size; 422 } 423 } else { 424 unsigned long rmask = 0xf000000000000000ul; 425 unsigned long rid = (address & rmask) | 0x1000000000000000ul; 426 unsigned long vpte = address & ~rmask; 427 428 vpte = (vpte >> (PAGE_SHIFT - 3)) & ~0xffful; 429 vpte |= rid; 430 __flush_tlb_page(tlb->mm, vpte, tsize, 0); 431 } 432} 433 434static void setup_page_sizes(void) 435{ 436 unsigned int tlb0cfg; 437 unsigned int tlb0ps; 438 unsigned int eptcfg; 439 int i, psize; 440 441#ifdef CONFIG_PPC_FSL_BOOK3E 442 unsigned int mmucfg = mfspr(SPRN_MMUCFG); 443 int fsl_mmu = mmu_has_feature(MMU_FTR_TYPE_FSL_E); 444 445 if (fsl_mmu && (mmucfg & MMUCFG_MAVN) == MMUCFG_MAVN_V1) { 446 unsigned int tlb1cfg = mfspr(SPRN_TLB1CFG); 447 unsigned int min_pg, max_pg; 448 449 min_pg = (tlb1cfg & TLBnCFG_MINSIZE) >> TLBnCFG_MINSIZE_SHIFT; 450 max_pg = (tlb1cfg & TLBnCFG_MAXSIZE) >> TLBnCFG_MAXSIZE_SHIFT; 451 452 for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) { 453 struct mmu_psize_def *def; 454 unsigned int shift; 455 456 def = &mmu_psize_defs[psize]; 457 shift = def->shift; 458 459 if (shift == 0 || shift & 1) 460 continue; 461 462 /* adjust to be in terms of 4^shift Kb */ 463 shift = (shift - 10) >> 1; 464 465 if ((shift >= min_pg) && (shift <= max_pg)) 466 def->flags |= MMU_PAGE_SIZE_DIRECT; 467 } 468 469 goto out; 470 } 471 472 if (fsl_mmu && (mmucfg & MMUCFG_MAVN) == MMUCFG_MAVN_V2) { 473 u32 tlb1cfg, tlb1ps; 474 475 tlb0cfg = mfspr(SPRN_TLB0CFG); 476 tlb1cfg = mfspr(SPRN_TLB1CFG); 477 tlb1ps = mfspr(SPRN_TLB1PS); 478 eptcfg = mfspr(SPRN_EPTCFG); 479 480 if ((tlb1cfg & TLBnCFG_IND) && (tlb0cfg & TLBnCFG_PT)) 481 book3e_htw_mode = PPC_HTW_E6500; 482 483 /* 484 * We expect 4K subpage size and unrestricted indirect size. 485 * The lack of a restriction on indirect size is a Freescale 486 * extension, indicated by PSn = 0 but SPSn != 0. 487 */ 488 if (eptcfg != 2) 489 book3e_htw_mode = PPC_HTW_NONE; 490 491 for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) { 492 struct mmu_psize_def *def = &mmu_psize_defs[psize]; 493 494 if (!def->shift) 495 continue; 496 497 if (tlb1ps & (1U << (def->shift - 10))) { 498 def->flags |= MMU_PAGE_SIZE_DIRECT; 499 500 if (book3e_htw_mode && psize == MMU_PAGE_2M) 501 def->flags |= MMU_PAGE_SIZE_INDIRECT; 502 } 503 } 504 505 goto out; 506 } 507#endif 508 509 tlb0cfg = mfspr(SPRN_TLB0CFG); 510 tlb0ps = mfspr(SPRN_TLB0PS); 511 eptcfg = mfspr(SPRN_EPTCFG); 512 513 /* Look for supported direct sizes */ 514 for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) { 515 struct mmu_psize_def *def = &mmu_psize_defs[psize]; 516 517 if (tlb0ps & (1U << (def->shift - 10))) 518 def->flags |= MMU_PAGE_SIZE_DIRECT; 519 } 520 521 /* Indirect page sizes supported ? */ 522 if ((tlb0cfg & TLBnCFG_IND) == 0 || 523 (tlb0cfg & TLBnCFG_PT) == 0) 524 goto out; 525 526 book3e_htw_mode = PPC_HTW_IBM; 527 528 /* Now, we only deal with one IND page size for each 529 * direct size. Hopefully all implementations today are 530 * unambiguous, but we might want to be careful in the 531 * future. 532 */ 533 for (i = 0; i < 3; i++) { 534 unsigned int ps, sps; 535 536 sps = eptcfg & 0x1f; 537 eptcfg >>= 5; 538 ps = eptcfg & 0x1f; 539 eptcfg >>= 5; 540 if (!ps || !sps) 541 continue; 542 for (psize = 0; psize < MMU_PAGE_COUNT; psize++) { 543 struct mmu_psize_def *def = &mmu_psize_defs[psize]; 544 545 if (ps == (def->shift - 10)) 546 def->flags |= MMU_PAGE_SIZE_INDIRECT; 547 if (sps == (def->shift - 10)) 548 def->ind = ps + 10; 549 } 550 } 551 552out: 553 /* Cleanup array and print summary */ 554 pr_info("MMU: Supported page sizes\n"); 555 for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) { 556 struct mmu_psize_def *def = &mmu_psize_defs[psize]; 557 const char *__page_type_names[] = { 558 "unsupported", 559 "direct", 560 "indirect", 561 "direct & indirect" 562 }; 563 if (def->flags == 0) { 564 def->shift = 0; 565 continue; 566 } 567 pr_info(" %8ld KB as %s\n", 1ul << (def->shift - 10), 568 __page_type_names[def->flags & 0x3]); 569 } 570} 571 572static void setup_mmu_htw(void) 573{ 574 /* 575 * If we want to use HW tablewalk, enable it by patching the TLB miss 576 * handlers to branch to the one dedicated to it. 577 */ 578 579 switch (book3e_htw_mode) { 580 case PPC_HTW_IBM: 581 patch_exception(0x1c0, exc_data_tlb_miss_htw_book3e); 582 patch_exception(0x1e0, exc_instruction_tlb_miss_htw_book3e); 583 break; 584#ifdef CONFIG_PPC_FSL_BOOK3E 585 case PPC_HTW_E6500: 586 extlb_level_exc = EX_TLB_SIZE; 587 patch_exception(0x1c0, exc_data_tlb_miss_e6500_book3e); 588 patch_exception(0x1e0, exc_instruction_tlb_miss_e6500_book3e); 589 break; 590#endif 591 } 592 pr_info("MMU: Book3E HW tablewalk %s\n", 593 book3e_htw_mode != PPC_HTW_NONE ? "enabled" : "not supported"); 594} 595 596/* 597 * Early initialization of the MMU TLB code 598 */ 599static void early_init_this_mmu(void) 600{ 601 unsigned int mas4; 602 603 /* Set MAS4 based on page table setting */ 604 605 mas4 = 0x4 << MAS4_WIMGED_SHIFT; 606 switch (book3e_htw_mode) { 607 case PPC_HTW_E6500: 608 mas4 |= MAS4_INDD; 609 mas4 |= BOOK3E_PAGESZ_2M << MAS4_TSIZED_SHIFT; 610 mas4 |= MAS4_TLBSELD(1); 611 mmu_pte_psize = MMU_PAGE_2M; 612 break; 613 614 case PPC_HTW_IBM: 615 mas4 |= MAS4_INDD; 616 mas4 |= BOOK3E_PAGESZ_1M << MAS4_TSIZED_SHIFT; 617 mmu_pte_psize = MMU_PAGE_1M; 618 break; 619 620 case PPC_HTW_NONE: 621 mas4 |= BOOK3E_PAGESZ_4K << MAS4_TSIZED_SHIFT; 622 mmu_pte_psize = mmu_virtual_psize; 623 break; 624 } 625 mtspr(SPRN_MAS4, mas4); 626 627#ifdef CONFIG_PPC_FSL_BOOK3E 628 if (mmu_has_feature(MMU_FTR_TYPE_FSL_E)) { 629 unsigned int num_cams; 630 bool map = true; 631 632 /* use a quarter of the TLBCAM for bolted linear map */ 633 num_cams = (mfspr(SPRN_TLB1CFG) & TLBnCFG_N_ENTRY) / 4; 634 635 /* 636 * Only do the mapping once per core, or else the 637 * transient mapping would cause problems. 638 */ 639#ifdef CONFIG_SMP 640 if (hweight32(get_tensr()) > 1) 641 map = false; 642#endif 643 644 if (map) 645 linear_map_top = map_mem_in_cams(linear_map_top, 646 num_cams, false); 647 } 648#endif 649 650 /* A sync won't hurt us after mucking around with 651 * the MMU configuration 652 */ 653 mb(); 654} 655 656static void __init early_init_mmu_global(void) 657{ 658 /* XXX This will have to be decided at runtime, but right 659 * now our boot and TLB miss code hard wires it. Ideally 660 * we should find out a suitable page size and patch the 661 * TLB miss code (either that or use the PACA to store 662 * the value we want) 663 */ 664 mmu_linear_psize = MMU_PAGE_1G; 665 666 /* XXX This should be decided at runtime based on supported 667 * page sizes in the TLB, but for now let's assume 16M is 668 * always there and a good fit (which it probably is) 669 * 670 * Freescale booke only supports 4K pages in TLB0, so use that. 671 */ 672 if (mmu_has_feature(MMU_FTR_TYPE_FSL_E)) 673 mmu_vmemmap_psize = MMU_PAGE_4K; 674 else 675 mmu_vmemmap_psize = MMU_PAGE_16M; 676 677 /* XXX This code only checks for TLB 0 capabilities and doesn't 678 * check what page size combos are supported by the HW. It 679 * also doesn't handle the case where a separate array holds 680 * the IND entries from the array loaded by the PT. 681 */ 682 /* Look for supported page sizes */ 683 setup_page_sizes(); 684 685 /* Look for HW tablewalk support */ 686 setup_mmu_htw(); 687 688#ifdef CONFIG_PPC_FSL_BOOK3E 689 if (mmu_has_feature(MMU_FTR_TYPE_FSL_E)) { 690 if (book3e_htw_mode == PPC_HTW_NONE) { 691 extlb_level_exc = EX_TLB_SIZE; 692 patch_exception(0x1c0, exc_data_tlb_miss_bolted_book3e); 693 patch_exception(0x1e0, 694 exc_instruction_tlb_miss_bolted_book3e); 695 } 696 } 697#endif 698 699 /* Set the global containing the top of the linear mapping 700 * for use by the TLB miss code 701 */ 702 linear_map_top = memblock_end_of_DRAM(); 703 704 ioremap_bot = IOREMAP_BASE; 705} 706 707static void __init early_mmu_set_memory_limit(void) 708{ 709#ifdef CONFIG_PPC_FSL_BOOK3E 710 if (mmu_has_feature(MMU_FTR_TYPE_FSL_E)) { 711 /* 712 * Limit memory so we dont have linear faults. 713 * Unlike memblock_set_current_limit, which limits 714 * memory available during early boot, this permanently 715 * reduces the memory available to Linux. We need to 716 * do this because highmem is not supported on 64-bit. 717 */ 718 memblock_enforce_memory_limit(linear_map_top); 719 } 720#endif 721 722 memblock_set_current_limit(linear_map_top); 723} 724 725/* boot cpu only */ 726void __init early_init_mmu(void) 727{ 728 early_init_mmu_global(); 729 early_init_this_mmu(); 730 early_mmu_set_memory_limit(); 731} 732 733void early_init_mmu_secondary(void) 734{ 735 early_init_this_mmu(); 736} 737 738void setup_initial_memory_limit(phys_addr_t first_memblock_base, 739 phys_addr_t first_memblock_size) 740{ 741 /* On non-FSL Embedded 64-bit, we adjust the RMA size to match 742 * the bolted TLB entry. We know for now that only 1G 743 * entries are supported though that may eventually 744 * change. 745 * 746 * on FSL Embedded 64-bit, usually all RAM is bolted, but with 747 * unusual memory sizes it's possible for some RAM to not be mapped 748 * (such RAM is not used at all by Linux, since we don't support 749 * highmem on 64-bit). We limit ppc64_rma_size to what would be 750 * mappable if this memblock is the only one. Additional memblocks 751 * can only increase, not decrease, the amount that ends up getting 752 * mapped. We still limit max to 1G even if we'll eventually map 753 * more. This is due to what the early init code is set up to do. 754 * 755 * We crop it to the size of the first MEMBLOCK to 756 * avoid going over total available memory just in case... 757 */ 758#ifdef CONFIG_PPC_FSL_BOOK3E 759 if (early_mmu_has_feature(MMU_FTR_TYPE_FSL_E)) { 760 unsigned long linear_sz; 761 unsigned int num_cams; 762 763 /* use a quarter of the TLBCAM for bolted linear map */ 764 num_cams = (mfspr(SPRN_TLB1CFG) & TLBnCFG_N_ENTRY) / 4; 765 766 linear_sz = map_mem_in_cams(first_memblock_size, num_cams, 767 true); 768 769 ppc64_rma_size = min_t(u64, linear_sz, 0x40000000); 770 } else 771#endif 772 ppc64_rma_size = min_t(u64, first_memblock_size, 0x40000000); 773 774 /* Finally limit subsequent allocations */ 775 memblock_set_current_limit(first_memblock_base + ppc64_rma_size); 776} 777#else /* ! CONFIG_PPC64 */ 778void __init early_init_mmu(void) 779{ 780#ifdef CONFIG_PPC_47x 781 early_init_mmu_47x(); 782#endif 783 784#ifdef CONFIG_PPC_MM_SLICES 785 mm_ctx_set_slb_addr_limit(&init_mm.context, SLB_ADDR_LIMIT_DEFAULT); 786#endif 787} 788#endif /* CONFIG_PPC64 */ 789