1// SPDX-License-Identifier: GPL-2.0-or-later 2/* 3 * PowerPC64 SLB support. 4 * 5 * Copyright (C) 2004 David Gibson <dwg@au.ibm.com>, IBM 6 * Based on earlier code written by: 7 * Dave Engebretsen and Mike Corrigan {engebret|mikejc}@us.ibm.com 8 * Copyright (c) 2001 Dave Engebretsen 9 * Copyright (C) 2002 Anton Blanchard <anton@au.ibm.com>, IBM 10 */ 11 12#include <asm/asm-prototypes.h> 13#include <asm/mmu.h> 14#include <asm/mmu_context.h> 15#include <asm/paca.h> 16#include <asm/lppaca.h> 17#include <asm/ppc-opcode.h> 18#include <asm/cputable.h> 19#include <asm/cacheflush.h> 20#include <asm/smp.h> 21#include <linux/compiler.h> 22#include <linux/context_tracking.h> 23#include <linux/mm_types.h> 24#include <linux/pgtable.h> 25 26#include <asm/udbg.h> 27#include <asm/code-patching.h> 28 29#include "internal.h" 30 31 32enum slb_index { 33 LINEAR_INDEX = 0, /* Kernel linear map (0xc000000000000000) */ 34 KSTACK_INDEX = 1, /* Kernel stack map */ 35}; 36 37static long slb_allocate_user(struct mm_struct *mm, unsigned long ea); 38 39#define slb_esid_mask(ssize) \ 40 (((ssize) == MMU_SEGSIZE_256M)? ESID_MASK: ESID_MASK_1T) 41 42static inline unsigned long mk_esid_data(unsigned long ea, int ssize, 43 enum slb_index index) 44{ 45 return (ea & slb_esid_mask(ssize)) | SLB_ESID_V | index; 46} 47 48static inline unsigned long __mk_vsid_data(unsigned long vsid, int ssize, 49 unsigned long flags) 50{ 51 return (vsid << slb_vsid_shift(ssize)) | flags | 52 ((unsigned long) ssize << SLB_VSID_SSIZE_SHIFT); 53} 54 55static inline unsigned long mk_vsid_data(unsigned long ea, int ssize, 56 unsigned long flags) 57{ 58 return __mk_vsid_data(get_kernel_vsid(ea, ssize), ssize, flags); 59} 60 61bool stress_slb_enabled __initdata; 62 63static int __init parse_stress_slb(char *p) 64{ 65 stress_slb_enabled = true; 66 return 0; 67} 68early_param("stress_slb", parse_stress_slb); 69 70__ro_after_init DEFINE_STATIC_KEY_FALSE(stress_slb_key); 71 72static void assert_slb_presence(bool present, unsigned long ea) 73{ 74#ifdef CONFIG_DEBUG_VM 75 unsigned long tmp; 76 77 WARN_ON_ONCE(mfmsr() & MSR_EE); 78 79 if (!cpu_has_feature(CPU_FTR_ARCH_206)) 80 return; 81 82 /* 83 * slbfee. requires bit 24 (PPC bit 39) be clear in RB. Hardware 84 * ignores all other bits from 0-27, so just clear them all. 85 */ 86 ea &= ~((1UL << SID_SHIFT) - 1); 87 asm volatile(__PPC_SLBFEE_DOT(%0, %1) : "=r"(tmp) : "r"(ea) : "cr0"); 88 89 WARN_ON(present == (tmp == 0)); 90#endif 91} 92 93static inline void slb_shadow_update(unsigned long ea, int ssize, 94 unsigned long flags, 95 enum slb_index index) 96{ 97 struct slb_shadow *p = get_slb_shadow(); 98 99 /* 100 * Clear the ESID first so the entry is not valid while we are 101 * updating it. No write barriers are needed here, provided 102 * we only update the current CPU's SLB shadow buffer. 103 */ 104 WRITE_ONCE(p->save_area[index].esid, 0); 105 WRITE_ONCE(p->save_area[index].vsid, cpu_to_be64(mk_vsid_data(ea, ssize, flags))); 106 WRITE_ONCE(p->save_area[index].esid, cpu_to_be64(mk_esid_data(ea, ssize, index))); 107} 108 109static inline void slb_shadow_clear(enum slb_index index) 110{ 111 WRITE_ONCE(get_slb_shadow()->save_area[index].esid, cpu_to_be64(index)); 112} 113 114static inline void create_shadowed_slbe(unsigned long ea, int ssize, 115 unsigned long flags, 116 enum slb_index index) 117{ 118 /* 119 * Updating the shadow buffer before writing the SLB ensures 120 * we don't get a stale entry here if we get preempted by PHYP 121 * between these two statements. 122 */ 123 slb_shadow_update(ea, ssize, flags, index); 124 125 assert_slb_presence(false, ea); 126 asm volatile("slbmte %0,%1" : 127 : "r" (mk_vsid_data(ea, ssize, flags)), 128 "r" (mk_esid_data(ea, ssize, index)) 129 : "memory" ); 130} 131 132/* 133 * Insert bolted entries into SLB (which may not be empty, so don't clear 134 * slb_cache_ptr). 135 */ 136void __slb_restore_bolted_realmode(void) 137{ 138 struct slb_shadow *p = get_slb_shadow(); 139 enum slb_index index; 140 141 /* No isync needed because realmode. */ 142 for (index = 0; index < SLB_NUM_BOLTED; index++) { 143 asm volatile("slbmte %0,%1" : 144 : "r" (be64_to_cpu(p->save_area[index].vsid)), 145 "r" (be64_to_cpu(p->save_area[index].esid))); 146 } 147 148 assert_slb_presence(true, local_paca->kstack); 149} 150 151/* 152 * Insert the bolted entries into an empty SLB. 153 */ 154void slb_restore_bolted_realmode(void) 155{ 156 __slb_restore_bolted_realmode(); 157 get_paca()->slb_cache_ptr = 0; 158 159 get_paca()->slb_kern_bitmap = (1U << SLB_NUM_BOLTED) - 1; 160 get_paca()->slb_used_bitmap = get_paca()->slb_kern_bitmap; 161} 162 163/* 164 * This flushes all SLB entries including 0, so it must be realmode. 165 */ 166void slb_flush_all_realmode(void) 167{ 168 asm volatile("slbmte %0,%0; slbia" : : "r" (0)); 169} 170 171static __always_inline void __slb_flush_and_restore_bolted(bool preserve_kernel_lookaside) 172{ 173 struct slb_shadow *p = get_slb_shadow(); 174 unsigned long ksp_esid_data, ksp_vsid_data; 175 u32 ih; 176 177 /* 178 * SLBIA IH=1 on ISA v2.05 and newer processors may preserve lookaside 179 * information created with Class=0 entries, which we use for kernel 180 * SLB entries (the SLB entries themselves are still invalidated). 181 * 182 * Older processors will ignore this optimisation. Over-invalidation 183 * is fine because we never rely on lookaside information existing. 184 */ 185 if (preserve_kernel_lookaside) 186 ih = 1; 187 else 188 ih = 0; 189 190 ksp_esid_data = be64_to_cpu(p->save_area[KSTACK_INDEX].esid); 191 ksp_vsid_data = be64_to_cpu(p->save_area[KSTACK_INDEX].vsid); 192 193 asm volatile(PPC_SLBIA(%0)" \n" 194 "slbmte %1, %2 \n" 195 :: "i" (ih), 196 "r" (ksp_vsid_data), 197 "r" (ksp_esid_data) 198 : "memory"); 199} 200 201/* 202 * This flushes non-bolted entries, it can be run in virtual mode. Must 203 * be called with interrupts disabled. 204 */ 205void slb_flush_and_restore_bolted(void) 206{ 207 BUILD_BUG_ON(SLB_NUM_BOLTED != 2); 208 209 WARN_ON(!irqs_disabled()); 210 211 /* 212 * We can't take a PMU exception in the following code, so hard 213 * disable interrupts. 214 */ 215 hard_irq_disable(); 216 217 isync(); 218 __slb_flush_and_restore_bolted(false); 219 isync(); 220 221 assert_slb_presence(true, get_paca()->kstack); 222 223 get_paca()->slb_cache_ptr = 0; 224 225 get_paca()->slb_kern_bitmap = (1U << SLB_NUM_BOLTED) - 1; 226 get_paca()->slb_used_bitmap = get_paca()->slb_kern_bitmap; 227} 228 229void slb_save_contents(struct slb_entry *slb_ptr) 230{ 231 int i; 232 unsigned long e, v; 233 234 /* Save slb_cache_ptr value. */ 235 get_paca()->slb_save_cache_ptr = get_paca()->slb_cache_ptr; 236 237 if (!slb_ptr) 238 return; 239 240 for (i = 0; i < mmu_slb_size; i++) { 241 asm volatile("slbmfee %0,%1" : "=r" (e) : "r" (i)); 242 asm volatile("slbmfev %0,%1" : "=r" (v) : "r" (i)); 243 slb_ptr->esid = e; 244 slb_ptr->vsid = v; 245 slb_ptr++; 246 } 247} 248 249void slb_dump_contents(struct slb_entry *slb_ptr) 250{ 251 int i, n; 252 unsigned long e, v; 253 unsigned long llp; 254 255 if (!slb_ptr) 256 return; 257 258 pr_err("SLB contents of cpu 0x%x\n", smp_processor_id()); 259 pr_err("Last SLB entry inserted at slot %d\n", get_paca()->stab_rr); 260 261 for (i = 0; i < mmu_slb_size; i++) { 262 e = slb_ptr->esid; 263 v = slb_ptr->vsid; 264 slb_ptr++; 265 266 if (!e && !v) 267 continue; 268 269 pr_err("%02d %016lx %016lx\n", i, e, v); 270 271 if (!(e & SLB_ESID_V)) { 272 pr_err("\n"); 273 continue; 274 } 275 llp = v & SLB_VSID_LLP; 276 if (v & SLB_VSID_B_1T) { 277 pr_err(" 1T ESID=%9lx VSID=%13lx LLP:%3lx\n", 278 GET_ESID_1T(e), 279 (v & ~SLB_VSID_B) >> SLB_VSID_SHIFT_1T, llp); 280 } else { 281 pr_err(" 256M ESID=%9lx VSID=%13lx LLP:%3lx\n", 282 GET_ESID(e), 283 (v & ~SLB_VSID_B) >> SLB_VSID_SHIFT, llp); 284 } 285 } 286 pr_err("----------------------------------\n"); 287 288 /* Dump slb cache entires as well. */ 289 pr_err("SLB cache ptr value = %d\n", get_paca()->slb_save_cache_ptr); 290 pr_err("Valid SLB cache entries:\n"); 291 n = min_t(int, get_paca()->slb_save_cache_ptr, SLB_CACHE_ENTRIES); 292 for (i = 0; i < n; i++) 293 pr_err("%02d EA[0-35]=%9x\n", i, get_paca()->slb_cache[i]); 294 pr_err("Rest of SLB cache entries:\n"); 295 for (i = n; i < SLB_CACHE_ENTRIES; i++) 296 pr_err("%02d EA[0-35]=%9x\n", i, get_paca()->slb_cache[i]); 297} 298 299void slb_vmalloc_update(void) 300{ 301 /* 302 * vmalloc is not bolted, so just have to flush non-bolted. 303 */ 304 slb_flush_and_restore_bolted(); 305} 306 307static bool preload_hit(struct thread_info *ti, unsigned long esid) 308{ 309 unsigned char i; 310 311 for (i = 0; i < ti->slb_preload_nr; i++) { 312 unsigned char idx; 313 314 idx = (ti->slb_preload_tail + i) % SLB_PRELOAD_NR; 315 if (esid == ti->slb_preload_esid[idx]) 316 return true; 317 } 318 return false; 319} 320 321static bool preload_add(struct thread_info *ti, unsigned long ea) 322{ 323 unsigned char idx; 324 unsigned long esid; 325 326 if (mmu_has_feature(MMU_FTR_1T_SEGMENT)) { 327 /* EAs are stored >> 28 so 256MB segments don't need clearing */ 328 if (ea & ESID_MASK_1T) 329 ea &= ESID_MASK_1T; 330 } 331 332 esid = ea >> SID_SHIFT; 333 334 if (preload_hit(ti, esid)) 335 return false; 336 337 idx = (ti->slb_preload_tail + ti->slb_preload_nr) % SLB_PRELOAD_NR; 338 ti->slb_preload_esid[idx] = esid; 339 if (ti->slb_preload_nr == SLB_PRELOAD_NR) 340 ti->slb_preload_tail = (ti->slb_preload_tail + 1) % SLB_PRELOAD_NR; 341 else 342 ti->slb_preload_nr++; 343 344 return true; 345} 346 347static void preload_age(struct thread_info *ti) 348{ 349 if (!ti->slb_preload_nr) 350 return; 351 ti->slb_preload_nr--; 352 ti->slb_preload_tail = (ti->slb_preload_tail + 1) % SLB_PRELOAD_NR; 353} 354 355void slb_setup_new_exec(void) 356{ 357 struct thread_info *ti = current_thread_info(); 358 struct mm_struct *mm = current->mm; 359 unsigned long exec = 0x10000000; 360 361 WARN_ON(irqs_disabled()); 362 363 /* 364 * preload cache can only be used to determine whether a SLB 365 * entry exists if it does not start to overflow. 366 */ 367 if (ti->slb_preload_nr + 2 > SLB_PRELOAD_NR) 368 return; 369 370 hard_irq_disable(); 371 372 /* 373 * We have no good place to clear the slb preload cache on exec, 374 * flush_thread is about the earliest arch hook but that happens 375 * after we switch to the mm and have aleady preloaded the SLBEs. 376 * 377 * For the most part that's probably okay to use entries from the 378 * previous exec, they will age out if unused. It may turn out to 379 * be an advantage to clear the cache before switching to it, 380 * however. 381 */ 382 383 /* 384 * preload some userspace segments into the SLB. 385 * Almost all 32 and 64bit PowerPC executables are linked at 386 * 0x10000000 so it makes sense to preload this segment. 387 */ 388 if (!is_kernel_addr(exec)) { 389 if (preload_add(ti, exec)) 390 slb_allocate_user(mm, exec); 391 } 392 393 /* Libraries and mmaps. */ 394 if (!is_kernel_addr(mm->mmap_base)) { 395 if (preload_add(ti, mm->mmap_base)) 396 slb_allocate_user(mm, mm->mmap_base); 397 } 398 399 /* see switch_slb */ 400 asm volatile("isync" : : : "memory"); 401 402 local_irq_enable(); 403} 404 405void preload_new_slb_context(unsigned long start, unsigned long sp) 406{ 407 struct thread_info *ti = current_thread_info(); 408 struct mm_struct *mm = current->mm; 409 unsigned long heap = mm->start_brk; 410 411 WARN_ON(irqs_disabled()); 412 413 /* see above */ 414 if (ti->slb_preload_nr + 3 > SLB_PRELOAD_NR) 415 return; 416 417 hard_irq_disable(); 418 419 /* Userspace entry address. */ 420 if (!is_kernel_addr(start)) { 421 if (preload_add(ti, start)) 422 slb_allocate_user(mm, start); 423 } 424 425 /* Top of stack, grows down. */ 426 if (!is_kernel_addr(sp)) { 427 if (preload_add(ti, sp)) 428 slb_allocate_user(mm, sp); 429 } 430 431 /* Bottom of heap, grows up. */ 432 if (heap && !is_kernel_addr(heap)) { 433 if (preload_add(ti, heap)) 434 slb_allocate_user(mm, heap); 435 } 436 437 /* see switch_slb */ 438 asm volatile("isync" : : : "memory"); 439 440 local_irq_enable(); 441} 442 443static void slb_cache_slbie_kernel(unsigned int index) 444{ 445 unsigned long slbie_data = get_paca()->slb_cache[index]; 446 unsigned long ksp = get_paca()->kstack; 447 448 slbie_data <<= SID_SHIFT; 449 slbie_data |= 0xc000000000000000ULL; 450 if ((ksp & slb_esid_mask(mmu_kernel_ssize)) == slbie_data) 451 return; 452 slbie_data |= mmu_kernel_ssize << SLBIE_SSIZE_SHIFT; 453 454 asm volatile("slbie %0" : : "r" (slbie_data)); 455} 456 457static void slb_cache_slbie_user(unsigned int index) 458{ 459 unsigned long slbie_data = get_paca()->slb_cache[index]; 460 461 slbie_data <<= SID_SHIFT; 462 slbie_data |= user_segment_size(slbie_data) << SLBIE_SSIZE_SHIFT; 463 slbie_data |= SLBIE_C; /* user slbs have C=1 */ 464 465 asm volatile("slbie %0" : : "r" (slbie_data)); 466} 467 468/* Flush all user entries from the segment table of the current processor. */ 469void switch_slb(struct task_struct *tsk, struct mm_struct *mm) 470{ 471 struct thread_info *ti = task_thread_info(tsk); 472 unsigned char i; 473 474 /* 475 * We need interrupts hard-disabled here, not just soft-disabled, 476 * so that a PMU interrupt can't occur, which might try to access 477 * user memory (to get a stack trace) and possible cause an SLB miss 478 * which would update the slb_cache/slb_cache_ptr fields in the PACA. 479 */ 480 hard_irq_disable(); 481 isync(); 482 if (stress_slb()) { 483 __slb_flush_and_restore_bolted(false); 484 isync(); 485 get_paca()->slb_cache_ptr = 0; 486 get_paca()->slb_kern_bitmap = (1U << SLB_NUM_BOLTED) - 1; 487 488 } else if (cpu_has_feature(CPU_FTR_ARCH_300)) { 489 /* 490 * SLBIA IH=3 invalidates all Class=1 SLBEs and their 491 * associated lookaside structures, which matches what 492 * switch_slb wants. So ARCH_300 does not use the slb 493 * cache. 494 */ 495 asm volatile(PPC_SLBIA(3)); 496 497 } else { 498 unsigned long offset = get_paca()->slb_cache_ptr; 499 500 if (!mmu_has_feature(MMU_FTR_NO_SLBIE_B) && 501 offset <= SLB_CACHE_ENTRIES) { 502 /* 503 * Could assert_slb_presence(true) here, but 504 * hypervisor or machine check could have come 505 * in and removed the entry at this point. 506 */ 507 508 for (i = 0; i < offset; i++) 509 slb_cache_slbie_user(i); 510 511 /* Workaround POWER5 < DD2.1 issue */ 512 if (!cpu_has_feature(CPU_FTR_ARCH_207S) && offset == 1) 513 slb_cache_slbie_user(0); 514 515 } else { 516 /* Flush but retain kernel lookaside information */ 517 __slb_flush_and_restore_bolted(true); 518 isync(); 519 520 get_paca()->slb_kern_bitmap = (1U << SLB_NUM_BOLTED) - 1; 521 } 522 523 get_paca()->slb_cache_ptr = 0; 524 } 525 get_paca()->slb_used_bitmap = get_paca()->slb_kern_bitmap; 526 527 copy_mm_to_paca(mm); 528 529 /* 530 * We gradually age out SLBs after a number of context switches to 531 * reduce reload overhead of unused entries (like we do with FP/VEC 532 * reload). Each time we wrap 256 switches, take an entry out of the 533 * SLB preload cache. 534 */ 535 tsk->thread.load_slb++; 536 if (!tsk->thread.load_slb) { 537 unsigned long pc = KSTK_EIP(tsk); 538 539 preload_age(ti); 540 preload_add(ti, pc); 541 } 542 543 for (i = 0; i < ti->slb_preload_nr; i++) { 544 unsigned char idx; 545 unsigned long ea; 546 547 idx = (ti->slb_preload_tail + i) % SLB_PRELOAD_NR; 548 ea = (unsigned long)ti->slb_preload_esid[idx] << SID_SHIFT; 549 550 slb_allocate_user(mm, ea); 551 } 552 553 /* 554 * Synchronize slbmte preloads with possible subsequent user memory 555 * address accesses by the kernel (user mode won't happen until 556 * rfid, which is safe). 557 */ 558 isync(); 559} 560 561void slb_set_size(u16 size) 562{ 563 mmu_slb_size = size; 564} 565 566void slb_initialize(void) 567{ 568 unsigned long linear_llp, vmalloc_llp, io_llp; 569 unsigned long lflags; 570 static int slb_encoding_inited; 571#ifdef CONFIG_SPARSEMEM_VMEMMAP 572 unsigned long vmemmap_llp; 573#endif 574 575 /* Prepare our SLB miss handler based on our page size */ 576 linear_llp = mmu_psize_defs[mmu_linear_psize].sllp; 577 io_llp = mmu_psize_defs[mmu_io_psize].sllp; 578 vmalloc_llp = mmu_psize_defs[mmu_vmalloc_psize].sllp; 579 get_paca()->vmalloc_sllp = SLB_VSID_KERNEL | vmalloc_llp; 580#ifdef CONFIG_SPARSEMEM_VMEMMAP 581 vmemmap_llp = mmu_psize_defs[mmu_vmemmap_psize].sllp; 582#endif 583 if (!slb_encoding_inited) { 584 slb_encoding_inited = 1; 585 pr_devel("SLB: linear LLP = %04lx\n", linear_llp); 586 pr_devel("SLB: io LLP = %04lx\n", io_llp); 587#ifdef CONFIG_SPARSEMEM_VMEMMAP 588 pr_devel("SLB: vmemmap LLP = %04lx\n", vmemmap_llp); 589#endif 590 } 591 592 get_paca()->stab_rr = SLB_NUM_BOLTED - 1; 593 get_paca()->slb_kern_bitmap = (1U << SLB_NUM_BOLTED) - 1; 594 get_paca()->slb_used_bitmap = get_paca()->slb_kern_bitmap; 595 596 lflags = SLB_VSID_KERNEL | linear_llp; 597 598 /* Invalidate the entire SLB (even entry 0) & all the ERATS */ 599 asm volatile("isync":::"memory"); 600 asm volatile("slbmte %0,%0"::"r" (0) : "memory"); 601 asm volatile("isync; slbia; isync":::"memory"); 602 create_shadowed_slbe(PAGE_OFFSET, mmu_kernel_ssize, lflags, LINEAR_INDEX); 603 604 /* 605 * For the boot cpu, we're running on the stack in init_thread_union, 606 * which is in the first segment of the linear mapping, and also 607 * get_paca()->kstack hasn't been initialized yet. 608 * For secondary cpus, we need to bolt the kernel stack entry now. 609 */ 610 slb_shadow_clear(KSTACK_INDEX); 611 if (raw_smp_processor_id() != boot_cpuid && 612 (get_paca()->kstack & slb_esid_mask(mmu_kernel_ssize)) > PAGE_OFFSET) 613 create_shadowed_slbe(get_paca()->kstack, 614 mmu_kernel_ssize, lflags, KSTACK_INDEX); 615 616 asm volatile("isync":::"memory"); 617} 618 619static void slb_cache_update(unsigned long esid_data) 620{ 621 int slb_cache_index; 622 623 if (cpu_has_feature(CPU_FTR_ARCH_300)) 624 return; /* ISAv3.0B and later does not use slb_cache */ 625 626 if (stress_slb()) 627 return; 628 629 /* 630 * Now update slb cache entries 631 */ 632 slb_cache_index = local_paca->slb_cache_ptr; 633 if (slb_cache_index < SLB_CACHE_ENTRIES) { 634 /* 635 * We have space in slb cache for optimized switch_slb(). 636 * Top 36 bits from esid_data as per ISA 637 */ 638 local_paca->slb_cache[slb_cache_index++] = esid_data >> SID_SHIFT; 639 local_paca->slb_cache_ptr++; 640 } else { 641 /* 642 * Our cache is full and the current cache content strictly 643 * doesn't indicate the active SLB conents. Bump the ptr 644 * so that switch_slb() will ignore the cache. 645 */ 646 local_paca->slb_cache_ptr = SLB_CACHE_ENTRIES + 1; 647 } 648} 649 650static enum slb_index alloc_slb_index(bool kernel) 651{ 652 enum slb_index index; 653 654 /* 655 * The allocation bitmaps can become out of synch with the SLB 656 * when the _switch code does slbie when bolting a new stack 657 * segment and it must not be anywhere else in the SLB. This leaves 658 * a kernel allocated entry that is unused in the SLB. With very 659 * large systems or small segment sizes, the bitmaps could slowly 660 * fill with these entries. They will eventually be cleared out 661 * by the round robin allocator in that case, so it's probably not 662 * worth accounting for. 663 */ 664 665 /* 666 * SLBs beyond 32 entries are allocated with stab_rr only 667 * POWER7/8/9 have 32 SLB entries, this could be expanded if a 668 * future CPU has more. 669 */ 670 if (local_paca->slb_used_bitmap != U32_MAX) { 671 index = ffz(local_paca->slb_used_bitmap); 672 local_paca->slb_used_bitmap |= 1U << index; 673 if (kernel) 674 local_paca->slb_kern_bitmap |= 1U << index; 675 } else { 676 /* round-robin replacement of slb starting at SLB_NUM_BOLTED. */ 677 index = local_paca->stab_rr; 678 if (index < (mmu_slb_size - 1)) 679 index++; 680 else 681 index = SLB_NUM_BOLTED; 682 local_paca->stab_rr = index; 683 if (index < 32) { 684 if (kernel) 685 local_paca->slb_kern_bitmap |= 1U << index; 686 else 687 local_paca->slb_kern_bitmap &= ~(1U << index); 688 } 689 } 690 BUG_ON(index < SLB_NUM_BOLTED); 691 692 return index; 693} 694 695static long slb_insert_entry(unsigned long ea, unsigned long context, 696 unsigned long flags, int ssize, bool kernel) 697{ 698 unsigned long vsid; 699 unsigned long vsid_data, esid_data; 700 enum slb_index index; 701 702 vsid = get_vsid(context, ea, ssize); 703 if (!vsid) 704 return -EFAULT; 705 706 /* 707 * There must not be a kernel SLB fault in alloc_slb_index or before 708 * slbmte here or the allocation bitmaps could get out of whack with 709 * the SLB. 710 * 711 * User SLB faults or preloads take this path which might get inlined 712 * into the caller, so add compiler barriers here to ensure unsafe 713 * memory accesses do not come between. 714 */ 715 barrier(); 716 717 index = alloc_slb_index(kernel); 718 719 vsid_data = __mk_vsid_data(vsid, ssize, flags); 720 esid_data = mk_esid_data(ea, ssize, index); 721 722 /* 723 * No need for an isync before or after this slbmte. The exception 724 * we enter with and the rfid we exit with are context synchronizing. 725 * User preloads should add isync afterwards in case the kernel 726 * accesses user memory before it returns to userspace with rfid. 727 */ 728 assert_slb_presence(false, ea); 729 if (stress_slb()) { 730 int slb_cache_index = local_paca->slb_cache_ptr; 731 732 /* 733 * stress_slb() does not use slb cache, repurpose as a 734 * cache of inserted (non-bolted) kernel SLB entries. All 735 * non-bolted kernel entries are flushed on any user fault, 736 * or if there are already 3 non-boled kernel entries. 737 */ 738 BUILD_BUG_ON(SLB_CACHE_ENTRIES < 3); 739 if (!kernel || slb_cache_index == 3) { 740 int i; 741 742 for (i = 0; i < slb_cache_index; i++) 743 slb_cache_slbie_kernel(i); 744 slb_cache_index = 0; 745 } 746 747 if (kernel) 748 local_paca->slb_cache[slb_cache_index++] = esid_data >> SID_SHIFT; 749 local_paca->slb_cache_ptr = slb_cache_index; 750 } 751 asm volatile("slbmte %0, %1" : : "r" (vsid_data), "r" (esid_data)); 752 753 barrier(); 754 755 if (!kernel) 756 slb_cache_update(esid_data); 757 758 return 0; 759} 760 761static long slb_allocate_kernel(unsigned long ea, unsigned long id) 762{ 763 unsigned long context; 764 unsigned long flags; 765 int ssize; 766 767 if (id == LINEAR_MAP_REGION_ID) { 768 769 /* We only support upto H_MAX_PHYSMEM_BITS */ 770 if ((ea & EA_MASK) > (1UL << H_MAX_PHYSMEM_BITS)) 771 return -EFAULT; 772 773 flags = SLB_VSID_KERNEL | mmu_psize_defs[mmu_linear_psize].sllp; 774 775#ifdef CONFIG_SPARSEMEM_VMEMMAP 776 } else if (id == VMEMMAP_REGION_ID) { 777 778 if (ea >= H_VMEMMAP_END) 779 return -EFAULT; 780 781 flags = SLB_VSID_KERNEL | mmu_psize_defs[mmu_vmemmap_psize].sllp; 782#endif 783 } else if (id == VMALLOC_REGION_ID) { 784 785 if (ea >= H_VMALLOC_END) 786 return -EFAULT; 787 788 flags = local_paca->vmalloc_sllp; 789 790 } else if (id == IO_REGION_ID) { 791 792 if (ea >= H_KERN_IO_END) 793 return -EFAULT; 794 795 flags = SLB_VSID_KERNEL | mmu_psize_defs[mmu_io_psize].sllp; 796 797 } else { 798 return -EFAULT; 799 } 800 801 ssize = MMU_SEGSIZE_1T; 802 if (!mmu_has_feature(MMU_FTR_1T_SEGMENT)) 803 ssize = MMU_SEGSIZE_256M; 804 805 context = get_kernel_context(ea); 806 807 return slb_insert_entry(ea, context, flags, ssize, true); 808} 809 810static long slb_allocate_user(struct mm_struct *mm, unsigned long ea) 811{ 812 unsigned long context; 813 unsigned long flags; 814 int bpsize; 815 int ssize; 816 817 /* 818 * consider this as bad access if we take a SLB miss 819 * on an address above addr limit. 820 */ 821 if (ea >= mm_ctx_slb_addr_limit(&mm->context)) 822 return -EFAULT; 823 824 context = get_user_context(&mm->context, ea); 825 if (!context) 826 return -EFAULT; 827 828 if (unlikely(ea >= H_PGTABLE_RANGE)) { 829 WARN_ON(1); 830 return -EFAULT; 831 } 832 833 ssize = user_segment_size(ea); 834 835 bpsize = get_slice_psize(mm, ea); 836 flags = SLB_VSID_USER | mmu_psize_defs[bpsize].sllp; 837 838 return slb_insert_entry(ea, context, flags, ssize, false); 839} 840 841long do_slb_fault(struct pt_regs *regs, unsigned long ea) 842{ 843 unsigned long id = get_region_id(ea); 844 845 /* IRQs are not reconciled here, so can't check irqs_disabled */ 846 VM_WARN_ON(mfmsr() & MSR_EE); 847 848 if (unlikely(!(regs->msr & MSR_RI))) 849 return -EINVAL; 850 851 /* 852 * SLB kernel faults must be very careful not to touch anything 853 * that is not bolted. E.g., PACA and global variables are okay, 854 * mm->context stuff is not. 855 * 856 * SLB user faults can access all of kernel memory, but must be 857 * careful not to touch things like IRQ state because it is not 858 * "reconciled" here. The difficulty is that we must use 859 * fast_exception_return to return from kernel SLB faults without 860 * looking at possible non-bolted memory. We could test user vs 861 * kernel faults in the interrupt handler asm and do a full fault, 862 * reconcile, ret_from_except for user faults which would make them 863 * first class kernel code. But for performance it's probably nicer 864 * if they go via fast_exception_return too. 865 */ 866 if (id >= LINEAR_MAP_REGION_ID) { 867 long err; 868#ifdef CONFIG_DEBUG_VM 869 /* Catch recursive kernel SLB faults. */ 870 BUG_ON(local_paca->in_kernel_slb_handler); 871 local_paca->in_kernel_slb_handler = 1; 872#endif 873 err = slb_allocate_kernel(ea, id); 874#ifdef CONFIG_DEBUG_VM 875 local_paca->in_kernel_slb_handler = 0; 876#endif 877 return err; 878 } else { 879 struct mm_struct *mm = current->mm; 880 long err; 881 882 if (unlikely(!mm)) 883 return -EFAULT; 884 885 err = slb_allocate_user(mm, ea); 886 if (!err) 887 preload_add(current_thread_info(), ea); 888 889 return err; 890 } 891} 892 893void do_bad_slb_fault(struct pt_regs *regs, unsigned long ea, long err) 894{ 895 if (err == -EFAULT) { 896 if (user_mode(regs)) 897 _exception(SIGSEGV, regs, SEGV_BNDERR, ea); 898 else 899 bad_page_fault(regs, ea, SIGSEGV); 900 } else if (err == -EINVAL) { 901 unrecoverable_exception(regs); 902 } else { 903 BUG(); 904 } 905} 906