1// SPDX-License-Identifier: GPL-2.0-or-later 2/* 3 * This file contains the routines for handling the MMU on those 4 * PowerPC implementations where the MMU is not using the hash 5 * table, such as 8xx, 4xx, BookE's etc... 6 * 7 * Copyright 2008 Ben Herrenschmidt <benh@kernel.crashing.org> 8 * IBM Corp. 9 * 10 * Derived from previous arch/powerpc/mm/mmu_context.c 11 * and arch/powerpc/include/asm/mmu_context.h 12 * 13 * TODO: 14 * 15 * - The global context lock will not scale very well 16 * - The maps should be dynamically allocated to allow for processors 17 * that support more PID bits at runtime 18 * - Implement flush_tlb_mm() by making the context stale and picking 19 * a new one 20 * - More aggressively clear stale map bits and maybe find some way to 21 * also clear mm->cpu_vm_mask bits when processes are migrated 22 */ 23 24//#define DEBUG_MAP_CONSISTENCY 25//#define DEBUG_CLAMP_LAST_CONTEXT 31 26//#define DEBUG_HARDER 27 28/* We don't use DEBUG because it tends to be compiled in always nowadays 29 * and this would generate way too much output 30 */ 31#ifdef DEBUG_HARDER 32#define pr_hard(args...) printk(KERN_DEBUG args) 33#define pr_hardcont(args...) printk(KERN_CONT args) 34#else 35#define pr_hard(args...) do { } while(0) 36#define pr_hardcont(args...) do { } while(0) 37#endif 38 39#include <linux/kernel.h> 40#include <linux/mm.h> 41#include <linux/init.h> 42#include <linux/spinlock.h> 43#include <linux/memblock.h> 44#include <linux/notifier.h> 45#include <linux/cpu.h> 46#include <linux/slab.h> 47 48#include <asm/mmu_context.h> 49#include <asm/tlbflush.h> 50 51#include <mm/mmu_decl.h> 52 53/* 54 * The MPC8xx has only 16 contexts. We rotate through them on each task switch. 55 * A better way would be to keep track of tasks that own contexts, and implement 56 * an LRU usage. That way very active tasks don't always have to pay the TLB 57 * reload overhead. The kernel pages are mapped shared, so the kernel can run on 58 * behalf of any task that makes a kernel entry. Shared does not mean they are 59 * not protected, just that the ASID comparison is not performed. -- Dan 60 * 61 * The IBM4xx has 256 contexts, so we can just rotate through these as a way of 62 * "switching" contexts. If the TID of the TLB is zero, the PID/TID comparison 63 * is disabled, so we can use a TID of zero to represent all kernel pages as 64 * shared among all contexts. -- Dan 65 * 66 * The IBM 47x core supports 16-bit PIDs, thus 65535 contexts. We should 67 * normally never have to steal though the facility is present if needed. 68 * -- BenH 69 */ 70#define FIRST_CONTEXT 1 71#ifdef DEBUG_CLAMP_LAST_CONTEXT 72#define LAST_CONTEXT DEBUG_CLAMP_LAST_CONTEXT 73#elif defined(CONFIG_PPC_8xx) 74#define LAST_CONTEXT 16 75#elif defined(CONFIG_PPC_47x) 76#define LAST_CONTEXT 65535 77#else 78#define LAST_CONTEXT 255 79#endif 80 81static unsigned int next_context, nr_free_contexts; 82static unsigned long *context_map; 83#ifdef CONFIG_SMP 84static unsigned long *stale_map[NR_CPUS]; 85#endif 86static struct mm_struct **context_mm; 87static DEFINE_RAW_SPINLOCK(context_lock); 88 89#define CTX_MAP_SIZE \ 90 (sizeof(unsigned long) * (LAST_CONTEXT / BITS_PER_LONG + 1)) 91 92 93/* Steal a context from a task that has one at the moment. 94 * 95 * This is used when we are running out of available PID numbers 96 * on the processors. 97 * 98 * This isn't an LRU system, it just frees up each context in 99 * turn (sort-of pseudo-random replacement :). This would be the 100 * place to implement an LRU scheme if anyone was motivated to do it. 101 * -- paulus 102 * 103 * For context stealing, we use a slightly different approach for 104 * SMP and UP. Basically, the UP one is simpler and doesn't use 105 * the stale map as we can just flush the local CPU 106 * -- benh 107 */ 108#ifdef CONFIG_SMP 109static unsigned int steal_context_smp(unsigned int id) 110{ 111 struct mm_struct *mm; 112 unsigned int cpu, max, i; 113 114 max = LAST_CONTEXT - FIRST_CONTEXT; 115 116 /* Attempt to free next_context first and then loop until we manage */ 117 while (max--) { 118 /* Pick up the victim mm */ 119 mm = context_mm[id]; 120 121 /* We have a candidate victim, check if it's active, on SMP 122 * we cannot steal active contexts 123 */ 124 if (mm->context.active) { 125 id++; 126 if (id > LAST_CONTEXT) 127 id = FIRST_CONTEXT; 128 continue; 129 } 130 pr_hardcont(" | steal %d from 0x%p", id, mm); 131 132 /* Mark this mm has having no context anymore */ 133 mm->context.id = MMU_NO_CONTEXT; 134 135 /* Mark it stale on all CPUs that used this mm. For threaded 136 * implementations, we set it on all threads on each core 137 * represented in the mask. A future implementation will use 138 * a core map instead but this will do for now. 139 */ 140 for_each_cpu(cpu, mm_cpumask(mm)) { 141 for (i = cpu_first_thread_sibling(cpu); 142 i <= cpu_last_thread_sibling(cpu); i++) { 143 if (stale_map[i]) 144 __set_bit(id, stale_map[i]); 145 } 146 cpu = i - 1; 147 } 148 return id; 149 } 150 151 /* This will happen if you have more CPUs than available contexts, 152 * all we can do here is wait a bit and try again 153 */ 154 raw_spin_unlock(&context_lock); 155 cpu_relax(); 156 raw_spin_lock(&context_lock); 157 158 /* This will cause the caller to try again */ 159 return MMU_NO_CONTEXT; 160} 161#endif /* CONFIG_SMP */ 162 163static unsigned int steal_all_contexts(void) 164{ 165 struct mm_struct *mm; 166#ifdef CONFIG_SMP 167 int cpu = smp_processor_id(); 168#endif 169 unsigned int id; 170 171 for (id = FIRST_CONTEXT; id <= LAST_CONTEXT; id++) { 172 /* Pick up the victim mm */ 173 mm = context_mm[id]; 174 175 pr_hardcont(" | steal %d from 0x%p", id, mm); 176 177 /* Mark this mm as having no context anymore */ 178 mm->context.id = MMU_NO_CONTEXT; 179 if (id != FIRST_CONTEXT) { 180 context_mm[id] = NULL; 181 __clear_bit(id, context_map); 182#ifdef DEBUG_MAP_CONSISTENCY 183 mm->context.active = 0; 184#endif 185 } 186#ifdef CONFIG_SMP 187 __clear_bit(id, stale_map[cpu]); 188#endif 189 } 190 191 /* Flush the TLB for all contexts (not to be used on SMP) */ 192 _tlbil_all(); 193 194 nr_free_contexts = LAST_CONTEXT - FIRST_CONTEXT; 195 196 return FIRST_CONTEXT; 197} 198 199/* Note that this will also be called on SMP if all other CPUs are 200 * offlined, which means that it may be called for cpu != 0. For 201 * this to work, we somewhat assume that CPUs that are onlined 202 * come up with a fully clean TLB (or are cleaned when offlined) 203 */ 204static unsigned int steal_context_up(unsigned int id) 205{ 206 struct mm_struct *mm; 207#ifdef CONFIG_SMP 208 int cpu = smp_processor_id(); 209#endif 210 211 /* Pick up the victim mm */ 212 mm = context_mm[id]; 213 214 pr_hardcont(" | steal %d from 0x%p", id, mm); 215 216 /* Flush the TLB for that context */ 217 local_flush_tlb_mm(mm); 218 219 /* Mark this mm has having no context anymore */ 220 mm->context.id = MMU_NO_CONTEXT; 221 222 /* XXX This clear should ultimately be part of local_flush_tlb_mm */ 223#ifdef CONFIG_SMP 224 __clear_bit(id, stale_map[cpu]); 225#endif 226 227 return id; 228} 229 230#ifdef DEBUG_MAP_CONSISTENCY 231static void context_check_map(void) 232{ 233 unsigned int id, nrf, nact; 234 235 nrf = nact = 0; 236 for (id = FIRST_CONTEXT; id <= LAST_CONTEXT; id++) { 237 int used = test_bit(id, context_map); 238 if (!used) 239 nrf++; 240 if (used != (context_mm[id] != NULL)) 241 pr_err("MMU: Context %d is %s and MM is %p !\n", 242 id, used ? "used" : "free", context_mm[id]); 243 if (context_mm[id] != NULL) 244 nact += context_mm[id]->context.active; 245 } 246 if (nrf != nr_free_contexts) { 247 pr_err("MMU: Free context count out of sync ! (%d vs %d)\n", 248 nr_free_contexts, nrf); 249 nr_free_contexts = nrf; 250 } 251 if (nact > num_online_cpus()) 252 pr_err("MMU: More active contexts than CPUs ! (%d vs %d)\n", 253 nact, num_online_cpus()); 254 if (FIRST_CONTEXT > 0 && !test_bit(0, context_map)) 255 pr_err("MMU: Context 0 has been freed !!!\n"); 256} 257#else 258static void context_check_map(void) { } 259#endif 260 261void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next, 262 struct task_struct *tsk) 263{ 264 unsigned int id; 265#ifdef CONFIG_SMP 266 unsigned int i, cpu = smp_processor_id(); 267#endif 268 unsigned long *map; 269 270 /* No lockless fast path .. yet */ 271 raw_spin_lock(&context_lock); 272 273 pr_hard("[%d] activating context for mm @%p, active=%d, id=%d", 274 cpu, next, next->context.active, next->context.id); 275 276#ifdef CONFIG_SMP 277 /* Mark us active and the previous one not anymore */ 278 next->context.active++; 279 if (prev) { 280 pr_hardcont(" (old=0x%p a=%d)", prev, prev->context.active); 281 WARN_ON(prev->context.active < 1); 282 prev->context.active--; 283 } 284 285 again: 286#endif /* CONFIG_SMP */ 287 288 /* If we already have a valid assigned context, skip all that */ 289 id = next->context.id; 290 if (likely(id != MMU_NO_CONTEXT)) { 291#ifdef DEBUG_MAP_CONSISTENCY 292 if (context_mm[id] != next) 293 pr_err("MMU: mm 0x%p has id %d but context_mm[%d] says 0x%p\n", 294 next, id, id, context_mm[id]); 295#endif 296 goto ctxt_ok; 297 } 298 299 /* We really don't have a context, let's try to acquire one */ 300 id = next_context; 301 if (id > LAST_CONTEXT) 302 id = FIRST_CONTEXT; 303 map = context_map; 304 305 /* No more free contexts, let's try to steal one */ 306 if (nr_free_contexts == 0) { 307#ifdef CONFIG_SMP 308 if (num_online_cpus() > 1) { 309 id = steal_context_smp(id); 310 if (id == MMU_NO_CONTEXT) 311 goto again; 312 goto stolen; 313 } 314#endif /* CONFIG_SMP */ 315 if (IS_ENABLED(CONFIG_PPC_8xx)) 316 id = steal_all_contexts(); 317 else 318 id = steal_context_up(id); 319 goto stolen; 320 } 321 nr_free_contexts--; 322 323 /* We know there's at least one free context, try to find it */ 324 while (__test_and_set_bit(id, map)) { 325 id = find_next_zero_bit(map, LAST_CONTEXT+1, id); 326 if (id > LAST_CONTEXT) 327 id = FIRST_CONTEXT; 328 } 329 stolen: 330 next_context = id + 1; 331 context_mm[id] = next; 332 next->context.id = id; 333 pr_hardcont(" | new id=%d,nrf=%d", id, nr_free_contexts); 334 335 context_check_map(); 336 ctxt_ok: 337 338 /* If that context got marked stale on this CPU, then flush the 339 * local TLB for it and unmark it before we use it 340 */ 341#ifdef CONFIG_SMP 342 if (test_bit(id, stale_map[cpu])) { 343 pr_hardcont(" | stale flush %d [%d..%d]", 344 id, cpu_first_thread_sibling(cpu), 345 cpu_last_thread_sibling(cpu)); 346 347 local_flush_tlb_mm(next); 348 349 /* XXX This clear should ultimately be part of local_flush_tlb_mm */ 350 for (i = cpu_first_thread_sibling(cpu); 351 i <= cpu_last_thread_sibling(cpu); i++) { 352 if (stale_map[i]) 353 __clear_bit(id, stale_map[i]); 354 } 355 } 356#endif 357 358 /* Flick the MMU and release lock */ 359 pr_hardcont(" -> %d\n", id); 360 set_context(id, next->pgd); 361 raw_spin_unlock(&context_lock); 362} 363 364/* 365 * Set up the context for a new address space. 366 */ 367int init_new_context(struct task_struct *t, struct mm_struct *mm) 368{ 369 pr_hard("initing context for mm @%p\n", mm); 370 371 /* 372 * We have MMU_NO_CONTEXT set to be ~0. Hence check 373 * explicitly against context.id == 0. This ensures that we properly 374 * initialize context slice details for newly allocated mm's (which will 375 * have id == 0) and don't alter context slice inherited via fork (which 376 * will have id != 0). 377 */ 378 if (mm->context.id == 0) 379 slice_init_new_context_exec(mm); 380 mm->context.id = MMU_NO_CONTEXT; 381 mm->context.active = 0; 382 pte_frag_set(&mm->context, NULL); 383 return 0; 384} 385 386/* 387 * We're finished using the context for an address space. 388 */ 389void destroy_context(struct mm_struct *mm) 390{ 391 unsigned long flags; 392 unsigned int id; 393 394 if (mm->context.id == MMU_NO_CONTEXT) 395 return; 396 397 WARN_ON(mm->context.active != 0); 398 399 raw_spin_lock_irqsave(&context_lock, flags); 400 id = mm->context.id; 401 if (id != MMU_NO_CONTEXT) { 402 __clear_bit(id, context_map); 403 mm->context.id = MMU_NO_CONTEXT; 404#ifdef DEBUG_MAP_CONSISTENCY 405 mm->context.active = 0; 406#endif 407 context_mm[id] = NULL; 408 nr_free_contexts++; 409 } 410 raw_spin_unlock_irqrestore(&context_lock, flags); 411} 412 413#ifdef CONFIG_SMP 414static int mmu_ctx_cpu_prepare(unsigned int cpu) 415{ 416 /* We don't touch CPU 0 map, it's allocated at aboot and kept 417 * around forever 418 */ 419 if (cpu == boot_cpuid) 420 return 0; 421 422 pr_devel("MMU: Allocating stale context map for CPU %d\n", cpu); 423 stale_map[cpu] = kzalloc(CTX_MAP_SIZE, GFP_KERNEL); 424 return 0; 425} 426 427static int mmu_ctx_cpu_dead(unsigned int cpu) 428{ 429#ifdef CONFIG_HOTPLUG_CPU 430 if (cpu == boot_cpuid) 431 return 0; 432 433 pr_devel("MMU: Freeing stale context map for CPU %d\n", cpu); 434 kfree(stale_map[cpu]); 435 stale_map[cpu] = NULL; 436 437 /* We also clear the cpu_vm_mask bits of CPUs going away */ 438 clear_tasks_mm_cpumask(cpu); 439#endif 440 return 0; 441} 442 443#endif /* CONFIG_SMP */ 444 445/* 446 * Initialize the context management stuff. 447 */ 448void __init mmu_context_init(void) 449{ 450 /* Mark init_mm as being active on all possible CPUs since 451 * we'll get called with prev == init_mm the first time 452 * we schedule on a given CPU 453 */ 454 init_mm.context.active = NR_CPUS; 455 456 /* 457 * Allocate the maps used by context management 458 */ 459 context_map = memblock_alloc(CTX_MAP_SIZE, SMP_CACHE_BYTES); 460 if (!context_map) 461 panic("%s: Failed to allocate %zu bytes\n", __func__, 462 CTX_MAP_SIZE); 463 context_mm = memblock_alloc(sizeof(void *) * (LAST_CONTEXT + 1), 464 SMP_CACHE_BYTES); 465 if (!context_mm) 466 panic("%s: Failed to allocate %zu bytes\n", __func__, 467 sizeof(void *) * (LAST_CONTEXT + 1)); 468#ifdef CONFIG_SMP 469 stale_map[boot_cpuid] = memblock_alloc(CTX_MAP_SIZE, SMP_CACHE_BYTES); 470 if (!stale_map[boot_cpuid]) 471 panic("%s: Failed to allocate %zu bytes\n", __func__, 472 CTX_MAP_SIZE); 473 474 cpuhp_setup_state_nocalls(CPUHP_POWERPC_MMU_CTX_PREPARE, 475 "powerpc/mmu/ctx:prepare", 476 mmu_ctx_cpu_prepare, mmu_ctx_cpu_dead); 477#endif 478 479 printk(KERN_INFO 480 "MMU: Allocated %zu bytes of context maps for %d contexts\n", 481 2 * CTX_MAP_SIZE + (sizeof(void *) * (LAST_CONTEXT + 1)), 482 LAST_CONTEXT - FIRST_CONTEXT + 1); 483 484 /* 485 * Some processors have too few contexts to reserve one for 486 * init_mm, and require using context 0 for a normal task. 487 * Other processors reserve the use of context zero for the kernel. 488 * This code assumes FIRST_CONTEXT < 32. 489 */ 490 context_map[0] = (1 << FIRST_CONTEXT) - 1; 491 next_context = FIRST_CONTEXT; 492 nr_free_contexts = LAST_CONTEXT - FIRST_CONTEXT + 1; 493} 494