1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * ARC Cache Management 4 * 5 * Copyright (C) 2014-15 Synopsys, Inc. (www.synopsys.com) 6 * Copyright (C) 2004, 2007-2010, 2011-2012 Synopsys, Inc. (www.synopsys.com) 7 */ 8 9#include <linux/module.h> 10#include <linux/mm.h> 11#include <linux/sched.h> 12#include <linux/cache.h> 13#include <linux/mmu_context.h> 14#include <linux/syscalls.h> 15#include <linux/uaccess.h> 16#include <linux/pagemap.h> 17#include <asm/cacheflush.h> 18#include <asm/cachectl.h> 19#include <asm/setup.h> 20 21#ifdef CONFIG_ISA_ARCV2 22#define USE_RGN_FLSH 1 23#endif 24 25static int l2_line_sz; 26static int ioc_exists; 27int slc_enable = 1, ioc_enable = 1; 28unsigned long perip_base = ARC_UNCACHED_ADDR_SPACE; /* legacy value for boot */ 29unsigned long perip_end = 0xFFFFFFFF; /* legacy value */ 30 31static struct cpuinfo_arc_cache { 32 unsigned int sz_k, line_len, colors; 33} ic_info, dc_info, slc_info; 34 35void (*_cache_line_loop_ic_fn)(phys_addr_t paddr, unsigned long vaddr, 36 unsigned long sz, const int op, const int full_page); 37 38void (*__dma_cache_wback_inv)(phys_addr_t start, unsigned long sz); 39void (*__dma_cache_inv)(phys_addr_t start, unsigned long sz); 40void (*__dma_cache_wback)(phys_addr_t start, unsigned long sz); 41 42static int read_decode_cache_bcr_arcv2(int c, char *buf, int len) 43{ 44 struct cpuinfo_arc_cache *p_slc = &slc_info; 45 struct bcr_identity ident; 46 struct bcr_generic sbcr; 47 struct bcr_clust_cfg cbcr; 48 struct bcr_volatile vol; 49 int n = 0; 50 51 READ_BCR(ARC_REG_SLC_BCR, sbcr); 52 if (sbcr.ver) { 53 struct bcr_slc_cfg slc_cfg; 54 READ_BCR(ARC_REG_SLC_CFG, slc_cfg); 55 p_slc->sz_k = 128 << slc_cfg.sz; 56 l2_line_sz = p_slc->line_len = (slc_cfg.lsz == 0) ? 128 : 64; 57 n += scnprintf(buf + n, len - n, 58 "SLC\t\t: %uK, %uB Line%s\n", 59 p_slc->sz_k, p_slc->line_len, IS_USED_RUN(slc_enable)); 60 } 61 62 READ_BCR(ARC_REG_CLUSTER_BCR, cbcr); 63 if (cbcr.c) { 64 ioc_exists = 1; 65 66 /* 67 * As for today we don't support both IOC and ZONE_HIGHMEM enabled 68 * simultaneously. This happens because as of today IOC aperture covers 69 * only ZONE_NORMAL (low mem) and any dma transactions outside this 70 * region won't be HW coherent. 71 * If we want to use both IOC and ZONE_HIGHMEM we can use 72 * bounce_buffer to handle dma transactions to HIGHMEM. 73 * Also it is possible to modify dma_direct cache ops or increase IOC 74 * aperture size if we are planning to use HIGHMEM without PAE. 75 */ 76 if (IS_ENABLED(CONFIG_HIGHMEM) || is_pae40_enabled()) 77 ioc_enable = 0; 78 } else { 79 ioc_enable = 0; 80 } 81 82 READ_BCR(AUX_IDENTITY, ident); 83 84 /* HS 2.0 didn't have AUX_VOL */ 85 if (ident.family > 0x51) { 86 READ_BCR(AUX_VOL, vol); 87 perip_base = vol.start << 28; 88 /* HS 3.0 has limit and strict-ordering fields */ 89 if (ident.family > 0x52) 90 perip_end = (vol.limit << 28) - 1; 91 } 92 93 n += scnprintf(buf + n, len - n, "Peripherals\t: %#lx%s%s\n", 94 perip_base, 95 IS_AVAIL3(ioc_exists, ioc_enable, ", IO-Coherency (per-device) ")); 96 97 return n; 98} 99 100int arc_cache_mumbojumbo(int c, char *buf, int len) 101{ 102 struct cpuinfo_arc_cache *p_ic = &ic_info, *p_dc = &dc_info; 103 struct bcr_cache ibcr, dbcr; 104 int vipt, assoc; 105 int n = 0; 106 107 READ_BCR(ARC_REG_IC_BCR, ibcr); 108 if (!ibcr.ver) 109 goto dc_chk; 110 111 if (is_isa_arcompact() && (ibcr.ver <= 3)) { 112 BUG_ON(ibcr.config != 3); 113 assoc = 2; /* Fixed to 2w set assoc */ 114 } else if (is_isa_arcv2() && (ibcr.ver >= 4)) { 115 assoc = 1 << ibcr.config; /* 1,2,4,8 */ 116 } 117 118 p_ic->line_len = 8 << ibcr.line_len; 119 p_ic->sz_k = 1 << (ibcr.sz - 1); 120 p_ic->colors = p_ic->sz_k/assoc/TO_KB(PAGE_SIZE); 121 122 n += scnprintf(buf + n, len - n, 123 "I-Cache\t\t: %uK, %dway/set, %uB Line, VIPT%s%s\n", 124 p_ic->sz_k, assoc, p_ic->line_len, 125 p_ic->colors > 1 ? " aliasing" : "", 126 IS_USED_CFG(CONFIG_ARC_HAS_ICACHE)); 127 128dc_chk: 129 READ_BCR(ARC_REG_DC_BCR, dbcr); 130 if (!dbcr.ver) 131 goto slc_chk; 132 133 if (is_isa_arcompact() && (dbcr.ver <= 3)) { 134 BUG_ON(dbcr.config != 2); 135 vipt = 1; 136 assoc = 4; /* Fixed to 4w set assoc */ 137 p_dc->colors = p_dc->sz_k/assoc/TO_KB(PAGE_SIZE); 138 } else if (is_isa_arcv2() && (dbcr.ver >= 4)) { 139 vipt = 0; 140 assoc = 1 << dbcr.config; /* 1,2,4,8 */ 141 p_dc->colors = 1; /* PIPT so can't VIPT alias */ 142 } 143 144 p_dc->line_len = 16 << dbcr.line_len; 145 p_dc->sz_k = 1 << (dbcr.sz - 1); 146 147 n += scnprintf(buf + n, len - n, 148 "D-Cache\t\t: %uK, %dway/set, %uB Line, %s%s%s\n", 149 p_dc->sz_k, assoc, p_dc->line_len, 150 vipt ? "VIPT" : "PIPT", 151 p_dc->colors > 1 ? " aliasing" : "", 152 IS_USED_CFG(CONFIG_ARC_HAS_DCACHE)); 153 154slc_chk: 155 if (is_isa_arcv2()) 156 n += read_decode_cache_bcr_arcv2(c, buf + n, len - n); 157 158 return n; 159} 160 161/* 162 * Line Operation on {I,D}-Cache 163 */ 164 165#define OP_INV 0x1 166#define OP_FLUSH 0x2 167#define OP_FLUSH_N_INV 0x3 168#define OP_INV_IC 0x4 169 170/* 171 * Cache Flush programming model 172 * 173 * ARC700 MMUv3 I$ and D$ are both VIPT and can potentially alias. 174 * Programming model requires both paddr and vaddr irrespecive of aliasing 175 * considerations: 176 * - vaddr in {I,D}C_IV?L 177 * - paddr in {I,D}C_PTAG 178 * 179 * In HS38x (MMUv4), D$ is PIPT, I$ is VIPT and can still alias. 180 * Programming model is different for aliasing vs. non-aliasing I$ 181 * - D$ / Non-aliasing I$: only paddr in {I,D}C_IV?L 182 * - Aliasing I$: same as ARC700 above (so MMUv3 routine used for MMUv4 I$) 183 * 184 * - If PAE40 is enabled, independent of aliasing considerations, the higher 185 * bits needs to be written into PTAG_HI 186 */ 187 188static inline 189void __cache_line_loop_v3(phys_addr_t paddr, unsigned long vaddr, 190 unsigned long sz, const int op, const int full_page) 191{ 192 unsigned int aux_cmd, aux_tag; 193 int num_lines; 194 195 if (op == OP_INV_IC) { 196 aux_cmd = ARC_REG_IC_IVIL; 197 aux_tag = ARC_REG_IC_PTAG; 198 } else { 199 aux_cmd = op & OP_INV ? ARC_REG_DC_IVDL : ARC_REG_DC_FLDL; 200 aux_tag = ARC_REG_DC_PTAG; 201 } 202 203 /* Ensure we properly floor/ceil the non-line aligned/sized requests 204 * and have @paddr - aligned to cache line and integral @num_lines. 205 * This however can be avoided for page sized since: 206 * -@paddr will be cache-line aligned already (being page aligned) 207 * -@sz will be integral multiple of line size (being page sized). 208 */ 209 if (!full_page) { 210 sz += paddr & ~CACHE_LINE_MASK; 211 paddr &= CACHE_LINE_MASK; 212 vaddr &= CACHE_LINE_MASK; 213 } 214 num_lines = DIV_ROUND_UP(sz, L1_CACHE_BYTES); 215 216 /* 217 * MMUv3, cache ops require paddr in PTAG reg 218 * if V-P const for loop, PTAG can be written once outside loop 219 */ 220 if (full_page) 221 write_aux_reg(aux_tag, paddr); 222 223 /* 224 * This is technically for MMU v4, using the MMU v3 programming model 225 * Special work for HS38 aliasing I-cache configuration with PAE40 226 * - upper 8 bits of paddr need to be written into PTAG_HI 227 * - (and needs to be written before the lower 32 bits) 228 * Note that PTAG_HI is hoisted outside the line loop 229 */ 230 if (is_pae40_enabled() && op == OP_INV_IC) 231 write_aux_reg(ARC_REG_IC_PTAG_HI, (u64)paddr >> 32); 232 233 while (num_lines-- > 0) { 234 if (!full_page) { 235 write_aux_reg(aux_tag, paddr); 236 paddr += L1_CACHE_BYTES; 237 } 238 239 write_aux_reg(aux_cmd, vaddr); 240 vaddr += L1_CACHE_BYTES; 241 } 242} 243 244#ifndef USE_RGN_FLSH 245 246/* 247 */ 248static inline 249void __cache_line_loop_v4(phys_addr_t paddr, unsigned long vaddr, 250 unsigned long sz, const int op, const int full_page) 251{ 252 unsigned int aux_cmd; 253 int num_lines; 254 255 if (op == OP_INV_IC) { 256 aux_cmd = ARC_REG_IC_IVIL; 257 } else { 258 /* d$ cmd: INV (discard or wback-n-discard) OR FLUSH (wback) */ 259 aux_cmd = op & OP_INV ? ARC_REG_DC_IVDL : ARC_REG_DC_FLDL; 260 } 261 262 /* Ensure we properly floor/ceil the non-line aligned/sized requests 263 * and have @paddr - aligned to cache line and integral @num_lines. 264 * This however can be avoided for page sized since: 265 * -@paddr will be cache-line aligned already (being page aligned) 266 * -@sz will be integral multiple of line size (being page sized). 267 */ 268 if (!full_page) { 269 sz += paddr & ~CACHE_LINE_MASK; 270 paddr &= CACHE_LINE_MASK; 271 } 272 273 num_lines = DIV_ROUND_UP(sz, L1_CACHE_BYTES); 274 275 /* 276 * For HS38 PAE40 configuration 277 * - upper 8 bits of paddr need to be written into PTAG_HI 278 * - (and needs to be written before the lower 32 bits) 279 */ 280 if (is_pae40_enabled()) { 281 if (op == OP_INV_IC) 282 /* 283 * Non aliasing I-cache in HS38, 284 * aliasing I-cache handled in __cache_line_loop_v3() 285 */ 286 write_aux_reg(ARC_REG_IC_PTAG_HI, (u64)paddr >> 32); 287 else 288 write_aux_reg(ARC_REG_DC_PTAG_HI, (u64)paddr >> 32); 289 } 290 291 while (num_lines-- > 0) { 292 write_aux_reg(aux_cmd, paddr); 293 paddr += L1_CACHE_BYTES; 294 } 295} 296 297#else 298 299/* 300 * optimized flush operation which takes a region as opposed to iterating per line 301 */ 302static inline 303void __cache_line_loop_v4(phys_addr_t paddr, unsigned long vaddr, 304 unsigned long sz, const int op, const int full_page) 305{ 306 unsigned int s, e; 307 308 /* Only for Non aliasing I-cache in HS38 */ 309 if (op == OP_INV_IC) { 310 s = ARC_REG_IC_IVIR; 311 e = ARC_REG_IC_ENDR; 312 } else { 313 s = ARC_REG_DC_STARTR; 314 e = ARC_REG_DC_ENDR; 315 } 316 317 if (!full_page) { 318 /* for any leading gap between @paddr and start of cache line */ 319 sz += paddr & ~CACHE_LINE_MASK; 320 paddr &= CACHE_LINE_MASK; 321 322 /* 323 * account for any trailing gap to end of cache line 324 * this is equivalent to DIV_ROUND_UP() in line ops above 325 */ 326 sz += L1_CACHE_BYTES - 1; 327 } 328 329 if (is_pae40_enabled()) { 330 /* TBD: check if crossing 4TB boundary */ 331 if (op == OP_INV_IC) 332 write_aux_reg(ARC_REG_IC_PTAG_HI, (u64)paddr >> 32); 333 else 334 write_aux_reg(ARC_REG_DC_PTAG_HI, (u64)paddr >> 32); 335 } 336 337 /* ENDR needs to be set ahead of START */ 338 write_aux_reg(e, paddr + sz); /* ENDR is exclusive */ 339 write_aux_reg(s, paddr); 340 341 /* caller waits on DC_CTRL.FS */ 342} 343 344#endif 345 346#ifdef CONFIG_ARC_MMU_V3 347#define __cache_line_loop __cache_line_loop_v3 348#else 349#define __cache_line_loop __cache_line_loop_v4 350#endif 351 352#ifdef CONFIG_ARC_HAS_DCACHE 353 354/*************************************************************** 355 * Machine specific helpers for Entire D-Cache or Per Line ops 356 */ 357 358#ifndef USE_RGN_FLSH 359/* 360 * this version avoids extra read/write of DC_CTRL for flush or invalid ops 361 * in the non region flush regime (such as for ARCompact) 362 */ 363static inline void __before_dc_op(const int op) 364{ 365 if (op == OP_FLUSH_N_INV) { 366 /* Dcache provides 2 cmd: FLUSH or INV 367 * INV in turn has sub-modes: DISCARD or FLUSH-BEFORE 368 * flush-n-inv is achieved by INV cmd but with IM=1 369 * So toggle INV sub-mode depending on op request and default 370 */ 371 const unsigned int ctl = ARC_REG_DC_CTRL; 372 write_aux_reg(ctl, read_aux_reg(ctl) | DC_CTRL_INV_MODE_FLUSH); 373 } 374} 375 376#else 377 378static inline void __before_dc_op(const int op) 379{ 380 const unsigned int ctl = ARC_REG_DC_CTRL; 381 unsigned int val = read_aux_reg(ctl); 382 383 if (op == OP_FLUSH_N_INV) { 384 val |= DC_CTRL_INV_MODE_FLUSH; 385 } 386 387 if (op != OP_INV_IC) { 388 /* 389 * Flush / Invalidate is provided by DC_CTRL.RNG_OP 0 or 1 390 * combined Flush-n-invalidate uses DC_CTRL.IM = 1 set above 391 */ 392 val &= ~DC_CTRL_RGN_OP_MSK; 393 if (op & OP_INV) 394 val |= DC_CTRL_RGN_OP_INV; 395 } 396 write_aux_reg(ctl, val); 397} 398 399#endif 400 401 402static inline void __after_dc_op(const int op) 403{ 404 if (op & OP_FLUSH) { 405 const unsigned int ctl = ARC_REG_DC_CTRL; 406 unsigned int reg; 407 408 /* flush / flush-n-inv both wait */ 409 while ((reg = read_aux_reg(ctl)) & DC_CTRL_FLUSH_STATUS) 410 ; 411 412 /* Switch back to default Invalidate mode */ 413 if (op == OP_FLUSH_N_INV) 414 write_aux_reg(ctl, reg & ~DC_CTRL_INV_MODE_FLUSH); 415 } 416} 417 418/* 419 * Operation on Entire D-Cache 420 * @op = {OP_INV, OP_FLUSH, OP_FLUSH_N_INV} 421 * Note that constant propagation ensures all the checks are gone 422 * in generated code 423 */ 424static inline void __dc_entire_op(const int op) 425{ 426 int aux; 427 428 __before_dc_op(op); 429 430 if (op & OP_INV) /* Inv or flush-n-inv use same cmd reg */ 431 aux = ARC_REG_DC_IVDC; 432 else 433 aux = ARC_REG_DC_FLSH; 434 435 write_aux_reg(aux, 0x1); 436 437 __after_dc_op(op); 438} 439 440static inline void __dc_disable(void) 441{ 442 const int r = ARC_REG_DC_CTRL; 443 444 __dc_entire_op(OP_FLUSH_N_INV); 445 write_aux_reg(r, read_aux_reg(r) | DC_CTRL_DIS); 446} 447 448static void __dc_enable(void) 449{ 450 const int r = ARC_REG_DC_CTRL; 451 452 write_aux_reg(r, read_aux_reg(r) & ~DC_CTRL_DIS); 453} 454 455/* For kernel mappings cache operation: index is same as paddr */ 456#define __dc_line_op_k(p, sz, op) __dc_line_op(p, p, sz, op) 457 458/* 459 * D-Cache Line ops: Per Line INV (discard or wback+discard) or FLUSH (wback) 460 */ 461static inline void __dc_line_op(phys_addr_t paddr, unsigned long vaddr, 462 unsigned long sz, const int op) 463{ 464 const int full_page = __builtin_constant_p(sz) && sz == PAGE_SIZE; 465 unsigned long flags; 466 467 local_irq_save(flags); 468 469 __before_dc_op(op); 470 471 __cache_line_loop(paddr, vaddr, sz, op, full_page); 472 473 __after_dc_op(op); 474 475 local_irq_restore(flags); 476} 477 478#else 479 480#define __dc_entire_op(op) 481#define __dc_disable() 482#define __dc_enable() 483#define __dc_line_op(paddr, vaddr, sz, op) 484#define __dc_line_op_k(paddr, sz, op) 485 486#endif /* CONFIG_ARC_HAS_DCACHE */ 487 488#ifdef CONFIG_ARC_HAS_ICACHE 489 490static inline void __ic_entire_inv(void) 491{ 492 write_aux_reg(ARC_REG_IC_IVIC, 1); 493 read_aux_reg(ARC_REG_IC_CTRL); /* blocks */ 494} 495 496static inline void 497__ic_line_inv_vaddr_local(phys_addr_t paddr, unsigned long vaddr, 498 unsigned long sz) 499{ 500 const int full_page = __builtin_constant_p(sz) && sz == PAGE_SIZE; 501 unsigned long flags; 502 503 local_irq_save(flags); 504 (*_cache_line_loop_ic_fn)(paddr, vaddr, sz, OP_INV_IC, full_page); 505 local_irq_restore(flags); 506} 507 508#ifndef CONFIG_SMP 509 510#define __ic_line_inv_vaddr(p, v, s) __ic_line_inv_vaddr_local(p, v, s) 511 512#else 513 514struct ic_inv_args { 515 phys_addr_t paddr, vaddr; 516 int sz; 517}; 518 519static void __ic_line_inv_vaddr_helper(void *info) 520{ 521 struct ic_inv_args *ic_inv = info; 522 523 __ic_line_inv_vaddr_local(ic_inv->paddr, ic_inv->vaddr, ic_inv->sz); 524} 525 526static void __ic_line_inv_vaddr(phys_addr_t paddr, unsigned long vaddr, 527 unsigned long sz) 528{ 529 struct ic_inv_args ic_inv = { 530 .paddr = paddr, 531 .vaddr = vaddr, 532 .sz = sz 533 }; 534 535 on_each_cpu(__ic_line_inv_vaddr_helper, &ic_inv, 1); 536} 537 538#endif /* CONFIG_SMP */ 539 540#else /* !CONFIG_ARC_HAS_ICACHE */ 541 542#define __ic_entire_inv() 543#define __ic_line_inv_vaddr(pstart, vstart, sz) 544 545#endif /* CONFIG_ARC_HAS_ICACHE */ 546 547static noinline void slc_op_rgn(phys_addr_t paddr, unsigned long sz, const int op) 548{ 549#ifdef CONFIG_ISA_ARCV2 550 /* 551 * SLC is shared between all cores and concurrent aux operations from 552 * multiple cores need to be serialized using a spinlock 553 * A concurrent operation can be silently ignored and/or the old/new 554 * operation can remain incomplete forever (lockup in SLC_CTRL_BUSY loop 555 * below) 556 */ 557 static DEFINE_SPINLOCK(lock); 558 unsigned long flags; 559 unsigned int ctrl; 560 phys_addr_t end; 561 562 spin_lock_irqsave(&lock, flags); 563 564 /* 565 * The Region Flush operation is specified by CTRL.RGN_OP[11..9] 566 * - b'000 (default) is Flush, 567 * - b'001 is Invalidate if CTRL.IM == 0 568 * - b'001 is Flush-n-Invalidate if CTRL.IM == 1 569 */ 570 ctrl = read_aux_reg(ARC_REG_SLC_CTRL); 571 572 /* Don't rely on default value of IM bit */ 573 if (!(op & OP_FLUSH)) /* i.e. OP_INV */ 574 ctrl &= ~SLC_CTRL_IM; /* clear IM: Disable flush before Inv */ 575 else 576 ctrl |= SLC_CTRL_IM; 577 578 if (op & OP_INV) 579 ctrl |= SLC_CTRL_RGN_OP_INV; /* Inv or flush-n-inv */ 580 else 581 ctrl &= ~SLC_CTRL_RGN_OP_INV; 582 583 write_aux_reg(ARC_REG_SLC_CTRL, ctrl); 584 585 /* 586 * Lower bits are ignored, no need to clip 587 * END needs to be setup before START (latter triggers the operation) 588 * END can't be same as START, so add (l2_line_sz - 1) to sz 589 */ 590 end = paddr + sz + l2_line_sz - 1; 591 if (is_pae40_enabled()) 592 write_aux_reg(ARC_REG_SLC_RGN_END1, upper_32_bits(end)); 593 594 write_aux_reg(ARC_REG_SLC_RGN_END, lower_32_bits(end)); 595 596 if (is_pae40_enabled()) 597 write_aux_reg(ARC_REG_SLC_RGN_START1, upper_32_bits(paddr)); 598 599 write_aux_reg(ARC_REG_SLC_RGN_START, lower_32_bits(paddr)); 600 601 /* Make sure "busy" bit reports correct stataus, see STAR 9001165532 */ 602 read_aux_reg(ARC_REG_SLC_CTRL); 603 604 while (read_aux_reg(ARC_REG_SLC_CTRL) & SLC_CTRL_BUSY); 605 606 spin_unlock_irqrestore(&lock, flags); 607#endif 608} 609 610static __maybe_unused noinline void slc_op_line(phys_addr_t paddr, unsigned long sz, const int op) 611{ 612#ifdef CONFIG_ISA_ARCV2 613 /* 614 * SLC is shared between all cores and concurrent aux operations from 615 * multiple cores need to be serialized using a spinlock 616 * A concurrent operation can be silently ignored and/or the old/new 617 * operation can remain incomplete forever (lockup in SLC_CTRL_BUSY loop 618 * below) 619 */ 620 static DEFINE_SPINLOCK(lock); 621 622 const unsigned long SLC_LINE_MASK = ~(l2_line_sz - 1); 623 unsigned int ctrl, cmd; 624 unsigned long flags; 625 int num_lines; 626 627 spin_lock_irqsave(&lock, flags); 628 629 ctrl = read_aux_reg(ARC_REG_SLC_CTRL); 630 631 /* Don't rely on default value of IM bit */ 632 if (!(op & OP_FLUSH)) /* i.e. OP_INV */ 633 ctrl &= ~SLC_CTRL_IM; /* clear IM: Disable flush before Inv */ 634 else 635 ctrl |= SLC_CTRL_IM; 636 637 write_aux_reg(ARC_REG_SLC_CTRL, ctrl); 638 639 cmd = op & OP_INV ? ARC_AUX_SLC_IVDL : ARC_AUX_SLC_FLDL; 640 641 sz += paddr & ~SLC_LINE_MASK; 642 paddr &= SLC_LINE_MASK; 643 644 num_lines = DIV_ROUND_UP(sz, l2_line_sz); 645 646 while (num_lines-- > 0) { 647 write_aux_reg(cmd, paddr); 648 paddr += l2_line_sz; 649 } 650 651 /* Make sure "busy" bit reports correct stataus, see STAR 9001165532 */ 652 read_aux_reg(ARC_REG_SLC_CTRL); 653 654 while (read_aux_reg(ARC_REG_SLC_CTRL) & SLC_CTRL_BUSY); 655 656 spin_unlock_irqrestore(&lock, flags); 657#endif 658} 659 660#define slc_op(paddr, sz, op) slc_op_rgn(paddr, sz, op) 661 662noinline static void slc_entire_op(const int op) 663{ 664 unsigned int ctrl, r = ARC_REG_SLC_CTRL; 665 666 ctrl = read_aux_reg(r); 667 668 if (!(op & OP_FLUSH)) /* i.e. OP_INV */ 669 ctrl &= ~SLC_CTRL_IM; /* clear IM: Disable flush before Inv */ 670 else 671 ctrl |= SLC_CTRL_IM; 672 673 write_aux_reg(r, ctrl); 674 675 if (op & OP_INV) /* Inv or flush-n-inv use same cmd reg */ 676 write_aux_reg(ARC_REG_SLC_INVALIDATE, 0x1); 677 else 678 write_aux_reg(ARC_REG_SLC_FLUSH, 0x1); 679 680 /* Make sure "busy" bit reports correct stataus, see STAR 9001165532 */ 681 read_aux_reg(r); 682 683 /* Important to wait for flush to complete */ 684 while (read_aux_reg(r) & SLC_CTRL_BUSY); 685} 686 687static inline void arc_slc_disable(void) 688{ 689 const int r = ARC_REG_SLC_CTRL; 690 691 slc_entire_op(OP_FLUSH_N_INV); 692 write_aux_reg(r, read_aux_reg(r) | SLC_CTRL_DIS); 693} 694 695static inline void arc_slc_enable(void) 696{ 697 const int r = ARC_REG_SLC_CTRL; 698 699 write_aux_reg(r, read_aux_reg(r) & ~SLC_CTRL_DIS); 700} 701 702/*********************************************************** 703 * Exported APIs 704 */ 705 706/* 707 * Handle cache congruency of kernel and userspace mappings of page when kernel 708 * writes-to/reads-from 709 * 710 * The idea is to defer flushing of kernel mapping after a WRITE, possible if: 711 * -dcache is NOT aliasing, hence any U/K-mappings of page are congruent 712 * -U-mapping doesn't exist yet for page (finalised in update_mmu_cache) 713 * -In SMP, if hardware caches are coherent 714 * 715 * There's a corollary case, where kernel READs from a userspace mapped page. 716 * If the U-mapping is not congruent to K-mapping, former needs flushing. 717 */ 718void flush_dcache_folio(struct folio *folio) 719{ 720 struct address_space *mapping; 721 722 if (!cache_is_vipt_aliasing()) { 723 clear_bit(PG_dc_clean, &folio->flags); 724 return; 725 } 726 727 /* don't handle anon pages here */ 728 mapping = folio_flush_mapping(folio); 729 if (!mapping) 730 return; 731 732 /* 733 * pagecache page, file not yet mapped to userspace 734 * Make a note that K-mapping is dirty 735 */ 736 if (!mapping_mapped(mapping)) { 737 clear_bit(PG_dc_clean, &folio->flags); 738 } else if (folio_mapped(folio)) { 739 /* kernel reading from page with U-mapping */ 740 phys_addr_t paddr = (unsigned long)folio_address(folio); 741 unsigned long vaddr = folio_pos(folio); 742 743 /* 744 * vaddr is not actually the virtual address, but is 745 * congruent to every user mapping. 746 */ 747 if (addr_not_cache_congruent(paddr, vaddr)) 748 __flush_dcache_pages(paddr, vaddr, 749 folio_nr_pages(folio)); 750 } 751} 752EXPORT_SYMBOL(flush_dcache_folio); 753 754void flush_dcache_page(struct page *page) 755{ 756 return flush_dcache_folio(page_folio(page)); 757} 758EXPORT_SYMBOL(flush_dcache_page); 759 760/* 761 * DMA ops for systems with L1 cache only 762 * Make memory coherent with L1 cache by flushing/invalidating L1 lines 763 */ 764static void __dma_cache_wback_inv_l1(phys_addr_t start, unsigned long sz) 765{ 766 __dc_line_op_k(start, sz, OP_FLUSH_N_INV); 767} 768 769static void __dma_cache_inv_l1(phys_addr_t start, unsigned long sz) 770{ 771 __dc_line_op_k(start, sz, OP_INV); 772} 773 774static void __dma_cache_wback_l1(phys_addr_t start, unsigned long sz) 775{ 776 __dc_line_op_k(start, sz, OP_FLUSH); 777} 778 779/* 780 * DMA ops for systems with both L1 and L2 caches, but without IOC 781 * Both L1 and L2 lines need to be explicitly flushed/invalidated 782 */ 783static void __dma_cache_wback_inv_slc(phys_addr_t start, unsigned long sz) 784{ 785 __dc_line_op_k(start, sz, OP_FLUSH_N_INV); 786 slc_op(start, sz, OP_FLUSH_N_INV); 787} 788 789static void __dma_cache_inv_slc(phys_addr_t start, unsigned long sz) 790{ 791 __dc_line_op_k(start, sz, OP_INV); 792 slc_op(start, sz, OP_INV); 793} 794 795static void __dma_cache_wback_slc(phys_addr_t start, unsigned long sz) 796{ 797 __dc_line_op_k(start, sz, OP_FLUSH); 798 slc_op(start, sz, OP_FLUSH); 799} 800 801/* 802 * Exported DMA API 803 */ 804void dma_cache_wback_inv(phys_addr_t start, unsigned long sz) 805{ 806 __dma_cache_wback_inv(start, sz); 807} 808EXPORT_SYMBOL(dma_cache_wback_inv); 809 810void dma_cache_inv(phys_addr_t start, unsigned long sz) 811{ 812 __dma_cache_inv(start, sz); 813} 814EXPORT_SYMBOL(dma_cache_inv); 815 816void dma_cache_wback(phys_addr_t start, unsigned long sz) 817{ 818 __dma_cache_wback(start, sz); 819} 820EXPORT_SYMBOL(dma_cache_wback); 821 822/* 823 * This is API for making I/D Caches consistent when modifying 824 * kernel code (loadable modules, kprobes, kgdb...) 825 * This is called on insmod, with kernel virtual address for CODE of 826 * the module. ARC cache maintenance ops require PHY address thus we 827 * need to convert vmalloc addr to PHY addr 828 */ 829void flush_icache_range(unsigned long kstart, unsigned long kend) 830{ 831 unsigned int tot_sz; 832 833 WARN(kstart < TASK_SIZE, "%s() can't handle user vaddr", __func__); 834 835 /* Shortcut for bigger flush ranges. 836 * Here we don't care if this was kernel virtual or phy addr 837 */ 838 tot_sz = kend - kstart; 839 if (tot_sz > PAGE_SIZE) { 840 flush_cache_all(); 841 return; 842 } 843 844 /* Case: Kernel Phy addr (0x8000_0000 onwards) */ 845 if (likely(kstart > PAGE_OFFSET)) { 846 /* 847 * The 2nd arg despite being paddr will be used to index icache 848 * This is OK since no alternate virtual mappings will exist 849 * given the callers for this case: kprobe/kgdb in built-in 850 * kernel code only. 851 */ 852 __sync_icache_dcache(kstart, kstart, kend - kstart); 853 return; 854 } 855 856 /* 857 * Case: Kernel Vaddr (0x7000_0000 to 0x7fff_ffff) 858 * (1) ARC Cache Maintenance ops only take Phy addr, hence special 859 * handling of kernel vaddr. 860 * 861 * (2) Despite @tot_sz being < PAGE_SIZE (bigger cases handled already), 862 * it still needs to handle a 2 page scenario, where the range 863 * straddles across 2 virtual pages and hence need for loop 864 */ 865 while (tot_sz > 0) { 866 unsigned int off, sz; 867 unsigned long phy, pfn; 868 869 off = kstart % PAGE_SIZE; 870 pfn = vmalloc_to_pfn((void *)kstart); 871 phy = (pfn << PAGE_SHIFT) + off; 872 sz = min_t(unsigned int, tot_sz, PAGE_SIZE - off); 873 __sync_icache_dcache(phy, kstart, sz); 874 kstart += sz; 875 tot_sz -= sz; 876 } 877} 878EXPORT_SYMBOL(flush_icache_range); 879 880/* 881 * General purpose helper to make I and D cache lines consistent. 882 * @paddr is phy addr of region 883 * @vaddr is typically user vaddr (breakpoint) or kernel vaddr (vmalloc) 884 * However in one instance, when called by kprobe (for a breakpt in 885 * builtin kernel code) @vaddr will be paddr only, meaning CDU operation will 886 * use a paddr to index the cache (despite VIPT). This is fine since a 887 * builtin kernel page will not have any virtual mappings. 888 * kprobe on loadable module will be kernel vaddr. 889 */ 890void __sync_icache_dcache(phys_addr_t paddr, unsigned long vaddr, int len) 891{ 892 __dc_line_op(paddr, vaddr, len, OP_FLUSH_N_INV); 893 __ic_line_inv_vaddr(paddr, vaddr, len); 894} 895 896/* wrapper to compile time eliminate alignment checks in flush loop */ 897void __inv_icache_pages(phys_addr_t paddr, unsigned long vaddr, unsigned nr) 898{ 899 __ic_line_inv_vaddr(paddr, vaddr, nr * PAGE_SIZE); 900} 901 902/* 903 * wrapper to clearout kernel or userspace mappings of a page 904 * For kernel mappings @vaddr == @paddr 905 */ 906void __flush_dcache_pages(phys_addr_t paddr, unsigned long vaddr, unsigned nr) 907{ 908 __dc_line_op(paddr, vaddr & PAGE_MASK, nr * PAGE_SIZE, OP_FLUSH_N_INV); 909} 910 911noinline void flush_cache_all(void) 912{ 913 unsigned long flags; 914 915 local_irq_save(flags); 916 917 __ic_entire_inv(); 918 __dc_entire_op(OP_FLUSH_N_INV); 919 920 local_irq_restore(flags); 921 922} 923 924#ifdef CONFIG_ARC_CACHE_VIPT_ALIASING 925 926void flush_cache_mm(struct mm_struct *mm) 927{ 928 flush_cache_all(); 929} 930 931void flush_cache_page(struct vm_area_struct *vma, unsigned long u_vaddr, 932 unsigned long pfn) 933{ 934 phys_addr_t paddr = pfn << PAGE_SHIFT; 935 936 u_vaddr &= PAGE_MASK; 937 938 __flush_dcache_pages(paddr, u_vaddr, 1); 939 940 if (vma->vm_flags & VM_EXEC) 941 __inv_icache_pages(paddr, u_vaddr, 1); 942} 943 944void flush_cache_range(struct vm_area_struct *vma, unsigned long start, 945 unsigned long end) 946{ 947 flush_cache_all(); 948} 949 950void flush_anon_page(struct vm_area_struct *vma, struct page *page, 951 unsigned long u_vaddr) 952{ 953 /* TBD: do we really need to clear the kernel mapping */ 954 __flush_dcache_pages((phys_addr_t)page_address(page), u_vaddr, 1); 955 __flush_dcache_pages((phys_addr_t)page_address(page), 956 (phys_addr_t)page_address(page), 1); 957 958} 959 960#endif 961 962void copy_user_highpage(struct page *to, struct page *from, 963 unsigned long u_vaddr, struct vm_area_struct *vma) 964{ 965 struct folio *src = page_folio(from); 966 struct folio *dst = page_folio(to); 967 void *kfrom = kmap_atomic(from); 968 void *kto = kmap_atomic(to); 969 int clean_src_k_mappings = 0; 970 971 /* 972 * If SRC page was already mapped in userspace AND it's U-mapping is 973 * not congruent with K-mapping, sync former to physical page so that 974 * K-mapping in memcpy below, sees the right data 975 * 976 * Note that while @u_vaddr refers to DST page's userspace vaddr, it is 977 * equally valid for SRC page as well 978 * 979 * For !VIPT cache, all of this gets compiled out as 980 * addr_not_cache_congruent() is 0 981 */ 982 if (page_mapcount(from) && addr_not_cache_congruent(kfrom, u_vaddr)) { 983 __flush_dcache_pages((unsigned long)kfrom, u_vaddr, 1); 984 clean_src_k_mappings = 1; 985 } 986 987 copy_page(kto, kfrom); 988 989 /* 990 * Mark DST page K-mapping as dirty for a later finalization by 991 * update_mmu_cache(). Although the finalization could have been done 992 * here as well (given that both vaddr/paddr are available). 993 * But update_mmu_cache() already has code to do that for other 994 * non copied user pages (e.g. read faults which wire in pagecache page 995 * directly). 996 */ 997 clear_bit(PG_dc_clean, &dst->flags); 998 999 /* 1000 * if SRC was already usermapped and non-congruent to kernel mapping 1001 * sync the kernel mapping back to physical page 1002 */ 1003 if (clean_src_k_mappings) { 1004 __flush_dcache_pages((unsigned long)kfrom, 1005 (unsigned long)kfrom, 1); 1006 } else { 1007 clear_bit(PG_dc_clean, &src->flags); 1008 } 1009 1010 kunmap_atomic(kto); 1011 kunmap_atomic(kfrom); 1012} 1013 1014void clear_user_page(void *to, unsigned long u_vaddr, struct page *page) 1015{ 1016 struct folio *folio = page_folio(page); 1017 clear_page(to); 1018 clear_bit(PG_dc_clean, &folio->flags); 1019} 1020EXPORT_SYMBOL(clear_user_page); 1021 1022/********************************************************************** 1023 * Explicit Cache flush request from user space via syscall 1024 * Needed for JITs which generate code on the fly 1025 */ 1026SYSCALL_DEFINE3(cacheflush, uint32_t, start, uint32_t, sz, uint32_t, flags) 1027{ 1028 /* TBD: optimize this */ 1029 flush_cache_all(); 1030 return 0; 1031} 1032 1033/* 1034 * IO-Coherency (IOC) setup rules: 1035 * 1036 * 1. Needs to be at system level, so only once by Master core 1037 * Non-Masters need not be accessing caches at that time 1038 * - They are either HALT_ON_RESET and kick started much later or 1039 * - if run on reset, need to ensure that arc_platform_smp_wait_to_boot() 1040 * doesn't perturb caches or coherency unit 1041 * 1042 * 2. caches (L1 and SLC) need to be purged (flush+inv) before setting up IOC, 1043 * otherwise any straggler data might behave strangely post IOC enabling 1044 * 1045 * 3. All Caches need to be disabled when setting up IOC to elide any in-flight 1046 * Coherency transactions 1047 */ 1048static noinline void __init arc_ioc_setup(void) 1049{ 1050 unsigned int ioc_base, mem_sz; 1051 1052 /* 1053 * If IOC was already enabled (due to bootloader) it technically needs to 1054 * be reconfigured with aperture base,size corresponding to Linux memory map 1055 * which will certainly be different than uboot's. But disabling and 1056 * reenabling IOC when DMA might be potentially active is tricky business. 1057 * To avoid random memory issues later, just panic here and ask user to 1058 * upgrade bootloader to one which doesn't enable IOC 1059 */ 1060 if (read_aux_reg(ARC_REG_IO_COH_ENABLE) & ARC_IO_COH_ENABLE_BIT) 1061 panic("IOC already enabled, please upgrade bootloader!\n"); 1062 1063 if (!ioc_enable) 1064 return; 1065 1066 /* Flush + invalidate + disable L1 dcache */ 1067 __dc_disable(); 1068 1069 /* Flush + invalidate SLC */ 1070 if (read_aux_reg(ARC_REG_SLC_BCR)) 1071 slc_entire_op(OP_FLUSH_N_INV); 1072 1073 /* 1074 * currently IOC Aperture covers entire DDR 1075 * TBD: fix for PGU + 1GB of low mem 1076 * TBD: fix for PAE 1077 */ 1078 mem_sz = arc_get_mem_sz(); 1079 1080 if (!is_power_of_2(mem_sz) || mem_sz < 4096) 1081 panic("IOC Aperture size must be power of 2 larger than 4KB"); 1082 1083 /* 1084 * IOC Aperture size decoded as 2 ^ (SIZE + 2) KB, 1085 * so setting 0x11 implies 512MB, 0x12 implies 1GB... 1086 */ 1087 write_aux_reg(ARC_REG_IO_COH_AP0_SIZE, order_base_2(mem_sz >> 10) - 2); 1088 1089 /* for now assume kernel base is start of IOC aperture */ 1090 ioc_base = CONFIG_LINUX_RAM_BASE; 1091 1092 if (ioc_base % mem_sz != 0) 1093 panic("IOC Aperture start must be aligned to the size of the aperture"); 1094 1095 write_aux_reg(ARC_REG_IO_COH_AP0_BASE, ioc_base >> 12); 1096 write_aux_reg(ARC_REG_IO_COH_PARTIAL, ARC_IO_COH_PARTIAL_BIT); 1097 write_aux_reg(ARC_REG_IO_COH_ENABLE, ARC_IO_COH_ENABLE_BIT); 1098 1099 /* Re-enable L1 dcache */ 1100 __dc_enable(); 1101} 1102 1103/* 1104 * Cache related boot time checks/setups only needed on master CPU: 1105 * - Geometry checks (kernel build and hardware agree: e.g. L1_CACHE_BYTES) 1106 * Assume SMP only, so all cores will have same cache config. A check on 1107 * one core suffices for all 1108 * - IOC setup / dma callbacks only need to be done once 1109 */ 1110static noinline void __init arc_cache_init_master(void) 1111{ 1112 if (IS_ENABLED(CONFIG_ARC_HAS_ICACHE)) { 1113 struct cpuinfo_arc_cache *ic = &ic_info; 1114 1115 if (!ic->line_len) 1116 panic("cache support enabled but non-existent cache\n"); 1117 1118 if (ic->line_len != L1_CACHE_BYTES) 1119 panic("ICache line [%d] != kernel Config [%d]", 1120 ic->line_len, L1_CACHE_BYTES); 1121 1122 /* 1123 * In MMU v4 (HS38x) the aliasing icache config uses IVIL/PTAG 1124 * pair to provide vaddr/paddr respectively, just as in MMU v3 1125 */ 1126 if (is_isa_arcv2() && ic->colors > 1) 1127 _cache_line_loop_ic_fn = __cache_line_loop_v3; 1128 else 1129 _cache_line_loop_ic_fn = __cache_line_loop; 1130 } 1131 1132 if (IS_ENABLED(CONFIG_ARC_HAS_DCACHE)) { 1133 struct cpuinfo_arc_cache *dc = &dc_info; 1134 1135 if (!dc->line_len) 1136 panic("cache support enabled but non-existent cache\n"); 1137 1138 if (dc->line_len != L1_CACHE_BYTES) 1139 panic("DCache line [%d] != kernel Config [%d]", 1140 dc->line_len, L1_CACHE_BYTES); 1141 1142 /* check for D-Cache aliasing on ARCompact: ARCv2 has PIPT */ 1143 if (is_isa_arcompact()) { 1144 int handled = IS_ENABLED(CONFIG_ARC_CACHE_VIPT_ALIASING); 1145 1146 if (dc->colors > 1) { 1147 if (!handled) 1148 panic("Enable CONFIG_ARC_CACHE_VIPT_ALIASING\n"); 1149 if (CACHE_COLORS_NUM != dc->colors) 1150 panic("CACHE_COLORS_NUM not optimized for config\n"); 1151 } else if (handled && dc->colors == 1) { 1152 panic("Disable CONFIG_ARC_CACHE_VIPT_ALIASING\n"); 1153 } 1154 } 1155 } 1156 1157 /* 1158 * Check that SMP_CACHE_BYTES (and hence ARCH_DMA_MINALIGN) is larger 1159 * or equal to any cache line length. 1160 */ 1161 BUILD_BUG_ON_MSG(L1_CACHE_BYTES > SMP_CACHE_BYTES, 1162 "SMP_CACHE_BYTES must be >= any cache line length"); 1163 if (is_isa_arcv2() && (l2_line_sz > SMP_CACHE_BYTES)) 1164 panic("L2 Cache line [%d] > kernel Config [%d]\n", 1165 l2_line_sz, SMP_CACHE_BYTES); 1166 1167 /* Note that SLC disable not formally supported till HS 3.0 */ 1168 if (is_isa_arcv2() && l2_line_sz && !slc_enable) 1169 arc_slc_disable(); 1170 1171 if (is_isa_arcv2() && ioc_exists) 1172 arc_ioc_setup(); 1173 1174 if (is_isa_arcv2() && l2_line_sz && slc_enable) { 1175 __dma_cache_wback_inv = __dma_cache_wback_inv_slc; 1176 __dma_cache_inv = __dma_cache_inv_slc; 1177 __dma_cache_wback = __dma_cache_wback_slc; 1178 } else { 1179 __dma_cache_wback_inv = __dma_cache_wback_inv_l1; 1180 __dma_cache_inv = __dma_cache_inv_l1; 1181 __dma_cache_wback = __dma_cache_wback_l1; 1182 } 1183 /* 1184 * In case of IOC (say IOC+SLC case), pointers above could still be set 1185 * but end up not being relevant as the first function in chain is not 1186 * called at all for devices using coherent DMA. 1187 * arch_sync_dma_for_cpu() -> dma_cache_*() -> __dma_cache_*() 1188 */ 1189} 1190 1191void __ref arc_cache_init(void) 1192{ 1193 unsigned int __maybe_unused cpu = smp_processor_id(); 1194 1195 if (!cpu) 1196 arc_cache_init_master(); 1197 1198 /* 1199 * In PAE regime, TLB and cache maintenance ops take wider addresses 1200 * And even if PAE is not enabled in kernel, the upper 32-bits still need 1201 * to be zeroed to keep the ops sane. 1202 * As an optimization for more common !PAE enabled case, zero them out 1203 * once at init, rather than checking/setting to 0 for every runtime op 1204 */ 1205 if (is_isa_arcv2() && pae40_exist_but_not_enab()) { 1206 1207 if (IS_ENABLED(CONFIG_ARC_HAS_ICACHE)) 1208 write_aux_reg(ARC_REG_IC_PTAG_HI, 0); 1209 1210 if (IS_ENABLED(CONFIG_ARC_HAS_DCACHE)) 1211 write_aux_reg(ARC_REG_DC_PTAG_HI, 0); 1212 1213 if (l2_line_sz) { 1214 write_aux_reg(ARC_REG_SLC_RGN_END1, 0); 1215 write_aux_reg(ARC_REG_SLC_RGN_START1, 0); 1216 } 1217 } 1218} 1219