1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * IOMMU API for ARM architected SMMU implementations. 4 * 5 * Copyright (C) 2013 ARM Limited 6 * 7 * Author: Will Deacon <will.deacon@arm.com> 8 * 9 * This driver currently supports: 10 * - SMMUv1 and v2 implementations 11 * - Stream-matching and stream-indexing 12 * - v7/v8 long-descriptor format 13 * - Non-secure access to the SMMU 14 * - Context fault reporting 15 * - Extended Stream ID (16 bit) 16 */ 17 18#define pr_fmt(fmt) "arm-smmu: " fmt 19 20#include <linux/acpi.h> 21#include <linux/acpi_iort.h> 22#include <linux/bitfield.h> 23#include <linux/delay.h> 24#include <linux/dma-iommu.h> 25#include <linux/dma-mapping.h> 26#include <linux/err.h> 27#include <linux/interrupt.h> 28#include <linux/io.h> 29#include <linux/iopoll.h> 30#include <linux/module.h> 31#include <linux/of.h> 32#include <linux/of_address.h> 33#include <linux/of_device.h> 34#include <linux/of_iommu.h> 35#include <linux/pci.h> 36#include <linux/platform_device.h> 37#include <linux/pm_runtime.h> 38#include <linux/ratelimit.h> 39#include <linux/slab.h> 40 41#include <linux/amba/bus.h> 42#include <linux/fsl/mc.h> 43 44#include "arm-smmu.h" 45 46/* 47 * Apparently, some Qualcomm arm64 platforms which appear to expose their SMMU 48 * global register space are still, in fact, using a hypervisor to mediate it 49 * by trapping and emulating register accesses. Sadly, some deployed versions 50 * of said trapping code have bugs wherein they go horribly wrong for stores 51 * using r31 (i.e. XZR/WZR) as the source register. 52 */ 53#define QCOM_DUMMY_VAL -1 54 55#define MSI_IOVA_BASE 0x8000000 56#define MSI_IOVA_LENGTH 0x100000 57 58static int force_stage; 59module_param(force_stage, int, S_IRUGO); 60MODULE_PARM_DESC(force_stage, 61 "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation."); 62static bool disable_bypass = 63 IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT); 64module_param(disable_bypass, bool, S_IRUGO); 65MODULE_PARM_DESC(disable_bypass, 66 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU."); 67 68#define s2cr_init_val (struct arm_smmu_s2cr){ \ 69 .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS, \ 70} 71 72static bool using_legacy_binding, using_generic_binding; 73 74static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu) 75{ 76 if (pm_runtime_enabled(smmu->dev)) 77 return pm_runtime_resume_and_get(smmu->dev); 78 79 return 0; 80} 81 82static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu) 83{ 84 if (pm_runtime_enabled(smmu->dev)) 85 pm_runtime_put_autosuspend(smmu->dev); 86} 87 88static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom) 89{ 90 return container_of(dom, struct arm_smmu_domain, domain); 91} 92 93static struct platform_driver arm_smmu_driver; 94static struct iommu_ops arm_smmu_ops; 95 96#ifdef CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS 97static int arm_smmu_bus_init(struct iommu_ops *ops); 98 99static struct device_node *dev_get_dev_node(struct device *dev) 100{ 101 if (dev_is_pci(dev)) { 102 struct pci_bus *bus = to_pci_dev(dev)->bus; 103 104 while (!pci_is_root_bus(bus)) 105 bus = bus->parent; 106 return of_node_get(bus->bridge->parent->of_node); 107 } 108 109 return of_node_get(dev->of_node); 110} 111 112static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data) 113{ 114 *((__be32 *)data) = cpu_to_be32(alias); 115 return 0; /* Continue walking */ 116} 117 118static int __find_legacy_master_phandle(struct device *dev, void *data) 119{ 120 struct of_phandle_iterator *it = *(void **)data; 121 struct device_node *np = it->node; 122 int err; 123 124 of_for_each_phandle(it, err, dev->of_node, "mmu-masters", 125 "#stream-id-cells", -1) 126 if (it->node == np) { 127 *(void **)data = dev; 128 return 1; 129 } 130 it->node = np; 131 return err == -ENOENT ? 0 : err; 132} 133 134static int arm_smmu_register_legacy_master(struct device *dev, 135 struct arm_smmu_device **smmu) 136{ 137 struct device *smmu_dev; 138 struct device_node *np; 139 struct of_phandle_iterator it; 140 void *data = ⁢ 141 u32 *sids; 142 __be32 pci_sid; 143 int err; 144 145 np = dev_get_dev_node(dev); 146 if (!np || !of_find_property(np, "#stream-id-cells", NULL)) { 147 of_node_put(np); 148 return -ENODEV; 149 } 150 151 it.node = np; 152 err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data, 153 __find_legacy_master_phandle); 154 smmu_dev = data; 155 of_node_put(np); 156 if (err == 0) 157 return -ENODEV; 158 if (err < 0) 159 return err; 160 161 if (dev_is_pci(dev)) { 162 /* "mmu-masters" assumes Stream ID == Requester ID */ 163 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid, 164 &pci_sid); 165 it.cur = &pci_sid; 166 it.cur_count = 1; 167 } 168 169 err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode, 170 &arm_smmu_ops); 171 if (err) 172 return err; 173 174 sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL); 175 if (!sids) 176 return -ENOMEM; 177 178 *smmu = dev_get_drvdata(smmu_dev); 179 of_phandle_iterator_args(&it, sids, it.cur_count); 180 err = iommu_fwspec_add_ids(dev, sids, it.cur_count); 181 kfree(sids); 182 return err; 183} 184 185/* 186 * With the legacy DT binding in play, we have no guarantees about 187 * probe order, but then we're also not doing default domains, so we can 188 * delay setting bus ops until we're sure every possible SMMU is ready, 189 * and that way ensure that no probe_device() calls get missed. 190 */ 191static int arm_smmu_legacy_bus_init(void) 192{ 193 if (using_legacy_binding) 194 return arm_smmu_bus_init(&arm_smmu_ops); 195 return 0; 196} 197device_initcall_sync(arm_smmu_legacy_bus_init); 198#else 199static int arm_smmu_register_legacy_master(struct device *dev, 200 struct arm_smmu_device **smmu) 201{ 202 return -ENODEV; 203} 204#endif /* CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS */ 205 206static void __arm_smmu_free_bitmap(unsigned long *map, int idx) 207{ 208 clear_bit(idx, map); 209} 210 211/* Wait for any pending TLB invalidations to complete */ 212static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page, 213 int sync, int status) 214{ 215 unsigned int spin_cnt, delay; 216 u32 reg; 217 218 if (smmu->impl && unlikely(smmu->impl->tlb_sync)) 219 return smmu->impl->tlb_sync(smmu, page, sync, status); 220 221 arm_smmu_writel(smmu, page, sync, QCOM_DUMMY_VAL); 222 for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) { 223 for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) { 224 reg = arm_smmu_readl(smmu, page, status); 225 if (!(reg & ARM_SMMU_sTLBGSTATUS_GSACTIVE)) 226 return; 227 cpu_relax(); 228 } 229 udelay(delay); 230 } 231 dev_err_ratelimited(smmu->dev, 232 "TLB sync timed out -- SMMU may be deadlocked\n"); 233} 234 235static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu) 236{ 237 unsigned long flags; 238 239 spin_lock_irqsave(&smmu->global_sync_lock, flags); 240 __arm_smmu_tlb_sync(smmu, ARM_SMMU_GR0, ARM_SMMU_GR0_sTLBGSYNC, 241 ARM_SMMU_GR0_sTLBGSTATUS); 242 spin_unlock_irqrestore(&smmu->global_sync_lock, flags); 243} 244 245static void arm_smmu_tlb_sync_context(struct arm_smmu_domain *smmu_domain) 246{ 247 struct arm_smmu_device *smmu = smmu_domain->smmu; 248 unsigned long flags; 249 250 spin_lock_irqsave(&smmu_domain->cb_lock, flags); 251 __arm_smmu_tlb_sync(smmu, ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx), 252 ARM_SMMU_CB_TLBSYNC, ARM_SMMU_CB_TLBSTATUS); 253 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); 254} 255 256static void arm_smmu_tlb_inv_context_s1(void *cookie) 257{ 258 struct arm_smmu_domain *smmu_domain = cookie; 259 /* 260 * The TLBI write may be relaxed, so ensure that PTEs cleared by the 261 * current CPU are visible beforehand. 262 */ 263 wmb(); 264 arm_smmu_cb_write(smmu_domain->smmu, smmu_domain->cfg.cbndx, 265 ARM_SMMU_CB_S1_TLBIASID, smmu_domain->cfg.asid); 266 arm_smmu_tlb_sync_context(smmu_domain); 267} 268 269static void arm_smmu_tlb_inv_context_s2(void *cookie) 270{ 271 struct arm_smmu_domain *smmu_domain = cookie; 272 struct arm_smmu_device *smmu = smmu_domain->smmu; 273 274 /* See above */ 275 wmb(); 276 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid); 277 arm_smmu_tlb_sync_global(smmu); 278} 279 280static void arm_smmu_tlb_inv_range_s1(unsigned long iova, size_t size, 281 size_t granule, void *cookie, int reg) 282{ 283 struct arm_smmu_domain *smmu_domain = cookie; 284 struct arm_smmu_device *smmu = smmu_domain->smmu; 285 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 286 int idx = cfg->cbndx; 287 288 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) 289 wmb(); 290 291 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) { 292 iova = (iova >> 12) << 12; 293 iova |= cfg->asid; 294 do { 295 arm_smmu_cb_write(smmu, idx, reg, iova); 296 iova += granule; 297 } while (size -= granule); 298 } else { 299 iova >>= 12; 300 iova |= (u64)cfg->asid << 48; 301 do { 302 arm_smmu_cb_writeq(smmu, idx, reg, iova); 303 iova += granule >> 12; 304 } while (size -= granule); 305 } 306} 307 308static void arm_smmu_tlb_inv_range_s2(unsigned long iova, size_t size, 309 size_t granule, void *cookie, int reg) 310{ 311 struct arm_smmu_domain *smmu_domain = cookie; 312 struct arm_smmu_device *smmu = smmu_domain->smmu; 313 int idx = smmu_domain->cfg.cbndx; 314 315 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) 316 wmb(); 317 318 iova >>= 12; 319 do { 320 if (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64) 321 arm_smmu_cb_writeq(smmu, idx, reg, iova); 322 else 323 arm_smmu_cb_write(smmu, idx, reg, iova); 324 iova += granule >> 12; 325 } while (size -= granule); 326} 327 328static void arm_smmu_tlb_inv_walk_s1(unsigned long iova, size_t size, 329 size_t granule, void *cookie) 330{ 331 arm_smmu_tlb_inv_range_s1(iova, size, granule, cookie, 332 ARM_SMMU_CB_S1_TLBIVA); 333 arm_smmu_tlb_sync_context(cookie); 334} 335 336static void arm_smmu_tlb_inv_leaf_s1(unsigned long iova, size_t size, 337 size_t granule, void *cookie) 338{ 339 arm_smmu_tlb_inv_range_s1(iova, size, granule, cookie, 340 ARM_SMMU_CB_S1_TLBIVAL); 341 arm_smmu_tlb_sync_context(cookie); 342} 343 344static void arm_smmu_tlb_add_page_s1(struct iommu_iotlb_gather *gather, 345 unsigned long iova, size_t granule, 346 void *cookie) 347{ 348 arm_smmu_tlb_inv_range_s1(iova, granule, granule, cookie, 349 ARM_SMMU_CB_S1_TLBIVAL); 350} 351 352static void arm_smmu_tlb_inv_walk_s2(unsigned long iova, size_t size, 353 size_t granule, void *cookie) 354{ 355 arm_smmu_tlb_inv_range_s2(iova, size, granule, cookie, 356 ARM_SMMU_CB_S2_TLBIIPAS2); 357 arm_smmu_tlb_sync_context(cookie); 358} 359 360static void arm_smmu_tlb_inv_leaf_s2(unsigned long iova, size_t size, 361 size_t granule, void *cookie) 362{ 363 arm_smmu_tlb_inv_range_s2(iova, size, granule, cookie, 364 ARM_SMMU_CB_S2_TLBIIPAS2L); 365 arm_smmu_tlb_sync_context(cookie); 366} 367 368static void arm_smmu_tlb_add_page_s2(struct iommu_iotlb_gather *gather, 369 unsigned long iova, size_t granule, 370 void *cookie) 371{ 372 arm_smmu_tlb_inv_range_s2(iova, granule, granule, cookie, 373 ARM_SMMU_CB_S2_TLBIIPAS2L); 374} 375 376static void arm_smmu_tlb_inv_any_s2_v1(unsigned long iova, size_t size, 377 size_t granule, void *cookie) 378{ 379 arm_smmu_tlb_inv_context_s2(cookie); 380} 381/* 382 * On MMU-401 at least, the cost of firing off multiple TLBIVMIDs appears 383 * almost negligible, but the benefit of getting the first one in as far ahead 384 * of the sync as possible is significant, hence we don't just make this a 385 * no-op and call arm_smmu_tlb_inv_context_s2() from .iotlb_sync as you might 386 * think. 387 */ 388static void arm_smmu_tlb_add_page_s2_v1(struct iommu_iotlb_gather *gather, 389 unsigned long iova, size_t granule, 390 void *cookie) 391{ 392 struct arm_smmu_domain *smmu_domain = cookie; 393 struct arm_smmu_device *smmu = smmu_domain->smmu; 394 395 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) 396 wmb(); 397 398 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid); 399} 400 401static const struct iommu_flush_ops arm_smmu_s1_tlb_ops = { 402 .tlb_flush_all = arm_smmu_tlb_inv_context_s1, 403 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s1, 404 .tlb_flush_leaf = arm_smmu_tlb_inv_leaf_s1, 405 .tlb_add_page = arm_smmu_tlb_add_page_s1, 406}; 407 408static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v2 = { 409 .tlb_flush_all = arm_smmu_tlb_inv_context_s2, 410 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2, 411 .tlb_flush_leaf = arm_smmu_tlb_inv_leaf_s2, 412 .tlb_add_page = arm_smmu_tlb_add_page_s2, 413}; 414 415static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v1 = { 416 .tlb_flush_all = arm_smmu_tlb_inv_context_s2, 417 .tlb_flush_walk = arm_smmu_tlb_inv_any_s2_v1, 418 .tlb_flush_leaf = arm_smmu_tlb_inv_any_s2_v1, 419 .tlb_add_page = arm_smmu_tlb_add_page_s2_v1, 420}; 421 422static irqreturn_t arm_smmu_context_fault(int irq, void *dev) 423{ 424 u32 fsr, fsynr, cbfrsynra; 425 unsigned long iova; 426 struct iommu_domain *domain = dev; 427 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 428 struct arm_smmu_device *smmu = smmu_domain->smmu; 429 int idx = smmu_domain->cfg.cbndx; 430 431 fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); 432 if (!(fsr & ARM_SMMU_FSR_FAULT)) 433 return IRQ_NONE; 434 435 fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0); 436 iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR); 437 cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx)); 438 439 dev_err_ratelimited(smmu->dev, 440 "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n", 441 fsr, iova, fsynr, cbfrsynra, idx); 442 443 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); 444 return IRQ_HANDLED; 445} 446 447static irqreturn_t arm_smmu_global_fault(int irq, void *dev) 448{ 449 u32 gfsr, gfsynr0, gfsynr1, gfsynr2; 450 struct arm_smmu_device *smmu = dev; 451 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL, 452 DEFAULT_RATELIMIT_BURST); 453 454 gfsr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR); 455 gfsynr0 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR0); 456 gfsynr1 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR1); 457 gfsynr2 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR2); 458 459 if (!gfsr) 460 return IRQ_NONE; 461 462 if (__ratelimit(&rs)) { 463 if (IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT) && 464 (gfsr & ARM_SMMU_sGFSR_USF)) 465 dev_err(smmu->dev, 466 "Blocked unknown Stream ID 0x%hx; boot with \"arm-smmu.disable_bypass=0\" to allow, but this may have security implications\n", 467 (u16)gfsynr1); 468 else 469 dev_err(smmu->dev, 470 "Unexpected global fault, this could be serious\n"); 471 dev_err(smmu->dev, 472 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n", 473 gfsr, gfsynr0, gfsynr1, gfsynr2); 474 } 475 476 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, gfsr); 477 return IRQ_HANDLED; 478} 479 480static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain, 481 struct io_pgtable_cfg *pgtbl_cfg) 482{ 483 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 484 struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx]; 485 bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS; 486 487 cb->cfg = cfg; 488 489 /* TCR */ 490 if (stage1) { 491 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) { 492 cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr; 493 } else { 494 cb->tcr[0] = arm_smmu_lpae_tcr(pgtbl_cfg); 495 cb->tcr[1] = arm_smmu_lpae_tcr2(pgtbl_cfg); 496 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) 497 cb->tcr[1] |= ARM_SMMU_TCR2_AS; 498 else 499 cb->tcr[0] |= ARM_SMMU_TCR_EAE; 500 } 501 } else { 502 cb->tcr[0] = arm_smmu_lpae_vtcr(pgtbl_cfg); 503 } 504 505 /* TTBRs */ 506 if (stage1) { 507 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) { 508 cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr; 509 cb->ttbr[1] = 0; 510 } else { 511 cb->ttbr[0] = FIELD_PREP(ARM_SMMU_TTBRn_ASID, 512 cfg->asid); 513 cb->ttbr[1] = FIELD_PREP(ARM_SMMU_TTBRn_ASID, 514 cfg->asid); 515 516 if (pgtbl_cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) 517 cb->ttbr[1] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr; 518 else 519 cb->ttbr[0] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr; 520 } 521 } else { 522 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr; 523 } 524 525 /* MAIRs (stage-1 only) */ 526 if (stage1) { 527 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) { 528 cb->mair[0] = pgtbl_cfg->arm_v7s_cfg.prrr; 529 cb->mair[1] = pgtbl_cfg->arm_v7s_cfg.nmrr; 530 } else { 531 cb->mair[0] = pgtbl_cfg->arm_lpae_s1_cfg.mair; 532 cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair >> 32; 533 } 534 } 535} 536 537void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx) 538{ 539 u32 reg; 540 bool stage1; 541 struct arm_smmu_cb *cb = &smmu->cbs[idx]; 542 struct arm_smmu_cfg *cfg = cb->cfg; 543 544 /* Unassigned context banks only need disabling */ 545 if (!cfg) { 546 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, 0); 547 return; 548 } 549 550 stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS; 551 552 /* CBA2R */ 553 if (smmu->version > ARM_SMMU_V1) { 554 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) 555 reg = ARM_SMMU_CBA2R_VA64; 556 else 557 reg = 0; 558 /* 16-bit VMIDs live in CBA2R */ 559 if (smmu->features & ARM_SMMU_FEAT_VMID16) 560 reg |= FIELD_PREP(ARM_SMMU_CBA2R_VMID16, cfg->vmid); 561 562 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBA2R(idx), reg); 563 } 564 565 /* CBAR */ 566 reg = FIELD_PREP(ARM_SMMU_CBAR_TYPE, cfg->cbar); 567 if (smmu->version < ARM_SMMU_V2) 568 reg |= FIELD_PREP(ARM_SMMU_CBAR_IRPTNDX, cfg->irptndx); 569 570 /* 571 * Use the weakest shareability/memory types, so they are 572 * overridden by the ttbcr/pte. 573 */ 574 if (stage1) { 575 reg |= FIELD_PREP(ARM_SMMU_CBAR_S1_BPSHCFG, 576 ARM_SMMU_CBAR_S1_BPSHCFG_NSH) | 577 FIELD_PREP(ARM_SMMU_CBAR_S1_MEMATTR, 578 ARM_SMMU_CBAR_S1_MEMATTR_WB); 579 } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) { 580 /* 8-bit VMIDs live in CBAR */ 581 reg |= FIELD_PREP(ARM_SMMU_CBAR_VMID, cfg->vmid); 582 } 583 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(idx), reg); 584 585 /* 586 * TCR 587 * We must write this before the TTBRs, since it determines the 588 * access behaviour of some fields (in particular, ASID[15:8]). 589 */ 590 if (stage1 && smmu->version > ARM_SMMU_V1) 591 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR2, cb->tcr[1]); 592 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR, cb->tcr[0]); 593 594 /* TTBRs */ 595 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) { 596 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_CONTEXTIDR, cfg->asid); 597 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]); 598 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR1, cb->ttbr[1]); 599 } else { 600 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]); 601 if (stage1) 602 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR1, 603 cb->ttbr[1]); 604 } 605 606 /* MAIRs (stage-1 only) */ 607 if (stage1) { 608 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR0, cb->mair[0]); 609 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR1, cb->mair[1]); 610 } 611 612 /* SCTLR */ 613 reg = ARM_SMMU_SCTLR_CFIE | ARM_SMMU_SCTLR_CFRE | ARM_SMMU_SCTLR_AFE | 614 ARM_SMMU_SCTLR_TRE | ARM_SMMU_SCTLR_M; 615 if (stage1) 616 reg |= ARM_SMMU_SCTLR_S1_ASIDPNE; 617 if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) 618 reg |= ARM_SMMU_SCTLR_E; 619 620 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg); 621} 622 623static int arm_smmu_alloc_context_bank(struct arm_smmu_domain *smmu_domain, 624 struct arm_smmu_device *smmu, 625 struct device *dev, unsigned int start) 626{ 627 if (smmu->impl && smmu->impl->alloc_context_bank) 628 return smmu->impl->alloc_context_bank(smmu_domain, smmu, dev, start); 629 630 return __arm_smmu_alloc_bitmap(smmu->context_map, start, smmu->num_context_banks); 631} 632 633static int arm_smmu_init_domain_context(struct iommu_domain *domain, 634 struct arm_smmu_device *smmu, 635 struct device *dev) 636{ 637 int irq, start, ret = 0; 638 unsigned long ias, oas; 639 struct io_pgtable_ops *pgtbl_ops; 640 struct io_pgtable_cfg pgtbl_cfg; 641 enum io_pgtable_fmt fmt; 642 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 643 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 644 irqreturn_t (*context_fault)(int irq, void *dev); 645 646 mutex_lock(&smmu_domain->init_mutex); 647 if (smmu_domain->smmu) 648 goto out_unlock; 649 650 if (domain->type == IOMMU_DOMAIN_IDENTITY) { 651 smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS; 652 smmu_domain->smmu = smmu; 653 goto out_unlock; 654 } 655 656 /* 657 * Mapping the requested stage onto what we support is surprisingly 658 * complicated, mainly because the spec allows S1+S2 SMMUs without 659 * support for nested translation. That means we end up with the 660 * following table: 661 * 662 * Requested Supported Actual 663 * S1 N S1 664 * S1 S1+S2 S1 665 * S1 S2 S2 666 * S1 S1 S1 667 * N N N 668 * N S1+S2 S2 669 * N S2 S2 670 * N S1 S1 671 * 672 * Note that you can't actually request stage-2 mappings. 673 */ 674 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1)) 675 smmu_domain->stage = ARM_SMMU_DOMAIN_S2; 676 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2)) 677 smmu_domain->stage = ARM_SMMU_DOMAIN_S1; 678 679 /* 680 * Choosing a suitable context format is even more fiddly. Until we 681 * grow some way for the caller to express a preference, and/or move 682 * the decision into the io-pgtable code where it arguably belongs, 683 * just aim for the closest thing to the rest of the system, and hope 684 * that the hardware isn't esoteric enough that we can't assume AArch64 685 * support to be a superset of AArch32 support... 686 */ 687 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L) 688 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L; 689 if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) && 690 !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) && 691 (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) && 692 (smmu_domain->stage == ARM_SMMU_DOMAIN_S1)) 693 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S; 694 if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) && 695 (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K | 696 ARM_SMMU_FEAT_FMT_AARCH64_16K | 697 ARM_SMMU_FEAT_FMT_AARCH64_4K))) 698 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64; 699 700 if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) { 701 ret = -EINVAL; 702 goto out_unlock; 703 } 704 705 switch (smmu_domain->stage) { 706 case ARM_SMMU_DOMAIN_S1: 707 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS; 708 start = smmu->num_s2_context_banks; 709 ias = smmu->va_size; 710 oas = smmu->ipa_size; 711 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) { 712 fmt = ARM_64_LPAE_S1; 713 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) { 714 fmt = ARM_32_LPAE_S1; 715 ias = min(ias, 32UL); 716 oas = min(oas, 40UL); 717 } else { 718 fmt = ARM_V7S; 719 ias = min(ias, 32UL); 720 oas = min(oas, 32UL); 721 } 722 smmu_domain->flush_ops = &arm_smmu_s1_tlb_ops; 723 break; 724 case ARM_SMMU_DOMAIN_NESTED: 725 /* 726 * We will likely want to change this if/when KVM gets 727 * involved. 728 */ 729 case ARM_SMMU_DOMAIN_S2: 730 cfg->cbar = CBAR_TYPE_S2_TRANS; 731 start = 0; 732 ias = smmu->ipa_size; 733 oas = smmu->pa_size; 734 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) { 735 fmt = ARM_64_LPAE_S2; 736 } else { 737 fmt = ARM_32_LPAE_S2; 738 ias = min(ias, 40UL); 739 oas = min(oas, 40UL); 740 } 741 if (smmu->version == ARM_SMMU_V2) 742 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v2; 743 else 744 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v1; 745 break; 746 default: 747 ret = -EINVAL; 748 goto out_unlock; 749 } 750 751 ret = arm_smmu_alloc_context_bank(smmu_domain, smmu, dev, start); 752 if (ret < 0) { 753 goto out_unlock; 754 } 755 756 smmu_domain->smmu = smmu; 757 758 cfg->cbndx = ret; 759 if (smmu->version < ARM_SMMU_V2) { 760 cfg->irptndx = atomic_inc_return(&smmu->irptndx); 761 cfg->irptndx %= smmu->num_context_irqs; 762 } else { 763 cfg->irptndx = cfg->cbndx; 764 } 765 766 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2) 767 cfg->vmid = cfg->cbndx + 1; 768 else 769 cfg->asid = cfg->cbndx; 770 771 pgtbl_cfg = (struct io_pgtable_cfg) { 772 .pgsize_bitmap = smmu->pgsize_bitmap, 773 .ias = ias, 774 .oas = oas, 775 .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK, 776 .tlb = smmu_domain->flush_ops, 777 .iommu_dev = smmu->dev, 778 }; 779 780 if (smmu->impl && smmu->impl->init_context) { 781 ret = smmu->impl->init_context(smmu_domain, &pgtbl_cfg, dev); 782 if (ret) 783 goto out_clear_smmu; 784 } 785 786 if (smmu_domain->non_strict) 787 pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_NON_STRICT; 788 789 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain); 790 if (!pgtbl_ops) { 791 ret = -ENOMEM; 792 goto out_clear_smmu; 793 } 794 795 /* Update the domain's page sizes to reflect the page table format */ 796 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap; 797 798 if (pgtbl_cfg.quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) { 799 domain->geometry.aperture_start = ~0UL << ias; 800 domain->geometry.aperture_end = ~0UL; 801 } else { 802 domain->geometry.aperture_end = (1UL << ias) - 1; 803 } 804 805 domain->geometry.force_aperture = true; 806 807 /* Initialise the context bank with our page table cfg */ 808 arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg); 809 arm_smmu_write_context_bank(smmu, cfg->cbndx); 810 811 /* 812 * Request context fault interrupt. Do this last to avoid the 813 * handler seeing a half-initialised domain state. 814 */ 815 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx]; 816 817 if (smmu->impl && smmu->impl->context_fault) 818 context_fault = smmu->impl->context_fault; 819 else 820 context_fault = arm_smmu_context_fault; 821 822 ret = devm_request_irq(smmu->dev, irq, context_fault, 823 IRQF_SHARED, "arm-smmu-context-fault", domain); 824 if (ret < 0) { 825 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n", 826 cfg->irptndx, irq); 827 cfg->irptndx = ARM_SMMU_INVALID_IRPTNDX; 828 } 829 830 mutex_unlock(&smmu_domain->init_mutex); 831 832 /* Publish page table ops for map/unmap */ 833 smmu_domain->pgtbl_ops = pgtbl_ops; 834 return 0; 835 836out_clear_smmu: 837 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx); 838 smmu_domain->smmu = NULL; 839out_unlock: 840 mutex_unlock(&smmu_domain->init_mutex); 841 return ret; 842} 843 844static void arm_smmu_destroy_domain_context(struct iommu_domain *domain) 845{ 846 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 847 struct arm_smmu_device *smmu = smmu_domain->smmu; 848 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 849 int ret, irq; 850 851 if (!smmu || domain->type == IOMMU_DOMAIN_IDENTITY) 852 return; 853 854 ret = arm_smmu_rpm_get(smmu); 855 if (ret < 0) 856 return; 857 858 /* 859 * Disable the context bank and free the page tables before freeing 860 * it. 861 */ 862 smmu->cbs[cfg->cbndx].cfg = NULL; 863 arm_smmu_write_context_bank(smmu, cfg->cbndx); 864 865 if (cfg->irptndx != ARM_SMMU_INVALID_IRPTNDX) { 866 irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx]; 867 devm_free_irq(smmu->dev, irq, domain); 868 } 869 870 free_io_pgtable_ops(smmu_domain->pgtbl_ops); 871 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx); 872 873 arm_smmu_rpm_put(smmu); 874} 875 876static struct iommu_domain *arm_smmu_domain_alloc(unsigned type) 877{ 878 struct arm_smmu_domain *smmu_domain; 879 880 if (type != IOMMU_DOMAIN_UNMANAGED && 881 type != IOMMU_DOMAIN_DMA && 882 type != IOMMU_DOMAIN_IDENTITY) 883 return NULL; 884 /* 885 * Allocate the domain and initialise some of its data structures. 886 * We can't really do anything meaningful until we've added a 887 * master. 888 */ 889 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL); 890 if (!smmu_domain) 891 return NULL; 892 893 if (type == IOMMU_DOMAIN_DMA && (using_legacy_binding || 894 iommu_get_dma_cookie(&smmu_domain->domain))) { 895 kfree(smmu_domain); 896 return NULL; 897 } 898 899 mutex_init(&smmu_domain->init_mutex); 900 spin_lock_init(&smmu_domain->cb_lock); 901 902 return &smmu_domain->domain; 903} 904 905static void arm_smmu_domain_free(struct iommu_domain *domain) 906{ 907 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 908 909 /* 910 * Free the domain resources. We assume that all devices have 911 * already been detached. 912 */ 913 iommu_put_dma_cookie(domain); 914 arm_smmu_destroy_domain_context(domain); 915 kfree(smmu_domain); 916} 917 918static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx) 919{ 920 struct arm_smmu_smr *smr = smmu->smrs + idx; 921 u32 reg = FIELD_PREP(ARM_SMMU_SMR_ID, smr->id) | 922 FIELD_PREP(ARM_SMMU_SMR_MASK, smr->mask); 923 924 if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid) 925 reg |= ARM_SMMU_SMR_VALID; 926 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(idx), reg); 927} 928 929static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx) 930{ 931 struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx; 932 u32 reg; 933 934 if (smmu->impl && smmu->impl->write_s2cr) { 935 smmu->impl->write_s2cr(smmu, idx); 936 return; 937 } 938 939 reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, s2cr->type) | 940 FIELD_PREP(ARM_SMMU_S2CR_CBNDX, s2cr->cbndx) | 941 FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg); 942 943 if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs && 944 smmu->smrs[idx].valid) 945 reg |= ARM_SMMU_S2CR_EXIDVALID; 946 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg); 947} 948 949static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx) 950{ 951 arm_smmu_write_s2cr(smmu, idx); 952 if (smmu->smrs) 953 arm_smmu_write_smr(smmu, idx); 954} 955 956/* 957 * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function 958 * should be called after sCR0 is written. 959 */ 960static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu) 961{ 962 u32 smr; 963 int i; 964 965 if (!smmu->smrs) 966 return; 967 /* 968 * If we've had to accommodate firmware memory regions, we may 969 * have live SMRs by now; tread carefully... 970 * 971 * Somewhat perversely, not having a free SMR for this test implies we 972 * can get away without it anyway, as we'll only be able to 'allocate' 973 * these SMRs for the ID/mask values we're already trusting to be OK. 974 */ 975 for (i = 0; i < smmu->num_mapping_groups; i++) 976 if (!smmu->smrs[i].valid) 977 goto smr_ok; 978 return; 979smr_ok: 980 /* 981 * SMR.ID bits may not be preserved if the corresponding MASK 982 * bits are set, so check each one separately. We can reject 983 * masters later if they try to claim IDs outside these masks. 984 */ 985 smr = FIELD_PREP(ARM_SMMU_SMR_ID, smmu->streamid_mask); 986 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr); 987 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i)); 988 smmu->streamid_mask = FIELD_GET(ARM_SMMU_SMR_ID, smr); 989 990 smr = FIELD_PREP(ARM_SMMU_SMR_MASK, smmu->streamid_mask); 991 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr); 992 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i)); 993 smmu->smr_mask_mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr); 994} 995 996static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask) 997{ 998 struct arm_smmu_smr *smrs = smmu->smrs; 999 int i, free_idx = -ENOSPC; 1000 1001 /* Stream indexing is blissfully easy */ 1002 if (!smrs) 1003 return id; 1004 1005 /* Validating SMRs is... less so */ 1006 for (i = 0; i < smmu->num_mapping_groups; ++i) { 1007 if (!smrs[i].valid) { 1008 /* 1009 * Note the first free entry we come across, which 1010 * we'll claim in the end if nothing else matches. 1011 */ 1012 if (free_idx < 0) 1013 free_idx = i; 1014 continue; 1015 } 1016 /* 1017 * If the new entry is _entirely_ matched by an existing entry, 1018 * then reuse that, with the guarantee that there also cannot 1019 * be any subsequent conflicting entries. In normal use we'd 1020 * expect simply identical entries for this case, but there's 1021 * no harm in accommodating the generalisation. 1022 */ 1023 if ((mask & smrs[i].mask) == mask && 1024 !((id ^ smrs[i].id) & ~smrs[i].mask)) 1025 return i; 1026 /* 1027 * If the new entry has any other overlap with an existing one, 1028 * though, then there always exists at least one stream ID 1029 * which would cause a conflict, and we can't allow that risk. 1030 */ 1031 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask))) 1032 return -EINVAL; 1033 } 1034 1035 return free_idx; 1036} 1037 1038static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx) 1039{ 1040 if (--smmu->s2crs[idx].count) 1041 return false; 1042 1043 smmu->s2crs[idx] = s2cr_init_val; 1044 if (smmu->smrs) 1045 smmu->smrs[idx].valid = false; 1046 1047 return true; 1048} 1049 1050static int arm_smmu_master_alloc_smes(struct device *dev) 1051{ 1052 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 1053 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev); 1054 struct arm_smmu_device *smmu = cfg->smmu; 1055 struct arm_smmu_smr *smrs = smmu->smrs; 1056 int i, idx, ret; 1057 1058 mutex_lock(&smmu->stream_map_mutex); 1059 /* Figure out a viable stream map entry allocation */ 1060 for_each_cfg_sme(cfg, fwspec, i, idx) { 1061 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]); 1062 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]); 1063 1064 if (idx != INVALID_SMENDX) { 1065 ret = -EEXIST; 1066 goto out_err; 1067 } 1068 1069 ret = arm_smmu_find_sme(smmu, sid, mask); 1070 if (ret < 0) 1071 goto out_err; 1072 1073 idx = ret; 1074 if (smrs && smmu->s2crs[idx].count == 0) { 1075 smrs[idx].id = sid; 1076 smrs[idx].mask = mask; 1077 smrs[idx].valid = true; 1078 } 1079 smmu->s2crs[idx].count++; 1080 cfg->smendx[i] = (s16)idx; 1081 } 1082 1083 /* It worked! Now, poke the actual hardware */ 1084 for_each_cfg_sme(cfg, fwspec, i, idx) 1085 arm_smmu_write_sme(smmu, idx); 1086 1087 mutex_unlock(&smmu->stream_map_mutex); 1088 return 0; 1089 1090out_err: 1091 while (i--) { 1092 arm_smmu_free_sme(smmu, cfg->smendx[i]); 1093 cfg->smendx[i] = INVALID_SMENDX; 1094 } 1095 mutex_unlock(&smmu->stream_map_mutex); 1096 return ret; 1097} 1098 1099static void arm_smmu_master_free_smes(struct arm_smmu_master_cfg *cfg, 1100 struct iommu_fwspec *fwspec) 1101{ 1102 struct arm_smmu_device *smmu = cfg->smmu; 1103 int i, idx; 1104 1105 mutex_lock(&smmu->stream_map_mutex); 1106 for_each_cfg_sme(cfg, fwspec, i, idx) { 1107 if (arm_smmu_free_sme(smmu, idx)) 1108 arm_smmu_write_sme(smmu, idx); 1109 cfg->smendx[i] = INVALID_SMENDX; 1110 } 1111 mutex_unlock(&smmu->stream_map_mutex); 1112} 1113 1114static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain, 1115 struct arm_smmu_master_cfg *cfg, 1116 struct iommu_fwspec *fwspec) 1117{ 1118 struct arm_smmu_device *smmu = smmu_domain->smmu; 1119 struct arm_smmu_s2cr *s2cr = smmu->s2crs; 1120 u8 cbndx = smmu_domain->cfg.cbndx; 1121 enum arm_smmu_s2cr_type type; 1122 int i, idx; 1123 1124 if (smmu_domain->stage == ARM_SMMU_DOMAIN_BYPASS) 1125 type = S2CR_TYPE_BYPASS; 1126 else 1127 type = S2CR_TYPE_TRANS; 1128 1129 for_each_cfg_sme(cfg, fwspec, i, idx) { 1130 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx) 1131 continue; 1132 1133 s2cr[idx].type = type; 1134 s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT; 1135 s2cr[idx].cbndx = cbndx; 1136 arm_smmu_write_s2cr(smmu, idx); 1137 } 1138 return 0; 1139} 1140 1141static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) 1142{ 1143 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1144 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 1145 struct arm_smmu_master_cfg *cfg; 1146 struct arm_smmu_device *smmu; 1147 int ret; 1148 1149 if (!fwspec || fwspec->ops != &arm_smmu_ops) { 1150 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n"); 1151 return -ENXIO; 1152 } 1153 1154 /* 1155 * FIXME: The arch/arm DMA API code tries to attach devices to its own 1156 * domains between of_xlate() and probe_device() - we have no way to cope 1157 * with that, so until ARM gets converted to rely on groups and default 1158 * domains, just say no (but more politely than by dereferencing NULL). 1159 * This should be at least a WARN_ON once that's sorted. 1160 */ 1161 cfg = dev_iommu_priv_get(dev); 1162 if (!cfg) 1163 return -ENODEV; 1164 1165 smmu = cfg->smmu; 1166 1167 ret = arm_smmu_rpm_get(smmu); 1168 if (ret < 0) 1169 return ret; 1170 1171 /* Ensure that the domain is finalised */ 1172 ret = arm_smmu_init_domain_context(domain, smmu, dev); 1173 if (ret < 0) 1174 goto rpm_put; 1175 1176 /* 1177 * Sanity check the domain. We don't support domains across 1178 * different SMMUs. 1179 */ 1180 if (smmu_domain->smmu != smmu) { 1181 dev_err(dev, 1182 "cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n", 1183 dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev)); 1184 ret = -EINVAL; 1185 goto rpm_put; 1186 } 1187 1188 /* Looks ok, so add the device to the domain */ 1189 ret = arm_smmu_domain_add_master(smmu_domain, cfg, fwspec); 1190 1191 /* 1192 * Setup an autosuspend delay to avoid bouncing runpm state. 1193 * Otherwise, if a driver for a suspended consumer device 1194 * unmaps buffers, it will runpm resume/suspend for each one. 1195 * 1196 * For example, when used by a GPU device, when an application 1197 * or game exits, it can trigger unmapping 100s or 1000s of 1198 * buffers. With a runpm cycle for each buffer, that adds up 1199 * to 5-10sec worth of reprogramming the context bank, while 1200 * the system appears to be locked up to the user. 1201 */ 1202 pm_runtime_set_autosuspend_delay(smmu->dev, 20); 1203 pm_runtime_use_autosuspend(smmu->dev); 1204 1205rpm_put: 1206 arm_smmu_rpm_put(smmu); 1207 return ret; 1208} 1209 1210static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova, 1211 phys_addr_t paddr, size_t size, int prot, gfp_t gfp) 1212{ 1213 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops; 1214 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu; 1215 int ret; 1216 1217 if (!ops) 1218 return -ENODEV; 1219 1220 arm_smmu_rpm_get(smmu); 1221 ret = ops->map(ops, iova, paddr, size, prot, gfp); 1222 arm_smmu_rpm_put(smmu); 1223 1224 return ret; 1225} 1226 1227static size_t arm_smmu_unmap(struct iommu_domain *domain, unsigned long iova, 1228 size_t size, struct iommu_iotlb_gather *gather) 1229{ 1230 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops; 1231 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu; 1232 size_t ret; 1233 1234 if (!ops) 1235 return 0; 1236 1237 arm_smmu_rpm_get(smmu); 1238 ret = ops->unmap(ops, iova, size, gather); 1239 arm_smmu_rpm_put(smmu); 1240 1241 return ret; 1242} 1243 1244static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain) 1245{ 1246 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1247 struct arm_smmu_device *smmu = smmu_domain->smmu; 1248 1249 if (smmu_domain->flush_ops) { 1250 arm_smmu_rpm_get(smmu); 1251 smmu_domain->flush_ops->tlb_flush_all(smmu_domain); 1252 arm_smmu_rpm_put(smmu); 1253 } 1254} 1255 1256static void arm_smmu_iotlb_sync(struct iommu_domain *domain, 1257 struct iommu_iotlb_gather *gather) 1258{ 1259 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1260 struct arm_smmu_device *smmu = smmu_domain->smmu; 1261 1262 if (!smmu) 1263 return; 1264 1265 arm_smmu_rpm_get(smmu); 1266 if (smmu->version == ARM_SMMU_V2 || 1267 smmu_domain->stage == ARM_SMMU_DOMAIN_S1) 1268 arm_smmu_tlb_sync_context(smmu_domain); 1269 else 1270 arm_smmu_tlb_sync_global(smmu); 1271 arm_smmu_rpm_put(smmu); 1272} 1273 1274static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, 1275 dma_addr_t iova) 1276{ 1277 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1278 struct arm_smmu_device *smmu = smmu_domain->smmu; 1279 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 1280 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops; 1281 struct device *dev = smmu->dev; 1282 void __iomem *reg; 1283 u32 tmp; 1284 u64 phys; 1285 unsigned long va, flags; 1286 int ret, idx = cfg->cbndx; 1287 phys_addr_t addr = 0; 1288 1289 ret = arm_smmu_rpm_get(smmu); 1290 if (ret < 0) 1291 return 0; 1292 1293 spin_lock_irqsave(&smmu_domain->cb_lock, flags); 1294 va = iova & ~0xfffUL; 1295 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) 1296 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_ATS1PR, va); 1297 else 1298 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ATS1PR, va); 1299 1300 reg = arm_smmu_page(smmu, ARM_SMMU_CB(smmu, idx)) + ARM_SMMU_CB_ATSR; 1301 if (readl_poll_timeout_atomic(reg, tmp, !(tmp & ARM_SMMU_ATSR_ACTIVE), 1302 5, 50)) { 1303 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); 1304 dev_err(dev, 1305 "iova to phys timed out on %pad. Falling back to software table walk.\n", 1306 &iova); 1307 arm_smmu_rpm_put(smmu); 1308 return ops->iova_to_phys(ops, iova); 1309 } 1310 1311 phys = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_PAR); 1312 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); 1313 if (phys & ARM_SMMU_CB_PAR_F) { 1314 dev_err(dev, "translation fault!\n"); 1315 dev_err(dev, "PAR = 0x%llx\n", phys); 1316 goto out; 1317 } 1318 1319 addr = (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff); 1320out: 1321 arm_smmu_rpm_put(smmu); 1322 1323 return addr; 1324} 1325 1326static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain, 1327 dma_addr_t iova) 1328{ 1329 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1330 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops; 1331 1332 if (domain->type == IOMMU_DOMAIN_IDENTITY) 1333 return iova; 1334 1335 if (!ops) 1336 return 0; 1337 1338 if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS && 1339 smmu_domain->stage == ARM_SMMU_DOMAIN_S1) 1340 return arm_smmu_iova_to_phys_hard(domain, iova); 1341 1342 return ops->iova_to_phys(ops, iova); 1343} 1344 1345static bool arm_smmu_capable(enum iommu_cap cap) 1346{ 1347 switch (cap) { 1348 case IOMMU_CAP_CACHE_COHERENCY: 1349 /* 1350 * Return true here as the SMMU can always send out coherent 1351 * requests. 1352 */ 1353 return true; 1354 case IOMMU_CAP_NOEXEC: 1355 return true; 1356 default: 1357 return false; 1358 } 1359} 1360 1361static 1362struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode) 1363{ 1364 struct device *dev = driver_find_device_by_fwnode(&arm_smmu_driver.driver, 1365 fwnode); 1366 put_device(dev); 1367 return dev ? dev_get_drvdata(dev) : NULL; 1368} 1369 1370static struct iommu_device *arm_smmu_probe_device(struct device *dev) 1371{ 1372 struct arm_smmu_device *smmu = NULL; 1373 struct arm_smmu_master_cfg *cfg; 1374 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 1375 int i, ret; 1376 1377 if (using_legacy_binding) { 1378 ret = arm_smmu_register_legacy_master(dev, &smmu); 1379 1380 /* 1381 * If dev->iommu_fwspec is initally NULL, arm_smmu_register_legacy_master() 1382 * will allocate/initialise a new one. Thus we need to update fwspec for 1383 * later use. 1384 */ 1385 fwspec = dev_iommu_fwspec_get(dev); 1386 if (ret) 1387 goto out_free; 1388 } else if (fwspec && fwspec->ops == &arm_smmu_ops) { 1389 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode); 1390 } else { 1391 return ERR_PTR(-ENODEV); 1392 } 1393 1394 ret = -EINVAL; 1395 for (i = 0; i < fwspec->num_ids; i++) { 1396 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]); 1397 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]); 1398 1399 if (sid & ~smmu->streamid_mask) { 1400 dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n", 1401 sid, smmu->streamid_mask); 1402 goto out_free; 1403 } 1404 if (mask & ~smmu->smr_mask_mask) { 1405 dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n", 1406 mask, smmu->smr_mask_mask); 1407 goto out_free; 1408 } 1409 } 1410 1411 ret = -ENOMEM; 1412 cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]), 1413 GFP_KERNEL); 1414 if (!cfg) 1415 goto out_free; 1416 1417 cfg->smmu = smmu; 1418 dev_iommu_priv_set(dev, cfg); 1419 while (i--) 1420 cfg->smendx[i] = INVALID_SMENDX; 1421 1422 ret = arm_smmu_rpm_get(smmu); 1423 if (ret < 0) 1424 goto out_cfg_free; 1425 1426 ret = arm_smmu_master_alloc_smes(dev); 1427 arm_smmu_rpm_put(smmu); 1428 1429 if (ret) 1430 goto out_cfg_free; 1431 1432 device_link_add(dev, smmu->dev, 1433 DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER); 1434 1435 return &smmu->iommu; 1436 1437out_cfg_free: 1438 kfree(cfg); 1439out_free: 1440 iommu_fwspec_free(dev); 1441 return ERR_PTR(ret); 1442} 1443 1444static void arm_smmu_release_device(struct device *dev) 1445{ 1446 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 1447 struct arm_smmu_master_cfg *cfg; 1448 struct arm_smmu_device *smmu; 1449 int ret; 1450 1451 if (!fwspec || fwspec->ops != &arm_smmu_ops) 1452 return; 1453 1454 cfg = dev_iommu_priv_get(dev); 1455 smmu = cfg->smmu; 1456 1457 ret = arm_smmu_rpm_get(smmu); 1458 if (ret < 0) 1459 return; 1460 1461 arm_smmu_master_free_smes(cfg, fwspec); 1462 1463 arm_smmu_rpm_put(smmu); 1464 1465 dev_iommu_priv_set(dev, NULL); 1466 kfree(cfg); 1467 iommu_fwspec_free(dev); 1468} 1469 1470static struct iommu_group *arm_smmu_device_group(struct device *dev) 1471{ 1472 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev); 1473 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 1474 struct arm_smmu_device *smmu = cfg->smmu; 1475 struct iommu_group *group = NULL; 1476 int i, idx; 1477 1478 for_each_cfg_sme(cfg, fwspec, i, idx) { 1479 if (group && smmu->s2crs[idx].group && 1480 group != smmu->s2crs[idx].group) 1481 return ERR_PTR(-EINVAL); 1482 1483 group = smmu->s2crs[idx].group; 1484 } 1485 1486 if (group) 1487 return iommu_group_ref_get(group); 1488 1489 if (dev_is_pci(dev)) 1490 group = pci_device_group(dev); 1491 else if (dev_is_fsl_mc(dev)) 1492 group = fsl_mc_device_group(dev); 1493 else 1494 group = generic_device_group(dev); 1495 1496 /* Remember group for faster lookups */ 1497 if (!IS_ERR(group)) 1498 for_each_cfg_sme(cfg, fwspec, i, idx) 1499 smmu->s2crs[idx].group = group; 1500 1501 return group; 1502} 1503 1504static int arm_smmu_domain_get_attr(struct iommu_domain *domain, 1505 enum iommu_attr attr, void *data) 1506{ 1507 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1508 1509 switch(domain->type) { 1510 case IOMMU_DOMAIN_UNMANAGED: 1511 switch (attr) { 1512 case DOMAIN_ATTR_NESTING: 1513 *(int *)data = (smmu_domain->stage == ARM_SMMU_DOMAIN_NESTED); 1514 return 0; 1515 default: 1516 return -ENODEV; 1517 } 1518 break; 1519 case IOMMU_DOMAIN_DMA: 1520 switch (attr) { 1521 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE: 1522 *(int *)data = smmu_domain->non_strict; 1523 return 0; 1524 default: 1525 return -ENODEV; 1526 } 1527 break; 1528 default: 1529 return -EINVAL; 1530 } 1531} 1532 1533static int arm_smmu_domain_set_attr(struct iommu_domain *domain, 1534 enum iommu_attr attr, void *data) 1535{ 1536 int ret = 0; 1537 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1538 1539 mutex_lock(&smmu_domain->init_mutex); 1540 1541 switch(domain->type) { 1542 case IOMMU_DOMAIN_UNMANAGED: 1543 switch (attr) { 1544 case DOMAIN_ATTR_NESTING: 1545 if (smmu_domain->smmu) { 1546 ret = -EPERM; 1547 goto out_unlock; 1548 } 1549 1550 if (*(int *)data) 1551 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED; 1552 else 1553 smmu_domain->stage = ARM_SMMU_DOMAIN_S1; 1554 break; 1555 default: 1556 ret = -ENODEV; 1557 } 1558 break; 1559 case IOMMU_DOMAIN_DMA: 1560 switch (attr) { 1561 case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE: 1562 smmu_domain->non_strict = *(int *)data; 1563 break; 1564 default: 1565 ret = -ENODEV; 1566 } 1567 break; 1568 default: 1569 ret = -EINVAL; 1570 } 1571out_unlock: 1572 mutex_unlock(&smmu_domain->init_mutex); 1573 return ret; 1574} 1575 1576static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args) 1577{ 1578 u32 mask, fwid = 0; 1579 1580 if (args->args_count > 0) 1581 fwid |= FIELD_PREP(ARM_SMMU_SMR_ID, args->args[0]); 1582 1583 if (args->args_count > 1) 1584 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, args->args[1]); 1585 else if (!of_property_read_u32(args->np, "stream-match-mask", &mask)) 1586 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, mask); 1587 1588 return iommu_fwspec_add_ids(dev, &fwid, 1); 1589} 1590 1591static void arm_smmu_get_resv_regions(struct device *dev, 1592 struct list_head *head) 1593{ 1594 struct iommu_resv_region *region; 1595 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO; 1596 1597 region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH, 1598 prot, IOMMU_RESV_SW_MSI); 1599 if (!region) 1600 return; 1601 1602 list_add_tail(®ion->list, head); 1603 1604 iommu_dma_get_resv_regions(dev, head); 1605} 1606 1607static int arm_smmu_def_domain_type(struct device *dev) 1608{ 1609 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev); 1610 const struct arm_smmu_impl *impl = cfg->smmu->impl; 1611 1612 if (impl && impl->def_domain_type) 1613 return impl->def_domain_type(dev); 1614 1615 return 0; 1616} 1617 1618static struct iommu_ops arm_smmu_ops = { 1619 .capable = arm_smmu_capable, 1620 .domain_alloc = arm_smmu_domain_alloc, 1621 .domain_free = arm_smmu_domain_free, 1622 .attach_dev = arm_smmu_attach_dev, 1623 .map = arm_smmu_map, 1624 .unmap = arm_smmu_unmap, 1625 .flush_iotlb_all = arm_smmu_flush_iotlb_all, 1626 .iotlb_sync = arm_smmu_iotlb_sync, 1627 .iova_to_phys = arm_smmu_iova_to_phys, 1628 .probe_device = arm_smmu_probe_device, 1629 .release_device = arm_smmu_release_device, 1630 .device_group = arm_smmu_device_group, 1631 .domain_get_attr = arm_smmu_domain_get_attr, 1632 .domain_set_attr = arm_smmu_domain_set_attr, 1633 .of_xlate = arm_smmu_of_xlate, 1634 .get_resv_regions = arm_smmu_get_resv_regions, 1635 .put_resv_regions = generic_iommu_put_resv_regions, 1636 .def_domain_type = arm_smmu_def_domain_type, 1637 .pgsize_bitmap = -1UL, /* Restricted during device attach */ 1638}; 1639 1640static void arm_smmu_device_reset(struct arm_smmu_device *smmu) 1641{ 1642 int i; 1643 u32 reg; 1644 1645 /* clear global FSR */ 1646 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR); 1647 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, reg); 1648 1649 /* 1650 * Reset stream mapping groups: Initial values mark all SMRn as 1651 * invalid and all S2CRn as bypass unless overridden. 1652 */ 1653 for (i = 0; i < smmu->num_mapping_groups; ++i) 1654 arm_smmu_write_sme(smmu, i); 1655 1656 /* Make sure all context banks are disabled and clear CB_FSR */ 1657 for (i = 0; i < smmu->num_context_banks; ++i) { 1658 arm_smmu_write_context_bank(smmu, i); 1659 arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, ARM_SMMU_FSR_FAULT); 1660 } 1661 1662 /* Invalidate the TLB, just in case */ 1663 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLH, QCOM_DUMMY_VAL); 1664 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLNSNH, QCOM_DUMMY_VAL); 1665 1666 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0); 1667 1668 /* Enable fault reporting */ 1669 reg |= (ARM_SMMU_sCR0_GFRE | ARM_SMMU_sCR0_GFIE | 1670 ARM_SMMU_sCR0_GCFGFRE | ARM_SMMU_sCR0_GCFGFIE); 1671 1672 /* Disable TLB broadcasting. */ 1673 reg |= (ARM_SMMU_sCR0_VMIDPNE | ARM_SMMU_sCR0_PTM); 1674 1675 /* Enable client access, handling unmatched streams as appropriate */ 1676 reg &= ~ARM_SMMU_sCR0_CLIENTPD; 1677 if (disable_bypass) 1678 reg |= ARM_SMMU_sCR0_USFCFG; 1679 else 1680 reg &= ~ARM_SMMU_sCR0_USFCFG; 1681 1682 /* Disable forced broadcasting */ 1683 reg &= ~ARM_SMMU_sCR0_FB; 1684 1685 /* Don't upgrade barriers */ 1686 reg &= ~(ARM_SMMU_sCR0_BSU); 1687 1688 if (smmu->features & ARM_SMMU_FEAT_VMID16) 1689 reg |= ARM_SMMU_sCR0_VMID16EN; 1690 1691 if (smmu->features & ARM_SMMU_FEAT_EXIDS) 1692 reg |= ARM_SMMU_sCR0_EXIDENABLE; 1693 1694 if (smmu->impl && smmu->impl->reset) 1695 smmu->impl->reset(smmu); 1696 1697 /* Push the button */ 1698 arm_smmu_tlb_sync_global(smmu); 1699 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg); 1700} 1701 1702static int arm_smmu_id_size_to_bits(int size) 1703{ 1704 switch (size) { 1705 case 0: 1706 return 32; 1707 case 1: 1708 return 36; 1709 case 2: 1710 return 40; 1711 case 3: 1712 return 42; 1713 case 4: 1714 return 44; 1715 case 5: 1716 default: 1717 return 48; 1718 } 1719} 1720 1721static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) 1722{ 1723 unsigned int size; 1724 u32 id; 1725 bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK; 1726 int i, ret; 1727 1728 dev_notice(smmu->dev, "probing hardware configuration...\n"); 1729 dev_notice(smmu->dev, "SMMUv%d with:\n", 1730 smmu->version == ARM_SMMU_V2 ? 2 : 1); 1731 1732 /* ID0 */ 1733 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID0); 1734 1735 /* Restrict available stages based on module parameter */ 1736 if (force_stage == 1) 1737 id &= ~(ARM_SMMU_ID0_S2TS | ARM_SMMU_ID0_NTS); 1738 else if (force_stage == 2) 1739 id &= ~(ARM_SMMU_ID0_S1TS | ARM_SMMU_ID0_NTS); 1740 1741 if (id & ARM_SMMU_ID0_S1TS) { 1742 smmu->features |= ARM_SMMU_FEAT_TRANS_S1; 1743 dev_notice(smmu->dev, "\tstage 1 translation\n"); 1744 } 1745 1746 if (id & ARM_SMMU_ID0_S2TS) { 1747 smmu->features |= ARM_SMMU_FEAT_TRANS_S2; 1748 dev_notice(smmu->dev, "\tstage 2 translation\n"); 1749 } 1750 1751 if (id & ARM_SMMU_ID0_NTS) { 1752 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED; 1753 dev_notice(smmu->dev, "\tnested translation\n"); 1754 } 1755 1756 if (!(smmu->features & 1757 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) { 1758 dev_err(smmu->dev, "\tno translation support!\n"); 1759 return -ENODEV; 1760 } 1761 1762 if ((id & ARM_SMMU_ID0_S1TS) && 1763 ((smmu->version < ARM_SMMU_V2) || !(id & ARM_SMMU_ID0_ATOSNS))) { 1764 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS; 1765 dev_notice(smmu->dev, "\taddress translation ops\n"); 1766 } 1767 1768 /* 1769 * In order for DMA API calls to work properly, we must defer to what 1770 * the FW says about coherency, regardless of what the hardware claims. 1771 * Fortunately, this also opens up a workaround for systems where the 1772 * ID register value has ended up configured incorrectly. 1773 */ 1774 cttw_reg = !!(id & ARM_SMMU_ID0_CTTW); 1775 if (cttw_fw || cttw_reg) 1776 dev_notice(smmu->dev, "\t%scoherent table walk\n", 1777 cttw_fw ? "" : "non-"); 1778 if (cttw_fw != cttw_reg) 1779 dev_notice(smmu->dev, 1780 "\t(IDR0.CTTW overridden by FW configuration)\n"); 1781 1782 /* Max. number of entries we have for stream matching/indexing */ 1783 if (smmu->version == ARM_SMMU_V2 && id & ARM_SMMU_ID0_EXIDS) { 1784 smmu->features |= ARM_SMMU_FEAT_EXIDS; 1785 size = 1 << 16; 1786 } else { 1787 size = 1 << FIELD_GET(ARM_SMMU_ID0_NUMSIDB, id); 1788 } 1789 smmu->streamid_mask = size - 1; 1790 if (id & ARM_SMMU_ID0_SMS) { 1791 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH; 1792 size = FIELD_GET(ARM_SMMU_ID0_NUMSMRG, id); 1793 if (size == 0) { 1794 dev_err(smmu->dev, 1795 "stream-matching supported, but no SMRs present!\n"); 1796 return -ENODEV; 1797 } 1798 1799 /* Zero-initialised to mark as invalid */ 1800 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs), 1801 GFP_KERNEL); 1802 if (!smmu->smrs) 1803 return -ENOMEM; 1804 1805 dev_notice(smmu->dev, 1806 "\tstream matching with %u register groups", size); 1807 } 1808 /* s2cr->type == 0 means translation, so initialise explicitly */ 1809 smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs), 1810 GFP_KERNEL); 1811 if (!smmu->s2crs) 1812 return -ENOMEM; 1813 for (i = 0; i < size; i++) 1814 smmu->s2crs[i] = s2cr_init_val; 1815 1816 smmu->num_mapping_groups = size; 1817 mutex_init(&smmu->stream_map_mutex); 1818 spin_lock_init(&smmu->global_sync_lock); 1819 1820 if (smmu->version < ARM_SMMU_V2 || 1821 !(id & ARM_SMMU_ID0_PTFS_NO_AARCH32)) { 1822 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L; 1823 if (!(id & ARM_SMMU_ID0_PTFS_NO_AARCH32S)) 1824 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S; 1825 } 1826 1827 /* ID1 */ 1828 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID1); 1829 smmu->pgshift = (id & ARM_SMMU_ID1_PAGESIZE) ? 16 : 12; 1830 1831 /* Check for size mismatch of SMMU address space from mapped region */ 1832 size = 1 << (FIELD_GET(ARM_SMMU_ID1_NUMPAGENDXB, id) + 1); 1833 if (smmu->numpage != 2 * size << smmu->pgshift) 1834 dev_warn(smmu->dev, 1835 "SMMU address space size (0x%x) differs from mapped region size (0x%x)!\n", 1836 2 * size << smmu->pgshift, smmu->numpage); 1837 /* Now properly encode NUMPAGE to subsequently derive SMMU_CB_BASE */ 1838 smmu->numpage = size; 1839 1840 smmu->num_s2_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMS2CB, id); 1841 smmu->num_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMCB, id); 1842 if (smmu->num_s2_context_banks > smmu->num_context_banks) { 1843 dev_err(smmu->dev, "impossible number of S2 context banks!\n"); 1844 return -ENODEV; 1845 } 1846 dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n", 1847 smmu->num_context_banks, smmu->num_s2_context_banks); 1848 smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks, 1849 sizeof(*smmu->cbs), GFP_KERNEL); 1850 if (!smmu->cbs) 1851 return -ENOMEM; 1852 1853 /* ID2 */ 1854 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID2); 1855 size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_IAS, id)); 1856 smmu->ipa_size = size; 1857 1858 /* The output mask is also applied for bypass */ 1859 size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_OAS, id)); 1860 smmu->pa_size = size; 1861 1862 if (id & ARM_SMMU_ID2_VMID16) 1863 smmu->features |= ARM_SMMU_FEAT_VMID16; 1864 1865 /* 1866 * What the page table walker can address actually depends on which 1867 * descriptor format is in use, but since a) we don't know that yet, 1868 * and b) it can vary per context bank, this will have to do... 1869 */ 1870 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size))) 1871 dev_warn(smmu->dev, 1872 "failed to set DMA mask for table walker\n"); 1873 1874 if (smmu->version < ARM_SMMU_V2) { 1875 smmu->va_size = smmu->ipa_size; 1876 if (smmu->version == ARM_SMMU_V1_64K) 1877 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K; 1878 } else { 1879 size = FIELD_GET(ARM_SMMU_ID2_UBS, id); 1880 smmu->va_size = arm_smmu_id_size_to_bits(size); 1881 if (id & ARM_SMMU_ID2_PTFS_4K) 1882 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K; 1883 if (id & ARM_SMMU_ID2_PTFS_16K) 1884 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K; 1885 if (id & ARM_SMMU_ID2_PTFS_64K) 1886 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K; 1887 } 1888 1889 if (smmu->impl && smmu->impl->cfg_probe) { 1890 ret = smmu->impl->cfg_probe(smmu); 1891 if (ret) 1892 return ret; 1893 } 1894 1895 /* Now we've corralled the various formats, what'll it do? */ 1896 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) 1897 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M; 1898 if (smmu->features & 1899 (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K)) 1900 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G; 1901 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K) 1902 smmu->pgsize_bitmap |= SZ_16K | SZ_32M; 1903 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K) 1904 smmu->pgsize_bitmap |= SZ_64K | SZ_512M; 1905 1906 if (arm_smmu_ops.pgsize_bitmap == -1UL) 1907 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap; 1908 else 1909 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap; 1910 dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n", 1911 smmu->pgsize_bitmap); 1912 1913 1914 if (smmu->features & ARM_SMMU_FEAT_TRANS_S1) 1915 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n", 1916 smmu->va_size, smmu->ipa_size); 1917 1918 if (smmu->features & ARM_SMMU_FEAT_TRANS_S2) 1919 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n", 1920 smmu->ipa_size, smmu->pa_size); 1921 1922 return 0; 1923} 1924 1925struct arm_smmu_match_data { 1926 enum arm_smmu_arch_version version; 1927 enum arm_smmu_implementation model; 1928}; 1929 1930#define ARM_SMMU_MATCH_DATA(name, ver, imp) \ 1931static const struct arm_smmu_match_data name = { .version = ver, .model = imp } 1932 1933ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU); 1934ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU); 1935ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU); 1936ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500); 1937ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2); 1938ARM_SMMU_MATCH_DATA(qcom_smmuv2, ARM_SMMU_V2, QCOM_SMMUV2); 1939 1940static const struct of_device_id arm_smmu_of_match[] = { 1941 { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 }, 1942 { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 }, 1943 { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 }, 1944 { .compatible = "arm,mmu-401", .data = &arm_mmu401 }, 1945 { .compatible = "arm,mmu-500", .data = &arm_mmu500 }, 1946 { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 }, 1947 { .compatible = "nvidia,smmu-500", .data = &arm_mmu500 }, 1948 { .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 }, 1949 { }, 1950}; 1951MODULE_DEVICE_TABLE(of, arm_smmu_of_match); 1952 1953#ifdef CONFIG_ACPI 1954static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu) 1955{ 1956 int ret = 0; 1957 1958 switch (model) { 1959 case ACPI_IORT_SMMU_V1: 1960 case ACPI_IORT_SMMU_CORELINK_MMU400: 1961 smmu->version = ARM_SMMU_V1; 1962 smmu->model = GENERIC_SMMU; 1963 break; 1964 case ACPI_IORT_SMMU_CORELINK_MMU401: 1965 smmu->version = ARM_SMMU_V1_64K; 1966 smmu->model = GENERIC_SMMU; 1967 break; 1968 case ACPI_IORT_SMMU_V2: 1969 smmu->version = ARM_SMMU_V2; 1970 smmu->model = GENERIC_SMMU; 1971 break; 1972 case ACPI_IORT_SMMU_CORELINK_MMU500: 1973 smmu->version = ARM_SMMU_V2; 1974 smmu->model = ARM_MMU500; 1975 break; 1976 case ACPI_IORT_SMMU_CAVIUM_THUNDERX: 1977 smmu->version = ARM_SMMU_V2; 1978 smmu->model = CAVIUM_SMMUV2; 1979 break; 1980 default: 1981 ret = -ENODEV; 1982 } 1983 1984 return ret; 1985} 1986 1987static int arm_smmu_device_acpi_probe(struct platform_device *pdev, 1988 struct arm_smmu_device *smmu) 1989{ 1990 struct device *dev = smmu->dev; 1991 struct acpi_iort_node *node = 1992 *(struct acpi_iort_node **)dev_get_platdata(dev); 1993 struct acpi_iort_smmu *iort_smmu; 1994 int ret; 1995 1996 /* Retrieve SMMU1/2 specific data */ 1997 iort_smmu = (struct acpi_iort_smmu *)node->node_data; 1998 1999 ret = acpi_smmu_get_data(iort_smmu->model, smmu); 2000 if (ret < 0) 2001 return ret; 2002 2003 /* Ignore the configuration access interrupt */ 2004 smmu->num_global_irqs = 1; 2005 2006 if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK) 2007 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK; 2008 2009 return 0; 2010} 2011#else 2012static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev, 2013 struct arm_smmu_device *smmu) 2014{ 2015 return -ENODEV; 2016} 2017#endif 2018 2019static int arm_smmu_device_dt_probe(struct platform_device *pdev, 2020 struct arm_smmu_device *smmu) 2021{ 2022 const struct arm_smmu_match_data *data; 2023 struct device *dev = &pdev->dev; 2024 bool legacy_binding; 2025 2026 if (of_property_read_u32(dev->of_node, "#global-interrupts", 2027 &smmu->num_global_irqs)) { 2028 dev_err(dev, "missing #global-interrupts property\n"); 2029 return -ENODEV; 2030 } 2031 2032 data = of_device_get_match_data(dev); 2033 smmu->version = data->version; 2034 smmu->model = data->model; 2035 2036 legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL); 2037 if (legacy_binding && !using_generic_binding) { 2038 if (!using_legacy_binding) { 2039 pr_notice("deprecated \"mmu-masters\" DT property in use; %s support unavailable\n", 2040 IS_ENABLED(CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS) ? "DMA API" : "SMMU"); 2041 } 2042 using_legacy_binding = true; 2043 } else if (!legacy_binding && !using_legacy_binding) { 2044 using_generic_binding = true; 2045 } else { 2046 dev_err(dev, "not probing due to mismatched DT properties\n"); 2047 return -ENODEV; 2048 } 2049 2050 if (of_dma_is_coherent(dev->of_node)) 2051 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK; 2052 2053 return 0; 2054} 2055 2056static int arm_smmu_bus_init(struct iommu_ops *ops) 2057{ 2058 int err; 2059 2060 /* Oh, for a proper bus abstraction */ 2061 if (!iommu_present(&platform_bus_type)) { 2062 err = bus_set_iommu(&platform_bus_type, ops); 2063 if (err) 2064 return err; 2065 } 2066#ifdef CONFIG_ARM_AMBA 2067 if (!iommu_present(&amba_bustype)) { 2068 err = bus_set_iommu(&amba_bustype, ops); 2069 if (err) 2070 goto err_reset_platform_ops; 2071 } 2072#endif 2073#ifdef CONFIG_PCI 2074 if (!iommu_present(&pci_bus_type)) { 2075 err = bus_set_iommu(&pci_bus_type, ops); 2076 if (err) 2077 goto err_reset_amba_ops; 2078 } 2079#endif 2080#ifdef CONFIG_FSL_MC_BUS 2081 if (!iommu_present(&fsl_mc_bus_type)) { 2082 err = bus_set_iommu(&fsl_mc_bus_type, ops); 2083 if (err) 2084 goto err_reset_pci_ops; 2085 } 2086#endif 2087 return 0; 2088 2089err_reset_pci_ops: __maybe_unused; 2090#ifdef CONFIG_PCI 2091 bus_set_iommu(&pci_bus_type, NULL); 2092#endif 2093err_reset_amba_ops: __maybe_unused; 2094#ifdef CONFIG_ARM_AMBA 2095 bus_set_iommu(&amba_bustype, NULL); 2096#endif 2097err_reset_platform_ops: __maybe_unused; 2098 bus_set_iommu(&platform_bus_type, NULL); 2099 return err; 2100} 2101 2102static int arm_smmu_device_probe(struct platform_device *pdev) 2103{ 2104 struct resource *res; 2105 resource_size_t ioaddr; 2106 struct arm_smmu_device *smmu; 2107 struct device *dev = &pdev->dev; 2108 int num_irqs, i, err; 2109 irqreturn_t (*global_fault)(int irq, void *dev); 2110 2111 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL); 2112 if (!smmu) { 2113 dev_err(dev, "failed to allocate arm_smmu_device\n"); 2114 return -ENOMEM; 2115 } 2116 smmu->dev = dev; 2117 2118 if (dev->of_node) 2119 err = arm_smmu_device_dt_probe(pdev, smmu); 2120 else 2121 err = arm_smmu_device_acpi_probe(pdev, smmu); 2122 2123 if (err) 2124 return err; 2125 2126 smmu->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res); 2127 if (IS_ERR(smmu->base)) 2128 return PTR_ERR(smmu->base); 2129 ioaddr = res->start; 2130 /* 2131 * The resource size should effectively match the value of SMMU_TOP; 2132 * stash that temporarily until we know PAGESIZE to validate it with. 2133 */ 2134 smmu->numpage = resource_size(res); 2135 2136 smmu = arm_smmu_impl_init(smmu); 2137 if (IS_ERR(smmu)) 2138 return PTR_ERR(smmu); 2139 2140 num_irqs = 0; 2141 while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) { 2142 num_irqs++; 2143 if (num_irqs > smmu->num_global_irqs) 2144 smmu->num_context_irqs++; 2145 } 2146 2147 if (!smmu->num_context_irqs) { 2148 dev_err(dev, "found %d interrupts but expected at least %d\n", 2149 num_irqs, smmu->num_global_irqs + 1); 2150 return -ENODEV; 2151 } 2152 2153 smmu->irqs = devm_kcalloc(dev, num_irqs, sizeof(*smmu->irqs), 2154 GFP_KERNEL); 2155 if (!smmu->irqs) { 2156 dev_err(dev, "failed to allocate %d irqs\n", num_irqs); 2157 return -ENOMEM; 2158 } 2159 2160 for (i = 0; i < num_irqs; ++i) { 2161 int irq = platform_get_irq(pdev, i); 2162 2163 if (irq < 0) 2164 return -ENODEV; 2165 smmu->irqs[i] = irq; 2166 } 2167 2168 err = devm_clk_bulk_get_all(dev, &smmu->clks); 2169 if (err < 0) { 2170 dev_err(dev, "failed to get clocks %d\n", err); 2171 return err; 2172 } 2173 smmu->num_clks = err; 2174 2175 err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks); 2176 if (err) 2177 return err; 2178 2179 err = arm_smmu_device_cfg_probe(smmu); 2180 if (err) 2181 return err; 2182 2183 if (smmu->version == ARM_SMMU_V2) { 2184 if (smmu->num_context_banks > smmu->num_context_irqs) { 2185 dev_err(dev, 2186 "found only %d context irq(s) but %d required\n", 2187 smmu->num_context_irqs, smmu->num_context_banks); 2188 return -ENODEV; 2189 } 2190 2191 /* Ignore superfluous interrupts */ 2192 smmu->num_context_irqs = smmu->num_context_banks; 2193 } 2194 2195 if (smmu->impl && smmu->impl->global_fault) 2196 global_fault = smmu->impl->global_fault; 2197 else 2198 global_fault = arm_smmu_global_fault; 2199 2200 for (i = 0; i < smmu->num_global_irqs; ++i) { 2201 err = devm_request_irq(smmu->dev, smmu->irqs[i], 2202 global_fault, 2203 IRQF_SHARED, 2204 "arm-smmu global fault", 2205 smmu); 2206 if (err) { 2207 dev_err(dev, "failed to request global IRQ %d (%u)\n", 2208 i, smmu->irqs[i]); 2209 return err; 2210 } 2211 } 2212 2213 err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL, 2214 "smmu.%pa", &ioaddr); 2215 if (err) { 2216 dev_err(dev, "Failed to register iommu in sysfs\n"); 2217 return err; 2218 } 2219 2220 iommu_device_set_ops(&smmu->iommu, &arm_smmu_ops); 2221 iommu_device_set_fwnode(&smmu->iommu, dev->fwnode); 2222 2223 err = iommu_device_register(&smmu->iommu); 2224 if (err) { 2225 dev_err(dev, "Failed to register iommu\n"); 2226 return err; 2227 } 2228 2229 platform_set_drvdata(pdev, smmu); 2230 arm_smmu_device_reset(smmu); 2231 arm_smmu_test_smr_masks(smmu); 2232 2233 /* 2234 * We want to avoid touching dev->power.lock in fastpaths unless 2235 * it's really going to do something useful - pm_runtime_enabled() 2236 * can serve as an ideal proxy for that decision. So, conditionally 2237 * enable pm_runtime. 2238 */ 2239 if (dev->pm_domain) { 2240 pm_runtime_set_active(dev); 2241 pm_runtime_enable(dev); 2242 } 2243 2244 /* 2245 * For ACPI and generic DT bindings, an SMMU will be probed before 2246 * any device which might need it, so we want the bus ops in place 2247 * ready to handle default domain setup as soon as any SMMU exists. 2248 */ 2249 if (!using_legacy_binding) 2250 return arm_smmu_bus_init(&arm_smmu_ops); 2251 2252 return 0; 2253} 2254 2255static int arm_smmu_device_remove(struct platform_device *pdev) 2256{ 2257 struct arm_smmu_device *smmu = platform_get_drvdata(pdev); 2258 2259 if (!smmu) 2260 return -ENODEV; 2261 2262 if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS)) 2263 dev_notice(&pdev->dev, "disabling translation\n"); 2264 2265 arm_smmu_bus_init(NULL); 2266 iommu_device_unregister(&smmu->iommu); 2267 iommu_device_sysfs_remove(&smmu->iommu); 2268 2269 arm_smmu_rpm_get(smmu); 2270 /* Turn the thing off */ 2271 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, ARM_SMMU_sCR0_CLIENTPD); 2272 arm_smmu_rpm_put(smmu); 2273 2274 if (pm_runtime_enabled(smmu->dev)) 2275 pm_runtime_force_suspend(smmu->dev); 2276 else 2277 clk_bulk_disable(smmu->num_clks, smmu->clks); 2278 2279 clk_bulk_unprepare(smmu->num_clks, smmu->clks); 2280 return 0; 2281} 2282 2283static void arm_smmu_device_shutdown(struct platform_device *pdev) 2284{ 2285 arm_smmu_device_remove(pdev); 2286} 2287 2288static int __maybe_unused arm_smmu_runtime_resume(struct device *dev) 2289{ 2290 struct arm_smmu_device *smmu = dev_get_drvdata(dev); 2291 int ret; 2292 2293 ret = clk_bulk_enable(smmu->num_clks, smmu->clks); 2294 if (ret) 2295 return ret; 2296 2297 arm_smmu_device_reset(smmu); 2298 2299 return 0; 2300} 2301 2302static int __maybe_unused arm_smmu_runtime_suspend(struct device *dev) 2303{ 2304 struct arm_smmu_device *smmu = dev_get_drvdata(dev); 2305 2306 clk_bulk_disable(smmu->num_clks, smmu->clks); 2307 2308 return 0; 2309} 2310 2311static int __maybe_unused arm_smmu_pm_resume(struct device *dev) 2312{ 2313 if (pm_runtime_suspended(dev)) 2314 return 0; 2315 2316 return arm_smmu_runtime_resume(dev); 2317} 2318 2319static int __maybe_unused arm_smmu_pm_suspend(struct device *dev) 2320{ 2321 if (pm_runtime_suspended(dev)) 2322 return 0; 2323 2324 return arm_smmu_runtime_suspend(dev); 2325} 2326 2327static const struct dev_pm_ops arm_smmu_pm_ops = { 2328 SET_SYSTEM_SLEEP_PM_OPS(arm_smmu_pm_suspend, arm_smmu_pm_resume) 2329 SET_RUNTIME_PM_OPS(arm_smmu_runtime_suspend, 2330 arm_smmu_runtime_resume, NULL) 2331}; 2332 2333static struct platform_driver arm_smmu_driver = { 2334 .driver = { 2335 .name = "arm-smmu", 2336 .of_match_table = arm_smmu_of_match, 2337 .pm = &arm_smmu_pm_ops, 2338 .suppress_bind_attrs = true, 2339 }, 2340 .probe = arm_smmu_device_probe, 2341 .remove = arm_smmu_device_remove, 2342 .shutdown = arm_smmu_device_shutdown, 2343}; 2344module_platform_driver(arm_smmu_driver); 2345 2346MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations"); 2347MODULE_AUTHOR("Will Deacon <will@kernel.org>"); 2348MODULE_ALIAS("platform:arm-smmu"); 2349MODULE_LICENSE("GPL v2"); 2350