1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * IOMMU API for ARM architected SMMU implementations. 4 * 5 * Copyright (C) 2013 ARM Limited 6 * 7 * Author: Will Deacon <will.deacon@arm.com> 8 * 9 * This driver currently supports: 10 * - SMMUv1 and v2 implementations 11 * - Stream-matching and stream-indexing 12 * - v7/v8 long-descriptor format 13 * - Non-secure access to the SMMU 14 * - Context fault reporting 15 * - Extended Stream ID (16 bit) 16 */ 17 18#define pr_fmt(fmt) "arm-smmu: " fmt 19 20#include <linux/acpi.h> 21#include <linux/acpi_iort.h> 22#include <linux/bitfield.h> 23#include <linux/delay.h> 24#include <linux/dma-mapping.h> 25#include <linux/err.h> 26#include <linux/interrupt.h> 27#include <linux/io.h> 28#include <linux/iopoll.h> 29#include <linux/module.h> 30#include <linux/of.h> 31#include <linux/of_address.h> 32#include <linux/pci.h> 33#include <linux/platform_device.h> 34#include <linux/pm_runtime.h> 35#include <linux/ratelimit.h> 36#include <linux/slab.h> 37 38#include <linux/fsl/mc.h> 39 40#include "arm-smmu.h" 41#include "../../dma-iommu.h" 42 43/* 44 * Apparently, some Qualcomm arm64 platforms which appear to expose their SMMU 45 * global register space are still, in fact, using a hypervisor to mediate it 46 * by trapping and emulating register accesses. Sadly, some deployed versions 47 * of said trapping code have bugs wherein they go horribly wrong for stores 48 * using r31 (i.e. XZR/WZR) as the source register. 49 */ 50#define QCOM_DUMMY_VAL -1 51 52#define MSI_IOVA_BASE 0x8000000 53#define MSI_IOVA_LENGTH 0x100000 54 55static int force_stage; 56module_param(force_stage, int, S_IRUGO); 57MODULE_PARM_DESC(force_stage, 58 "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation."); 59static bool disable_bypass = 60 IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT); 61module_param(disable_bypass, bool, S_IRUGO); 62MODULE_PARM_DESC(disable_bypass, 63 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU."); 64 65#define s2cr_init_val (struct arm_smmu_s2cr){ \ 66 .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS, \ 67} 68 69static bool using_legacy_binding, using_generic_binding; 70 71static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu) 72{ 73 if (pm_runtime_enabled(smmu->dev)) 74 return pm_runtime_resume_and_get(smmu->dev); 75 76 return 0; 77} 78 79static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu) 80{ 81 if (pm_runtime_enabled(smmu->dev)) 82 pm_runtime_put_autosuspend(smmu->dev); 83} 84 85static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom) 86{ 87 return container_of(dom, struct arm_smmu_domain, domain); 88} 89 90static struct platform_driver arm_smmu_driver; 91static struct iommu_ops arm_smmu_ops; 92 93#ifdef CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS 94static struct device_node *dev_get_dev_node(struct device *dev) 95{ 96 if (dev_is_pci(dev)) { 97 struct pci_bus *bus = to_pci_dev(dev)->bus; 98 99 while (!pci_is_root_bus(bus)) 100 bus = bus->parent; 101 return of_node_get(bus->bridge->parent->of_node); 102 } 103 104 return of_node_get(dev->of_node); 105} 106 107static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data) 108{ 109 *((__be32 *)data) = cpu_to_be32(alias); 110 return 0; /* Continue walking */ 111} 112 113static int __find_legacy_master_phandle(struct device *dev, void *data) 114{ 115 struct of_phandle_iterator *it = *(void **)data; 116 struct device_node *np = it->node; 117 int err; 118 119 of_for_each_phandle(it, err, dev->of_node, "mmu-masters", 120 "#stream-id-cells", -1) 121 if (it->node == np) { 122 *(void **)data = dev; 123 return 1; 124 } 125 it->node = np; 126 return err == -ENOENT ? 0 : err; 127} 128 129static int arm_smmu_register_legacy_master(struct device *dev, 130 struct arm_smmu_device **smmu) 131{ 132 struct device *smmu_dev; 133 struct device_node *np; 134 struct of_phandle_iterator it; 135 void *data = ⁢ 136 u32 *sids; 137 __be32 pci_sid; 138 int err; 139 140 np = dev_get_dev_node(dev); 141 if (!np || !of_property_present(np, "#stream-id-cells")) { 142 of_node_put(np); 143 return -ENODEV; 144 } 145 146 it.node = np; 147 err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data, 148 __find_legacy_master_phandle); 149 smmu_dev = data; 150 of_node_put(np); 151 if (err == 0) 152 return -ENODEV; 153 if (err < 0) 154 return err; 155 156 if (dev_is_pci(dev)) { 157 /* "mmu-masters" assumes Stream ID == Requester ID */ 158 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid, 159 &pci_sid); 160 it.cur = &pci_sid; 161 it.cur_count = 1; 162 } 163 164 err = iommu_fwspec_init(dev, &smmu_dev->of_node->fwnode, 165 &arm_smmu_ops); 166 if (err) 167 return err; 168 169 sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL); 170 if (!sids) 171 return -ENOMEM; 172 173 *smmu = dev_get_drvdata(smmu_dev); 174 of_phandle_iterator_args(&it, sids, it.cur_count); 175 err = iommu_fwspec_add_ids(dev, sids, it.cur_count); 176 kfree(sids); 177 return err; 178} 179#else 180static int arm_smmu_register_legacy_master(struct device *dev, 181 struct arm_smmu_device **smmu) 182{ 183 return -ENODEV; 184} 185#endif /* CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS */ 186 187static void __arm_smmu_free_bitmap(unsigned long *map, int idx) 188{ 189 clear_bit(idx, map); 190} 191 192/* Wait for any pending TLB invalidations to complete */ 193static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page, 194 int sync, int status) 195{ 196 unsigned int spin_cnt, delay; 197 u32 reg; 198 199 if (smmu->impl && unlikely(smmu->impl->tlb_sync)) 200 return smmu->impl->tlb_sync(smmu, page, sync, status); 201 202 arm_smmu_writel(smmu, page, sync, QCOM_DUMMY_VAL); 203 for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) { 204 for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) { 205 reg = arm_smmu_readl(smmu, page, status); 206 if (!(reg & ARM_SMMU_sTLBGSTATUS_GSACTIVE)) 207 return; 208 cpu_relax(); 209 } 210 udelay(delay); 211 } 212 dev_err_ratelimited(smmu->dev, 213 "TLB sync timed out -- SMMU may be deadlocked\n"); 214} 215 216static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu) 217{ 218 unsigned long flags; 219 220 spin_lock_irqsave(&smmu->global_sync_lock, flags); 221 __arm_smmu_tlb_sync(smmu, ARM_SMMU_GR0, ARM_SMMU_GR0_sTLBGSYNC, 222 ARM_SMMU_GR0_sTLBGSTATUS); 223 spin_unlock_irqrestore(&smmu->global_sync_lock, flags); 224} 225 226static void arm_smmu_tlb_sync_context(struct arm_smmu_domain *smmu_domain) 227{ 228 struct arm_smmu_device *smmu = smmu_domain->smmu; 229 unsigned long flags; 230 231 spin_lock_irqsave(&smmu_domain->cb_lock, flags); 232 __arm_smmu_tlb_sync(smmu, ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx), 233 ARM_SMMU_CB_TLBSYNC, ARM_SMMU_CB_TLBSTATUS); 234 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); 235} 236 237static void arm_smmu_tlb_inv_context_s1(void *cookie) 238{ 239 struct arm_smmu_domain *smmu_domain = cookie; 240 /* 241 * The TLBI write may be relaxed, so ensure that PTEs cleared by the 242 * current CPU are visible beforehand. 243 */ 244 wmb(); 245 arm_smmu_cb_write(smmu_domain->smmu, smmu_domain->cfg.cbndx, 246 ARM_SMMU_CB_S1_TLBIASID, smmu_domain->cfg.asid); 247 arm_smmu_tlb_sync_context(smmu_domain); 248} 249 250static void arm_smmu_tlb_inv_context_s2(void *cookie) 251{ 252 struct arm_smmu_domain *smmu_domain = cookie; 253 struct arm_smmu_device *smmu = smmu_domain->smmu; 254 255 /* See above */ 256 wmb(); 257 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid); 258 arm_smmu_tlb_sync_global(smmu); 259} 260 261static void arm_smmu_tlb_inv_range_s1(unsigned long iova, size_t size, 262 size_t granule, void *cookie, int reg) 263{ 264 struct arm_smmu_domain *smmu_domain = cookie; 265 struct arm_smmu_device *smmu = smmu_domain->smmu; 266 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 267 int idx = cfg->cbndx; 268 269 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) 270 wmb(); 271 272 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) { 273 iova = (iova >> 12) << 12; 274 iova |= cfg->asid; 275 do { 276 arm_smmu_cb_write(smmu, idx, reg, iova); 277 iova += granule; 278 } while (size -= granule); 279 } else { 280 iova >>= 12; 281 iova |= (u64)cfg->asid << 48; 282 do { 283 arm_smmu_cb_writeq(smmu, idx, reg, iova); 284 iova += granule >> 12; 285 } while (size -= granule); 286 } 287} 288 289static void arm_smmu_tlb_inv_range_s2(unsigned long iova, size_t size, 290 size_t granule, void *cookie, int reg) 291{ 292 struct arm_smmu_domain *smmu_domain = cookie; 293 struct arm_smmu_device *smmu = smmu_domain->smmu; 294 int idx = smmu_domain->cfg.cbndx; 295 296 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) 297 wmb(); 298 299 iova >>= 12; 300 do { 301 if (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64) 302 arm_smmu_cb_writeq(smmu, idx, reg, iova); 303 else 304 arm_smmu_cb_write(smmu, idx, reg, iova); 305 iova += granule >> 12; 306 } while (size -= granule); 307} 308 309static void arm_smmu_tlb_inv_walk_s1(unsigned long iova, size_t size, 310 size_t granule, void *cookie) 311{ 312 struct arm_smmu_domain *smmu_domain = cookie; 313 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 314 315 if (cfg->flush_walk_prefer_tlbiasid) { 316 arm_smmu_tlb_inv_context_s1(cookie); 317 } else { 318 arm_smmu_tlb_inv_range_s1(iova, size, granule, cookie, 319 ARM_SMMU_CB_S1_TLBIVA); 320 arm_smmu_tlb_sync_context(cookie); 321 } 322} 323 324static void arm_smmu_tlb_add_page_s1(struct iommu_iotlb_gather *gather, 325 unsigned long iova, size_t granule, 326 void *cookie) 327{ 328 arm_smmu_tlb_inv_range_s1(iova, granule, granule, cookie, 329 ARM_SMMU_CB_S1_TLBIVAL); 330} 331 332static void arm_smmu_tlb_inv_walk_s2(unsigned long iova, size_t size, 333 size_t granule, void *cookie) 334{ 335 arm_smmu_tlb_inv_range_s2(iova, size, granule, cookie, 336 ARM_SMMU_CB_S2_TLBIIPAS2); 337 arm_smmu_tlb_sync_context(cookie); 338} 339 340static void arm_smmu_tlb_add_page_s2(struct iommu_iotlb_gather *gather, 341 unsigned long iova, size_t granule, 342 void *cookie) 343{ 344 arm_smmu_tlb_inv_range_s2(iova, granule, granule, cookie, 345 ARM_SMMU_CB_S2_TLBIIPAS2L); 346} 347 348static void arm_smmu_tlb_inv_walk_s2_v1(unsigned long iova, size_t size, 349 size_t granule, void *cookie) 350{ 351 arm_smmu_tlb_inv_context_s2(cookie); 352} 353/* 354 * On MMU-401 at least, the cost of firing off multiple TLBIVMIDs appears 355 * almost negligible, but the benefit of getting the first one in as far ahead 356 * of the sync as possible is significant, hence we don't just make this a 357 * no-op and call arm_smmu_tlb_inv_context_s2() from .iotlb_sync as you might 358 * think. 359 */ 360static void arm_smmu_tlb_add_page_s2_v1(struct iommu_iotlb_gather *gather, 361 unsigned long iova, size_t granule, 362 void *cookie) 363{ 364 struct arm_smmu_domain *smmu_domain = cookie; 365 struct arm_smmu_device *smmu = smmu_domain->smmu; 366 367 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK) 368 wmb(); 369 370 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid); 371} 372 373static const struct iommu_flush_ops arm_smmu_s1_tlb_ops = { 374 .tlb_flush_all = arm_smmu_tlb_inv_context_s1, 375 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s1, 376 .tlb_add_page = arm_smmu_tlb_add_page_s1, 377}; 378 379static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v2 = { 380 .tlb_flush_all = arm_smmu_tlb_inv_context_s2, 381 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2, 382 .tlb_add_page = arm_smmu_tlb_add_page_s2, 383}; 384 385static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v1 = { 386 .tlb_flush_all = arm_smmu_tlb_inv_context_s2, 387 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2_v1, 388 .tlb_add_page = arm_smmu_tlb_add_page_s2_v1, 389}; 390 391static irqreturn_t arm_smmu_context_fault(int irq, void *dev) 392{ 393 u32 fsr, fsynr, cbfrsynra; 394 unsigned long iova; 395 struct iommu_domain *domain = dev; 396 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 397 struct arm_smmu_device *smmu = smmu_domain->smmu; 398 int idx = smmu_domain->cfg.cbndx; 399 int ret; 400 401 fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR); 402 if (!(fsr & ARM_SMMU_FSR_FAULT)) 403 return IRQ_NONE; 404 405 fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0); 406 iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR); 407 cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx)); 408 409 ret = report_iommu_fault(domain, NULL, iova, 410 fsynr & ARM_SMMU_FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ); 411 412 if (ret == -ENOSYS) 413 dev_err_ratelimited(smmu->dev, 414 "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n", 415 fsr, iova, fsynr, cbfrsynra, idx); 416 417 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr); 418 return IRQ_HANDLED; 419} 420 421static irqreturn_t arm_smmu_global_fault(int irq, void *dev) 422{ 423 u32 gfsr, gfsynr0, gfsynr1, gfsynr2; 424 struct arm_smmu_device *smmu = dev; 425 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL, 426 DEFAULT_RATELIMIT_BURST); 427 428 gfsr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR); 429 gfsynr0 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR0); 430 gfsynr1 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR1); 431 gfsynr2 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR2); 432 433 if (!gfsr) 434 return IRQ_NONE; 435 436 if (__ratelimit(&rs)) { 437 if (IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT) && 438 (gfsr & ARM_SMMU_sGFSR_USF)) 439 dev_err(smmu->dev, 440 "Blocked unknown Stream ID 0x%hx; boot with \"arm-smmu.disable_bypass=0\" to allow, but this may have security implications\n", 441 (u16)gfsynr1); 442 else 443 dev_err(smmu->dev, 444 "Unexpected global fault, this could be serious\n"); 445 dev_err(smmu->dev, 446 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n", 447 gfsr, gfsynr0, gfsynr1, gfsynr2); 448 } 449 450 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, gfsr); 451 return IRQ_HANDLED; 452} 453 454static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain, 455 struct io_pgtable_cfg *pgtbl_cfg) 456{ 457 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 458 struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx]; 459 bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS; 460 461 cb->cfg = cfg; 462 463 /* TCR */ 464 if (stage1) { 465 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) { 466 cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr; 467 } else { 468 cb->tcr[0] = arm_smmu_lpae_tcr(pgtbl_cfg); 469 cb->tcr[1] = arm_smmu_lpae_tcr2(pgtbl_cfg); 470 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) 471 cb->tcr[1] |= ARM_SMMU_TCR2_AS; 472 else 473 cb->tcr[0] |= ARM_SMMU_TCR_EAE; 474 } 475 } else { 476 cb->tcr[0] = arm_smmu_lpae_vtcr(pgtbl_cfg); 477 } 478 479 /* TTBRs */ 480 if (stage1) { 481 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) { 482 cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr; 483 cb->ttbr[1] = 0; 484 } else { 485 cb->ttbr[0] = FIELD_PREP(ARM_SMMU_TTBRn_ASID, 486 cfg->asid); 487 cb->ttbr[1] = FIELD_PREP(ARM_SMMU_TTBRn_ASID, 488 cfg->asid); 489 490 if (pgtbl_cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) 491 cb->ttbr[1] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr; 492 else 493 cb->ttbr[0] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr; 494 } 495 } else { 496 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr; 497 } 498 499 /* MAIRs (stage-1 only) */ 500 if (stage1) { 501 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) { 502 cb->mair[0] = pgtbl_cfg->arm_v7s_cfg.prrr; 503 cb->mair[1] = pgtbl_cfg->arm_v7s_cfg.nmrr; 504 } else { 505 cb->mair[0] = pgtbl_cfg->arm_lpae_s1_cfg.mair; 506 cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair >> 32; 507 } 508 } 509} 510 511void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx) 512{ 513 u32 reg; 514 bool stage1; 515 struct arm_smmu_cb *cb = &smmu->cbs[idx]; 516 struct arm_smmu_cfg *cfg = cb->cfg; 517 518 /* Unassigned context banks only need disabling */ 519 if (!cfg) { 520 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, 0); 521 return; 522 } 523 524 stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS; 525 526 /* CBA2R */ 527 if (smmu->version > ARM_SMMU_V1) { 528 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) 529 reg = ARM_SMMU_CBA2R_VA64; 530 else 531 reg = 0; 532 /* 16-bit VMIDs live in CBA2R */ 533 if (smmu->features & ARM_SMMU_FEAT_VMID16) 534 reg |= FIELD_PREP(ARM_SMMU_CBA2R_VMID16, cfg->vmid); 535 536 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBA2R(idx), reg); 537 } 538 539 /* CBAR */ 540 reg = FIELD_PREP(ARM_SMMU_CBAR_TYPE, cfg->cbar); 541 if (smmu->version < ARM_SMMU_V2) 542 reg |= FIELD_PREP(ARM_SMMU_CBAR_IRPTNDX, cfg->irptndx); 543 544 /* 545 * Use the weakest shareability/memory types, so they are 546 * overridden by the ttbcr/pte. 547 */ 548 if (stage1) { 549 reg |= FIELD_PREP(ARM_SMMU_CBAR_S1_BPSHCFG, 550 ARM_SMMU_CBAR_S1_BPSHCFG_NSH) | 551 FIELD_PREP(ARM_SMMU_CBAR_S1_MEMATTR, 552 ARM_SMMU_CBAR_S1_MEMATTR_WB); 553 } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) { 554 /* 8-bit VMIDs live in CBAR */ 555 reg |= FIELD_PREP(ARM_SMMU_CBAR_VMID, cfg->vmid); 556 } 557 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(idx), reg); 558 559 /* 560 * TCR 561 * We must write this before the TTBRs, since it determines the 562 * access behaviour of some fields (in particular, ASID[15:8]). 563 */ 564 if (stage1 && smmu->version > ARM_SMMU_V1) 565 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR2, cb->tcr[1]); 566 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR, cb->tcr[0]); 567 568 /* TTBRs */ 569 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) { 570 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_CONTEXTIDR, cfg->asid); 571 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]); 572 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR1, cb->ttbr[1]); 573 } else { 574 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]); 575 if (stage1) 576 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR1, 577 cb->ttbr[1]); 578 } 579 580 /* MAIRs (stage-1 only) */ 581 if (stage1) { 582 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR0, cb->mair[0]); 583 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR1, cb->mair[1]); 584 } 585 586 /* SCTLR */ 587 reg = ARM_SMMU_SCTLR_CFIE | ARM_SMMU_SCTLR_CFRE | ARM_SMMU_SCTLR_AFE | 588 ARM_SMMU_SCTLR_TRE | ARM_SMMU_SCTLR_M; 589 if (stage1) 590 reg |= ARM_SMMU_SCTLR_S1_ASIDPNE; 591 if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) 592 reg |= ARM_SMMU_SCTLR_E; 593 594 if (smmu->impl && smmu->impl->write_sctlr) 595 smmu->impl->write_sctlr(smmu, idx, reg); 596 else 597 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg); 598} 599 600static int arm_smmu_alloc_context_bank(struct arm_smmu_domain *smmu_domain, 601 struct arm_smmu_device *smmu, 602 struct device *dev, unsigned int start) 603{ 604 if (smmu->impl && smmu->impl->alloc_context_bank) 605 return smmu->impl->alloc_context_bank(smmu_domain, smmu, dev, start); 606 607 return __arm_smmu_alloc_bitmap(smmu->context_map, start, smmu->num_context_banks); 608} 609 610static int arm_smmu_init_domain_context(struct iommu_domain *domain, 611 struct arm_smmu_device *smmu, 612 struct device *dev) 613{ 614 int irq, start, ret = 0; 615 unsigned long ias, oas; 616 struct io_pgtable_ops *pgtbl_ops; 617 struct io_pgtable_cfg pgtbl_cfg; 618 enum io_pgtable_fmt fmt; 619 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 620 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 621 irqreturn_t (*context_fault)(int irq, void *dev); 622 623 mutex_lock(&smmu_domain->init_mutex); 624 if (smmu_domain->smmu) 625 goto out_unlock; 626 627 if (domain->type == IOMMU_DOMAIN_IDENTITY) { 628 smmu_domain->stage = ARM_SMMU_DOMAIN_BYPASS; 629 smmu_domain->smmu = smmu; 630 goto out_unlock; 631 } 632 633 /* 634 * Mapping the requested stage onto what we support is surprisingly 635 * complicated, mainly because the spec allows S1+S2 SMMUs without 636 * support for nested translation. That means we end up with the 637 * following table: 638 * 639 * Requested Supported Actual 640 * S1 N S1 641 * S1 S1+S2 S1 642 * S1 S2 S2 643 * S1 S1 S1 644 * N N N 645 * N S1+S2 S2 646 * N S2 S2 647 * N S1 S1 648 * 649 * Note that you can't actually request stage-2 mappings. 650 */ 651 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1)) 652 smmu_domain->stage = ARM_SMMU_DOMAIN_S2; 653 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2)) 654 smmu_domain->stage = ARM_SMMU_DOMAIN_S1; 655 656 /* 657 * Choosing a suitable context format is even more fiddly. Until we 658 * grow some way for the caller to express a preference, and/or move 659 * the decision into the io-pgtable code where it arguably belongs, 660 * just aim for the closest thing to the rest of the system, and hope 661 * that the hardware isn't esoteric enough that we can't assume AArch64 662 * support to be a superset of AArch32 support... 663 */ 664 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L) 665 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L; 666 if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) && 667 !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) && 668 (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) && 669 (smmu_domain->stage == ARM_SMMU_DOMAIN_S1)) 670 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S; 671 if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) && 672 (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K | 673 ARM_SMMU_FEAT_FMT_AARCH64_16K | 674 ARM_SMMU_FEAT_FMT_AARCH64_4K))) 675 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64; 676 677 if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) { 678 ret = -EINVAL; 679 goto out_unlock; 680 } 681 682 switch (smmu_domain->stage) { 683 case ARM_SMMU_DOMAIN_S1: 684 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS; 685 start = smmu->num_s2_context_banks; 686 ias = smmu->va_size; 687 oas = smmu->ipa_size; 688 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) { 689 fmt = ARM_64_LPAE_S1; 690 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) { 691 fmt = ARM_32_LPAE_S1; 692 ias = min(ias, 32UL); 693 oas = min(oas, 40UL); 694 } else { 695 fmt = ARM_V7S; 696 ias = min(ias, 32UL); 697 oas = min(oas, 32UL); 698 } 699 smmu_domain->flush_ops = &arm_smmu_s1_tlb_ops; 700 break; 701 case ARM_SMMU_DOMAIN_NESTED: 702 /* 703 * We will likely want to change this if/when KVM gets 704 * involved. 705 */ 706 case ARM_SMMU_DOMAIN_S2: 707 cfg->cbar = CBAR_TYPE_S2_TRANS; 708 start = 0; 709 ias = smmu->ipa_size; 710 oas = smmu->pa_size; 711 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) { 712 fmt = ARM_64_LPAE_S2; 713 } else { 714 fmt = ARM_32_LPAE_S2; 715 ias = min(ias, 40UL); 716 oas = min(oas, 40UL); 717 } 718 if (smmu->version == ARM_SMMU_V2) 719 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v2; 720 else 721 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v1; 722 break; 723 default: 724 ret = -EINVAL; 725 goto out_unlock; 726 } 727 728 ret = arm_smmu_alloc_context_bank(smmu_domain, smmu, dev, start); 729 if (ret < 0) { 730 goto out_unlock; 731 } 732 733 smmu_domain->smmu = smmu; 734 735 cfg->cbndx = ret; 736 if (smmu->version < ARM_SMMU_V2) { 737 cfg->irptndx = atomic_inc_return(&smmu->irptndx); 738 cfg->irptndx %= smmu->num_context_irqs; 739 } else { 740 cfg->irptndx = cfg->cbndx; 741 } 742 743 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2) 744 cfg->vmid = cfg->cbndx + 1; 745 else 746 cfg->asid = cfg->cbndx; 747 748 pgtbl_cfg = (struct io_pgtable_cfg) { 749 .pgsize_bitmap = smmu->pgsize_bitmap, 750 .ias = ias, 751 .oas = oas, 752 .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK, 753 .tlb = smmu_domain->flush_ops, 754 .iommu_dev = smmu->dev, 755 }; 756 757 if (smmu->impl && smmu->impl->init_context) { 758 ret = smmu->impl->init_context(smmu_domain, &pgtbl_cfg, dev); 759 if (ret) 760 goto out_clear_smmu; 761 } 762 763 if (smmu_domain->pgtbl_quirks) 764 pgtbl_cfg.quirks |= smmu_domain->pgtbl_quirks; 765 766 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain); 767 if (!pgtbl_ops) { 768 ret = -ENOMEM; 769 goto out_clear_smmu; 770 } 771 772 /* Update the domain's page sizes to reflect the page table format */ 773 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap; 774 775 if (pgtbl_cfg.quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) { 776 domain->geometry.aperture_start = ~0UL << ias; 777 domain->geometry.aperture_end = ~0UL; 778 } else { 779 domain->geometry.aperture_end = (1UL << ias) - 1; 780 } 781 782 domain->geometry.force_aperture = true; 783 784 /* Initialise the context bank with our page table cfg */ 785 arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg); 786 arm_smmu_write_context_bank(smmu, cfg->cbndx); 787 788 /* 789 * Request context fault interrupt. Do this last to avoid the 790 * handler seeing a half-initialised domain state. 791 */ 792 irq = smmu->irqs[cfg->irptndx]; 793 794 if (smmu->impl && smmu->impl->context_fault) 795 context_fault = smmu->impl->context_fault; 796 else 797 context_fault = arm_smmu_context_fault; 798 799 ret = devm_request_irq(smmu->dev, irq, context_fault, 800 IRQF_SHARED, "arm-smmu-context-fault", domain); 801 if (ret < 0) { 802 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n", 803 cfg->irptndx, irq); 804 cfg->irptndx = ARM_SMMU_INVALID_IRPTNDX; 805 } 806 807 mutex_unlock(&smmu_domain->init_mutex); 808 809 /* Publish page table ops for map/unmap */ 810 smmu_domain->pgtbl_ops = pgtbl_ops; 811 return 0; 812 813out_clear_smmu: 814 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx); 815 smmu_domain->smmu = NULL; 816out_unlock: 817 mutex_unlock(&smmu_domain->init_mutex); 818 return ret; 819} 820 821static void arm_smmu_destroy_domain_context(struct iommu_domain *domain) 822{ 823 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 824 struct arm_smmu_device *smmu = smmu_domain->smmu; 825 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 826 int ret, irq; 827 828 if (!smmu || domain->type == IOMMU_DOMAIN_IDENTITY) 829 return; 830 831 ret = arm_smmu_rpm_get(smmu); 832 if (ret < 0) 833 return; 834 835 /* 836 * Disable the context bank and free the page tables before freeing 837 * it. 838 */ 839 smmu->cbs[cfg->cbndx].cfg = NULL; 840 arm_smmu_write_context_bank(smmu, cfg->cbndx); 841 842 if (cfg->irptndx != ARM_SMMU_INVALID_IRPTNDX) { 843 irq = smmu->irqs[cfg->irptndx]; 844 devm_free_irq(smmu->dev, irq, domain); 845 } 846 847 free_io_pgtable_ops(smmu_domain->pgtbl_ops); 848 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx); 849 850 arm_smmu_rpm_put(smmu); 851} 852 853static struct iommu_domain *arm_smmu_domain_alloc(unsigned type) 854{ 855 struct arm_smmu_domain *smmu_domain; 856 857 if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_IDENTITY) { 858 if (using_legacy_binding || type != IOMMU_DOMAIN_DMA) 859 return NULL; 860 } 861 /* 862 * Allocate the domain and initialise some of its data structures. 863 * We can't really do anything meaningful until we've added a 864 * master. 865 */ 866 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL); 867 if (!smmu_domain) 868 return NULL; 869 870 mutex_init(&smmu_domain->init_mutex); 871 spin_lock_init(&smmu_domain->cb_lock); 872 873 return &smmu_domain->domain; 874} 875 876static void arm_smmu_domain_free(struct iommu_domain *domain) 877{ 878 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 879 880 /* 881 * Free the domain resources. We assume that all devices have 882 * already been detached. 883 */ 884 arm_smmu_destroy_domain_context(domain); 885 kfree(smmu_domain); 886} 887 888static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx) 889{ 890 struct arm_smmu_smr *smr = smmu->smrs + idx; 891 u32 reg = FIELD_PREP(ARM_SMMU_SMR_ID, smr->id) | 892 FIELD_PREP(ARM_SMMU_SMR_MASK, smr->mask); 893 894 if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid) 895 reg |= ARM_SMMU_SMR_VALID; 896 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(idx), reg); 897} 898 899static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx) 900{ 901 struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx; 902 u32 reg; 903 904 if (smmu->impl && smmu->impl->write_s2cr) { 905 smmu->impl->write_s2cr(smmu, idx); 906 return; 907 } 908 909 reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, s2cr->type) | 910 FIELD_PREP(ARM_SMMU_S2CR_CBNDX, s2cr->cbndx) | 911 FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg); 912 913 if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs && 914 smmu->smrs[idx].valid) 915 reg |= ARM_SMMU_S2CR_EXIDVALID; 916 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg); 917} 918 919static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx) 920{ 921 arm_smmu_write_s2cr(smmu, idx); 922 if (smmu->smrs) 923 arm_smmu_write_smr(smmu, idx); 924} 925 926/* 927 * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function 928 * should be called after sCR0 is written. 929 */ 930static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu) 931{ 932 u32 smr; 933 int i; 934 935 if (!smmu->smrs) 936 return; 937 /* 938 * If we've had to accommodate firmware memory regions, we may 939 * have live SMRs by now; tread carefully... 940 * 941 * Somewhat perversely, not having a free SMR for this test implies we 942 * can get away without it anyway, as we'll only be able to 'allocate' 943 * these SMRs for the ID/mask values we're already trusting to be OK. 944 */ 945 for (i = 0; i < smmu->num_mapping_groups; i++) 946 if (!smmu->smrs[i].valid) 947 goto smr_ok; 948 return; 949smr_ok: 950 /* 951 * SMR.ID bits may not be preserved if the corresponding MASK 952 * bits are set, so check each one separately. We can reject 953 * masters later if they try to claim IDs outside these masks. 954 */ 955 smr = FIELD_PREP(ARM_SMMU_SMR_ID, smmu->streamid_mask); 956 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr); 957 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i)); 958 smmu->streamid_mask = FIELD_GET(ARM_SMMU_SMR_ID, smr); 959 960 smr = FIELD_PREP(ARM_SMMU_SMR_MASK, smmu->streamid_mask); 961 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr); 962 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i)); 963 smmu->smr_mask_mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr); 964} 965 966static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask) 967{ 968 struct arm_smmu_smr *smrs = smmu->smrs; 969 int i, free_idx = -ENOSPC; 970 971 /* Stream indexing is blissfully easy */ 972 if (!smrs) 973 return id; 974 975 /* Validating SMRs is... less so */ 976 for (i = 0; i < smmu->num_mapping_groups; ++i) { 977 if (!smrs[i].valid) { 978 /* 979 * Note the first free entry we come across, which 980 * we'll claim in the end if nothing else matches. 981 */ 982 if (free_idx < 0) 983 free_idx = i; 984 continue; 985 } 986 /* 987 * If the new entry is _entirely_ matched by an existing entry, 988 * then reuse that, with the guarantee that there also cannot 989 * be any subsequent conflicting entries. In normal use we'd 990 * expect simply identical entries for this case, but there's 991 * no harm in accommodating the generalisation. 992 */ 993 if ((mask & smrs[i].mask) == mask && 994 !((id ^ smrs[i].id) & ~smrs[i].mask)) 995 return i; 996 /* 997 * If the new entry has any other overlap with an existing one, 998 * though, then there always exists at least one stream ID 999 * which would cause a conflict, and we can't allow that risk. 1000 */ 1001 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask))) 1002 return -EINVAL; 1003 } 1004 1005 return free_idx; 1006} 1007 1008static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx) 1009{ 1010 if (--smmu->s2crs[idx].count) 1011 return false; 1012 1013 smmu->s2crs[idx] = s2cr_init_val; 1014 if (smmu->smrs) 1015 smmu->smrs[idx].valid = false; 1016 1017 return true; 1018} 1019 1020static int arm_smmu_master_alloc_smes(struct device *dev) 1021{ 1022 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 1023 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev); 1024 struct arm_smmu_device *smmu = cfg->smmu; 1025 struct arm_smmu_smr *smrs = smmu->smrs; 1026 int i, idx, ret; 1027 1028 mutex_lock(&smmu->stream_map_mutex); 1029 /* Figure out a viable stream map entry allocation */ 1030 for_each_cfg_sme(cfg, fwspec, i, idx) { 1031 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]); 1032 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]); 1033 1034 if (idx != INVALID_SMENDX) { 1035 ret = -EEXIST; 1036 goto out_err; 1037 } 1038 1039 ret = arm_smmu_find_sme(smmu, sid, mask); 1040 if (ret < 0) 1041 goto out_err; 1042 1043 idx = ret; 1044 if (smrs && smmu->s2crs[idx].count == 0) { 1045 smrs[idx].id = sid; 1046 smrs[idx].mask = mask; 1047 smrs[idx].valid = true; 1048 } 1049 smmu->s2crs[idx].count++; 1050 cfg->smendx[i] = (s16)idx; 1051 } 1052 1053 /* It worked! Now, poke the actual hardware */ 1054 for_each_cfg_sme(cfg, fwspec, i, idx) 1055 arm_smmu_write_sme(smmu, idx); 1056 1057 mutex_unlock(&smmu->stream_map_mutex); 1058 return 0; 1059 1060out_err: 1061 while (i--) { 1062 arm_smmu_free_sme(smmu, cfg->smendx[i]); 1063 cfg->smendx[i] = INVALID_SMENDX; 1064 } 1065 mutex_unlock(&smmu->stream_map_mutex); 1066 return ret; 1067} 1068 1069static void arm_smmu_master_free_smes(struct arm_smmu_master_cfg *cfg, 1070 struct iommu_fwspec *fwspec) 1071{ 1072 struct arm_smmu_device *smmu = cfg->smmu; 1073 int i, idx; 1074 1075 mutex_lock(&smmu->stream_map_mutex); 1076 for_each_cfg_sme(cfg, fwspec, i, idx) { 1077 if (arm_smmu_free_sme(smmu, idx)) 1078 arm_smmu_write_sme(smmu, idx); 1079 cfg->smendx[i] = INVALID_SMENDX; 1080 } 1081 mutex_unlock(&smmu->stream_map_mutex); 1082} 1083 1084static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain, 1085 struct arm_smmu_master_cfg *cfg, 1086 struct iommu_fwspec *fwspec) 1087{ 1088 struct arm_smmu_device *smmu = smmu_domain->smmu; 1089 struct arm_smmu_s2cr *s2cr = smmu->s2crs; 1090 u8 cbndx = smmu_domain->cfg.cbndx; 1091 enum arm_smmu_s2cr_type type; 1092 int i, idx; 1093 1094 if (smmu_domain->stage == ARM_SMMU_DOMAIN_BYPASS) 1095 type = S2CR_TYPE_BYPASS; 1096 else 1097 type = S2CR_TYPE_TRANS; 1098 1099 for_each_cfg_sme(cfg, fwspec, i, idx) { 1100 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx) 1101 continue; 1102 1103 s2cr[idx].type = type; 1104 s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT; 1105 s2cr[idx].cbndx = cbndx; 1106 arm_smmu_write_s2cr(smmu, idx); 1107 } 1108 return 0; 1109} 1110 1111static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) 1112{ 1113 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1114 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 1115 struct arm_smmu_master_cfg *cfg; 1116 struct arm_smmu_device *smmu; 1117 int ret; 1118 1119 if (!fwspec || fwspec->ops != &arm_smmu_ops) { 1120 dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n"); 1121 return -ENXIO; 1122 } 1123 1124 /* 1125 * FIXME: The arch/arm DMA API code tries to attach devices to its own 1126 * domains between of_xlate() and probe_device() - we have no way to cope 1127 * with that, so until ARM gets converted to rely on groups and default 1128 * domains, just say no (but more politely than by dereferencing NULL). 1129 * This should be at least a WARN_ON once that's sorted. 1130 */ 1131 cfg = dev_iommu_priv_get(dev); 1132 if (!cfg) 1133 return -ENODEV; 1134 1135 smmu = cfg->smmu; 1136 1137 ret = arm_smmu_rpm_get(smmu); 1138 if (ret < 0) 1139 return ret; 1140 1141 /* Ensure that the domain is finalised */ 1142 ret = arm_smmu_init_domain_context(domain, smmu, dev); 1143 if (ret < 0) 1144 goto rpm_put; 1145 1146 /* 1147 * Sanity check the domain. We don't support domains across 1148 * different SMMUs. 1149 */ 1150 if (smmu_domain->smmu != smmu) { 1151 ret = -EINVAL; 1152 goto rpm_put; 1153 } 1154 1155 /* Looks ok, so add the device to the domain */ 1156 ret = arm_smmu_domain_add_master(smmu_domain, cfg, fwspec); 1157 1158 /* 1159 * Setup an autosuspend delay to avoid bouncing runpm state. 1160 * Otherwise, if a driver for a suspended consumer device 1161 * unmaps buffers, it will runpm resume/suspend for each one. 1162 * 1163 * For example, when used by a GPU device, when an application 1164 * or game exits, it can trigger unmapping 100s or 1000s of 1165 * buffers. With a runpm cycle for each buffer, that adds up 1166 * to 5-10sec worth of reprogramming the context bank, while 1167 * the system appears to be locked up to the user. 1168 */ 1169 pm_runtime_set_autosuspend_delay(smmu->dev, 20); 1170 pm_runtime_use_autosuspend(smmu->dev); 1171 1172rpm_put: 1173 arm_smmu_rpm_put(smmu); 1174 return ret; 1175} 1176 1177static int arm_smmu_map_pages(struct iommu_domain *domain, unsigned long iova, 1178 phys_addr_t paddr, size_t pgsize, size_t pgcount, 1179 int prot, gfp_t gfp, size_t *mapped) 1180{ 1181 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops; 1182 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu; 1183 int ret; 1184 1185 if (!ops) 1186 return -ENODEV; 1187 1188 arm_smmu_rpm_get(smmu); 1189 ret = ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, mapped); 1190 arm_smmu_rpm_put(smmu); 1191 1192 return ret; 1193} 1194 1195static size_t arm_smmu_unmap_pages(struct iommu_domain *domain, unsigned long iova, 1196 size_t pgsize, size_t pgcount, 1197 struct iommu_iotlb_gather *iotlb_gather) 1198{ 1199 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops; 1200 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu; 1201 size_t ret; 1202 1203 if (!ops) 1204 return 0; 1205 1206 arm_smmu_rpm_get(smmu); 1207 ret = ops->unmap_pages(ops, iova, pgsize, pgcount, iotlb_gather); 1208 arm_smmu_rpm_put(smmu); 1209 1210 return ret; 1211} 1212 1213static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain) 1214{ 1215 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1216 struct arm_smmu_device *smmu = smmu_domain->smmu; 1217 1218 if (smmu_domain->flush_ops) { 1219 arm_smmu_rpm_get(smmu); 1220 smmu_domain->flush_ops->tlb_flush_all(smmu_domain); 1221 arm_smmu_rpm_put(smmu); 1222 } 1223} 1224 1225static void arm_smmu_iotlb_sync(struct iommu_domain *domain, 1226 struct iommu_iotlb_gather *gather) 1227{ 1228 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1229 struct arm_smmu_device *smmu = smmu_domain->smmu; 1230 1231 if (!smmu) 1232 return; 1233 1234 arm_smmu_rpm_get(smmu); 1235 if (smmu->version == ARM_SMMU_V2 || 1236 smmu_domain->stage == ARM_SMMU_DOMAIN_S1) 1237 arm_smmu_tlb_sync_context(smmu_domain); 1238 else 1239 arm_smmu_tlb_sync_global(smmu); 1240 arm_smmu_rpm_put(smmu); 1241} 1242 1243static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain, 1244 dma_addr_t iova) 1245{ 1246 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1247 struct arm_smmu_device *smmu = smmu_domain->smmu; 1248 struct arm_smmu_cfg *cfg = &smmu_domain->cfg; 1249 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops; 1250 struct device *dev = smmu->dev; 1251 void __iomem *reg; 1252 u32 tmp; 1253 u64 phys; 1254 unsigned long va, flags; 1255 int ret, idx = cfg->cbndx; 1256 phys_addr_t addr = 0; 1257 1258 ret = arm_smmu_rpm_get(smmu); 1259 if (ret < 0) 1260 return 0; 1261 1262 spin_lock_irqsave(&smmu_domain->cb_lock, flags); 1263 va = iova & ~0xfffUL; 1264 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) 1265 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_ATS1PR, va); 1266 else 1267 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ATS1PR, va); 1268 1269 reg = arm_smmu_page(smmu, ARM_SMMU_CB(smmu, idx)) + ARM_SMMU_CB_ATSR; 1270 if (readl_poll_timeout_atomic(reg, tmp, !(tmp & ARM_SMMU_ATSR_ACTIVE), 1271 5, 50)) { 1272 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); 1273 dev_err(dev, 1274 "iova to phys timed out on %pad. Falling back to software table walk.\n", 1275 &iova); 1276 arm_smmu_rpm_put(smmu); 1277 return ops->iova_to_phys(ops, iova); 1278 } 1279 1280 phys = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_PAR); 1281 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags); 1282 if (phys & ARM_SMMU_CB_PAR_F) { 1283 dev_err(dev, "translation fault!\n"); 1284 dev_err(dev, "PAR = 0x%llx\n", phys); 1285 goto out; 1286 } 1287 1288 addr = (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff); 1289out: 1290 arm_smmu_rpm_put(smmu); 1291 1292 return addr; 1293} 1294 1295static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain, 1296 dma_addr_t iova) 1297{ 1298 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1299 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops; 1300 1301 if (!ops) 1302 return 0; 1303 1304 if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS && 1305 smmu_domain->stage == ARM_SMMU_DOMAIN_S1) 1306 return arm_smmu_iova_to_phys_hard(domain, iova); 1307 1308 return ops->iova_to_phys(ops, iova); 1309} 1310 1311static bool arm_smmu_capable(struct device *dev, enum iommu_cap cap) 1312{ 1313 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev); 1314 1315 switch (cap) { 1316 case IOMMU_CAP_CACHE_COHERENCY: 1317 /* 1318 * It's overwhelmingly the case in practice that when the pagetable 1319 * walk interface is connected to a coherent interconnect, all the 1320 * translation interfaces are too. Furthermore if the device is 1321 * natively coherent, then its translation interface must also be. 1322 */ 1323 return cfg->smmu->features & ARM_SMMU_FEAT_COHERENT_WALK || 1324 device_get_dma_attr(dev) == DEV_DMA_COHERENT; 1325 case IOMMU_CAP_NOEXEC: 1326 case IOMMU_CAP_DEFERRED_FLUSH: 1327 return true; 1328 default: 1329 return false; 1330 } 1331} 1332 1333static 1334struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode) 1335{ 1336 struct device *dev = driver_find_device_by_fwnode(&arm_smmu_driver.driver, 1337 fwnode); 1338 put_device(dev); 1339 return dev ? dev_get_drvdata(dev) : NULL; 1340} 1341 1342static struct iommu_device *arm_smmu_probe_device(struct device *dev) 1343{ 1344 struct arm_smmu_device *smmu = NULL; 1345 struct arm_smmu_master_cfg *cfg; 1346 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 1347 int i, ret; 1348 1349 if (using_legacy_binding) { 1350 ret = arm_smmu_register_legacy_master(dev, &smmu); 1351 1352 /* 1353 * If dev->iommu_fwspec is initally NULL, arm_smmu_register_legacy_master() 1354 * will allocate/initialise a new one. Thus we need to update fwspec for 1355 * later use. 1356 */ 1357 fwspec = dev_iommu_fwspec_get(dev); 1358 if (ret) 1359 goto out_free; 1360 } else if (fwspec && fwspec->ops == &arm_smmu_ops) { 1361 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode); 1362 } else { 1363 return ERR_PTR(-ENODEV); 1364 } 1365 1366 ret = -EINVAL; 1367 for (i = 0; i < fwspec->num_ids; i++) { 1368 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]); 1369 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]); 1370 1371 if (sid & ~smmu->streamid_mask) { 1372 dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n", 1373 sid, smmu->streamid_mask); 1374 goto out_free; 1375 } 1376 if (mask & ~smmu->smr_mask_mask) { 1377 dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n", 1378 mask, smmu->smr_mask_mask); 1379 goto out_free; 1380 } 1381 } 1382 1383 ret = -ENOMEM; 1384 cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]), 1385 GFP_KERNEL); 1386 if (!cfg) 1387 goto out_free; 1388 1389 cfg->smmu = smmu; 1390 dev_iommu_priv_set(dev, cfg); 1391 while (i--) 1392 cfg->smendx[i] = INVALID_SMENDX; 1393 1394 ret = arm_smmu_rpm_get(smmu); 1395 if (ret < 0) 1396 goto out_cfg_free; 1397 1398 ret = arm_smmu_master_alloc_smes(dev); 1399 arm_smmu_rpm_put(smmu); 1400 1401 if (ret) 1402 goto out_cfg_free; 1403 1404 device_link_add(dev, smmu->dev, 1405 DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER); 1406 1407 return &smmu->iommu; 1408 1409out_cfg_free: 1410 kfree(cfg); 1411out_free: 1412 iommu_fwspec_free(dev); 1413 return ERR_PTR(ret); 1414} 1415 1416static void arm_smmu_release_device(struct device *dev) 1417{ 1418 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 1419 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev); 1420 int ret; 1421 1422 ret = arm_smmu_rpm_get(cfg->smmu); 1423 if (ret < 0) 1424 return; 1425 1426 arm_smmu_master_free_smes(cfg, fwspec); 1427 1428 arm_smmu_rpm_put(cfg->smmu); 1429 1430 dev_iommu_priv_set(dev, NULL); 1431 kfree(cfg); 1432} 1433 1434static void arm_smmu_probe_finalize(struct device *dev) 1435{ 1436 struct arm_smmu_master_cfg *cfg; 1437 struct arm_smmu_device *smmu; 1438 1439 cfg = dev_iommu_priv_get(dev); 1440 smmu = cfg->smmu; 1441 1442 if (smmu->impl && smmu->impl->probe_finalize) 1443 smmu->impl->probe_finalize(smmu, dev); 1444} 1445 1446static struct iommu_group *arm_smmu_device_group(struct device *dev) 1447{ 1448 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev); 1449 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 1450 struct arm_smmu_device *smmu = cfg->smmu; 1451 struct iommu_group *group = NULL; 1452 int i, idx; 1453 1454 mutex_lock(&smmu->stream_map_mutex); 1455 for_each_cfg_sme(cfg, fwspec, i, idx) { 1456 if (group && smmu->s2crs[idx].group && 1457 group != smmu->s2crs[idx].group) { 1458 mutex_unlock(&smmu->stream_map_mutex); 1459 return ERR_PTR(-EINVAL); 1460 } 1461 1462 group = smmu->s2crs[idx].group; 1463 } 1464 1465 if (group) { 1466 mutex_unlock(&smmu->stream_map_mutex); 1467 return iommu_group_ref_get(group); 1468 } 1469 1470 if (dev_is_pci(dev)) 1471 group = pci_device_group(dev); 1472 else if (dev_is_fsl_mc(dev)) 1473 group = fsl_mc_device_group(dev); 1474 else 1475 group = generic_device_group(dev); 1476 1477 /* Remember group for faster lookups */ 1478 if (!IS_ERR(group)) 1479 for_each_cfg_sme(cfg, fwspec, i, idx) 1480 smmu->s2crs[idx].group = group; 1481 1482 mutex_unlock(&smmu->stream_map_mutex); 1483 return group; 1484} 1485 1486static int arm_smmu_enable_nesting(struct iommu_domain *domain) 1487{ 1488 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1489 int ret = 0; 1490 1491 mutex_lock(&smmu_domain->init_mutex); 1492 if (smmu_domain->smmu) 1493 ret = -EPERM; 1494 else 1495 smmu_domain->stage = ARM_SMMU_DOMAIN_NESTED; 1496 mutex_unlock(&smmu_domain->init_mutex); 1497 1498 return ret; 1499} 1500 1501static int arm_smmu_set_pgtable_quirks(struct iommu_domain *domain, 1502 unsigned long quirks) 1503{ 1504 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 1505 int ret = 0; 1506 1507 mutex_lock(&smmu_domain->init_mutex); 1508 if (smmu_domain->smmu) 1509 ret = -EPERM; 1510 else 1511 smmu_domain->pgtbl_quirks = quirks; 1512 mutex_unlock(&smmu_domain->init_mutex); 1513 1514 return ret; 1515} 1516 1517static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args) 1518{ 1519 u32 mask, fwid = 0; 1520 1521 if (args->args_count > 0) 1522 fwid |= FIELD_PREP(ARM_SMMU_SMR_ID, args->args[0]); 1523 1524 if (args->args_count > 1) 1525 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, args->args[1]); 1526 else if (!of_property_read_u32(args->np, "stream-match-mask", &mask)) 1527 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, mask); 1528 1529 return iommu_fwspec_add_ids(dev, &fwid, 1); 1530} 1531 1532static void arm_smmu_get_resv_regions(struct device *dev, 1533 struct list_head *head) 1534{ 1535 struct iommu_resv_region *region; 1536 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO; 1537 1538 region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH, 1539 prot, IOMMU_RESV_SW_MSI, GFP_KERNEL); 1540 if (!region) 1541 return; 1542 1543 list_add_tail(®ion->list, head); 1544 1545 iommu_dma_get_resv_regions(dev, head); 1546} 1547 1548static int arm_smmu_def_domain_type(struct device *dev) 1549{ 1550 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev); 1551 const struct arm_smmu_impl *impl = cfg->smmu->impl; 1552 1553 if (using_legacy_binding) 1554 return IOMMU_DOMAIN_IDENTITY; 1555 1556 if (impl && impl->def_domain_type) 1557 return impl->def_domain_type(dev); 1558 1559 return 0; 1560} 1561 1562static struct iommu_ops arm_smmu_ops = { 1563 .capable = arm_smmu_capable, 1564 .domain_alloc = arm_smmu_domain_alloc, 1565 .probe_device = arm_smmu_probe_device, 1566 .release_device = arm_smmu_release_device, 1567 .probe_finalize = arm_smmu_probe_finalize, 1568 .device_group = arm_smmu_device_group, 1569 .of_xlate = arm_smmu_of_xlate, 1570 .get_resv_regions = arm_smmu_get_resv_regions, 1571 .def_domain_type = arm_smmu_def_domain_type, 1572 .pgsize_bitmap = -1UL, /* Restricted during device attach */ 1573 .owner = THIS_MODULE, 1574 .default_domain_ops = &(const struct iommu_domain_ops) { 1575 .attach_dev = arm_smmu_attach_dev, 1576 .map_pages = arm_smmu_map_pages, 1577 .unmap_pages = arm_smmu_unmap_pages, 1578 .flush_iotlb_all = arm_smmu_flush_iotlb_all, 1579 .iotlb_sync = arm_smmu_iotlb_sync, 1580 .iova_to_phys = arm_smmu_iova_to_phys, 1581 .enable_nesting = arm_smmu_enable_nesting, 1582 .set_pgtable_quirks = arm_smmu_set_pgtable_quirks, 1583 .free = arm_smmu_domain_free, 1584 } 1585}; 1586 1587static void arm_smmu_device_reset(struct arm_smmu_device *smmu) 1588{ 1589 int i; 1590 u32 reg; 1591 1592 /* clear global FSR */ 1593 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR); 1594 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, reg); 1595 1596 /* 1597 * Reset stream mapping groups: Initial values mark all SMRn as 1598 * invalid and all S2CRn as bypass unless overridden. 1599 */ 1600 for (i = 0; i < smmu->num_mapping_groups; ++i) 1601 arm_smmu_write_sme(smmu, i); 1602 1603 /* Make sure all context banks are disabled and clear CB_FSR */ 1604 for (i = 0; i < smmu->num_context_banks; ++i) { 1605 arm_smmu_write_context_bank(smmu, i); 1606 arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, ARM_SMMU_FSR_FAULT); 1607 } 1608 1609 /* Invalidate the TLB, just in case */ 1610 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLH, QCOM_DUMMY_VAL); 1611 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLNSNH, QCOM_DUMMY_VAL); 1612 1613 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0); 1614 1615 /* Enable fault reporting */ 1616 reg |= (ARM_SMMU_sCR0_GFRE | ARM_SMMU_sCR0_GFIE | 1617 ARM_SMMU_sCR0_GCFGFRE | ARM_SMMU_sCR0_GCFGFIE); 1618 1619 /* Disable TLB broadcasting. */ 1620 reg |= (ARM_SMMU_sCR0_VMIDPNE | ARM_SMMU_sCR0_PTM); 1621 1622 /* Enable client access, handling unmatched streams as appropriate */ 1623 reg &= ~ARM_SMMU_sCR0_CLIENTPD; 1624 if (disable_bypass) 1625 reg |= ARM_SMMU_sCR0_USFCFG; 1626 else 1627 reg &= ~ARM_SMMU_sCR0_USFCFG; 1628 1629 /* Disable forced broadcasting */ 1630 reg &= ~ARM_SMMU_sCR0_FB; 1631 1632 /* Don't upgrade barriers */ 1633 reg &= ~(ARM_SMMU_sCR0_BSU); 1634 1635 if (smmu->features & ARM_SMMU_FEAT_VMID16) 1636 reg |= ARM_SMMU_sCR0_VMID16EN; 1637 1638 if (smmu->features & ARM_SMMU_FEAT_EXIDS) 1639 reg |= ARM_SMMU_sCR0_EXIDENABLE; 1640 1641 if (smmu->impl && smmu->impl->reset) 1642 smmu->impl->reset(smmu); 1643 1644 /* Push the button */ 1645 arm_smmu_tlb_sync_global(smmu); 1646 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg); 1647} 1648 1649static int arm_smmu_id_size_to_bits(int size) 1650{ 1651 switch (size) { 1652 case 0: 1653 return 32; 1654 case 1: 1655 return 36; 1656 case 2: 1657 return 40; 1658 case 3: 1659 return 42; 1660 case 4: 1661 return 44; 1662 case 5: 1663 default: 1664 return 48; 1665 } 1666} 1667 1668static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu) 1669{ 1670 unsigned int size; 1671 u32 id; 1672 bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK; 1673 int i, ret; 1674 1675 dev_notice(smmu->dev, "probing hardware configuration...\n"); 1676 dev_notice(smmu->dev, "SMMUv%d with:\n", 1677 smmu->version == ARM_SMMU_V2 ? 2 : 1); 1678 1679 /* ID0 */ 1680 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID0); 1681 1682 /* Restrict available stages based on module parameter */ 1683 if (force_stage == 1) 1684 id &= ~(ARM_SMMU_ID0_S2TS | ARM_SMMU_ID0_NTS); 1685 else if (force_stage == 2) 1686 id &= ~(ARM_SMMU_ID0_S1TS | ARM_SMMU_ID0_NTS); 1687 1688 if (id & ARM_SMMU_ID0_S1TS) { 1689 smmu->features |= ARM_SMMU_FEAT_TRANS_S1; 1690 dev_notice(smmu->dev, "\tstage 1 translation\n"); 1691 } 1692 1693 if (id & ARM_SMMU_ID0_S2TS) { 1694 smmu->features |= ARM_SMMU_FEAT_TRANS_S2; 1695 dev_notice(smmu->dev, "\tstage 2 translation\n"); 1696 } 1697 1698 if (id & ARM_SMMU_ID0_NTS) { 1699 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED; 1700 dev_notice(smmu->dev, "\tnested translation\n"); 1701 } 1702 1703 if (!(smmu->features & 1704 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) { 1705 dev_err(smmu->dev, "\tno translation support!\n"); 1706 return -ENODEV; 1707 } 1708 1709 if ((id & ARM_SMMU_ID0_S1TS) && 1710 ((smmu->version < ARM_SMMU_V2) || !(id & ARM_SMMU_ID0_ATOSNS))) { 1711 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS; 1712 dev_notice(smmu->dev, "\taddress translation ops\n"); 1713 } 1714 1715 /* 1716 * In order for DMA API calls to work properly, we must defer to what 1717 * the FW says about coherency, regardless of what the hardware claims. 1718 * Fortunately, this also opens up a workaround for systems where the 1719 * ID register value has ended up configured incorrectly. 1720 */ 1721 cttw_reg = !!(id & ARM_SMMU_ID0_CTTW); 1722 if (cttw_fw || cttw_reg) 1723 dev_notice(smmu->dev, "\t%scoherent table walk\n", 1724 cttw_fw ? "" : "non-"); 1725 if (cttw_fw != cttw_reg) 1726 dev_notice(smmu->dev, 1727 "\t(IDR0.CTTW overridden by FW configuration)\n"); 1728 1729 /* Max. number of entries we have for stream matching/indexing */ 1730 if (smmu->version == ARM_SMMU_V2 && id & ARM_SMMU_ID0_EXIDS) { 1731 smmu->features |= ARM_SMMU_FEAT_EXIDS; 1732 size = 1 << 16; 1733 } else { 1734 size = 1 << FIELD_GET(ARM_SMMU_ID0_NUMSIDB, id); 1735 } 1736 smmu->streamid_mask = size - 1; 1737 if (id & ARM_SMMU_ID0_SMS) { 1738 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH; 1739 size = FIELD_GET(ARM_SMMU_ID0_NUMSMRG, id); 1740 if (size == 0) { 1741 dev_err(smmu->dev, 1742 "stream-matching supported, but no SMRs present!\n"); 1743 return -ENODEV; 1744 } 1745 1746 /* Zero-initialised to mark as invalid */ 1747 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs), 1748 GFP_KERNEL); 1749 if (!smmu->smrs) 1750 return -ENOMEM; 1751 1752 dev_notice(smmu->dev, 1753 "\tstream matching with %u register groups", size); 1754 } 1755 /* s2cr->type == 0 means translation, so initialise explicitly */ 1756 smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs), 1757 GFP_KERNEL); 1758 if (!smmu->s2crs) 1759 return -ENOMEM; 1760 for (i = 0; i < size; i++) 1761 smmu->s2crs[i] = s2cr_init_val; 1762 1763 smmu->num_mapping_groups = size; 1764 mutex_init(&smmu->stream_map_mutex); 1765 spin_lock_init(&smmu->global_sync_lock); 1766 1767 if (smmu->version < ARM_SMMU_V2 || 1768 !(id & ARM_SMMU_ID0_PTFS_NO_AARCH32)) { 1769 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L; 1770 if (!(id & ARM_SMMU_ID0_PTFS_NO_AARCH32S)) 1771 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S; 1772 } 1773 1774 /* ID1 */ 1775 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID1); 1776 smmu->pgshift = (id & ARM_SMMU_ID1_PAGESIZE) ? 16 : 12; 1777 1778 /* Check for size mismatch of SMMU address space from mapped region */ 1779 size = 1 << (FIELD_GET(ARM_SMMU_ID1_NUMPAGENDXB, id) + 1); 1780 if (smmu->numpage != 2 * size << smmu->pgshift) 1781 dev_warn(smmu->dev, 1782 "SMMU address space size (0x%x) differs from mapped region size (0x%x)!\n", 1783 2 * size << smmu->pgshift, smmu->numpage); 1784 /* Now properly encode NUMPAGE to subsequently derive SMMU_CB_BASE */ 1785 smmu->numpage = size; 1786 1787 smmu->num_s2_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMS2CB, id); 1788 smmu->num_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMCB, id); 1789 if (smmu->num_s2_context_banks > smmu->num_context_banks) { 1790 dev_err(smmu->dev, "impossible number of S2 context banks!\n"); 1791 return -ENODEV; 1792 } 1793 dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n", 1794 smmu->num_context_banks, smmu->num_s2_context_banks); 1795 smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks, 1796 sizeof(*smmu->cbs), GFP_KERNEL); 1797 if (!smmu->cbs) 1798 return -ENOMEM; 1799 1800 /* ID2 */ 1801 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID2); 1802 size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_IAS, id)); 1803 smmu->ipa_size = size; 1804 1805 /* The output mask is also applied for bypass */ 1806 size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_OAS, id)); 1807 smmu->pa_size = size; 1808 1809 if (id & ARM_SMMU_ID2_VMID16) 1810 smmu->features |= ARM_SMMU_FEAT_VMID16; 1811 1812 /* 1813 * What the page table walker can address actually depends on which 1814 * descriptor format is in use, but since a) we don't know that yet, 1815 * and b) it can vary per context bank, this will have to do... 1816 */ 1817 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size))) 1818 dev_warn(smmu->dev, 1819 "failed to set DMA mask for table walker\n"); 1820 1821 if (smmu->version < ARM_SMMU_V2) { 1822 smmu->va_size = smmu->ipa_size; 1823 if (smmu->version == ARM_SMMU_V1_64K) 1824 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K; 1825 } else { 1826 size = FIELD_GET(ARM_SMMU_ID2_UBS, id); 1827 smmu->va_size = arm_smmu_id_size_to_bits(size); 1828 if (id & ARM_SMMU_ID2_PTFS_4K) 1829 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K; 1830 if (id & ARM_SMMU_ID2_PTFS_16K) 1831 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K; 1832 if (id & ARM_SMMU_ID2_PTFS_64K) 1833 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K; 1834 } 1835 1836 if (smmu->impl && smmu->impl->cfg_probe) { 1837 ret = smmu->impl->cfg_probe(smmu); 1838 if (ret) 1839 return ret; 1840 } 1841 1842 /* Now we've corralled the various formats, what'll it do? */ 1843 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) 1844 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M; 1845 if (smmu->features & 1846 (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K)) 1847 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G; 1848 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K) 1849 smmu->pgsize_bitmap |= SZ_16K | SZ_32M; 1850 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K) 1851 smmu->pgsize_bitmap |= SZ_64K | SZ_512M; 1852 1853 if (arm_smmu_ops.pgsize_bitmap == -1UL) 1854 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap; 1855 else 1856 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap; 1857 dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n", 1858 smmu->pgsize_bitmap); 1859 1860 1861 if (smmu->features & ARM_SMMU_FEAT_TRANS_S1) 1862 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n", 1863 smmu->va_size, smmu->ipa_size); 1864 1865 if (smmu->features & ARM_SMMU_FEAT_TRANS_S2) 1866 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n", 1867 smmu->ipa_size, smmu->pa_size); 1868 1869 return 0; 1870} 1871 1872struct arm_smmu_match_data { 1873 enum arm_smmu_arch_version version; 1874 enum arm_smmu_implementation model; 1875}; 1876 1877#define ARM_SMMU_MATCH_DATA(name, ver, imp) \ 1878static const struct arm_smmu_match_data name = { .version = ver, .model = imp } 1879 1880ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU); 1881ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU); 1882ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU); 1883ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500); 1884ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2); 1885ARM_SMMU_MATCH_DATA(qcom_smmuv2, ARM_SMMU_V2, QCOM_SMMUV2); 1886 1887static const struct of_device_id arm_smmu_of_match[] = { 1888 { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 }, 1889 { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 }, 1890 { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 }, 1891 { .compatible = "arm,mmu-401", .data = &arm_mmu401 }, 1892 { .compatible = "arm,mmu-500", .data = &arm_mmu500 }, 1893 { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 }, 1894 { .compatible = "nvidia,smmu-500", .data = &arm_mmu500 }, 1895 { .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 }, 1896 { }, 1897}; 1898MODULE_DEVICE_TABLE(of, arm_smmu_of_match); 1899 1900#ifdef CONFIG_ACPI 1901static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu) 1902{ 1903 int ret = 0; 1904 1905 switch (model) { 1906 case ACPI_IORT_SMMU_V1: 1907 case ACPI_IORT_SMMU_CORELINK_MMU400: 1908 smmu->version = ARM_SMMU_V1; 1909 smmu->model = GENERIC_SMMU; 1910 break; 1911 case ACPI_IORT_SMMU_CORELINK_MMU401: 1912 smmu->version = ARM_SMMU_V1_64K; 1913 smmu->model = GENERIC_SMMU; 1914 break; 1915 case ACPI_IORT_SMMU_V2: 1916 smmu->version = ARM_SMMU_V2; 1917 smmu->model = GENERIC_SMMU; 1918 break; 1919 case ACPI_IORT_SMMU_CORELINK_MMU500: 1920 smmu->version = ARM_SMMU_V2; 1921 smmu->model = ARM_MMU500; 1922 break; 1923 case ACPI_IORT_SMMU_CAVIUM_THUNDERX: 1924 smmu->version = ARM_SMMU_V2; 1925 smmu->model = CAVIUM_SMMUV2; 1926 break; 1927 default: 1928 ret = -ENODEV; 1929 } 1930 1931 return ret; 1932} 1933 1934static int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu, 1935 u32 *global_irqs, u32 *pmu_irqs) 1936{ 1937 struct device *dev = smmu->dev; 1938 struct acpi_iort_node *node = 1939 *(struct acpi_iort_node **)dev_get_platdata(dev); 1940 struct acpi_iort_smmu *iort_smmu; 1941 int ret; 1942 1943 /* Retrieve SMMU1/2 specific data */ 1944 iort_smmu = (struct acpi_iort_smmu *)node->node_data; 1945 1946 ret = acpi_smmu_get_data(iort_smmu->model, smmu); 1947 if (ret < 0) 1948 return ret; 1949 1950 /* Ignore the configuration access interrupt */ 1951 *global_irqs = 1; 1952 *pmu_irqs = 0; 1953 1954 if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK) 1955 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK; 1956 1957 return 0; 1958} 1959#else 1960static inline int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu, 1961 u32 *global_irqs, u32 *pmu_irqs) 1962{ 1963 return -ENODEV; 1964} 1965#endif 1966 1967static int arm_smmu_device_dt_probe(struct arm_smmu_device *smmu, 1968 u32 *global_irqs, u32 *pmu_irqs) 1969{ 1970 const struct arm_smmu_match_data *data; 1971 struct device *dev = smmu->dev; 1972 bool legacy_binding; 1973 1974 if (of_property_read_u32(dev->of_node, "#global-interrupts", global_irqs)) 1975 return dev_err_probe(dev, -ENODEV, 1976 "missing #global-interrupts property\n"); 1977 *pmu_irqs = 0; 1978 1979 data = of_device_get_match_data(dev); 1980 smmu->version = data->version; 1981 smmu->model = data->model; 1982 1983 legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL); 1984 if (legacy_binding && !using_generic_binding) { 1985 if (!using_legacy_binding) { 1986 pr_notice("deprecated \"mmu-masters\" DT property in use; %s support unavailable\n", 1987 IS_ENABLED(CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS) ? "DMA API" : "SMMU"); 1988 } 1989 using_legacy_binding = true; 1990 } else if (!legacy_binding && !using_legacy_binding) { 1991 using_generic_binding = true; 1992 } else { 1993 dev_err(dev, "not probing due to mismatched DT properties\n"); 1994 return -ENODEV; 1995 } 1996 1997 if (of_dma_is_coherent(dev->of_node)) 1998 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK; 1999 2000 return 0; 2001} 2002 2003static void arm_smmu_rmr_install_bypass_smr(struct arm_smmu_device *smmu) 2004{ 2005 struct list_head rmr_list; 2006 struct iommu_resv_region *e; 2007 int idx, cnt = 0; 2008 u32 reg; 2009 2010 INIT_LIST_HEAD(&rmr_list); 2011 iort_get_rmr_sids(dev_fwnode(smmu->dev), &rmr_list); 2012 2013 /* 2014 * Rather than trying to look at existing mappings that 2015 * are setup by the firmware and then invalidate the ones 2016 * that do no have matching RMR entries, just disable the 2017 * SMMU until it gets enabled again in the reset routine. 2018 */ 2019 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0); 2020 reg |= ARM_SMMU_sCR0_CLIENTPD; 2021 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg); 2022 2023 list_for_each_entry(e, &rmr_list, list) { 2024 struct iommu_iort_rmr_data *rmr; 2025 int i; 2026 2027 rmr = container_of(e, struct iommu_iort_rmr_data, rr); 2028 for (i = 0; i < rmr->num_sids; i++) { 2029 idx = arm_smmu_find_sme(smmu, rmr->sids[i], ~0); 2030 if (idx < 0) 2031 continue; 2032 2033 if (smmu->s2crs[idx].count == 0) { 2034 smmu->smrs[idx].id = rmr->sids[i]; 2035 smmu->smrs[idx].mask = 0; 2036 smmu->smrs[idx].valid = true; 2037 } 2038 smmu->s2crs[idx].count++; 2039 smmu->s2crs[idx].type = S2CR_TYPE_BYPASS; 2040 smmu->s2crs[idx].privcfg = S2CR_PRIVCFG_DEFAULT; 2041 2042 cnt++; 2043 } 2044 } 2045 2046 dev_notice(smmu->dev, "\tpreserved %d boot mapping%s\n", cnt, 2047 cnt == 1 ? "" : "s"); 2048 iort_put_rmr_sids(dev_fwnode(smmu->dev), &rmr_list); 2049} 2050 2051static int arm_smmu_device_probe(struct platform_device *pdev) 2052{ 2053 struct resource *res; 2054 struct arm_smmu_device *smmu; 2055 struct device *dev = &pdev->dev; 2056 int num_irqs, i, err; 2057 u32 global_irqs, pmu_irqs; 2058 irqreturn_t (*global_fault)(int irq, void *dev); 2059 2060 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL); 2061 if (!smmu) { 2062 dev_err(dev, "failed to allocate arm_smmu_device\n"); 2063 return -ENOMEM; 2064 } 2065 smmu->dev = dev; 2066 2067 if (dev->of_node) 2068 err = arm_smmu_device_dt_probe(smmu, &global_irqs, &pmu_irqs); 2069 else 2070 err = arm_smmu_device_acpi_probe(smmu, &global_irqs, &pmu_irqs); 2071 if (err) 2072 return err; 2073 2074 smmu->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res); 2075 if (IS_ERR(smmu->base)) 2076 return PTR_ERR(smmu->base); 2077 smmu->ioaddr = res->start; 2078 2079 /* 2080 * The resource size should effectively match the value of SMMU_TOP; 2081 * stash that temporarily until we know PAGESIZE to validate it with. 2082 */ 2083 smmu->numpage = resource_size(res); 2084 2085 smmu = arm_smmu_impl_init(smmu); 2086 if (IS_ERR(smmu)) 2087 return PTR_ERR(smmu); 2088 2089 num_irqs = platform_irq_count(pdev); 2090 2091 smmu->num_context_irqs = num_irqs - global_irqs - pmu_irqs; 2092 if (smmu->num_context_irqs <= 0) 2093 return dev_err_probe(dev, -ENODEV, 2094 "found %d interrupts but expected at least %d\n", 2095 num_irqs, global_irqs + pmu_irqs + 1); 2096 2097 smmu->irqs = devm_kcalloc(dev, smmu->num_context_irqs, 2098 sizeof(*smmu->irqs), GFP_KERNEL); 2099 if (!smmu->irqs) 2100 return dev_err_probe(dev, -ENOMEM, "failed to allocate %d irqs\n", 2101 smmu->num_context_irqs); 2102 2103 for (i = 0; i < smmu->num_context_irqs; i++) { 2104 int irq = platform_get_irq(pdev, global_irqs + pmu_irqs + i); 2105 2106 if (irq < 0) 2107 return irq; 2108 smmu->irqs[i] = irq; 2109 } 2110 2111 err = devm_clk_bulk_get_all(dev, &smmu->clks); 2112 if (err < 0) { 2113 dev_err(dev, "failed to get clocks %d\n", err); 2114 return err; 2115 } 2116 smmu->num_clks = err; 2117 2118 err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks); 2119 if (err) 2120 return err; 2121 2122 err = arm_smmu_device_cfg_probe(smmu); 2123 if (err) 2124 return err; 2125 2126 if (smmu->version == ARM_SMMU_V2) { 2127 if (smmu->num_context_banks > smmu->num_context_irqs) { 2128 dev_err(dev, 2129 "found only %d context irq(s) but %d required\n", 2130 smmu->num_context_irqs, smmu->num_context_banks); 2131 return -ENODEV; 2132 } 2133 2134 /* Ignore superfluous interrupts */ 2135 smmu->num_context_irqs = smmu->num_context_banks; 2136 } 2137 2138 if (smmu->impl && smmu->impl->global_fault) 2139 global_fault = smmu->impl->global_fault; 2140 else 2141 global_fault = arm_smmu_global_fault; 2142 2143 for (i = 0; i < global_irqs; i++) { 2144 int irq = platform_get_irq(pdev, i); 2145 2146 if (irq < 0) 2147 return irq; 2148 2149 err = devm_request_irq(dev, irq, global_fault, IRQF_SHARED, 2150 "arm-smmu global fault", smmu); 2151 if (err) 2152 return dev_err_probe(dev, err, 2153 "failed to request global IRQ %d (%u)\n", 2154 i, irq); 2155 } 2156 2157 err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL, 2158 "smmu.%pa", &smmu->ioaddr); 2159 if (err) { 2160 dev_err(dev, "Failed to register iommu in sysfs\n"); 2161 return err; 2162 } 2163 2164 err = iommu_device_register(&smmu->iommu, &arm_smmu_ops, dev); 2165 if (err) { 2166 dev_err(dev, "Failed to register iommu\n"); 2167 iommu_device_sysfs_remove(&smmu->iommu); 2168 return err; 2169 } 2170 2171 platform_set_drvdata(pdev, smmu); 2172 2173 /* Check for RMRs and install bypass SMRs if any */ 2174 arm_smmu_rmr_install_bypass_smr(smmu); 2175 2176 arm_smmu_device_reset(smmu); 2177 arm_smmu_test_smr_masks(smmu); 2178 2179 /* 2180 * We want to avoid touching dev->power.lock in fastpaths unless 2181 * it's really going to do something useful - pm_runtime_enabled() 2182 * can serve as an ideal proxy for that decision. So, conditionally 2183 * enable pm_runtime. 2184 */ 2185 if (dev->pm_domain) { 2186 pm_runtime_set_active(dev); 2187 pm_runtime_enable(dev); 2188 } 2189 2190 return 0; 2191} 2192 2193static void arm_smmu_device_shutdown(struct platform_device *pdev) 2194{ 2195 struct arm_smmu_device *smmu = platform_get_drvdata(pdev); 2196 2197 if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS)) 2198 dev_notice(&pdev->dev, "disabling translation\n"); 2199 2200 arm_smmu_rpm_get(smmu); 2201 /* Turn the thing off */ 2202 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, ARM_SMMU_sCR0_CLIENTPD); 2203 arm_smmu_rpm_put(smmu); 2204 2205 if (pm_runtime_enabled(smmu->dev)) 2206 pm_runtime_force_suspend(smmu->dev); 2207 else 2208 clk_bulk_disable(smmu->num_clks, smmu->clks); 2209 2210 clk_bulk_unprepare(smmu->num_clks, smmu->clks); 2211} 2212 2213static void arm_smmu_device_remove(struct platform_device *pdev) 2214{ 2215 struct arm_smmu_device *smmu = platform_get_drvdata(pdev); 2216 2217 iommu_device_unregister(&smmu->iommu); 2218 iommu_device_sysfs_remove(&smmu->iommu); 2219 2220 arm_smmu_device_shutdown(pdev); 2221} 2222 2223static int __maybe_unused arm_smmu_runtime_resume(struct device *dev) 2224{ 2225 struct arm_smmu_device *smmu = dev_get_drvdata(dev); 2226 int ret; 2227 2228 ret = clk_bulk_enable(smmu->num_clks, smmu->clks); 2229 if (ret) 2230 return ret; 2231 2232 arm_smmu_device_reset(smmu); 2233 2234 return 0; 2235} 2236 2237static int __maybe_unused arm_smmu_runtime_suspend(struct device *dev) 2238{ 2239 struct arm_smmu_device *smmu = dev_get_drvdata(dev); 2240 2241 clk_bulk_disable(smmu->num_clks, smmu->clks); 2242 2243 return 0; 2244} 2245 2246static int __maybe_unused arm_smmu_pm_resume(struct device *dev) 2247{ 2248 int ret; 2249 struct arm_smmu_device *smmu = dev_get_drvdata(dev); 2250 2251 ret = clk_bulk_prepare(smmu->num_clks, smmu->clks); 2252 if (ret) 2253 return ret; 2254 2255 if (pm_runtime_suspended(dev)) 2256 return 0; 2257 2258 ret = arm_smmu_runtime_resume(dev); 2259 if (ret) 2260 clk_bulk_unprepare(smmu->num_clks, smmu->clks); 2261 2262 return ret; 2263} 2264 2265static int __maybe_unused arm_smmu_pm_suspend(struct device *dev) 2266{ 2267 int ret = 0; 2268 struct arm_smmu_device *smmu = dev_get_drvdata(dev); 2269 2270 if (pm_runtime_suspended(dev)) 2271 goto clk_unprepare; 2272 2273 ret = arm_smmu_runtime_suspend(dev); 2274 if (ret) 2275 return ret; 2276 2277clk_unprepare: 2278 clk_bulk_unprepare(smmu->num_clks, smmu->clks); 2279 return ret; 2280} 2281 2282static const struct dev_pm_ops arm_smmu_pm_ops = { 2283 SET_SYSTEM_SLEEP_PM_OPS(arm_smmu_pm_suspend, arm_smmu_pm_resume) 2284 SET_RUNTIME_PM_OPS(arm_smmu_runtime_suspend, 2285 arm_smmu_runtime_resume, NULL) 2286}; 2287 2288static struct platform_driver arm_smmu_driver = { 2289 .driver = { 2290 .name = "arm-smmu", 2291 .of_match_table = arm_smmu_of_match, 2292 .pm = &arm_smmu_pm_ops, 2293 .suppress_bind_attrs = true, 2294 }, 2295 .probe = arm_smmu_device_probe, 2296 .remove_new = arm_smmu_device_remove, 2297 .shutdown = arm_smmu_device_shutdown, 2298}; 2299module_platform_driver(arm_smmu_driver); 2300 2301MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations"); 2302MODULE_AUTHOR("Will Deacon <will@kernel.org>"); 2303MODULE_ALIAS("platform:arm-smmu"); 2304MODULE_LICENSE("GPL v2"); 2305