1// SPDX-License-Identifier: GPL-2.0-only 2/* 3 * Copyright (C) 2011-2014 NVIDIA CORPORATION. All rights reserved. 4 */ 5 6#include <linux/bitops.h> 7#include <linux/debugfs.h> 8#include <linux/err.h> 9#include <linux/iommu.h> 10#include <linux/kernel.h> 11#include <linux/of.h> 12#include <linux/of_device.h> 13#include <linux/platform_device.h> 14#include <linux/slab.h> 15#include <linux/spinlock.h> 16#include <linux/dma-mapping.h> 17 18#include <soc/tegra/ahb.h> 19#include <soc/tegra/mc.h> 20 21struct tegra_smmu_group { 22 struct list_head list; 23 struct tegra_smmu *smmu; 24 const struct tegra_smmu_group_soc *soc; 25 struct iommu_group *group; 26 unsigned int swgroup; 27}; 28 29struct tegra_smmu { 30 void __iomem *regs; 31 struct device *dev; 32 33 struct tegra_mc *mc; 34 const struct tegra_smmu_soc *soc; 35 36 struct list_head groups; 37 38 unsigned long pfn_mask; 39 unsigned long tlb_mask; 40 41 unsigned long *asids; 42 struct mutex lock; 43 44 struct list_head list; 45 46 struct dentry *debugfs; 47 48 struct iommu_device iommu; /* IOMMU Core code handle */ 49}; 50 51struct tegra_smmu_as { 52 struct iommu_domain domain; 53 struct tegra_smmu *smmu; 54 unsigned int use_count; 55 spinlock_t lock; 56 u32 *count; 57 struct page **pts; 58 struct page *pd; 59 dma_addr_t pd_dma; 60 unsigned id; 61 u32 attr; 62}; 63 64static struct tegra_smmu_as *to_smmu_as(struct iommu_domain *dom) 65{ 66 return container_of(dom, struct tegra_smmu_as, domain); 67} 68 69static inline void smmu_writel(struct tegra_smmu *smmu, u32 value, 70 unsigned long offset) 71{ 72 writel(value, smmu->regs + offset); 73} 74 75static inline u32 smmu_readl(struct tegra_smmu *smmu, unsigned long offset) 76{ 77 return readl(smmu->regs + offset); 78} 79 80#define SMMU_CONFIG 0x010 81#define SMMU_CONFIG_ENABLE (1 << 0) 82 83#define SMMU_TLB_CONFIG 0x14 84#define SMMU_TLB_CONFIG_HIT_UNDER_MISS (1 << 29) 85#define SMMU_TLB_CONFIG_ROUND_ROBIN_ARBITRATION (1 << 28) 86#define SMMU_TLB_CONFIG_ACTIVE_LINES(smmu) \ 87 ((smmu)->soc->num_tlb_lines & (smmu)->tlb_mask) 88 89#define SMMU_PTC_CONFIG 0x18 90#define SMMU_PTC_CONFIG_ENABLE (1 << 29) 91#define SMMU_PTC_CONFIG_REQ_LIMIT(x) (((x) & 0x0f) << 24) 92#define SMMU_PTC_CONFIG_INDEX_MAP(x) ((x) & 0x3f) 93 94#define SMMU_PTB_ASID 0x01c 95#define SMMU_PTB_ASID_VALUE(x) ((x) & 0x7f) 96 97#define SMMU_PTB_DATA 0x020 98#define SMMU_PTB_DATA_VALUE(dma, attr) ((dma) >> 12 | (attr)) 99 100#define SMMU_MK_PDE(dma, attr) ((dma) >> SMMU_PTE_SHIFT | (attr)) 101 102#define SMMU_TLB_FLUSH 0x030 103#define SMMU_TLB_FLUSH_VA_MATCH_ALL (0 << 0) 104#define SMMU_TLB_FLUSH_VA_MATCH_SECTION (2 << 0) 105#define SMMU_TLB_FLUSH_VA_MATCH_GROUP (3 << 0) 106#define SMMU_TLB_FLUSH_VA_SECTION(addr) ((((addr) & 0xffc00000) >> 12) | \ 107 SMMU_TLB_FLUSH_VA_MATCH_SECTION) 108#define SMMU_TLB_FLUSH_VA_GROUP(addr) ((((addr) & 0xffffc000) >> 12) | \ 109 SMMU_TLB_FLUSH_VA_MATCH_GROUP) 110#define SMMU_TLB_FLUSH_ASID_MATCH (1 << 31) 111 112#define SMMU_PTC_FLUSH 0x034 113#define SMMU_PTC_FLUSH_TYPE_ALL (0 << 0) 114#define SMMU_PTC_FLUSH_TYPE_ADR (1 << 0) 115 116#define SMMU_PTC_FLUSH_HI 0x9b8 117#define SMMU_PTC_FLUSH_HI_MASK 0x3 118 119/* per-SWGROUP SMMU_*_ASID register */ 120#define SMMU_ASID_ENABLE (1 << 31) 121#define SMMU_ASID_MASK 0x7f 122#define SMMU_ASID_VALUE(x) ((x) & SMMU_ASID_MASK) 123 124/* page table definitions */ 125#define SMMU_NUM_PDE 1024 126#define SMMU_NUM_PTE 1024 127 128#define SMMU_SIZE_PD (SMMU_NUM_PDE * 4) 129#define SMMU_SIZE_PT (SMMU_NUM_PTE * 4) 130 131#define SMMU_PDE_SHIFT 22 132#define SMMU_PTE_SHIFT 12 133 134#define SMMU_PAGE_MASK (~(SMMU_SIZE_PT-1)) 135#define SMMU_OFFSET_IN_PAGE(x) ((unsigned long)(x) & ~SMMU_PAGE_MASK) 136#define SMMU_PFN_PHYS(x) ((phys_addr_t)(x) << SMMU_PTE_SHIFT) 137#define SMMU_PHYS_PFN(x) ((unsigned long)((x) >> SMMU_PTE_SHIFT)) 138 139#define SMMU_PD_READABLE (1 << 31) 140#define SMMU_PD_WRITABLE (1 << 30) 141#define SMMU_PD_NONSECURE (1 << 29) 142 143#define SMMU_PDE_READABLE (1 << 31) 144#define SMMU_PDE_WRITABLE (1 << 30) 145#define SMMU_PDE_NONSECURE (1 << 29) 146#define SMMU_PDE_NEXT (1 << 28) 147 148#define SMMU_PTE_READABLE (1 << 31) 149#define SMMU_PTE_WRITABLE (1 << 30) 150#define SMMU_PTE_NONSECURE (1 << 29) 151 152#define SMMU_PDE_ATTR (SMMU_PDE_READABLE | SMMU_PDE_WRITABLE | \ 153 SMMU_PDE_NONSECURE) 154 155static unsigned int iova_pd_index(unsigned long iova) 156{ 157 return (iova >> SMMU_PDE_SHIFT) & (SMMU_NUM_PDE - 1); 158} 159 160static unsigned int iova_pt_index(unsigned long iova) 161{ 162 return (iova >> SMMU_PTE_SHIFT) & (SMMU_NUM_PTE - 1); 163} 164 165static bool smmu_dma_addr_valid(struct tegra_smmu *smmu, dma_addr_t addr) 166{ 167 addr >>= 12; 168 return (addr & smmu->pfn_mask) == addr; 169} 170 171static dma_addr_t smmu_pde_to_dma(struct tegra_smmu *smmu, u32 pde) 172{ 173 return (dma_addr_t)(pde & smmu->pfn_mask) << 12; 174} 175 176static void smmu_flush_ptc_all(struct tegra_smmu *smmu) 177{ 178 smmu_writel(smmu, SMMU_PTC_FLUSH_TYPE_ALL, SMMU_PTC_FLUSH); 179} 180 181static inline void smmu_flush_ptc(struct tegra_smmu *smmu, dma_addr_t dma, 182 unsigned long offset) 183{ 184 u32 value; 185 186 offset &= ~(smmu->mc->soc->atom_size - 1); 187 188 if (smmu->mc->soc->num_address_bits > 32) { 189#ifdef CONFIG_ARCH_DMA_ADDR_T_64BIT 190 value = (dma >> 32) & SMMU_PTC_FLUSH_HI_MASK; 191#else 192 value = 0; 193#endif 194 smmu_writel(smmu, value, SMMU_PTC_FLUSH_HI); 195 } 196 197 value = (dma + offset) | SMMU_PTC_FLUSH_TYPE_ADR; 198 smmu_writel(smmu, value, SMMU_PTC_FLUSH); 199} 200 201static inline void smmu_flush_tlb(struct tegra_smmu *smmu) 202{ 203 smmu_writel(smmu, SMMU_TLB_FLUSH_VA_MATCH_ALL, SMMU_TLB_FLUSH); 204} 205 206static inline void smmu_flush_tlb_asid(struct tegra_smmu *smmu, 207 unsigned long asid) 208{ 209 u32 value; 210 211 if (smmu->soc->num_asids == 4) 212 value = (asid & 0x3) << 29; 213 else 214 value = (asid & 0x7f) << 24; 215 216 value |= SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_VA_MATCH_ALL; 217 smmu_writel(smmu, value, SMMU_TLB_FLUSH); 218} 219 220static inline void smmu_flush_tlb_section(struct tegra_smmu *smmu, 221 unsigned long asid, 222 unsigned long iova) 223{ 224 u32 value; 225 226 if (smmu->soc->num_asids == 4) 227 value = (asid & 0x3) << 29; 228 else 229 value = (asid & 0x7f) << 24; 230 231 value |= SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_VA_SECTION(iova); 232 smmu_writel(smmu, value, SMMU_TLB_FLUSH); 233} 234 235static inline void smmu_flush_tlb_group(struct tegra_smmu *smmu, 236 unsigned long asid, 237 unsigned long iova) 238{ 239 u32 value; 240 241 if (smmu->soc->num_asids == 4) 242 value = (asid & 0x3) << 29; 243 else 244 value = (asid & 0x7f) << 24; 245 246 value |= SMMU_TLB_FLUSH_ASID_MATCH | SMMU_TLB_FLUSH_VA_GROUP(iova); 247 smmu_writel(smmu, value, SMMU_TLB_FLUSH); 248} 249 250static inline void smmu_flush(struct tegra_smmu *smmu) 251{ 252 smmu_readl(smmu, SMMU_PTB_ASID); 253} 254 255static int tegra_smmu_alloc_asid(struct tegra_smmu *smmu, unsigned int *idp) 256{ 257 unsigned long id; 258 259 mutex_lock(&smmu->lock); 260 261 id = find_first_zero_bit(smmu->asids, smmu->soc->num_asids); 262 if (id >= smmu->soc->num_asids) { 263 mutex_unlock(&smmu->lock); 264 return -ENOSPC; 265 } 266 267 set_bit(id, smmu->asids); 268 *idp = id; 269 270 mutex_unlock(&smmu->lock); 271 return 0; 272} 273 274static void tegra_smmu_free_asid(struct tegra_smmu *smmu, unsigned int id) 275{ 276 mutex_lock(&smmu->lock); 277 clear_bit(id, smmu->asids); 278 mutex_unlock(&smmu->lock); 279} 280 281static bool tegra_smmu_capable(enum iommu_cap cap) 282{ 283 return false; 284} 285 286static struct iommu_domain *tegra_smmu_domain_alloc(unsigned type) 287{ 288 struct tegra_smmu_as *as; 289 290 if (type != IOMMU_DOMAIN_UNMANAGED) 291 return NULL; 292 293 as = kzalloc(sizeof(*as), GFP_KERNEL); 294 if (!as) 295 return NULL; 296 297 as->attr = SMMU_PD_READABLE | SMMU_PD_WRITABLE | SMMU_PD_NONSECURE; 298 299 as->pd = alloc_page(GFP_KERNEL | __GFP_DMA | __GFP_ZERO); 300 if (!as->pd) { 301 kfree(as); 302 return NULL; 303 } 304 305 as->count = kcalloc(SMMU_NUM_PDE, sizeof(u32), GFP_KERNEL); 306 if (!as->count) { 307 __free_page(as->pd); 308 kfree(as); 309 return NULL; 310 } 311 312 as->pts = kcalloc(SMMU_NUM_PDE, sizeof(*as->pts), GFP_KERNEL); 313 if (!as->pts) { 314 kfree(as->count); 315 __free_page(as->pd); 316 kfree(as); 317 return NULL; 318 } 319 320 spin_lock_init(&as->lock); 321 322 /* setup aperture */ 323 as->domain.geometry.aperture_start = 0; 324 as->domain.geometry.aperture_end = 0xffffffff; 325 as->domain.geometry.force_aperture = true; 326 327 return &as->domain; 328} 329 330static void tegra_smmu_domain_free(struct iommu_domain *domain) 331{ 332 struct tegra_smmu_as *as = to_smmu_as(domain); 333 334 /* TODO: free page directory and page tables */ 335 336 WARN_ON_ONCE(as->use_count); 337 kfree(as->count); 338 kfree(as->pts); 339 kfree(as); 340} 341 342static const struct tegra_smmu_swgroup * 343tegra_smmu_find_swgroup(struct tegra_smmu *smmu, unsigned int swgroup) 344{ 345 const struct tegra_smmu_swgroup *group = NULL; 346 unsigned int i; 347 348 for (i = 0; i < smmu->soc->num_swgroups; i++) { 349 if (smmu->soc->swgroups[i].swgroup == swgroup) { 350 group = &smmu->soc->swgroups[i]; 351 break; 352 } 353 } 354 355 return group; 356} 357 358static void tegra_smmu_enable(struct tegra_smmu *smmu, unsigned int swgroup, 359 unsigned int asid) 360{ 361 const struct tegra_smmu_swgroup *group; 362 unsigned int i; 363 u32 value; 364 365 group = tegra_smmu_find_swgroup(smmu, swgroup); 366 if (group) { 367 value = smmu_readl(smmu, group->reg); 368 value &= ~SMMU_ASID_MASK; 369 value |= SMMU_ASID_VALUE(asid); 370 value |= SMMU_ASID_ENABLE; 371 smmu_writel(smmu, value, group->reg); 372 } else { 373 pr_warn("%s group from swgroup %u not found\n", __func__, 374 swgroup); 375 /* No point moving ahead if group was not found */ 376 return; 377 } 378 379 for (i = 0; i < smmu->soc->num_clients; i++) { 380 const struct tegra_mc_client *client = &smmu->soc->clients[i]; 381 382 if (client->swgroup != swgroup) 383 continue; 384 385 value = smmu_readl(smmu, client->smmu.reg); 386 value |= BIT(client->smmu.bit); 387 smmu_writel(smmu, value, client->smmu.reg); 388 } 389} 390 391static void tegra_smmu_disable(struct tegra_smmu *smmu, unsigned int swgroup, 392 unsigned int asid) 393{ 394 const struct tegra_smmu_swgroup *group; 395 unsigned int i; 396 u32 value; 397 398 group = tegra_smmu_find_swgroup(smmu, swgroup); 399 if (group) { 400 value = smmu_readl(smmu, group->reg); 401 value &= ~SMMU_ASID_MASK; 402 value |= SMMU_ASID_VALUE(asid); 403 value &= ~SMMU_ASID_ENABLE; 404 smmu_writel(smmu, value, group->reg); 405 } 406 407 for (i = 0; i < smmu->soc->num_clients; i++) { 408 const struct tegra_mc_client *client = &smmu->soc->clients[i]; 409 410 if (client->swgroup != swgroup) 411 continue; 412 413 value = smmu_readl(smmu, client->smmu.reg); 414 value &= ~BIT(client->smmu.bit); 415 smmu_writel(smmu, value, client->smmu.reg); 416 } 417} 418 419static int tegra_smmu_as_prepare(struct tegra_smmu *smmu, 420 struct tegra_smmu_as *as) 421{ 422 u32 value; 423 int err; 424 425 if (as->use_count > 0) { 426 as->use_count++; 427 return 0; 428 } 429 430 as->pd_dma = dma_map_page(smmu->dev, as->pd, 0, SMMU_SIZE_PD, 431 DMA_TO_DEVICE); 432 if (dma_mapping_error(smmu->dev, as->pd_dma)) 433 return -ENOMEM; 434 435 /* We can't handle 64-bit DMA addresses */ 436 if (!smmu_dma_addr_valid(smmu, as->pd_dma)) { 437 err = -ENOMEM; 438 goto err_unmap; 439 } 440 441 err = tegra_smmu_alloc_asid(smmu, &as->id); 442 if (err < 0) 443 goto err_unmap; 444 445 smmu_flush_ptc(smmu, as->pd_dma, 0); 446 smmu_flush_tlb_asid(smmu, as->id); 447 448 smmu_writel(smmu, as->id & 0x7f, SMMU_PTB_ASID); 449 value = SMMU_PTB_DATA_VALUE(as->pd_dma, as->attr); 450 smmu_writel(smmu, value, SMMU_PTB_DATA); 451 smmu_flush(smmu); 452 453 as->smmu = smmu; 454 as->use_count++; 455 456 return 0; 457 458err_unmap: 459 dma_unmap_page(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE); 460 return err; 461} 462 463static void tegra_smmu_as_unprepare(struct tegra_smmu *smmu, 464 struct tegra_smmu_as *as) 465{ 466 if (--as->use_count > 0) 467 return; 468 469 tegra_smmu_free_asid(smmu, as->id); 470 471 dma_unmap_page(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE); 472 473 as->smmu = NULL; 474} 475 476static int tegra_smmu_attach_dev(struct iommu_domain *domain, 477 struct device *dev) 478{ 479 struct tegra_smmu *smmu = dev_iommu_priv_get(dev); 480 struct tegra_smmu_as *as = to_smmu_as(domain); 481 struct device_node *np = dev->of_node; 482 struct of_phandle_args args; 483 unsigned int index = 0; 484 int err = 0; 485 486 while (!of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index, 487 &args)) { 488 unsigned int swgroup = args.args[0]; 489 490 if (args.np != smmu->dev->of_node) { 491 of_node_put(args.np); 492 continue; 493 } 494 495 of_node_put(args.np); 496 497 err = tegra_smmu_as_prepare(smmu, as); 498 if (err < 0) 499 return err; 500 501 tegra_smmu_enable(smmu, swgroup, as->id); 502 index++; 503 } 504 505 if (index == 0) 506 return -ENODEV; 507 508 return 0; 509} 510 511static void tegra_smmu_detach_dev(struct iommu_domain *domain, struct device *dev) 512{ 513 struct tegra_smmu_as *as = to_smmu_as(domain); 514 struct device_node *np = dev->of_node; 515 struct tegra_smmu *smmu = as->smmu; 516 struct of_phandle_args args; 517 unsigned int index = 0; 518 519 while (!of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index, 520 &args)) { 521 unsigned int swgroup = args.args[0]; 522 523 if (args.np != smmu->dev->of_node) { 524 of_node_put(args.np); 525 continue; 526 } 527 528 of_node_put(args.np); 529 530 tegra_smmu_disable(smmu, swgroup, as->id); 531 tegra_smmu_as_unprepare(smmu, as); 532 index++; 533 } 534} 535 536static void tegra_smmu_set_pde(struct tegra_smmu_as *as, unsigned long iova, 537 u32 value) 538{ 539 unsigned int pd_index = iova_pd_index(iova); 540 struct tegra_smmu *smmu = as->smmu; 541 u32 *pd = page_address(as->pd); 542 unsigned long offset = pd_index * sizeof(*pd); 543 544 /* Set the page directory entry first */ 545 pd[pd_index] = value; 546 547 /* The flush the page directory entry from caches */ 548 dma_sync_single_range_for_device(smmu->dev, as->pd_dma, offset, 549 sizeof(*pd), DMA_TO_DEVICE); 550 551 /* And flush the iommu */ 552 smmu_flush_ptc(smmu, as->pd_dma, offset); 553 smmu_flush_tlb_section(smmu, as->id, iova); 554 smmu_flush(smmu); 555} 556 557static u32 *tegra_smmu_pte_offset(struct page *pt_page, unsigned long iova) 558{ 559 u32 *pt = page_address(pt_page); 560 561 return pt + iova_pt_index(iova); 562} 563 564static u32 *tegra_smmu_pte_lookup(struct tegra_smmu_as *as, unsigned long iova, 565 dma_addr_t *dmap) 566{ 567 unsigned int pd_index = iova_pd_index(iova); 568 struct tegra_smmu *smmu = as->smmu; 569 struct page *pt_page; 570 u32 *pd; 571 572 pt_page = as->pts[pd_index]; 573 if (!pt_page) 574 return NULL; 575 576 pd = page_address(as->pd); 577 *dmap = smmu_pde_to_dma(smmu, pd[pd_index]); 578 579 return tegra_smmu_pte_offset(pt_page, iova); 580} 581 582static u32 *as_get_pte(struct tegra_smmu_as *as, dma_addr_t iova, 583 dma_addr_t *dmap, struct page *page) 584{ 585 unsigned int pde = iova_pd_index(iova); 586 struct tegra_smmu *smmu = as->smmu; 587 588 if (!as->pts[pde]) { 589 dma_addr_t dma; 590 591 dma = dma_map_page(smmu->dev, page, 0, SMMU_SIZE_PT, 592 DMA_TO_DEVICE); 593 if (dma_mapping_error(smmu->dev, dma)) { 594 __free_page(page); 595 return NULL; 596 } 597 598 if (!smmu_dma_addr_valid(smmu, dma)) { 599 dma_unmap_page(smmu->dev, dma, SMMU_SIZE_PT, 600 DMA_TO_DEVICE); 601 __free_page(page); 602 return NULL; 603 } 604 605 as->pts[pde] = page; 606 607 tegra_smmu_set_pde(as, iova, SMMU_MK_PDE(dma, SMMU_PDE_ATTR | 608 SMMU_PDE_NEXT)); 609 610 *dmap = dma; 611 } else { 612 u32 *pd = page_address(as->pd); 613 614 *dmap = smmu_pde_to_dma(smmu, pd[pde]); 615 } 616 617 return tegra_smmu_pte_offset(as->pts[pde], iova); 618} 619 620static void tegra_smmu_pte_get_use(struct tegra_smmu_as *as, unsigned long iova) 621{ 622 unsigned int pd_index = iova_pd_index(iova); 623 624 as->count[pd_index]++; 625} 626 627static void tegra_smmu_pte_put_use(struct tegra_smmu_as *as, unsigned long iova) 628{ 629 unsigned int pde = iova_pd_index(iova); 630 struct page *page = as->pts[pde]; 631 632 /* 633 * When no entries in this page table are used anymore, return the 634 * memory page to the system. 635 */ 636 if (--as->count[pde] == 0) { 637 struct tegra_smmu *smmu = as->smmu; 638 u32 *pd = page_address(as->pd); 639 dma_addr_t pte_dma = smmu_pde_to_dma(smmu, pd[pde]); 640 641 tegra_smmu_set_pde(as, iova, 0); 642 643 dma_unmap_page(smmu->dev, pte_dma, SMMU_SIZE_PT, DMA_TO_DEVICE); 644 __free_page(page); 645 as->pts[pde] = NULL; 646 } 647} 648 649static void tegra_smmu_set_pte(struct tegra_smmu_as *as, unsigned long iova, 650 u32 *pte, dma_addr_t pte_dma, u32 val) 651{ 652 struct tegra_smmu *smmu = as->smmu; 653 unsigned long offset = SMMU_OFFSET_IN_PAGE(pte); 654 655 *pte = val; 656 657 dma_sync_single_range_for_device(smmu->dev, pte_dma, offset, 658 4, DMA_TO_DEVICE); 659 smmu_flush_ptc(smmu, pte_dma, offset); 660 smmu_flush_tlb_group(smmu, as->id, iova); 661 smmu_flush(smmu); 662} 663 664static struct page *as_get_pde_page(struct tegra_smmu_as *as, 665 unsigned long iova, gfp_t gfp, 666 unsigned long *flags) 667{ 668 unsigned int pde = iova_pd_index(iova); 669 struct page *page = as->pts[pde]; 670 671 /* at first check whether allocation needs to be done at all */ 672 if (page) 673 return page; 674 675 /* 676 * In order to prevent exhaustion of the atomic memory pool, we 677 * allocate page in a sleeping context if GFP flags permit. Hence 678 * spinlock needs to be unlocked and re-locked after allocation. 679 */ 680 if (!(gfp & __GFP_ATOMIC)) 681 spin_unlock_irqrestore(&as->lock, *flags); 682 683 page = alloc_page(gfp | __GFP_DMA | __GFP_ZERO); 684 685 if (!(gfp & __GFP_ATOMIC)) 686 spin_lock_irqsave(&as->lock, *flags); 687 688 /* 689 * In a case of blocking allocation, a concurrent mapping may win 690 * the PDE allocation. In this case the allocated page isn't needed 691 * if allocation succeeded and the allocation failure isn't fatal. 692 */ 693 if (as->pts[pde]) { 694 if (page) 695 __free_page(page); 696 697 page = as->pts[pde]; 698 } 699 700 return page; 701} 702 703static int 704__tegra_smmu_map(struct iommu_domain *domain, unsigned long iova, 705 phys_addr_t paddr, size_t size, int prot, gfp_t gfp, 706 unsigned long *flags) 707{ 708 struct tegra_smmu_as *as = to_smmu_as(domain); 709 dma_addr_t pte_dma; 710 struct page *page; 711 u32 pte_attrs; 712 u32 *pte; 713 714 page = as_get_pde_page(as, iova, gfp, flags); 715 if (!page) 716 return -ENOMEM; 717 718 pte = as_get_pte(as, iova, &pte_dma, page); 719 if (!pte) 720 return -ENOMEM; 721 722 /* If we aren't overwriting a pre-existing entry, increment use */ 723 if (*pte == 0) 724 tegra_smmu_pte_get_use(as, iova); 725 726 pte_attrs = SMMU_PTE_NONSECURE; 727 728 if (prot & IOMMU_READ) 729 pte_attrs |= SMMU_PTE_READABLE; 730 731 if (prot & IOMMU_WRITE) 732 pte_attrs |= SMMU_PTE_WRITABLE; 733 734 tegra_smmu_set_pte(as, iova, pte, pte_dma, 735 SMMU_PHYS_PFN(paddr) | pte_attrs); 736 737 return 0; 738} 739 740static size_t 741__tegra_smmu_unmap(struct iommu_domain *domain, unsigned long iova, 742 size_t size, struct iommu_iotlb_gather *gather) 743{ 744 struct tegra_smmu_as *as = to_smmu_as(domain); 745 dma_addr_t pte_dma; 746 u32 *pte; 747 748 pte = tegra_smmu_pte_lookup(as, iova, &pte_dma); 749 if (!pte || !*pte) 750 return 0; 751 752 tegra_smmu_set_pte(as, iova, pte, pte_dma, 0); 753 tegra_smmu_pte_put_use(as, iova); 754 755 return size; 756} 757 758static int tegra_smmu_map(struct iommu_domain *domain, unsigned long iova, 759 phys_addr_t paddr, size_t size, int prot, gfp_t gfp) 760{ 761 struct tegra_smmu_as *as = to_smmu_as(domain); 762 unsigned long flags; 763 int ret; 764 765 spin_lock_irqsave(&as->lock, flags); 766 ret = __tegra_smmu_map(domain, iova, paddr, size, prot, gfp, &flags); 767 spin_unlock_irqrestore(&as->lock, flags); 768 769 return ret; 770} 771 772static size_t tegra_smmu_unmap(struct iommu_domain *domain, unsigned long iova, 773 size_t size, struct iommu_iotlb_gather *gather) 774{ 775 struct tegra_smmu_as *as = to_smmu_as(domain); 776 unsigned long flags; 777 778 spin_lock_irqsave(&as->lock, flags); 779 size = __tegra_smmu_unmap(domain, iova, size, gather); 780 spin_unlock_irqrestore(&as->lock, flags); 781 782 return size; 783} 784 785static phys_addr_t tegra_smmu_iova_to_phys(struct iommu_domain *domain, 786 dma_addr_t iova) 787{ 788 struct tegra_smmu_as *as = to_smmu_as(domain); 789 unsigned long pfn; 790 dma_addr_t pte_dma; 791 u32 *pte; 792 793 pte = tegra_smmu_pte_lookup(as, iova, &pte_dma); 794 if (!pte || !*pte) 795 return 0; 796 797 pfn = *pte & as->smmu->pfn_mask; 798 799 return SMMU_PFN_PHYS(pfn) + SMMU_OFFSET_IN_PAGE(iova); 800} 801 802static struct tegra_smmu *tegra_smmu_find(struct device_node *np) 803{ 804 struct platform_device *pdev; 805 struct tegra_mc *mc; 806 807 pdev = of_find_device_by_node(np); 808 if (!pdev) 809 return NULL; 810 811 mc = platform_get_drvdata(pdev); 812 if (!mc) 813 return NULL; 814 815 return mc->smmu; 816} 817 818static int tegra_smmu_configure(struct tegra_smmu *smmu, struct device *dev, 819 struct of_phandle_args *args) 820{ 821 const struct iommu_ops *ops = smmu->iommu.ops; 822 int err; 823 824 err = iommu_fwspec_init(dev, &dev->of_node->fwnode, ops); 825 if (err < 0) { 826 dev_err(dev, "failed to initialize fwspec: %d\n", err); 827 return err; 828 } 829 830 err = ops->of_xlate(dev, args); 831 if (err < 0) { 832 dev_err(dev, "failed to parse SW group ID: %d\n", err); 833 iommu_fwspec_free(dev); 834 return err; 835 } 836 837 return 0; 838} 839 840static struct iommu_device *tegra_smmu_probe_device(struct device *dev) 841{ 842 struct device_node *np = dev->of_node; 843 struct tegra_smmu *smmu = NULL; 844 struct of_phandle_args args; 845 unsigned int index = 0; 846 int err; 847 848 while (of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index, 849 &args) == 0) { 850 smmu = tegra_smmu_find(args.np); 851 if (smmu) { 852 err = tegra_smmu_configure(smmu, dev, &args); 853 of_node_put(args.np); 854 855 if (err < 0) 856 return ERR_PTR(err); 857 858 /* 859 * Only a single IOMMU master interface is currently 860 * supported by the Linux kernel, so abort after the 861 * first match. 862 */ 863 dev_iommu_priv_set(dev, smmu); 864 865 break; 866 } 867 868 of_node_put(args.np); 869 index++; 870 } 871 872 if (!smmu) 873 return ERR_PTR(-ENODEV); 874 875 return &smmu->iommu; 876} 877 878static void tegra_smmu_release_device(struct device *dev) 879{ 880 dev_iommu_priv_set(dev, NULL); 881} 882 883static const struct tegra_smmu_group_soc * 884tegra_smmu_find_group(struct tegra_smmu *smmu, unsigned int swgroup) 885{ 886 unsigned int i, j; 887 888 for (i = 0; i < smmu->soc->num_groups; i++) 889 for (j = 0; j < smmu->soc->groups[i].num_swgroups; j++) 890 if (smmu->soc->groups[i].swgroups[j] == swgroup) 891 return &smmu->soc->groups[i]; 892 893 return NULL; 894} 895 896static void tegra_smmu_group_release(void *iommu_data) 897{ 898 struct tegra_smmu_group *group = iommu_data; 899 struct tegra_smmu *smmu = group->smmu; 900 901 mutex_lock(&smmu->lock); 902 list_del(&group->list); 903 mutex_unlock(&smmu->lock); 904} 905 906static struct iommu_group *tegra_smmu_group_get(struct tegra_smmu *smmu, 907 unsigned int swgroup) 908{ 909 const struct tegra_smmu_group_soc *soc; 910 struct tegra_smmu_group *group; 911 struct iommu_group *grp; 912 913 /* Find group_soc associating with swgroup */ 914 soc = tegra_smmu_find_group(smmu, swgroup); 915 916 mutex_lock(&smmu->lock); 917 918 /* Find existing iommu_group associating with swgroup or group_soc */ 919 list_for_each_entry(group, &smmu->groups, list) 920 if ((group->swgroup == swgroup) || (soc && group->soc == soc)) { 921 grp = iommu_group_ref_get(group->group); 922 mutex_unlock(&smmu->lock); 923 return grp; 924 } 925 926 group = devm_kzalloc(smmu->dev, sizeof(*group), GFP_KERNEL); 927 if (!group) { 928 mutex_unlock(&smmu->lock); 929 return NULL; 930 } 931 932 INIT_LIST_HEAD(&group->list); 933 group->swgroup = swgroup; 934 group->smmu = smmu; 935 group->soc = soc; 936 937 group->group = iommu_group_alloc(); 938 if (IS_ERR(group->group)) { 939 devm_kfree(smmu->dev, group); 940 mutex_unlock(&smmu->lock); 941 return NULL; 942 } 943 944 iommu_group_set_iommudata(group->group, group, tegra_smmu_group_release); 945 if (soc) 946 iommu_group_set_name(group->group, soc->name); 947 list_add_tail(&group->list, &smmu->groups); 948 mutex_unlock(&smmu->lock); 949 950 return group->group; 951} 952 953static struct iommu_group *tegra_smmu_device_group(struct device *dev) 954{ 955 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev); 956 struct tegra_smmu *smmu = dev_iommu_priv_get(dev); 957 struct iommu_group *group; 958 959 group = tegra_smmu_group_get(smmu, fwspec->ids[0]); 960 if (!group) 961 group = generic_device_group(dev); 962 963 return group; 964} 965 966static int tegra_smmu_of_xlate(struct device *dev, 967 struct of_phandle_args *args) 968{ 969 u32 id = args->args[0]; 970 971 return iommu_fwspec_add_ids(dev, &id, 1); 972} 973 974static const struct iommu_ops tegra_smmu_ops = { 975 .capable = tegra_smmu_capable, 976 .domain_alloc = tegra_smmu_domain_alloc, 977 .domain_free = tegra_smmu_domain_free, 978 .attach_dev = tegra_smmu_attach_dev, 979 .detach_dev = tegra_smmu_detach_dev, 980 .probe_device = tegra_smmu_probe_device, 981 .release_device = tegra_smmu_release_device, 982 .device_group = tegra_smmu_device_group, 983 .map = tegra_smmu_map, 984 .unmap = tegra_smmu_unmap, 985 .iova_to_phys = tegra_smmu_iova_to_phys, 986 .of_xlate = tegra_smmu_of_xlate, 987 .pgsize_bitmap = SZ_4K, 988}; 989 990static void tegra_smmu_ahb_enable(void) 991{ 992 static const struct of_device_id ahb_match[] = { 993 { .compatible = "nvidia,tegra30-ahb", }, 994 { } 995 }; 996 struct device_node *ahb; 997 998 ahb = of_find_matching_node(NULL, ahb_match); 999 if (ahb) { 1000 tegra_ahb_enable_smmu(ahb); 1001 of_node_put(ahb); 1002 } 1003} 1004 1005static int tegra_smmu_swgroups_show(struct seq_file *s, void *data) 1006{ 1007 struct tegra_smmu *smmu = s->private; 1008 unsigned int i; 1009 u32 value; 1010 1011 seq_printf(s, "swgroup enabled ASID\n"); 1012 seq_printf(s, "------------------------\n"); 1013 1014 for (i = 0; i < smmu->soc->num_swgroups; i++) { 1015 const struct tegra_smmu_swgroup *group = &smmu->soc->swgroups[i]; 1016 const char *status; 1017 unsigned int asid; 1018 1019 value = smmu_readl(smmu, group->reg); 1020 1021 if (value & SMMU_ASID_ENABLE) 1022 status = "yes"; 1023 else 1024 status = "no"; 1025 1026 asid = value & SMMU_ASID_MASK; 1027 1028 seq_printf(s, "%-9s %-7s %#04x\n", group->name, status, 1029 asid); 1030 } 1031 1032 return 0; 1033} 1034 1035DEFINE_SHOW_ATTRIBUTE(tegra_smmu_swgroups); 1036 1037static int tegra_smmu_clients_show(struct seq_file *s, void *data) 1038{ 1039 struct tegra_smmu *smmu = s->private; 1040 unsigned int i; 1041 u32 value; 1042 1043 seq_printf(s, "client enabled\n"); 1044 seq_printf(s, "--------------------\n"); 1045 1046 for (i = 0; i < smmu->soc->num_clients; i++) { 1047 const struct tegra_mc_client *client = &smmu->soc->clients[i]; 1048 const char *status; 1049 1050 value = smmu_readl(smmu, client->smmu.reg); 1051 1052 if (value & BIT(client->smmu.bit)) 1053 status = "yes"; 1054 else 1055 status = "no"; 1056 1057 seq_printf(s, "%-12s %s\n", client->name, status); 1058 } 1059 1060 return 0; 1061} 1062 1063DEFINE_SHOW_ATTRIBUTE(tegra_smmu_clients); 1064 1065static void tegra_smmu_debugfs_init(struct tegra_smmu *smmu) 1066{ 1067 smmu->debugfs = debugfs_create_dir("smmu", NULL); 1068 if (!smmu->debugfs) 1069 return; 1070 1071 debugfs_create_file("swgroups", S_IRUGO, smmu->debugfs, smmu, 1072 &tegra_smmu_swgroups_fops); 1073 debugfs_create_file("clients", S_IRUGO, smmu->debugfs, smmu, 1074 &tegra_smmu_clients_fops); 1075} 1076 1077static void tegra_smmu_debugfs_exit(struct tegra_smmu *smmu) 1078{ 1079 debugfs_remove_recursive(smmu->debugfs); 1080} 1081 1082struct tegra_smmu *tegra_smmu_probe(struct device *dev, 1083 const struct tegra_smmu_soc *soc, 1084 struct tegra_mc *mc) 1085{ 1086 struct tegra_smmu *smmu; 1087 size_t size; 1088 u32 value; 1089 int err; 1090 1091 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL); 1092 if (!smmu) 1093 return ERR_PTR(-ENOMEM); 1094 1095 /* 1096 * This is a bit of a hack. Ideally we'd want to simply return this 1097 * value. However the IOMMU registration process will attempt to add 1098 * all devices to the IOMMU when bus_set_iommu() is called. In order 1099 * not to rely on global variables to track the IOMMU instance, we 1100 * set it here so that it can be looked up from the .probe_device() 1101 * callback via the IOMMU device's .drvdata field. 1102 */ 1103 mc->smmu = smmu; 1104 1105 size = BITS_TO_LONGS(soc->num_asids) * sizeof(long); 1106 1107 smmu->asids = devm_kzalloc(dev, size, GFP_KERNEL); 1108 if (!smmu->asids) 1109 return ERR_PTR(-ENOMEM); 1110 1111 INIT_LIST_HEAD(&smmu->groups); 1112 mutex_init(&smmu->lock); 1113 1114 smmu->regs = mc->regs; 1115 smmu->soc = soc; 1116 smmu->dev = dev; 1117 smmu->mc = mc; 1118 1119 smmu->pfn_mask = 1120 BIT_MASK(mc->soc->num_address_bits - SMMU_PTE_SHIFT) - 1; 1121 dev_dbg(dev, "address bits: %u, PFN mask: %#lx\n", 1122 mc->soc->num_address_bits, smmu->pfn_mask); 1123 smmu->tlb_mask = (1 << fls(smmu->soc->num_tlb_lines)) - 1; 1124 dev_dbg(dev, "TLB lines: %u, mask: %#lx\n", smmu->soc->num_tlb_lines, 1125 smmu->tlb_mask); 1126 1127 value = SMMU_PTC_CONFIG_ENABLE | SMMU_PTC_CONFIG_INDEX_MAP(0x3f); 1128 1129 if (soc->supports_request_limit) 1130 value |= SMMU_PTC_CONFIG_REQ_LIMIT(8); 1131 1132 smmu_writel(smmu, value, SMMU_PTC_CONFIG); 1133 1134 value = SMMU_TLB_CONFIG_HIT_UNDER_MISS | 1135 SMMU_TLB_CONFIG_ACTIVE_LINES(smmu); 1136 1137 if (soc->supports_round_robin_arbitration) 1138 value |= SMMU_TLB_CONFIG_ROUND_ROBIN_ARBITRATION; 1139 1140 smmu_writel(smmu, value, SMMU_TLB_CONFIG); 1141 1142 smmu_flush_ptc_all(smmu); 1143 smmu_flush_tlb(smmu); 1144 smmu_writel(smmu, SMMU_CONFIG_ENABLE, SMMU_CONFIG); 1145 smmu_flush(smmu); 1146 1147 tegra_smmu_ahb_enable(); 1148 1149 err = iommu_device_sysfs_add(&smmu->iommu, dev, NULL, dev_name(dev)); 1150 if (err) 1151 return ERR_PTR(err); 1152 1153 iommu_device_set_ops(&smmu->iommu, &tegra_smmu_ops); 1154 iommu_device_set_fwnode(&smmu->iommu, dev->fwnode); 1155 1156 err = iommu_device_register(&smmu->iommu); 1157 if (err) { 1158 iommu_device_sysfs_remove(&smmu->iommu); 1159 return ERR_PTR(err); 1160 } 1161 1162 err = bus_set_iommu(&platform_bus_type, &tegra_smmu_ops); 1163 if (err < 0) { 1164 iommu_device_unregister(&smmu->iommu); 1165 iommu_device_sysfs_remove(&smmu->iommu); 1166 return ERR_PTR(err); 1167 } 1168 1169 if (IS_ENABLED(CONFIG_DEBUG_FS)) 1170 tegra_smmu_debugfs_init(smmu); 1171 1172 return smmu; 1173} 1174 1175void tegra_smmu_remove(struct tegra_smmu *smmu) 1176{ 1177 iommu_device_unregister(&smmu->iommu); 1178 iommu_device_sysfs_remove(&smmu->iommu); 1179 1180 if (IS_ENABLED(CONFIG_DEBUG_FS)) 1181 tegra_smmu_debugfs_exit(smmu); 1182} 1183