Lines Matching refs:data
160 struct arm_lpae_io_pgtable *data)
169 struct arm_lpae_io_pgtable *data)
173 if (ARM_LPAE_GRANULE(data) < SZ_64K)
251 static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
256 static void __arm_lpae_init_pte(struct arm_lpae_io_pgtable *data,
262 if (data->iop.fmt != ARM_MALI_LPAE && lvl == ARM_LPAE_MAX_LEVELS - 1)
267 pte |= paddr_to_iopte(paddr, data);
269 __arm_lpae_set_pte(ptep, pte, &data->iop.cfg);
272 static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data,
279 if (iopte_leaf(pte, lvl, data->iop.fmt)) {
289 size_t sz = ARM_LPAE_BLOCK_SIZE(lvl, data);
291 tblp = ptep - ARM_LPAE_LVL_IDX(iova, lvl, data);
292 if (__arm_lpae_unmap(data, NULL, iova, sz, lvl, tblp) != sz) {
298 __arm_lpae_init_pte(data, paddr, prot, lvl, ptep);
305 struct arm_lpae_io_pgtable *data)
308 struct io_pgtable_cfg *cfg = &data->iop.cfg;
310 new = paddr_to_iopte(__pa(table), data) | ARM_LPAE_PTE_TYPE_TABLE;
334 static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova,
339 size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data);
340 size_t tblsz = ARM_LPAE_GRANULE(data);
341 struct io_pgtable_cfg *cfg = &data->iop.cfg;
344 ptep += ARM_LPAE_LVL_IDX(iova, lvl, data);
348 return arm_lpae_init_pte(data, iova, paddr, prot, lvl, ptep);
361 pte = arm_lpae_install_table(cptep, ptep, 0, data);
368 if (pte && !iopte_leaf(pte, lvl, data->iop.fmt)) {
369 cptep = iopte_deref(pte, data);
377 return __arm_lpae_map(data, iova, paddr, size, prot, lvl + 1, cptep, gfp);
380 static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data,
385 if (data->iop.fmt == ARM_64_LPAE_S1 ||
386 data->iop.fmt == ARM_32_LPAE_S1) {
404 if (data->iop.fmt == ARM_64_LPAE_S2 ||
405 data->iop.fmt == ARM_32_LPAE_S2) {
427 if (prot & IOMMU_CACHE && data->iop.fmt != ARM_MALI_LPAE)
435 if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS)
438 if (data->iop.fmt != ARM_MALI_LPAE)
447 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
448 struct io_pgtable_cfg *cfg = &data->iop.cfg;
449 arm_lpae_iopte *ptep = data->pgd;
450 int ret, lvl = data->start_level;
466 prot = arm_lpae_prot_to_pte(data, iommu_prot);
467 ret = __arm_lpae_map(data, iova, paddr, size, prot, lvl, ptep, gfp);
477 static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl,
483 if (lvl == data->start_level)
484 table_size = ARM_LPAE_PGD_SIZE(data);
486 table_size = ARM_LPAE_GRANULE(data);
499 if (!pte || iopte_leaf(pte, lvl, data->iop.fmt))
502 __arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data));
505 __arm_lpae_free_pages(start, table_size, &data->iop.cfg);
510 struct arm_lpae_io_pgtable *data = io_pgtable_to_data(iop);
512 __arm_lpae_free_pgtable(data, data->start_level, data->pgd);
513 kfree(data);
516 static size_t arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data,
522 struct io_pgtable_cfg *cfg = &data->iop.cfg;
525 size_t tablesz = ARM_LPAE_GRANULE(data);
526 size_t split_sz = ARM_LPAE_BLOCK_SIZE(lvl, data);
537 unmap_idx = ARM_LPAE_LVL_IDX(iova, lvl, data);
539 blk_paddr = iopte_to_paddr(blk_pte, data);
547 __arm_lpae_init_pte(data, blk_paddr, pte, lvl, &tablep[i]);
550 pte = arm_lpae_install_table(tablep, ptep, blk_pte, data);
561 tablep = iopte_deref(pte, data);
563 io_pgtable_tlb_add_page(&data->iop, gather, iova, size);
567 return __arm_lpae_unmap(data, gather, iova, size, lvl, tablep);
570 static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
576 struct io_pgtable *iop = &data->iop;
582 ptep += ARM_LPAE_LVL_IDX(iova, lvl, data);
588 if (size == ARM_LPAE_BLOCK_SIZE(lvl, data)) {
594 ARM_LPAE_GRANULE(data));
595 ptep = iopte_deref(pte, data);
596 __arm_lpae_free_pgtable(data, lvl + 1, ptep);
614 return arm_lpae_split_blk_unmap(data, gather, iova, size, pte,
619 ptep = iopte_deref(pte, data);
620 return __arm_lpae_unmap(data, gather, iova, size, lvl + 1, ptep);
626 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
627 struct io_pgtable_cfg *cfg = &data->iop.cfg;
628 arm_lpae_iopte *ptep = data->pgd;
639 return __arm_lpae_unmap(data, gather, iova, size, data->start_level, ptep);
645 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
646 arm_lpae_iopte pte, *ptep = data->pgd;
647 int lvl = data->start_level;
655 ptep += ARM_LPAE_LVL_IDX(iova, lvl, data);
663 if (iopte_leaf(pte, lvl, data->iop.fmt))
667 ptep = iopte_deref(pte, data);
674 iova &= (ARM_LPAE_BLOCK_SIZE(lvl, data) - 1);
675 return iopte_to_paddr(pte, data) | iova;
724 struct arm_lpae_io_pgtable *data;
738 data = kmalloc(sizeof(*data), GFP_KERNEL);
739 if (!data)
743 data->bits_per_level = pg_shift - ilog2(sizeof(arm_lpae_iopte));
746 levels = DIV_ROUND_UP(va_bits, data->bits_per_level);
747 data->start_level = ARM_LPAE_MAX_LEVELS - levels;
750 data->pgd_bits = va_bits - (data->bits_per_level * (levels - 1));
752 data->iop.ops = (struct io_pgtable_ops) {
758 return data;
765 struct arm_lpae_io_pgtable *data;
774 data = arm_lpae_alloc_pgtable(cfg);
775 if (!data)
790 switch (ARM_LPAE_GRANULE(data)) {
843 data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data),
845 if (!data->pgd)
852 cfg->arm_lpae_s1_cfg.ttbr = virt_to_phys(data->pgd);
853 return &data->iop;
856 kfree(data);
864 struct arm_lpae_io_pgtable *data;
871 data = arm_lpae_alloc_pgtable(cfg);
872 if (!data)
879 if (data->start_level == 0) {
882 pgd_pages = ARM_LPAE_PGD_SIZE(data) / sizeof(arm_lpae_iopte);
884 data->pgd_bits += data->bits_per_level;
885 data->start_level++;
900 sl = data->start_level;
902 switch (ARM_LPAE_GRANULE(data)) {
945 data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data),
947 if (!data->pgd)
954 cfg->arm_lpae_s2_cfg.vttbr = virt_to_phys(data->pgd);
955 return &data->iop;
958 kfree(data);
985 struct arm_lpae_io_pgtable *data;
996 data = arm_lpae_alloc_pgtable(cfg);
997 if (!data)
1001 if (data->start_level > 0) {
1002 data->start_level = 0;
1003 data->pgd_bits = 0;
1020 data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data), GFP_KERNEL,
1022 if (!data->pgd)
1028 cfg->arm_mali_lpae_cfg.transtab = virt_to_phys(data->pgd) |
1034 return &data->iop;
1037 kfree(data);
1098 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
1099 struct io_pgtable_cfg *cfg = &data->iop.cfg;
1103 pr_err("data: %d levels, 0x%zx pgd_size, %u pg_shift, %u bits_per_level, pgd @ %p\n",
1104 ARM_LPAE_MAX_LEVELS - data->start_level, ARM_LPAE_PGD_SIZE(data),
1105 ilog2(ARM_LPAE_GRANULE(data)), data->bits_per_level, data->pgd);