Lines Matching refs:data
163 struct arm_lpae_io_pgtable *data)
172 struct arm_lpae_io_pgtable *data)
176 if (ARM_LPAE_GRANULE(data) < SZ_64K)
253 static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
258 static void __arm_lpae_init_pte(struct arm_lpae_io_pgtable *data,
263 struct io_pgtable_cfg *cfg = &data->iop.cfg;
264 size_t sz = ARM_LPAE_BLOCK_SIZE(lvl, data);
267 if (data->iop.fmt != ARM_MALI_LPAE && lvl == ARM_LPAE_MAX_LEVELS - 1)
273 ptep[i] = pte | paddr_to_iopte(paddr + i * sz, data);
279 static int arm_lpae_init_pte(struct arm_lpae_io_pgtable *data,
287 if (iopte_leaf(ptep[i], lvl, data->iop.fmt)) {
297 size_t sz = ARM_LPAE_BLOCK_SIZE(lvl, data);
299 tblp = ptep - ARM_LPAE_LVL_IDX(iova, lvl, data);
300 if (__arm_lpae_unmap(data, NULL, iova + i * sz, sz, 1,
307 __arm_lpae_init_pte(data, paddr, prot, lvl, num_entries, ptep);
314 struct arm_lpae_io_pgtable *data)
317 struct io_pgtable_cfg *cfg = &data->iop.cfg;
319 new = paddr_to_iopte(__pa(table), data) | ARM_LPAE_PTE_TYPE_TABLE;
343 static int __arm_lpae_map(struct arm_lpae_io_pgtable *data, unsigned long iova,
349 size_t block_size = ARM_LPAE_BLOCK_SIZE(lvl, data);
350 size_t tblsz = ARM_LPAE_GRANULE(data);
351 struct io_pgtable_cfg *cfg = &data->iop.cfg;
355 map_idx_start = ARM_LPAE_LVL_IDX(iova, lvl, data);
360 max_entries = ARM_LPAE_PTES_PER_TABLE(data) - map_idx_start;
362 ret = arm_lpae_init_pte(data, iova, paddr, prot, lvl, num_entries, ptep);
380 pte = arm_lpae_install_table(cptep, ptep, 0, data);
387 if (pte && !iopte_leaf(pte, lvl, data->iop.fmt)) {
388 cptep = iopte_deref(pte, data);
396 return __arm_lpae_map(data, iova, paddr, size, pgcount, prot, lvl + 1,
400 static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data,
405 if (data->iop.fmt == ARM_64_LPAE_S1 ||
406 data->iop.fmt == ARM_32_LPAE_S1) {
424 if (data->iop.fmt == ARM_64_LPAE_S2 ||
425 data->iop.fmt == ARM_32_LPAE_S2) {
447 if (prot & IOMMU_CACHE && data->iop.fmt != ARM_MALI_LPAE)
455 if (data->iop.cfg.quirks & IO_PGTABLE_QUIRK_ARM_NS)
458 if (data->iop.fmt != ARM_MALI_LPAE)
468 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
469 struct io_pgtable_cfg *cfg = &data->iop.cfg;
470 arm_lpae_iopte *ptep = data->pgd;
471 int ret, lvl = data->start_level;
487 prot = arm_lpae_prot_to_pte(data, iommu_prot);
488 ret = __arm_lpae_map(data, iova, paddr, pgsize, pgcount, prot, lvl,
499 static void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl,
505 if (lvl == data->start_level)
506 table_size = ARM_LPAE_PGD_SIZE(data);
508 table_size = ARM_LPAE_GRANULE(data);
521 if (!pte || iopte_leaf(pte, lvl, data->iop.fmt))
524 __arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data));
527 __arm_lpae_free_pages(start, table_size, &data->iop.cfg);
532 struct arm_lpae_io_pgtable *data = io_pgtable_to_data(iop);
534 __arm_lpae_free_pgtable(data, data->start_level, data->pgd);
535 kfree(data);
538 static size_t arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data,
544 struct io_pgtable_cfg *cfg = &data->iop.cfg;
547 size_t tablesz = ARM_LPAE_GRANULE(data);
548 size_t split_sz = ARM_LPAE_BLOCK_SIZE(lvl, data);
549 int ptes_per_table = ARM_LPAE_PTES_PER_TABLE(data);
560 unmap_idx_start = ARM_LPAE_LVL_IDX(iova, lvl, data);
565 blk_paddr = iopte_to_paddr(blk_pte, data);
573 __arm_lpae_init_pte(data, blk_paddr, pte, lvl, 1, &tablep[i]);
576 pte = arm_lpae_install_table(tablep, ptep, blk_pte, data);
587 tablep = iopte_deref(pte, data);
590 io_pgtable_tlb_add_page(&data->iop, gather, iova + i * size, size);
595 return __arm_lpae_unmap(data, gather, iova, size, pgcount, lvl, tablep);
598 static size_t __arm_lpae_unmap(struct arm_lpae_io_pgtable *data,
604 struct io_pgtable *iop = &data->iop;
611 unmap_idx_start = ARM_LPAE_LVL_IDX(iova, lvl, data);
618 if (size == ARM_LPAE_BLOCK_SIZE(lvl, data)) {
619 max_entries = ARM_LPAE_PTES_PER_TABLE(data) - unmap_idx_start;
632 ARM_LPAE_GRANULE(data));
633 __arm_lpae_free_pgtable(data, lvl + 1, iopte_deref(pte, data));
648 return arm_lpae_split_blk_unmap(data, gather, iova, size, pte,
653 ptep = iopte_deref(pte, data);
654 return __arm_lpae_unmap(data, gather, iova, size, pgcount, lvl + 1, ptep);
661 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
662 struct io_pgtable_cfg *cfg = &data->iop.cfg;
663 arm_lpae_iopte *ptep = data->pgd;
674 return __arm_lpae_unmap(data, gather, iova, pgsize, pgcount,
675 data->start_level, ptep);
681 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
682 arm_lpae_iopte pte, *ptep = data->pgd;
683 int lvl = data->start_level;
691 ptep += ARM_LPAE_LVL_IDX(iova, lvl, data);
699 if (iopte_leaf(pte, lvl, data->iop.fmt))
703 ptep = iopte_deref(pte, data);
710 iova &= (ARM_LPAE_BLOCK_SIZE(lvl, data) - 1);
711 return iopte_to_paddr(pte, data) | iova;
760 struct arm_lpae_io_pgtable *data;
774 data = kmalloc(sizeof(*data), GFP_KERNEL);
775 if (!data)
779 data->bits_per_level = pg_shift - ilog2(sizeof(arm_lpae_iopte));
782 levels = DIV_ROUND_UP(va_bits, data->bits_per_level);
783 data->start_level = ARM_LPAE_MAX_LEVELS - levels;
786 data->pgd_bits = va_bits - (data->bits_per_level * (levels - 1));
788 data->iop.ops = (struct io_pgtable_ops) {
794 return data;
801 struct arm_lpae_io_pgtable *data;
810 data = arm_lpae_alloc_pgtable(cfg);
811 if (!data)
831 switch (ARM_LPAE_GRANULE(data)) {
884 data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data),
886 if (!data->pgd)
893 cfg->arm_lpae_s1_cfg.ttbr = virt_to_phys(data->pgd);
894 return &data->iop;
897 kfree(data);
905 struct arm_lpae_io_pgtable *data;
912 data = arm_lpae_alloc_pgtable(cfg);
913 if (!data)
920 if (data->start_level == 0) {
923 pgd_pages = ARM_LPAE_PGD_SIZE(data) / sizeof(arm_lpae_iopte);
925 data->pgd_bits += data->bits_per_level;
926 data->start_level++;
941 sl = data->start_level;
943 switch (ARM_LPAE_GRANULE(data)) {
986 data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data),
988 if (!data->pgd)
995 cfg->arm_lpae_s2_cfg.vttbr = virt_to_phys(data->pgd);
996 return &data->iop;
999 kfree(data);
1026 struct arm_lpae_io_pgtable *data;
1037 data = arm_lpae_alloc_pgtable(cfg);
1038 if (!data)
1042 if (data->start_level > 0) {
1043 data->start_level = 0;
1044 data->pgd_bits = 0;
1061 data->pgd = __arm_lpae_alloc_pages(ARM_LPAE_PGD_SIZE(data), GFP_KERNEL,
1063 if (!data->pgd)
1069 cfg->arm_mali_lpae_cfg.transtab = virt_to_phys(data->pgd) |
1075 return &data->iop;
1078 kfree(data);
1138 struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops);
1139 struct io_pgtable_cfg *cfg = &data->iop.cfg;
1143 pr_err("data: %d levels, 0x%zx pgd_size, %u pg_shift, %u bits_per_level, pgd @ %p\n",
1144 ARM_LPAE_MAX_LEVELS - data->start_level, ARM_LPAE_PGD_SIZE(data),
1145 ilog2(ARM_LPAE_GRANULE(data)), data->bits_per_level, data->pgd);