Lines Matching refs:lvl
52 #define _ARM_V7S_LVL_BITS(lvl) (16 - (lvl) * 4)
53 #define ARM_V7S_LVL_SHIFT(lvl) (ARM_V7S_ADDR_BITS - (4 + 8 * (lvl)))
56 #define ARM_V7S_PTES_PER_LVL(lvl) (1 << _ARM_V7S_LVL_BITS(lvl))
57 #define ARM_V7S_TABLE_SIZE(lvl) \
58 (ARM_V7S_PTES_PER_LVL(lvl) * sizeof(arm_v7s_iopte))
60 #define ARM_V7S_BLOCK_SIZE(lvl) (1UL << ARM_V7S_LVL_SHIFT(lvl))
61 #define ARM_V7S_LVL_MASK(lvl) ((u32)(~0U << ARM_V7S_LVL_SHIFT(lvl)))
63 #define _ARM_V7S_IDX_MASK(lvl) (ARM_V7S_PTES_PER_LVL(lvl) - 1)
64 #define ARM_V7S_LVL_IDX(addr, lvl) ({ \
65 int _l = lvl; \
85 #define ARM_V7S_PTE_IS_TABLE(pte, lvl) \
86 ((lvl) == 1 && (((pte) & 0x3) == ARM_V7S_PTE_TYPE_TABLE))
89 #define ARM_V7S_ATTR_XN(lvl) BIT(4 * (2 - (lvl)))
103 #define ARM_V7S_ATTR_SHIFT(lvl) (16 - (lvl) * 6)
172 static bool arm_v7s_pte_is_cont(arm_v7s_iopte pte, int lvl);
185 static arm_v7s_iopte paddr_to_iopte(phys_addr_t paddr, int lvl,
188 arm_v7s_iopte pte = paddr & ARM_V7S_LVL_MASK(lvl);
200 static phys_addr_t iopte_to_paddr(arm_v7s_iopte pte, int lvl,
206 if (ARM_V7S_PTE_IS_TABLE(pte, lvl))
208 else if (arm_v7s_pte_is_cont(pte, lvl))
209 mask = ARM_V7S_LVL_MASK(lvl) * ARM_V7S_CONT_PAGES;
211 mask = ARM_V7S_LVL_MASK(lvl);
224 static arm_v7s_iopte *iopte_deref(arm_v7s_iopte pte, int lvl,
227 return phys_to_virt(iopte_to_paddr(pte, lvl, &data->iop.cfg));
230 static void *__arm_v7s_alloc_table(int lvl, gfp_t gfp,
237 size_t size = ARM_V7S_TABLE_SIZE(lvl);
240 if (lvl == 1)
243 else if (lvl == 2)
267 if (lvl == 2)
275 if (lvl == 1)
282 static void __arm_v7s_free_table(void *table, int lvl,
287 size_t size = ARM_V7S_TABLE_SIZE(lvl);
292 if (lvl == 1)
318 static arm_v7s_iopte arm_v7s_prot_to_pte(int prot, int lvl,
333 pte <<= ARM_V7S_ATTR_SHIFT(lvl);
336 pte |= ARM_V7S_ATTR_XN(lvl);
343 if (lvl == 1 && (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS))
349 static int arm_v7s_pte_to_prot(arm_v7s_iopte pte, int lvl)
352 arm_v7s_iopte attr = pte >> ARM_V7S_ATTR_SHIFT(lvl);
362 if (pte & ARM_V7S_ATTR_XN(lvl))
368 static arm_v7s_iopte arm_v7s_pte_to_cont(arm_v7s_iopte pte, int lvl)
370 if (lvl == 1) {
372 } else if (lvl == 2) {
373 arm_v7s_iopte xn = pte & ARM_V7S_ATTR_XN(lvl);
384 static arm_v7s_iopte arm_v7s_cont_to_pte(arm_v7s_iopte pte, int lvl)
386 if (lvl == 1) {
388 } else if (lvl == 2) {
401 static bool arm_v7s_pte_is_cont(arm_v7s_iopte pte, int lvl)
403 if (lvl == 1 && !ARM_V7S_PTE_IS_TABLE(pte, lvl))
405 else if (lvl == 2)
416 int lvl, int num_entries, arm_v7s_iopte *ptep)
423 if (ARM_V7S_PTE_IS_TABLE(ptep[i], lvl)) {
429 size_t sz = ARM_V7S_BLOCK_SIZE(lvl);
431 tblp = ptep - ARM_V7S_LVL_IDX(iova, lvl);
433 sz, lvl, tblp) != sz))
441 pte = arm_v7s_prot_to_pte(prot, lvl, cfg);
443 pte = arm_v7s_pte_to_cont(pte, lvl);
445 pte |= paddr_to_iopte(paddr, lvl, cfg);
477 int lvl, arm_v7s_iopte *ptep, gfp_t gfp)
481 int num_entries = size >> ARM_V7S_LVL_SHIFT(lvl);
484 ptep += ARM_V7S_LVL_IDX(iova, lvl);
489 lvl, num_entries, ptep);
492 if (WARN_ON(lvl == 2))
498 cptep = __arm_v7s_alloc_table(lvl + 1, gfp, data);
504 __arm_v7s_free_table(cptep, lvl + 1, data);
510 if (ARM_V7S_PTE_IS_TABLE(pte, lvl)) {
511 cptep = iopte_deref(pte, lvl, data);
519 return __arm_v7s_map(data, iova, paddr, size, prot, lvl + 1, cptep, gfp);
570 unsigned long iova, int idx, int lvl,
575 size_t size = ARM_V7S_BLOCK_SIZE(lvl);
580 if (!arm_v7s_pte_is_cont(pte, lvl))
584 pte = arm_v7s_cont_to_pte(pte, lvl);
642 unsigned long iova, size_t size, int lvl,
647 int idx, i = 0, num_entries = size >> ARM_V7S_LVL_SHIFT(lvl);
650 if (WARN_ON(lvl > 2))
653 idx = ARM_V7S_LVL_IDX(iova, lvl);
671 if (num_entries <= 1 && arm_v7s_pte_is_cont(pte[0], lvl)) {
675 pte[0] = arm_v7s_split_cont(data, iova, idx, lvl, ptep);
681 size_t blk_size = ARM_V7S_BLOCK_SIZE(lvl);
686 if (ARM_V7S_PTE_IS_TABLE(pte[i], lvl)) {
689 ARM_V7S_BLOCK_SIZE(lvl + 1));
690 ptep = iopte_deref(pte[i], lvl, data);
691 __arm_v7s_free_table(ptep, lvl + 1, data);
705 } else if (lvl == 1 && !ARM_V7S_PTE_IS_TABLE(pte[0], lvl)) {
715 ptep = iopte_deref(pte[0], lvl, data);
716 return __arm_v7s_unmap(data, gather, iova, size, lvl + 1, ptep);
735 int lvl = 0;
739 ptep += ARM_V7S_LVL_IDX(iova, ++lvl);
741 ptep = iopte_deref(pte, lvl, data);
742 } while (ARM_V7S_PTE_IS_TABLE(pte, lvl));
747 mask = ARM_V7S_LVL_MASK(lvl);
748 if (arm_v7s_pte_is_cont(pte, lvl))
750 return iopte_to_paddr(pte, lvl, &data->iop.cfg) | (iova & ~mask);