| /kernel/linux/linux-5.10/drivers/staging/media/tegra-vde/ |
| H A D | iommu.c | 9 #include <linux/iova.h>
21 struct iova **iovap, in tegra_vde_iommu_map()
24 struct iova *iova; in tegra_vde_iommu_map() local
30 size = iova_align(&vde->iova, size); in tegra_vde_iommu_map()
31 shift = iova_shift(&vde->iova); in tegra_vde_iommu_map()
33 iova = alloc_iova(&vde->iova, size >> shift, end >> shift, true); in tegra_vde_iommu_map()
34 if (!iova) in tegra_vde_iommu_map()
37 addr = iova_dma_addr(&vde->iova, iov in tegra_vde_iommu_map()
51 tegra_vde_iommu_unmap(struct tegra_vde *vde, struct iova *iova) tegra_vde_iommu_unmap() argument
64 struct iova *iova; tegra_vde_iommu_init() local
[all...] |
| /kernel/linux/linux-6.6/drivers/media/platform/nvidia/tegra-vde/ |
| H A D | iommu.c | 9 #include <linux/iova.h>
21 struct iova **iovap, in tegra_vde_iommu_map()
24 struct iova *iova; in tegra_vde_iommu_map() local
30 size = iova_align(&vde->iova, size); in tegra_vde_iommu_map()
31 shift = iova_shift(&vde->iova); in tegra_vde_iommu_map()
33 iova = alloc_iova(&vde->iova, size >> shift, end >> shift, true); in tegra_vde_iommu_map()
34 if (!iova) in tegra_vde_iommu_map()
37 addr = iova_dma_addr(&vde->iova, iov in tegra_vde_iommu_map()
51 tegra_vde_iommu_unmap(struct tegra_vde *vde, struct iova *iova) tegra_vde_iommu_unmap() argument
64 struct iova *iova; tegra_vde_iommu_init() local
[all...] |
| /kernel/linux/linux-5.10/include/linux/ |
| H A D | iova.h | 18 /* iova structure */
19 struct iova { struct
67 /* holds all the iova translations for a domain */
70 struct rb_root rbroot; /* iova domain rbtree root */
85 struct iova anchor; /* rbtree lookup anchor */
92 iova entry */
100 static inline unsigned long iova_size(struct iova *iova) in iova_size() argument
102 return iova->pfn_hi - iova in iova_size()
115 iova_offset(struct iova_domain *iovad, dma_addr_t iova) iova_offset() argument
125 iova_dma_addr(struct iova_domain *iovad, struct iova *iova) iova_dma_addr() argument
130 iova_pfn(struct iova_domain *iovad, dma_addr_t iova) iova_pfn() argument
181 free_iova_mem(struct iova *iova) free_iova_mem() argument
189 __free_iova(struct iova_domain *iovad, struct iova *iova) __free_iova() argument
261 split_and_remove_iova(struct iova_domain *iovad, struct iova *iova, unsigned long pfn_lo, unsigned long pfn_hi) split_and_remove_iova() argument
[all...] |
| H A D | io-pgtable.h | 41 void (*tlb_flush_walk)(unsigned long iova, size_t size, size_t granule,
43 void (*tlb_flush_leaf)(unsigned long iova, size_t size, size_t granule,
46 unsigned long iova, size_t granule, void *cookie);
56 * @ias: Input address (iova) size, in bits.
151 * @iova_to_phys: Translate iova to physical address.
157 int (*map)(struct io_pgtable_ops *ops, unsigned long iova,
159 size_t (*unmap)(struct io_pgtable_ops *ops, unsigned long iova,
162 unsigned long iova);
217 io_pgtable_tlb_flush_walk(struct io_pgtable *iop, unsigned long iova, in io_pgtable_tlb_flush_walk() argument
220 iop->cfg.tlb->tlb_flush_walk(iova, siz in io_pgtable_tlb_flush_walk()
224 io_pgtable_tlb_flush_leaf(struct io_pgtable *iop, unsigned long iova, size_t size, size_t granule) io_pgtable_tlb_flush_leaf() argument
231 io_pgtable_tlb_add_page(struct io_pgtable *iop, struct iommu_iotlb_gather * gather, unsigned long iova, size_t granule) io_pgtable_tlb_add_page() argument
[all...] |
| /kernel/linux/linux-5.10/drivers/iommu/ |
| H A D | iova.c | 8 #include <linux/iova.h>
125 __cached_rbnode_insert_update(struct iova_domain *iovad, struct iova *new) in __cached_rbnode_insert_update()
134 __cached_rbnode_delete_update(struct iova_domain *iovad, struct iova *free) in __cached_rbnode_delete_update()
136 struct iova *cached_iova; in __cached_rbnode_delete_update()
138 cached_iova = rb_entry(iovad->cached32_node, struct iova, node); in __cached_rbnode_delete_update()
147 cached_iova = rb_entry(iovad->cached_node, struct iova, node); in __cached_rbnode_delete_update()
152 /* Insert the iova into domain rbtree by holding writer lock */
154 iova_insert_rbtree(struct rb_root *root, struct iova *iova, in iova_insert_rbtree() argument
162 struct iova *thi in iova_insert_rbtree()
240 free_iova_mem(struct iova *iova) free_iova_mem() argument
325 struct iova *iova = rb_entry(node, struct iova, node); private_find_iova() local
338 private_free_iova(struct iova_domain *iovad, struct iova *iova) private_free_iova() argument
356 struct iova *iova; find_iova() local
373 __free_iova(struct iova_domain *iovad, struct iova *iova) __free_iova() argument
393 struct iova *iova = find_iova(iovad, pfn); free_iova() local
595 struct iova *iova, *tmp; put_iova_domain() local
608 struct iova *iova = rb_entry(node, struct iova, node); __is_range_overlap() local
618 struct iova *iova; alloc_and_init_iova() local
633 struct iova *iova; __insert_new_range() local
643 __adjust_overlap_range(struct iova *iova, unsigned long *pfn_lo, unsigned long *pfn_hi) __adjust_overlap_range() argument
666 struct iova *iova; reserve_iova() local
713 struct iova *iova = rb_entry(node, struct iova, node); copy_reserved_iova() local
729 split_and_remove_iova(struct iova_domain *iovad, struct iova *iova, unsigned long pfn_lo, unsigned long pfn_hi) split_and_remove_iova() argument
812 struct iova *iova = private_find_iova(iovad, mag->pfns[i]); iova_magazine_free_pfns() local
[all...] |
| H A D | tegra-gart.c | 60 #define for_each_gart_pte(gart, iova) \
61 for (iova = gart->iovmm_base; \
62 iova < gart->iovmm_end; \
63 iova += GART_PAGE_SIZE)
66 unsigned long iova, unsigned long pte) in gart_set_pte()
68 writel_relaxed(iova, gart->regs + GART_ENTRY_ADDR); in gart_set_pte()
73 unsigned long iova) in gart_read_pte()
77 writel_relaxed(iova, gart->regs + GART_ENTRY_ADDR); in gart_read_pte()
85 unsigned long iova; in do_gart_setup() local
87 for_each_gart_pte(gart, iova) in do_gart_setup()
65 gart_set_pte(struct gart_device *gart, unsigned long iova, unsigned long pte) gart_set_pte() argument
72 gart_read_pte(struct gart_device *gart, unsigned long iova) gart_read_pte() argument
94 gart_iova_range_invalid(struct gart_device *gart, unsigned long iova, size_t bytes) gart_iova_range_invalid() argument
101 gart_pte_valid(struct gart_device *gart, unsigned long iova) gart_pte_valid() argument
167 __gart_iommu_map(struct gart_device *gart, unsigned long iova, unsigned long pa) __gart_iommu_map() argument
180 gart_iommu_map(struct iommu_domain *domain, unsigned long iova, phys_addr_t pa, size_t bytes, int prot, gfp_t gfp) gart_iommu_map() argument
196 __gart_iommu_unmap(struct gart_device *gart, unsigned long iova) __gart_iommu_unmap() argument
209 gart_iommu_unmap(struct iommu_domain *domain, unsigned long iova, size_t bytes, struct iommu_iotlb_gather *gather) gart_iommu_unmap() argument
225 gart_iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) gart_iommu_iova_to_phys() argument
296 unsigned long iova; tegra_gart_suspend() local
[all...] |
| H A D | io-pgtable-arm-v7s.c | 415 unsigned long iova, phys_addr_t paddr, int prot, in arm_v7s_init_pte()
431 tblp = ptep - ARM_V7S_LVL_IDX(iova, lvl); in arm_v7s_init_pte()
432 if (WARN_ON(__arm_v7s_unmap(data, NULL, iova + i * sz, in arm_v7s_init_pte()
475 static int __arm_v7s_map(struct arm_v7s_io_pgtable *data, unsigned long iova, in __arm_v7s_map() argument
484 ptep += ARM_V7S_LVL_IDX(iova, lvl); in __arm_v7s_map()
488 return arm_v7s_init_pte(data, iova, paddr, prot, in __arm_v7s_map()
519 return __arm_v7s_map(data, iova, paddr, size, prot, lvl + 1, cptep, gfp); in __arm_v7s_map()
522 static int arm_v7s_map(struct io_pgtable_ops *ops, unsigned long iova, in arm_v7s_map() argument
533 if (WARN_ON(iova >= (1ULL << data->iop.cfg.ias) || in arm_v7s_map()
537 ret = __arm_v7s_map(data, iova, padd in arm_v7s_map()
414 arm_v7s_init_pte(struct arm_v7s_io_pgtable *data, unsigned long iova, phys_addr_t paddr, int prot, int lvl, int num_entries, arm_v7s_iopte *ptep) arm_v7s_init_pte() argument
569 arm_v7s_split_cont(struct arm_v7s_io_pgtable *data, unsigned long iova, int idx, int lvl, arm_v7s_iopte *ptep) arm_v7s_split_cont() argument
595 arm_v7s_split_blk_unmap(struct arm_v7s_io_pgtable *data, struct iommu_iotlb_gather *gather, unsigned long iova, size_t size, arm_v7s_iopte blk_pte, arm_v7s_iopte *ptep) arm_v7s_split_blk_unmap() argument
640 __arm_v7s_unmap(struct arm_v7s_io_pgtable *data, struct iommu_iotlb_gather *gather, unsigned long iova, size_t size, int lvl, arm_v7s_iopte *ptep) __arm_v7s_unmap() argument
719 arm_v7s_unmap(struct io_pgtable_ops *ops, unsigned long iova, size_t size, struct iommu_iotlb_gather *gather) arm_v7s_unmap() argument
730 arm_v7s_iova_to_phys(struct io_pgtable_ops *ops, unsigned long iova) arm_v7s_iova_to_phys() argument
856 dummy_tlb_flush(unsigned long iova, size_t size, size_t granule, void *cookie) dummy_tlb_flush() argument
863 dummy_tlb_add_page(struct iommu_iotlb_gather *gather, unsigned long iova, size_t granule, void *cookie) dummy_tlb_add_page() argument
894 unsigned int iova, size, iova_start; arm_v7s_do_selftests() local
[all...] |
| /kernel/linux/linux-6.6/include/linux/ |
| H A D | iova.h | 17 /* iova structure */
18 struct iova { struct
27 /* holds all the iova translations for a domain */
30 struct rb_root rbroot; /* iova domain rbtree root */
37 struct iova anchor; /* rbtree lookup anchor */
43 static inline unsigned long iova_size(struct iova *iova) in iova_size() argument
45 return iova->pfn_hi - iova->pfn_lo + 1; in iova_size()
58 static inline size_t iova_offset(struct iova_domain *iovad, dma_addr_t iova) in iova_offset() argument
68 iova_dma_addr(struct iova_domain *iovad, struct iova *iova) iova_dma_addr() argument
73 iova_pfn(struct iova_domain *iovad, dma_addr_t iova) iova_pfn() argument
114 __free_iova(struct iova_domain *iovad, struct iova *iova) __free_iova() argument
[all...] |
| /kernel/linux/linux-6.6/drivers/iommu/ |
| H A D | iova.c | 8 #include <linux/iova.h>
46 static struct iova *to_iova(struct rb_node *node) in to_iova()
48 return rb_entry(node, struct iova, node); in to_iova()
86 __cached_rbnode_insert_update(struct iova_domain *iovad, struct iova *new) in __cached_rbnode_insert_update()
95 __cached_rbnode_delete_update(struct iova_domain *iovad, struct iova *free) in __cached_rbnode_delete_update()
97 struct iova *cached_iova; in __cached_rbnode_delete_update()
150 /* Insert the iova into domain rbtree by holding writer lock */
152 iova_insert_rbtree(struct rb_root *root, struct iova *iova, in iova_insert_rbtree() argument
160 struct iova *thi in iova_insert_rbtree()
248 free_iova_mem(struct iova *iova) free_iova_mem() argument
345 struct iova *iova = to_iova(node); private_find_iova() local
358 remove_iova(struct iova_domain *iovad, struct iova *iova) remove_iova() argument
375 struct iova *iova; find_iova() local
392 __free_iova(struct iova_domain *iovad, struct iova *iova) __free_iova() argument
414 struct iova *iova; free_iova() local
510 struct iova *iova, *tmp; put_iova_domain() local
524 struct iova *iova = to_iova(node); __is_range_overlap() local
534 struct iova *iova; alloc_and_init_iova() local
549 struct iova *iova; __insert_new_range() local
559 __adjust_overlap_range(struct iova *iova, unsigned long *pfn_lo, unsigned long *pfn_hi) __adjust_overlap_range() argument
582 struct iova *iova; reserve_iova() local
673 struct iova *iova = private_find_iova(iovad, mag->pfns[i]); iova_magazine_free_pfns() local
[all...] |
| H A D | tegra-gart.c | 60 #define for_each_gart_pte(gart, iova) \
61 for (iova = gart->iovmm_base; \
62 iova < gart->iovmm_end; \
63 iova += GART_PAGE_SIZE)
66 unsigned long iova, unsigned long pte) in gart_set_pte()
68 writel_relaxed(iova, gart->regs + GART_ENTRY_ADDR); in gart_set_pte()
73 unsigned long iova) in gart_read_pte()
77 writel_relaxed(iova, gart->regs + GART_ENTRY_ADDR); in gart_read_pte()
85 unsigned long iova; in do_gart_setup() local
87 for_each_gart_pte(gart, iova) in do_gart_setup()
65 gart_set_pte(struct gart_device *gart, unsigned long iova, unsigned long pte) gart_set_pte() argument
72 gart_read_pte(struct gart_device *gart, unsigned long iova) gart_read_pte() argument
94 gart_iova_range_invalid(struct gart_device *gart, unsigned long iova, size_t bytes) gart_iova_range_invalid() argument
101 gart_pte_valid(struct gart_device *gart, unsigned long iova) gart_pte_valid() argument
167 __gart_iommu_map(struct gart_device *gart, unsigned long iova, unsigned long pa) __gart_iommu_map() argument
180 gart_iommu_map(struct iommu_domain *domain, unsigned long iova, phys_addr_t pa, size_t bytes, int prot, gfp_t gfp) gart_iommu_map() argument
196 __gart_iommu_unmap(struct gart_device *gart, unsigned long iova) __gart_iommu_unmap() argument
209 gart_iommu_unmap(struct iommu_domain *domain, unsigned long iova, size_t bytes, struct iommu_iotlb_gather *gather) gart_iommu_unmap() argument
225 gart_iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) gart_iommu_iova_to_phys() argument
255 gart_iommu_sync_map(struct iommu_domain *domain, unsigned long iova, size_t size) gart_iommu_sync_map() argument
290 unsigned long iova; tegra_gart_suspend() local
[all...] |
| H A D | io-pgtable-arm-v7s.c | 432 unsigned long iova, phys_addr_t paddr, int prot, in arm_v7s_init_pte()
448 tblp = ptep - ARM_V7S_LVL_IDX(iova, lvl, cfg); in arm_v7s_init_pte()
449 if (WARN_ON(__arm_v7s_unmap(data, NULL, iova + i * sz, in arm_v7s_init_pte()
497 static int __arm_v7s_map(struct arm_v7s_io_pgtable *data, unsigned long iova, in __arm_v7s_map() argument
506 ptep += ARM_V7S_LVL_IDX(iova, lvl, cfg); in __arm_v7s_map()
510 return arm_v7s_init_pte(data, iova, paddr, prot, in __arm_v7s_map()
541 return __arm_v7s_map(data, iova, paddr, size, prot, lvl + 1, cptep, gfp); in __arm_v7s_map()
544 static int arm_v7s_map_pages(struct io_pgtable_ops *ops, unsigned long iova, in arm_v7s_map_pages() argument
551 if (WARN_ON(iova >= (1ULL << data->iop.cfg.ias) || in arm_v7s_map_pages()
560 ret = __arm_v7s_map(data, iova, padd in arm_v7s_map_pages()
431 arm_v7s_init_pte(struct arm_v7s_io_pgtable *data, unsigned long iova, phys_addr_t paddr, int prot, int lvl, int num_entries, arm_v7s_iopte *ptep) arm_v7s_init_pte() argument
595 arm_v7s_split_cont(struct arm_v7s_io_pgtable *data, unsigned long iova, int idx, int lvl, arm_v7s_iopte *ptep) arm_v7s_split_cont() argument
621 arm_v7s_split_blk_unmap(struct arm_v7s_io_pgtable *data, struct iommu_iotlb_gather *gather, unsigned long iova, size_t size, arm_v7s_iopte blk_pte, arm_v7s_iopte *ptep) arm_v7s_split_blk_unmap() argument
666 __arm_v7s_unmap(struct arm_v7s_io_pgtable *data, struct iommu_iotlb_gather *gather, unsigned long iova, size_t size, int lvl, arm_v7s_iopte *ptep) __arm_v7s_unmap() argument
738 arm_v7s_unmap_pages(struct io_pgtable_ops *ops, unsigned long iova, size_t pgsize, size_t pgcount, struct iommu_iotlb_gather *gather) arm_v7s_unmap_pages() argument
760 arm_v7s_iova_to_phys(struct io_pgtable_ops *ops, unsigned long iova) arm_v7s_iova_to_phys() argument
903 dummy_tlb_flush(unsigned long iova, size_t size, size_t granule, void *cookie) dummy_tlb_flush() argument
910 dummy_tlb_add_page(struct iommu_iotlb_gather *gather, unsigned long iova, size_t granule, void *cookie) dummy_tlb_add_page() argument
940 unsigned int iova, size, iova_start; arm_v7s_do_selftests() local
[all...] |
| H A D | sun50i-iommu.c | 165 static u32 sun50i_iova_get_dte_index(dma_addr_t iova) in sun50i_iova_get_dte_index() argument
167 return FIELD_GET(SUN50I_IOVA_DTE_MASK, iova); in sun50i_iova_get_dte_index()
170 static u32 sun50i_iova_get_pte_index(dma_addr_t iova) in sun50i_iova_get_pte_index() argument
172 return FIELD_GET(SUN50I_IOVA_PTE_MASK, iova); in sun50i_iova_get_pte_index()
175 static u32 sun50i_iova_get_page_offset(dma_addr_t iova) in sun50i_iova_get_page_offset() argument
177 return FIELD_GET(SUN50I_IOVA_PAGE_MASK, iova); in sun50i_iova_get_page_offset()
301 unsigned long iova) in sun50i_iommu_zap_iova()
306 iommu_write(iommu, IOMMU_TLB_IVLD_ADDR_REG, iova); in sun50i_iommu_zap_iova()
318 unsigned long iova) in sun50i_iommu_zap_ptw_cache()
323 iommu_write(iommu, IOMMU_PC_IVLD_ADDR_REG, iova); in sun50i_iommu_zap_ptw_cache()
300 sun50i_iommu_zap_iova(struct sun50i_iommu *iommu, unsigned long iova) sun50i_iommu_zap_iova() argument
317 sun50i_iommu_zap_ptw_cache(struct sun50i_iommu *iommu, unsigned long iova) sun50i_iommu_zap_ptw_cache() argument
333 sun50i_iommu_zap_range(struct sun50i_iommu *iommu, unsigned long iova, size_t size) sun50i_iommu_zap_range() argument
405 sun50i_iommu_iotlb_sync_map(struct iommu_domain *domain, unsigned long iova, size_t size) sun50i_iommu_iotlb_sync_map() argument
552 sun50i_dte_get_page_table(struct sun50i_iommu_domain *sun50i_domain, dma_addr_t iova, gfp_t gfp) sun50i_dte_get_page_table() argument
591 sun50i_iommu_map(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t size, int prot, gfp_t gfp) sun50i_iommu_map() argument
624 sun50i_iommu_unmap(struct iommu_domain *domain, unsigned long iova, size_t size, struct iommu_iotlb_gather *gather) sun50i_iommu_unmap() argument
648 sun50i_iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) sun50i_iommu_iova_to_phys() argument
847 sun50i_iommu_report_fault(struct sun50i_iommu *iommu, unsigned master, phys_addr_t iova, unsigned prot) sun50i_iommu_report_fault() argument
866 phys_addr_t iova; sun50i_iommu_handle_pt_irq() local
889 phys_addr_t iova; sun50i_iommu_handle_perm_irq() local
[all...] |
| /kernel/linux/linux-6.6/include/trace/events/ |
| H A D | iommu.h | 81 TP_PROTO(unsigned long iova, phys_addr_t paddr, size_t size),
83 TP_ARGS(iova, paddr, size),
86 __field(u64, iova)
92 __entry->iova = iova;
97 TP_printk("IOMMU: iova=0x%016llx - 0x%016llx paddr=0x%016llx size=%zu",
98 __entry->iova, __entry->iova + __entry->size, __entry->paddr,
105 TP_PROTO(unsigned long iova, size_t size, size_t unmapped_size),
107 TP_ARGS(iova, siz
[all...] |
| /kernel/linux/linux-5.10/drivers/fpga/ |
| H A D | dfl-afu-dma-region.c | 118 * @iova: address of the dma memory area
121 * Compare the dma memory area defined by @iova and @size with given dma region.
125 u64 iova, u64 size) in dma_region_check_iova()
127 if (!size && region->iova != iova) in dma_region_check_iova()
130 return (region->iova <= iova) && in dma_region_check_iova()
131 (region->length + region->iova >= iova + size); in dma_region_check_iova()
149 dev_dbg(&pdata->dev->dev, "add region (iova in afu_dma_region_add()
124 dma_region_check_iova(struct dfl_afu_dma_region *region, u64 iova, u64 size) dma_region_check_iova() argument
245 afu_dma_region_find(struct dfl_feature_platform_data *pdata, u64 iova, u64 size) afu_dma_region_find() argument
285 afu_dma_region_find_iova(struct dfl_feature_platform_data *pdata, u64 iova) afu_dma_region_find_iova() argument
301 afu_dma_map_region(struct dfl_feature_platform_data *pdata, u64 user_addr, u64 length, u64 *iova) afu_dma_map_region() argument
380 afu_dma_unmap_region(struct dfl_feature_platform_data *pdata, u64 iova) afu_dma_unmap_region() argument
[all...] |
| /kernel/linux/linux-6.6/drivers/fpga/ |
| H A D | dfl-afu-dma-region.c | 118 * @iova: address of the dma memory area
121 * Compare the dma memory area defined by @iova and @size with given dma region.
125 u64 iova, u64 size) in dma_region_check_iova()
127 if (!size && region->iova != iova) in dma_region_check_iova()
130 return (region->iova <= iova) && in dma_region_check_iova()
131 (region->length + region->iova >= iova + size); in dma_region_check_iova()
149 dev_dbg(&pdata->dev->dev, "add region (iova in afu_dma_region_add()
124 dma_region_check_iova(struct dfl_afu_dma_region *region, u64 iova, u64 size) dma_region_check_iova() argument
245 afu_dma_region_find(struct dfl_feature_platform_data *pdata, u64 iova, u64 size) afu_dma_region_find() argument
285 afu_dma_region_find_iova(struct dfl_feature_platform_data *pdata, u64 iova) afu_dma_region_find_iova() argument
301 afu_dma_map_region(struct dfl_feature_platform_data *pdata, u64 user_addr, u64 length, u64 *iova) afu_dma_map_region() argument
380 afu_dma_unmap_region(struct dfl_feature_platform_data *pdata, u64 iova) afu_dma_unmap_region() argument
[all...] |
| /kernel/linux/linux-5.10/include/trace/events/ |
| H A D | iommu.h | 88 TP_PROTO(unsigned long iova, phys_addr_t paddr, size_t size),
90 TP_ARGS(iova, paddr, size),
93 __field(u64, iova)
99 __entry->iova = iova;
104 TP_printk("IOMMU: iova=0x%016llx paddr=0x%016llx size=%zu",
105 __entry->iova, __entry->paddr, __entry->size
111 TP_PROTO(unsigned long iova, size_t size, size_t unmapped_size),
113 TP_ARGS(iova, size, unmapped_size),
116 __field(u64, iova)
[all...] |
| /kernel/linux/linux-5.10/drivers/staging/media/ipu3/ |
| H A D | ipu3-dmamap.c | 102 struct iova *iova; in imgu_dmamap_alloc() local
107 iova = alloc_iova(&imgu->iova_domain, size >> shift, in imgu_dmamap_alloc()
109 if (!iova) in imgu_dmamap_alloc()
117 iovaddr = iova_dma_addr(&imgu->iova_domain, iova); in imgu_dmamap_alloc()
133 map->daddr = iova_dma_addr(&imgu->iova_domain, iova); in imgu_dmamap_alloc()
142 imgu_mmu_unmap(imgu->mmu, iova_dma_addr(&imgu->iova_domain, iova), in imgu_dmamap_alloc()
146 __free_iova(&imgu->iova_domain, iova); in imgu_dmamap_alloc()
153 struct iova *iova; in imgu_dmamap_unmap() local
189 struct iova *iova; imgu_dmamap_map_sg() local
[all...] |
| /kernel/linux/linux-6.6/drivers/staging/media/ipu3/ |
| H A D | ipu3-dmamap.c | 102 struct iova *iova; in imgu_dmamap_alloc() local
107 iova = alloc_iova(&imgu->iova_domain, size >> shift, in imgu_dmamap_alloc()
109 if (!iova) in imgu_dmamap_alloc()
117 iovaddr = iova_dma_addr(&imgu->iova_domain, iova); in imgu_dmamap_alloc()
133 map->daddr = iova_dma_addr(&imgu->iova_domain, iova); in imgu_dmamap_alloc()
142 imgu_mmu_unmap(imgu->mmu, iova_dma_addr(&imgu->iova_domain, iova), in imgu_dmamap_alloc()
146 __free_iova(&imgu->iova_domain, iova); in imgu_dmamap_alloc()
153 struct iova *iova; in imgu_dmamap_unmap() local
189 struct iova *iova; imgu_dmamap_map_sg() local
[all...] |
| /kernel/linux/linux-6.6/drivers/vfio/ |
| H A D | vfio_iommu_type1.c | 90 dma_addr_t iova; /* Device address */ member
129 dma_addr_t iova; /* Device address */ member
136 dma_addr_t iova; member
173 if (start + size <= dma->iova) in vfio_find_dma()
175 else if (start >= dma->iova + dma->size) in vfio_find_dma()
194 if (start < dma->iova + dma->size) { in vfio_find_dma_first_node()
197 if (start >= dma->iova) in vfio_find_dma_first_node()
204 if (res && size && dma_res->iova >= start + size) in vfio_find_dma_first_node()
218 if (new->iova + new->size <= dma->iova) in vfio_link_dma()
323 vfio_find_vpfn(struct vfio_dma *dma, dma_addr_t iova) vfio_find_vpfn() argument
367 vfio_add_to_pfn_list(struct vfio_dma *dma, dma_addr_t iova, unsigned long pfn) vfio_add_to_pfn_list() argument
390 vfio_iova_get_vfio_pfn(struct vfio_dma *dma, unsigned long iova) vfio_iova_get_vfio_pfn() argument
621 dma_addr_t iova = vaddr - dma->vaddr + dma->iova; vfio_pin_pages_remote() local
724 vfio_unpin_pages_remote(struct vfio_dma *dma, dma_addr_t iova, unsigned long pfn, long npage, bool do_accounting) vfio_unpin_pages_remote() argument
779 vfio_unpin_page_external(struct vfio_dma *dma, dma_addr_t iova, bool do_accounting) vfio_unpin_page_external() argument
839 dma_addr_t iova; vfio_iommu_type1_pin_pages() local
904 dma_addr_t iova; vfio_iommu_type1_pin_pages() local
931 dma_addr_t iova = user_iova + PAGE_SIZE * i; vfio_iommu_type1_unpin_pages() local
979 unmap_unpin_fast(struct vfio_domain *domain, struct vfio_dma *dma, dma_addr_t *iova, size_t len, phys_addr_t phys, long *unlocked, struct list_head *unmapped_list, int *unmapped_cnt, struct iommu_iotlb_gather *iotlb_gather) unmap_unpin_fast() argument
1019 unmap_unpin_slow(struct vfio_domain *domain, struct vfio_dma *dma, dma_addr_t *iova, size_t len, phys_addr_t phys, long *unlocked) unmap_unpin_slow() argument
1040 dma_addr_t iova = dma->iova, end = dma->iova + dma->size; vfio_unmap_unpin() local
1195 vfio_iova_dirty_bitmap(u64 __user *bitmap, struct vfio_iommu *iommu, dma_addr_t iova, size_t size, size_t pgsize) vfio_iova_dirty_bitmap() argument
1287 dma_addr_t iova = unmap->iova; vfio_dma_do_unmap() local
1430 vfio_iommu_map(struct vfio_iommu *iommu, dma_addr_t iova, unsigned long pfn, long npage, int prot) vfio_iommu_map() argument
1460 dma_addr_t iova = dma->iova; vfio_pin_map_dma() local
1510 struct list_head *iova = &iommu->iova_list; vfio_iommu_iova_dma_valid() local
1561 dma_addr_t iova = map->iova; vfio_dma_do_map() local
1697 dma_addr_t iova; vfio_iommu_replay() local
1782 dma_addr_t iova; vfio_iommu_replay() local
1955 struct list_head *iova = &iommu->iova_list; vfio_iommu_aper_conflict() local
1985 vfio_iommu_aper_resize(struct list_head *iova, dma_addr_t start, dma_addr_t end) vfio_iommu_aper_resize() argument
2046 vfio_iommu_resv_exclude(struct list_head *iova, struct list_head *resv_regions) vfio_iommu_resv_exclude() argument
2104 vfio_iommu_iova_free(struct list_head *iova) vfio_iommu_iova_free() argument
2117 struct list_head *iova = &iommu->iova_list; vfio_iommu_iova_get_copy() local
2137 struct list_head *iova = &iommu->iova_list; vfio_iommu_iova_insert_copy() local
2697 struct vfio_iova *iova; vfio_iommu_iova_build_caps() local
[all...] |
| H A D | iova_bitmap.c | 32 * data[(iova / page_size) / 64] & (1ULL << (iova % 64))
36 unsigned long iova; member
74 * bitmap = iova_bitmap_alloc(iova, length, page_size, data);
82 * Each iteration of the @dirty_reporter_fn is called with a unique @iova
87 * iova_bitmap_set(bitmap, iova, iova_length);
112 unsigned long iova; member
123 * (stored in mapped::iova). All computations in this file are done using
124 * relative IOVAs and thus avoid an extra subtraction against mapped::iova.
128 unsigned long iova) in iova_bitmap_offset_to_index()
127 iova_bitmap_offset_to_index(struct iova_bitmap *bitmap, unsigned long iova) iova_bitmap_offset_to_index() argument
238 iova_bitmap_alloc(unsigned long iova, size_t length, unsigned long page_size, u64 __user *data) iova_bitmap_alloc() argument
317 unsigned long iova = iova_bitmap_mapped_iova(bitmap); iova_bitmap_mapped_length() local
350 unsigned long iova = iova_bitmap_mapped_length(bitmap) - 1; iova_bitmap_advance() local
402 iova_bitmap_set(struct iova_bitmap *bitmap, unsigned long iova, size_t length) iova_bitmap_set() argument
[all...] |
| /kernel/linux/linux-5.10/drivers/vfio/ |
| H A D | vfio_iommu_type1.c | 90 dma_addr_t iova; /* Device address */ member
126 dma_addr_t iova; /* Device address */ member
133 dma_addr_t iova; member
173 if (start + size <= dma->iova) in vfio_find_dma()
175 else if (start >= dma->iova + dma->size) in vfio_find_dma()
193 if (new->iova + new->size <= dma->iova) in vfio_link_dma()
243 bitmap_set(dma->bitmap, (vpfn->iova - dma->iova) >> pgshift, 1); in vfio_dma_populate_bitmap()
296 * Helper Functions for host iova
298 vfio_find_vpfn(struct vfio_dma *dma, dma_addr_t iova) vfio_find_vpfn() argument
342 vfio_add_to_pfn_list(struct vfio_dma *dma, dma_addr_t iova, unsigned long pfn) vfio_add_to_pfn_list() argument
365 vfio_iova_get_vfio_pfn(struct vfio_dma *dma, unsigned long iova) vfio_iova_get_vfio_pfn() argument
568 dma_addr_t iova = vaddr - dma->vaddr + dma->iova; vfio_pin_pages_remote() local
642 vfio_unpin_pages_remote(struct vfio_dma *dma, dma_addr_t iova, unsigned long pfn, long npage, bool do_accounting) vfio_unpin_pages_remote() argument
697 vfio_unpin_page_external(struct vfio_dma *dma, dma_addr_t iova, bool do_accounting) vfio_unpin_page_external() argument
750 dma_addr_t iova; vfio_iommu_type1_pin_pages() local
808 dma_addr_t iova; vfio_iommu_type1_pin_pages() local
840 dma_addr_t iova; vfio_iommu_type1_unpin_pages() local
887 unmap_unpin_fast(struct vfio_domain *domain, struct vfio_dma *dma, dma_addr_t *iova, size_t len, phys_addr_t phys, long *unlocked, struct list_head *unmapped_list, int *unmapped_cnt, struct iommu_iotlb_gather *iotlb_gather) unmap_unpin_fast() argument
927 unmap_unpin_slow(struct vfio_domain *domain, struct vfio_dma *dma, dma_addr_t *iova, size_t len, phys_addr_t phys, long *unlocked) unmap_unpin_slow() argument
948 dma_addr_t iova = dma->iova, end = dma->iova + dma->size; vfio_unmap_unpin() local
1101 vfio_iova_dirty_bitmap(u64 __user *bitmap, struct vfio_iommu *iommu, dma_addr_t iova, size_t size, size_t pgsize) vfio_iova_dirty_bitmap() argument
1296 vfio_iommu_map(struct vfio_iommu *iommu, dma_addr_t iova, unsigned long pfn, long npage, int prot) vfio_iommu_map() argument
1325 dma_addr_t iova = dma->iova; vfio_pin_map_dma() local
1374 struct list_head *iova = &iommu->iova_list; vfio_iommu_iova_dma_valid() local
1392 dma_addr_t iova = map->iova; vfio_dma_do_map() local
1521 dma_addr_t iova; vfio_iommu_replay() local
1604 dma_addr_t iova; vfio_iommu_replay() local
1890 struct list_head *iova = &iommu->iova_list; vfio_iommu_aper_conflict() local
1920 vfio_iommu_aper_resize(struct list_head *iova, dma_addr_t start, dma_addr_t end) vfio_iommu_aper_resize() argument
1981 vfio_iommu_resv_exclude(struct list_head *iova, struct list_head *resv_regions) vfio_iommu_resv_exclude() argument
2039 vfio_iommu_iova_free(struct list_head *iova) vfio_iommu_iova_free() argument
2052 struct list_head *iova = &iommu->iova_list; vfio_iommu_iova_get_copy() local
2072 struct list_head *iova = &iommu->iova_list; vfio_iommu_iova_insert_copy() local
2624 struct vfio_iova *iova; vfio_iommu_iova_build_caps() local
[all...] |
| /kernel/linux/linux-6.6/drivers/infiniband/sw/rxe/ |
| H A D | rxe_mr.c | 27 int mr_check_range(struct rxe_mr *mr, u64 iova, size_t length) in mr_check_range() argument
35 if (iova < mr->ibmr.iova || in mr_check_range()
36 iova + length > mr->ibmr.iova + mr->ibmr.length) { in mr_check_range()
37 rxe_dbg_mr(mr, "iova/length out of range"); in mr_check_range()
75 static unsigned long rxe_mr_iova_to_index(struct rxe_mr *mr, u64 iova) in rxe_mr_iova_to_index() argument
77 return (iova >> mr->page_shift) - (mr->ibmr.iova >> mr->page_shift); in rxe_mr_iova_to_index()
80 static unsigned long rxe_mr_iova_to_page_offset(struct rxe_mr *mr, u64 iova) in rxe_mr_iova_to_page_offset() argument
129 rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova, int access, struct rxe_mr *mr) rxe_mr_init_user() argument
245 rxe_mr_copy_xarray(struct rxe_mr *mr, u64 iova, void *addr, unsigned int length, enum rxe_mr_copy_dir dir) rxe_mr_copy_xarray() argument
304 rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, unsigned int length, enum rxe_mr_copy_dir dir) rxe_mr_copy() argument
345 u64 iova; copy_data() local
424 rxe_flush_pmem_iova(struct rxe_mr *mr, u64 iova, unsigned int length) rxe_flush_pmem_iova() argument
471 rxe_mr_do_atomic_op(struct rxe_mr *mr, u64 iova, int opcode, u64 compare, u64 swap_add, u64 *orig_val) rxe_mr_do_atomic_op() argument
529 rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value) rxe_mr_do_atomic_write() argument
577 rxe_mr_do_atomic_write(struct rxe_mr *mr, u64 iova, u64 value) rxe_mr_do_atomic_write() argument
[all...] |
| /kernel/linux/linux-6.6/drivers/vdpa/vdpa_user/ |
| H A D | iova_domain.c | 104 u64 iova, u64 size, u64 paddr) in vduse_domain_map_bounce_page()
107 u64 last = iova + size - 1; in vduse_domain_map_bounce_page()
109 while (iova <= last) { in vduse_domain_map_bounce_page()
110 map = &domain->bounce_maps[iova >> PAGE_SHIFT]; in vduse_domain_map_bounce_page()
118 iova += PAGE_SIZE; in vduse_domain_map_bounce_page()
124 u64 iova, u64 size) in vduse_domain_unmap_bounce_page()
127 u64 last = iova + size - 1; in vduse_domain_unmap_bounce_page()
129 while (iova <= last) { in vduse_domain_unmap_bounce_page()
130 map = &domain->bounce_maps[iova >> PAGE_SHIFT]; in vduse_domain_unmap_bounce_page()
132 iova in vduse_domain_unmap_bounce_page()
103 vduse_domain_map_bounce_page(struct vduse_iova_domain *domain, u64 iova, u64 size, u64 paddr) vduse_domain_map_bounce_page() argument
123 vduse_domain_unmap_bounce_page(struct vduse_iova_domain *domain, u64 iova, u64 size) vduse_domain_unmap_bounce_page() argument
160 vduse_domain_bounce(struct vduse_iova_domain *domain, dma_addr_t iova, size_t size, enum dma_data_direction dir) vduse_domain_bounce() argument
190 vduse_domain_get_coherent_page(struct vduse_iova_domain *domain, u64 iova) vduse_domain_get_coherent_page() argument
211 vduse_domain_get_bounce_page(struct vduse_iova_domain *domain, u64 iova) vduse_domain_get_bounce_page() argument
367 vduse_domain_free_iova(struct iova_domain *iovad, dma_addr_t iova, size_t size) vduse_domain_free_iova() argument
384 dma_addr_t iova = vduse_domain_alloc_iova(iovad, size, limit); vduse_domain_map_page() local
430 dma_addr_t iova = vduse_domain_alloc_iova(iovad, size, limit); vduse_domain_alloc_coherent() local
488 unsigned long iova = vmf->pgoff << PAGE_SHIFT; vduse_domain_mmap_fault() local
[all...] |
| /kernel/linux/linux-6.6/drivers/iommu/iommufd/ |
| H A D | selftest.c | 42 * Syzkaller has trouble randomizing the correct iova to use since it is linked
48 u64 *iova) in iommufd_test_syz_conv_iova()
54 struct syz_layout *syz = (void *)iova; in iommufd_test_syz_conv_iova()
73 unsigned int ioas_id, u64 *iova, u32 *flags) in iommufd_test_syz_conv_iova_id()
84 *iova = iommufd_test_syz_conv_iova(&ioas->iopt, iova); in iommufd_test_syz_conv_iova_id()
179 unsigned long iova, phys_addr_t paddr, in mock_domain_map_pages()
186 unsigned long start_iova = iova; in mock_domain_map_pages()
195 WARN_ON(iova % MOCK_IO_PAGE_SIZE); in mock_domain_map_pages()
205 old = xa_store(&mock->pfns, iova / MOCK_IO_PAGE_SIZ in mock_domain_map_pages()
47 iommufd_test_syz_conv_iova(struct io_pagetable *iopt, u64 *iova) iommufd_test_syz_conv_iova() argument
72 iommufd_test_syz_conv_iova_id(struct iommufd_ucmd *ucmd, unsigned int ioas_id, u64 *iova, u32 *flags) iommufd_test_syz_conv_iova_id() argument
178 mock_domain_map_pages(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t pgsize, size_t pgcount, int prot, gfp_t gfp, size_t *mapped) mock_domain_map_pages() argument
227 mock_domain_unmap_pages(struct iommu_domain *domain, unsigned long iova, size_t pgsize, size_t pgcount, struct iommu_iotlb_gather *iotlb_gather) mock_domain_unmap_pages() argument
271 mock_domain_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova) mock_domain_iova_to_phys() argument
514 iommufd_test_md_check_pa(struct iommufd_ucmd *ucmd, unsigned int mockpt_id, unsigned long iova, size_t length, void __user *uptr) iommufd_test_md_check_pa() argument
614 unsigned long iova; global() member
636 iommufd_test_access_unmap(void *data, unsigned long iova, unsigned long length) iommufd_test_access_unmap() argument
826 iommufd_test_access_pages(struct iommufd_ucmd *ucmd, unsigned int access_id, unsigned long iova, size_t length, void __user *uptr, u32 flags) iommufd_test_access_pages() argument
921 iommufd_test_access_rw(struct iommufd_ucmd *ucmd, unsigned int access_id, unsigned long iova, size_t length, void __user *ubuf, unsigned int flags) iommufd_test_access_rw() argument
[all...] |
| /kernel/linux/linux-6.6/drivers/iommu/amd/ |
| H A D | io_pgtable_v2.c | 135 static u64 *v2_alloc_pte(int nid, u64 *pgd, unsigned long iova, in v2_alloc_pte() argument
143 pte = &pgd[PM_LEVEL_INDEX(level, iova)]; in v2_alloc_pte()
144 iova = PAGE_SIZE_ALIGN(iova, PAGE_SIZE); in v2_alloc_pte()
175 pte = &pte[PM_LEVEL_INDEX(level, iova)]; in v2_alloc_pte()
199 unsigned long iova, unsigned long *page_size) in fetch_pte()
205 pte = &pgtable->pgd[PM_LEVEL_INDEX(level, iova)]; in fetch_pte()
216 pte = &pte[PM_LEVEL_INDEX(level - 1, iova)]; in fetch_pte()
236 static int iommu_v2_map_pages(struct io_pgtable_ops *ops, unsigned long iova, in iommu_v2_map_pages() argument
245 unsigned long o_iova = iova; in iommu_v2_map_pages()
198 fetch_pte(struct amd_io_pgtable *pgtable, unsigned long iova, unsigned long *page_size) fetch_pte() argument
288 iommu_v2_unmap_pages(struct io_pgtable_ops *ops, unsigned long iova, size_t pgsize, size_t pgcount, struct iommu_iotlb_gather *gather) iommu_v2_unmap_pages() argument
317 iommu_v2_iova_to_phys(struct io_pgtable_ops *ops, unsigned long iova) iommu_v2_iova_to_phys() argument
340 v2_tlb_flush_walk(unsigned long iova, size_t size, size_t granule, void *cookie) v2_tlb_flush_walk() argument
345 v2_tlb_add_page(struct iommu_iotlb_gather *gather, unsigned long iova, size_t granule, void *cookie) v2_tlb_add_page() argument
[all...] |