| /kernel/linux/linux-5.10/drivers/iommu/ |
| H A D | iova.c | 18 static bool iova_rcache_insert(struct iova_domain *iovad,
21 static unsigned long iova_rcache_get(struct iova_domain *iovad,
24 static void init_iova_rcaches(struct iova_domain *iovad);
25 static void free_iova_rcaches(struct iova_domain *iovad);
26 static void fq_destroy_all_entries(struct iova_domain *iovad);
30 init_iova_domain(struct iova_domain *iovad, unsigned long granule, in init_iova_domain() argument
40 spin_lock_init(&iovad->iova_rbtree_lock); in init_iova_domain()
41 iovad->rbroot = RB_ROOT; in init_iova_domain()
42 iovad->cached_node = &iovad in init_iova_domain()
57 has_iova_flush_queue(struct iova_domain *iovad) has_iova_flush_queue() argument
62 free_iova_flush_queue(struct iova_domain *iovad) free_iova_flush_queue() argument
78 init_iova_flush_queue(struct iova_domain *iovad, iova_flush_cb flush_cb, iova_entry_dtor entry_dtor) init_iova_flush_queue() argument
116 __get_cached_rbnode(struct iova_domain *iovad, unsigned long limit_pfn) __get_cached_rbnode() argument
125 __cached_rbnode_insert_update(struct iova_domain *iovad, struct iova *new) __cached_rbnode_insert_update() argument
134 __cached_rbnode_delete_update(struct iova_domain *iovad, struct iova *free) __cached_rbnode_delete_update() argument
180 __alloc_and_insert_iova_range(struct iova_domain *iovad, unsigned long size, unsigned long limit_pfn, struct iova *new, bool size_aligned) __alloc_and_insert_iova_range() argument
294 alloc_iova(struct iova_domain *iovad, unsigned long size, unsigned long limit_pfn, bool size_aligned) alloc_iova() argument
318 private_find_iova(struct iova_domain *iovad, unsigned long pfn) private_find_iova() argument
338 private_free_iova(struct iova_domain *iovad, struct iova *iova) private_free_iova() argument
353 find_iova(struct iova_domain *iovad, unsigned long pfn) find_iova() argument
373 __free_iova(struct iova_domain *iovad, struct iova *iova) __free_iova() argument
391 free_iova(struct iova_domain *iovad, unsigned long pfn) free_iova() argument
412 alloc_iova_fast(struct iova_domain *iovad, unsigned long size, unsigned long limit_pfn, bool flush_rcache) alloc_iova_fast() argument
450 free_iova_fast(struct iova_domain *iovad, unsigned long pfn, unsigned long size) free_iova_fast() argument
479 fq_ring_free(struct iova_domain *iovad, struct iova_fq *fq) fq_ring_free() argument
502 iova_domain_flush(struct iova_domain *iovad) iova_domain_flush() argument
509 fq_destroy_all_entries(struct iova_domain *iovad) fq_destroy_all_entries() argument
532 struct iova_domain *iovad = from_timer(iovad, t, fq_timer); fq_flush_timeout() local
549 queue_iova(struct iova_domain *iovad, unsigned long pfn, unsigned long pages, unsigned long data) queue_iova() argument
593 put_iova_domain(struct iova_domain *iovad) put_iova_domain() argument
630 __insert_new_range(struct iova_domain *iovad, unsigned long pfn_lo, unsigned long pfn_hi) __insert_new_range() argument
661 reserve_iova(struct iova_domain *iovad, unsigned long pfn_lo, unsigned long pfn_hi) reserve_iova() argument
729 split_and_remove_iova(struct iova_domain *iovad, struct iova *iova, unsigned long pfn_lo, unsigned long pfn_hi) split_and_remove_iova() argument
801 iova_magazine_free_pfns(struct iova_magazine *mag, struct iova_domain *iovad) iova_magazine_free_pfns() argument
862 init_iova_rcaches(struct iova_domain *iovad) init_iova_rcaches() argument
891 __iova_rcache_insert(struct iova_domain *iovad, struct iova_rcache *rcache, unsigned long iova_pfn) __iova_rcache_insert() argument
939 iova_rcache_insert(struct iova_domain *iovad, unsigned long pfn, unsigned long size) iova_rcache_insert() argument
994 iova_rcache_get(struct iova_domain *iovad, unsigned long size, unsigned long limit_pfn) iova_rcache_get() argument
1009 free_iova_rcaches(struct iova_domain *iovad) free_iova_rcaches() argument
1032 free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad) free_cpu_cached_iovas() argument
[all...] |
| H A D | dma-iommu.c | 42 struct iova_domain iovad; member
55 return cookie->iovad.granule; in cookie_msi_granule()
138 if (cookie->type == IOMMU_DMA_IOVA_COOKIE && cookie->iovad.granule) in iommu_put_dma_cookie()
139 put_iova_domain(&cookie->iovad); in iommu_put_dma_cookie()
172 struct iova_domain *iovad = &cookie->iovad; in cookie_init_hw_msi_region() local
176 start -= iova_offset(iovad, start); in cookie_init_hw_msi_region()
177 num_pages = iova_align(iovad, end - start) >> iova_shift(iovad); in cookie_init_hw_msi_region()
188 start += iovad in cookie_init_hw_msi_region()
194 iova_reserve_pci_windows(struct pci_dev *dev, struct iova_domain *iovad) iova_reserve_pci_windows() argument
243 struct iova_domain *iovad = &cookie->iovad; iova_reserve_iommu_regions() local
277 iommu_dma_flush_iotlb_all(struct iova_domain *iovad) iommu_dma_flush_iotlb_all() argument
308 struct iova_domain *iovad; iommu_dma_init_domain() local
408 struct iova_domain *iovad = &cookie->iovad; iommu_dma_alloc_iova() local
447 struct iova_domain *iovad = &cookie->iovad; iommu_dma_free_iova() local
465 struct iova_domain *iovad = &cookie->iovad; __iommu_dma_unmap() local
487 struct iova_domain *iovad = &cookie->iovad; __iommu_dma_map() local
590 struct iova_domain *iovad = &cookie->iovad; iommu_dma_alloc_remap() local
838 struct iova_domain *iovad = &cookie->iovad; iommu_dma_map_sg() local
[all...] |
| /kernel/linux/linux-5.10/include/linux/ |
| H A D | iova.h | 105 static inline unsigned long iova_shift(struct iova_domain *iovad) in iova_shift() argument
107 return __ffs(iovad->granule); in iova_shift()
110 static inline unsigned long iova_mask(struct iova_domain *iovad) in iova_mask() argument
112 return iovad->granule - 1; in iova_mask()
115 static inline size_t iova_offset(struct iova_domain *iovad, dma_addr_t iova) in iova_offset() argument
117 return iova & iova_mask(iovad); in iova_offset()
120 static inline size_t iova_align(struct iova_domain *iovad, size_t size) in iova_align() argument
122 return ALIGN(size, iovad->granule); in iova_align()
125 static inline dma_addr_t iova_dma_addr(struct iova_domain *iovad, struct iova *iova) in iova_dma_addr() argument
127 return (dma_addr_t)iova->pfn_lo << iova_shift(iovad); in iova_dma_addr()
130 iova_pfn(struct iova_domain *iovad, dma_addr_t iova) iova_pfn() argument
185 free_iova(struct iova_domain *iovad, unsigned long pfn) free_iova() argument
189 __free_iova(struct iova_domain *iovad, struct iova *iova) __free_iova() argument
193 alloc_iova(struct iova_domain *iovad, unsigned long size, unsigned long limit_pfn, bool size_aligned) alloc_iova() argument
201 free_iova_fast(struct iova_domain *iovad, unsigned long pfn, unsigned long size) free_iova_fast() argument
207 queue_iova(struct iova_domain *iovad, unsigned long pfn, unsigned long pages, unsigned long data) queue_iova() argument
213 alloc_iova_fast(struct iova_domain *iovad, unsigned long size, unsigned long limit_pfn, bool flush_rcache) alloc_iova_fast() argument
221 reserve_iova(struct iova_domain *iovad, unsigned long pfn_lo, unsigned long pfn_hi) reserve_iova() argument
233 init_iova_domain(struct iova_domain *iovad, unsigned long granule, unsigned long start_pfn) init_iova_domain() argument
239 has_iova_flush_queue(struct iova_domain *iovad) has_iova_flush_queue() argument
244 init_iova_flush_queue(struct iova_domain *iovad, iova_flush_cb flush_cb, iova_entry_dtor entry_dtor) init_iova_flush_queue() argument
251 find_iova(struct iova_domain *iovad, unsigned long pfn) find_iova() argument
257 put_iova_domain(struct iova_domain *iovad) put_iova_domain() argument
261 split_and_remove_iova(struct iova_domain *iovad, struct iova *iova, unsigned long pfn_lo, unsigned long pfn_hi) split_and_remove_iova() argument
269 free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad) free_cpu_cached_iovas() argument
[all...] |
| H A D | intel-iommu.h | 544 struct iova_domain iovad; /* iova's that belong to this domain */ member
|
| /kernel/linux/linux-6.6/drivers/iommu/ |
| H A D | iova.c | 20 static bool iova_rcache_insert(struct iova_domain *iovad,
23 static unsigned long iova_rcache_get(struct iova_domain *iovad,
26 static void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad);
27 static void free_iova_rcaches(struct iova_domain *iovad);
36 struct iova_domain *iovad; in iova_cpuhp_dead() local
38 iovad = hlist_entry_safe(node, struct iova_domain, cpuhp_dead); in iova_cpuhp_dead()
40 free_cpu_cached_iovas(cpu, iovad); in iova_cpuhp_dead()
44 static void free_global_cached_iovas(struct iova_domain *iovad);
52 init_iova_domain(struct iova_domain *iovad, unsigned long granule, in init_iova_domain() argument
62 spin_lock_init(&iovad in init_iova_domain()
77 __get_cached_rbnode(struct iova_domain *iovad, unsigned long limit_pfn) __get_cached_rbnode() argument
86 __cached_rbnode_insert_update(struct iova_domain *iovad, struct iova *new) __cached_rbnode_insert_update() argument
95 __cached_rbnode_delete_update(struct iova_domain *iovad, struct iova *free) __cached_rbnode_delete_update() argument
113 iova_find_limit(struct iova_domain *iovad, unsigned long limit_pfn) iova_find_limit() argument
178 __alloc_and_insert_iova_range(struct iova_domain *iovad, unsigned long size, unsigned long limit_pfn, struct iova *new, bool size_aligned) __alloc_and_insert_iova_range() argument
314 alloc_iova(struct iova_domain *iovad, unsigned long size, unsigned long limit_pfn, bool size_aligned) alloc_iova() argument
338 private_find_iova(struct iova_domain *iovad, unsigned long pfn) private_find_iova() argument
358 remove_iova(struct iova_domain *iovad, struct iova *iova) remove_iova() argument
372 find_iova(struct iova_domain *iovad, unsigned long pfn) find_iova() argument
392 __free_iova(struct iova_domain *iovad, struct iova *iova) __free_iova() argument
411 free_iova(struct iova_domain *iovad, unsigned long pfn) free_iova() argument
439 alloc_iova_fast(struct iova_domain *iovad, unsigned long size, unsigned long limit_pfn, bool flush_rcache) alloc_iova_fast() argument
487 free_iova_fast(struct iova_domain *iovad, unsigned long pfn, unsigned long size) free_iova_fast() argument
496 iova_domain_free_rcaches(struct iova_domain *iovad) iova_domain_free_rcaches() argument
508 put_iova_domain(struct iova_domain *iovad) put_iova_domain() argument
546 __insert_new_range(struct iova_domain *iovad, unsigned long pfn_lo, unsigned long pfn_hi) __insert_new_range() argument
577 reserve_iova(struct iova_domain *iovad, unsigned long pfn_lo, unsigned long pfn_hi) reserve_iova() argument
665 iova_magazine_free_pfns(struct iova_magazine *mag, struct iova_domain *iovad) iova_magazine_free_pfns() argument
720 iova_domain_init_rcaches(struct iova_domain *iovad) iova_domain_init_rcaches() argument
775 __iova_rcache_insert(struct iova_domain *iovad, struct iova_rcache *rcache, unsigned long iova_pfn) __iova_rcache_insert() argument
823 iova_rcache_insert(struct iova_domain *iovad, unsigned long pfn, unsigned long size) iova_rcache_insert() argument
878 iova_rcache_get(struct iova_domain *iovad, unsigned long size, unsigned long limit_pfn) iova_rcache_get() argument
893 free_iova_rcaches(struct iova_domain *iovad) free_iova_rcaches() argument
921 free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad) free_cpu_cached_iovas() argument
941 free_global_cached_iovas(struct iova_domain *iovad) free_global_cached_iovas() argument
[all...] |
| H A D | dma-iommu.c | 52 struct iova_domain iovad; member
141 free_iova_fast(&cookie->iovad, in fq_ring_free()
289 return cookie->iovad.granule; in cookie_msi_granule()
367 if (cookie->type == IOMMU_DMA_IOVA_COOKIE && cookie->iovad.granule) { in iommu_put_dma_cookie()
369 put_iova_domain(&cookie->iovad); in iommu_put_dma_cookie()
404 struct iova_domain *iovad = &cookie->iovad; in cookie_init_hw_msi_region() local
408 start -= iova_offset(iovad, start); in cookie_init_hw_msi_region()
409 num_pages = iova_align(iovad, end - start) >> iova_shift(iovad); in cookie_init_hw_msi_region()
435 iova_reserve_pci_windows(struct pci_dev *dev, struct iova_domain *iovad) iova_reserve_pci_windows() argument
485 struct iova_domain *iovad = &cookie->iovad; iova_reserve_iommu_regions() local
575 struct iova_domain *iovad; iommu_dma_init_domain() local
663 struct iova_domain *iovad = &cookie->iovad; iommu_dma_alloc_iova() local
708 struct iova_domain *iovad = &cookie->iovad; iommu_dma_free_iova() local
727 struct iova_domain *iovad = &cookie->iovad; __iommu_dma_unmap() local
750 struct iova_domain *iovad = &cookie->iovad; __iommu_dma_map() local
840 struct iova_domain *iovad = &cookie->iovad; __iommu_dma_alloc_noncontiguous() local
1039 struct iova_domain *iovad = &cookie->iovad; iommu_dma_map_page() local
1246 struct iova_domain *iovad = &cookie->iovad; iommu_dma_map_sg() local
[all...] |
| /kernel/linux/linux-6.6/include/linux/ |
| H A D | iova.h | 48 static inline unsigned long iova_shift(struct iova_domain *iovad) in iova_shift() argument
50 return __ffs(iovad->granule); in iova_shift()
53 static inline unsigned long iova_mask(struct iova_domain *iovad) in iova_mask() argument
55 return iovad->granule - 1; in iova_mask()
58 static inline size_t iova_offset(struct iova_domain *iovad, dma_addr_t iova) in iova_offset() argument
60 return iova & iova_mask(iovad); in iova_offset()
63 static inline size_t iova_align(struct iova_domain *iovad, size_t size) in iova_align() argument
65 return ALIGN(size, iovad->granule); in iova_align()
68 static inline dma_addr_t iova_dma_addr(struct iova_domain *iovad, struct iova *iova) in iova_dma_addr() argument
70 return (dma_addr_t)iova->pfn_lo << iova_shift(iovad); in iova_dma_addr()
73 iova_pfn(struct iova_domain *iovad, dma_addr_t iova) iova_pfn() argument
110 free_iova(struct iova_domain *iovad, unsigned long pfn) free_iova() argument
114 __free_iova(struct iova_domain *iovad, struct iova *iova) __free_iova() argument
118 alloc_iova(struct iova_domain *iovad, unsigned long size, unsigned long limit_pfn, bool size_aligned) alloc_iova() argument
126 free_iova_fast(struct iova_domain *iovad, unsigned long pfn, unsigned long size) free_iova_fast() argument
132 alloc_iova_fast(struct iova_domain *iovad, unsigned long size, unsigned long limit_pfn, bool flush_rcache) alloc_iova_fast() argument
140 reserve_iova(struct iova_domain *iovad, unsigned long pfn_lo, unsigned long pfn_hi) reserve_iova() argument
147 init_iova_domain(struct iova_domain *iovad, unsigned long granule, unsigned long start_pfn) init_iova_domain() argument
153 find_iova(struct iova_domain *iovad, unsigned long pfn) find_iova() argument
159 put_iova_domain(struct iova_domain *iovad) put_iova_domain() argument
[all...] |
| /kernel/linux/linux-6.6/drivers/vdpa/vdpa_user/ |
| H A D | iova_domain.c | 355 vduse_domain_alloc_iova(struct iova_domain *iovad, in vduse_domain_alloc_iova() argument
358 unsigned long shift = iova_shift(iovad); in vduse_domain_alloc_iova()
359 unsigned long iova_len = iova_align(iovad, size) >> shift; in vduse_domain_alloc_iova()
362 iova_pfn = alloc_iova_fast(iovad, iova_len, limit >> shift, true); in vduse_domain_alloc_iova()
367 static void vduse_domain_free_iova(struct iova_domain *iovad, in vduse_domain_free_iova() argument
370 unsigned long shift = iova_shift(iovad); in vduse_domain_free_iova()
371 unsigned long iova_len = iova_align(iovad, size) >> shift; in vduse_domain_free_iova()
373 free_iova_fast(iovad, iova >> shift, iova_len); in vduse_domain_free_iova()
381 struct iova_domain *iovad = &domain->stream_iovad; in vduse_domain_map_page() local
384 dma_addr_t iova = vduse_domain_alloc_iova(iovad, siz in vduse_domain_map_page()
413 struct iova_domain *iovad = &domain->stream_iovad; vduse_domain_unmap_page() local
428 struct iova_domain *iovad = &domain->consistent_iovad; vduse_domain_alloc_coherent() local
462 struct iova_domain *iovad = &domain->consistent_iovad; vduse_domain_free_coherent() local
[all...] |
| /kernel/linux/linux-5.10/drivers/iommu/intel/ |
| H A D | iommu.c | 1721 static void iommu_flush_iova(struct iova_domain *iovad) in iommu_flush_iova() argument
1726 domain = container_of(iovad, struct dmar_domain, iovad); in iommu_flush_iova()
2079 put_iova_domain(&domain->iovad); in domain_exit()
3551 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages, in intel_alloc_iova()
3556 iova_pfn = alloc_iova_fast(&domain->iovad, nrpages, in intel_alloc_iova()
3623 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size)); in __intel_map_single()
3671 !has_iova_flush_queue(&domain->iovad)) { in intel_unmap()
3675 free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(nrpages)); in intel_unmap()
3678 queue_iova(&domain->iovad, iova_pf in intel_unmap()
[all...] |