Lines Matching defs:iovad
18 static bool iova_rcache_insert(struct iova_domain *iovad,
21 static unsigned long iova_rcache_get(struct iova_domain *iovad,
24 static void init_iova_rcaches(struct iova_domain *iovad);
25 static void free_iova_rcaches(struct iova_domain *iovad);
26 static void fq_destroy_all_entries(struct iova_domain *iovad);
30 init_iova_domain(struct iova_domain *iovad, unsigned long granule,
40 spin_lock_init(&iovad->iova_rbtree_lock);
41 iovad->rbroot = RB_ROOT;
42 iovad->cached_node = &iovad->anchor.node;
43 iovad->cached32_node = &iovad->anchor.node;
44 iovad->granule = granule;
45 iovad->start_pfn = start_pfn;
46 iovad->dma_32bit_pfn = 1UL << (32 - iova_shift(iovad));
47 iovad->max32_alloc_size = iovad->dma_32bit_pfn;
48 iovad->flush_cb = NULL;
49 iovad->fq = NULL;
50 iovad->anchor.pfn_lo = iovad->anchor.pfn_hi = IOVA_ANCHOR;
51 rb_link_node(&iovad->anchor.node, NULL, &iovad->rbroot.rb_node);
52 rb_insert_color(&iovad->anchor.node, &iovad->rbroot);
53 init_iova_rcaches(iovad);
57 bool has_iova_flush_queue(struct iova_domain *iovad)
59 return !!iovad->fq;
62 static void free_iova_flush_queue(struct iova_domain *iovad)
64 if (!has_iova_flush_queue(iovad))
67 del_timer_sync(&iovad->fq_timer);
69 fq_destroy_all_entries(iovad);
71 free_percpu(iovad->fq);
73 iovad->fq = NULL;
74 iovad->flush_cb = NULL;
75 iovad->entry_dtor = NULL;
78 int init_iova_flush_queue(struct iova_domain *iovad,
84 atomic64_set(&iovad->fq_flush_start_cnt, 0);
85 atomic64_set(&iovad->fq_flush_finish_cnt, 0);
91 iovad->flush_cb = flush_cb;
92 iovad->entry_dtor = entry_dtor;
106 iovad->fq = queue;
108 timer_setup(&iovad->fq_timer, fq_flush_timeout, 0);
109 atomic_set(&iovad->fq_timer_on, 0);
116 __get_cached_rbnode(struct iova_domain *iovad, unsigned long limit_pfn)
118 if (limit_pfn <= iovad->dma_32bit_pfn)
119 return iovad->cached32_node;
121 return iovad->cached_node;
125 __cached_rbnode_insert_update(struct iova_domain *iovad, struct iova *new)
127 if (new->pfn_hi < iovad->dma_32bit_pfn)
128 iovad->cached32_node = &new->node;
130 iovad->cached_node = &new->node;
134 __cached_rbnode_delete_update(struct iova_domain *iovad, struct iova *free)
138 cached_iova = rb_entry(iovad->cached32_node, struct iova, node);
140 (free->pfn_hi < iovad->dma_32bit_pfn &&
142 iovad->cached32_node = rb_next(&free->node);
144 if (free->pfn_lo < iovad->dma_32bit_pfn)
145 iovad->max32_alloc_size = iovad->dma_32bit_pfn;
147 cached_iova = rb_entry(iovad->cached_node, struct iova, node);
149 iovad->cached_node = rb_next(&free->node);
180 static int __alloc_and_insert_iova_range(struct iova_domain *iovad,
194 spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);
195 if (limit_pfn <= iovad->dma_32bit_pfn &&
196 size >= iovad->max32_alloc_size)
199 curr = __get_cached_rbnode(iovad, limit_pfn);
209 if (limit_pfn < size || new_pfn < iovad->start_pfn) {
210 iovad->max32_alloc_size = size;
219 iova_insert_rbtree(&iovad->rbroot, new, prev);
220 __cached_rbnode_insert_update(iovad, new);
222 spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
226 spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
284 * @iovad: - iova domain in question
288 * This function allocates an iova in the range iovad->start_pfn to limit_pfn,
289 * searching top-down from limit_pfn to iovad->start_pfn. If the size_aligned
294 alloc_iova(struct iova_domain *iovad, unsigned long size,
305 ret = __alloc_and_insert_iova_range(iovad, size, limit_pfn + 1,
318 private_find_iova(struct iova_domain *iovad, unsigned long pfn)
320 struct rb_node *node = iovad->rbroot.rb_node;
322 assert_spin_locked(&iovad->iova_rbtree_lock);
338 static void private_free_iova(struct iova_domain *iovad, struct iova *iova)
340 assert_spin_locked(&iovad->iova_rbtree_lock);
341 __cached_rbnode_delete_update(iovad, iova);
342 rb_erase(&iova->node, &iovad->rbroot);
348 * @iovad: - iova domain in question.
353 struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn)
359 spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);
360 iova = private_find_iova(iovad, pfn);
361 spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
368 * @iovad: iova domain in question.
373 __free_iova(struct iova_domain *iovad, struct iova *iova)
377 spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);
378 private_free_iova(iovad, iova);
379 spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
385 * @iovad: - iova domain in question.
391 free_iova(struct iova_domain *iovad, unsigned long pfn)
393 struct iova *iova = find_iova(iovad, pfn);
396 __free_iova(iovad, iova);
403 * @iovad: - iova domain in question
412 alloc_iova_fast(struct iova_domain *iovad, unsigned long size,
418 iova_pfn = iova_rcache_get(iovad, size, limit_pfn + 1);
423 new_iova = alloc_iova(iovad, size, limit_pfn, true);
433 free_cpu_cached_iovas(cpu, iovad);
443 * @iovad: - iova domain in question.
450 free_iova_fast(struct iova_domain *iovad, unsigned long pfn, unsigned long size)
452 if (iova_rcache_insert(iovad, pfn, size))
455 free_iova(iovad, pfn);
479 static void fq_ring_free(struct iova_domain *iovad, struct iova_fq *fq)
481 u64 counter = atomic64_read(&iovad->fq_flush_finish_cnt);
491 if (iovad->entry_dtor)
492 iovad->entry_dtor(fq->entries[idx].data);
494 free_iova_fast(iovad,
502 static void iova_domain_flush(struct iova_domain *iovad)
504 atomic64_inc(&iovad->fq_flush_start_cnt);
505 iovad->flush_cb(iovad);
506 atomic64_inc(&iovad->fq_flush_finish_cnt);
509 static void fq_destroy_all_entries(struct iova_domain *iovad)
518 if (!iovad->entry_dtor)
522 struct iova_fq *fq = per_cpu_ptr(iovad->fq, cpu);
526 iovad->entry_dtor(fq->entries[idx].data);
532 struct iova_domain *iovad = from_timer(iovad, t, fq_timer);
535 atomic_set(&iovad->fq_timer_on, 0);
536 iova_domain_flush(iovad);
542 fq = per_cpu_ptr(iovad->fq, cpu);
544 fq_ring_free(iovad, fq);
549 void queue_iova(struct iova_domain *iovad,
553 struct iova_fq *fq = raw_cpu_ptr(iovad->fq);
564 fq_ring_free(iovad, fq);
567 iova_domain_flush(iovad);
568 fq_ring_free(iovad, fq);
576 fq->entries[idx].counter = atomic64_read(&iovad->fq_flush_start_cnt);
581 if (!atomic_read(&iovad->fq_timer_on) &&
582 !atomic_xchg(&iovad->fq_timer_on, 1))
583 mod_timer(&iovad->fq_timer,
590 * @iovad: - iova domain in question.
593 void put_iova_domain(struct iova_domain *iovad)
597 free_iova_flush_queue(iovad);
598 free_iova_rcaches(iovad);
599 rbtree_postorder_for_each_entry_safe(iova, tmp, &iovad->rbroot, node)
630 __insert_new_range(struct iova_domain *iovad,
637 iova_insert_rbtree(&iovad->rbroot, iova, NULL);
654 * @iovad: - iova domain pointer
661 reserve_iova(struct iova_domain *iovad,
670 if (WARN_ON((pfn_hi | pfn_lo) > (ULLONG_MAX >> iova_shift(iovad))))
673 spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);
674 for (node = rb_first(&iovad->rbroot); node; node = rb_next(node)) {
690 iova = __insert_new_range(iovad, pfn_lo, pfn_hi);
693 spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
729 split_and_remove_iova(struct iova_domain *iovad, struct iova *iova,
735 spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);
747 __cached_rbnode_delete_update(iovad, iova);
748 rb_erase(&iova->node, &iovad->rbroot);
751 iova_insert_rbtree(&iovad->rbroot, prev, NULL);
755 iova_insert_rbtree(&iovad->rbroot, next, NULL);
758 spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
763 spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
801 iova_magazine_free_pfns(struct iova_magazine *mag, struct iova_domain *iovad)
809 spin_lock_irqsave(&iovad->iova_rbtree_lock, flags);
812 struct iova *iova = private_find_iova(iovad, mag->pfns[i]);
817 private_free_iova(iovad, iova);
820 spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags);
862 static void init_iova_rcaches(struct iova_domain *iovad)
870 rcache = &iovad->rcaches[i];
891 static bool __iova_rcache_insert(struct iova_domain *iovad,
932 iova_magazine_free_pfns(mag_to_free, iovad);
939 static bool iova_rcache_insert(struct iova_domain *iovad, unsigned long pfn,
947 return __iova_rcache_insert(iovad, &iovad->rcaches[log_size], pfn);
994 static unsigned long iova_rcache_get(struct iova_domain *iovad,
1003 return __iova_rcache_get(&iovad->rcaches[log_size], limit_pfn - size);
1009 static void free_iova_rcaches(struct iova_domain *iovad)
1017 rcache = &iovad->rcaches[i];
1032 void free_cpu_cached_iovas(unsigned int cpu, struct iova_domain *iovad)
1040 rcache = &iovad->rcaches[i];
1043 iova_magazine_free_pfns(cpu_rcache->loaded, iovad);
1044 iova_magazine_free_pfns(cpu_rcache->prev, iovad);