Lines Matching refs:cookie
128 static void fq_ring_free(struct iommu_dma_cookie *cookie, struct iova_fq *fq)
130 u64 counter = atomic64_read(&cookie->fq_flush_finish_cnt);
141 free_iova_fast(&cookie->iovad,
149 static void fq_flush_iotlb(struct iommu_dma_cookie *cookie)
151 atomic64_inc(&cookie->fq_flush_start_cnt);
152 cookie->fq_domain->ops->flush_iotlb_all(cookie->fq_domain);
153 atomic64_inc(&cookie->fq_flush_finish_cnt);
158 struct iommu_dma_cookie *cookie = from_timer(cookie, t, fq_timer);
161 atomic_set(&cookie->fq_timer_on, 0);
162 fq_flush_iotlb(cookie);
168 fq = per_cpu_ptr(cookie->fq, cpu);
170 fq_ring_free(cookie, fq);
175 static void queue_iova(struct iommu_dma_cookie *cookie,
192 fq = raw_cpu_ptr(cookie->fq);
200 fq_ring_free(cookie, fq);
203 fq_flush_iotlb(cookie);
204 fq_ring_free(cookie, fq);
211 fq->entries[idx].counter = atomic64_read(&cookie->fq_flush_start_cnt);
217 if (!atomic_read(&cookie->fq_timer_on) &&
218 !atomic_xchg(&cookie->fq_timer_on, 1))
219 mod_timer(&cookie->fq_timer,
223 static void iommu_dma_free_fq(struct iommu_dma_cookie *cookie)
227 if (!cookie->fq)
230 del_timer_sync(&cookie->fq_timer);
233 struct iova_fq *fq = per_cpu_ptr(cookie->fq, cpu);
239 free_percpu(cookie->fq);
245 struct iommu_dma_cookie *cookie = domain->iova_cookie;
249 if (cookie->fq_domain)
252 atomic64_set(&cookie->fq_flush_start_cnt, 0);
253 atomic64_set(&cookie->fq_flush_finish_cnt, 0);
273 cookie->fq = queue;
275 timer_setup(&cookie->fq_timer, fq_flush_timeout, 0);
276 atomic_set(&cookie->fq_timer_on, 0);
282 WRITE_ONCE(cookie->fq_domain, domain);
286 static inline size_t cookie_msi_granule(struct iommu_dma_cookie *cookie)
288 if (cookie->type == IOMMU_DMA_IOVA_COOKIE)
289 return cookie->iovad.granule;
295 struct iommu_dma_cookie *cookie;
297 cookie = kzalloc(sizeof(*cookie), GFP_KERNEL);
298 if (cookie) {
299 INIT_LIST_HEAD(&cookie->msi_page_list);
300 cookie->type = type;
302 return cookie;
336 struct iommu_dma_cookie *cookie;
344 cookie = cookie_alloc(IOMMU_DMA_MSI_COOKIE);
345 if (!cookie)
348 cookie->msi_iova = base;
349 domain->iova_cookie = cookie;
361 struct iommu_dma_cookie *cookie = domain->iova_cookie;
364 if (!cookie)
367 if (cookie->type == IOMMU_DMA_IOVA_COOKIE && cookie->iovad.granule) {
368 iommu_dma_free_fq(cookie);
369 put_iova_domain(&cookie->iovad);
372 list_for_each_entry_safe(msi, tmp, &cookie->msi_page_list, list) {
376 kfree(cookie);
401 static int cookie_init_hw_msi_region(struct iommu_dma_cookie *cookie,
404 struct iova_domain *iovad = &cookie->iovad;
419 list_add(&msi_page->list, &cookie->msi_page_list);
484 struct iommu_dma_cookie *cookie = domain->iova_cookie;
485 struct iova_domain *iovad = &cookie->iovad;
509 ret = cookie_init_hw_msi_region(cookie, region->start,
573 struct iommu_dma_cookie *cookie = domain->iova_cookie;
578 if (!cookie || cookie->type != IOMMU_DMA_IOVA_COOKIE)
581 iovad = &cookie->iovad;
600 mutex_lock(&cookie->mutex);
626 mutex_unlock(&cookie->mutex);
662 struct iommu_dma_cookie *cookie = domain->iova_cookie;
663 struct iova_domain *iovad = &cookie->iovad;
666 if (cookie->type == IOMMU_DMA_MSI_COOKIE) {
667 cookie->msi_iova += size;
668 return cookie->msi_iova - size;
705 static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie,
708 struct iova_domain *iovad = &cookie->iovad;
711 if (cookie->type == IOMMU_DMA_MSI_COOKIE)
712 cookie->msi_iova -= size;
714 queue_iova(cookie, iova_pfn(iovad, iova),
726 struct iommu_dma_cookie *cookie = domain->iova_cookie;
727 struct iova_domain *iovad = &cookie->iovad;
735 iotlb_gather.queued = READ_ONCE(cookie->fq_domain);
742 iommu_dma_free_iova(cookie, dma_addr, size, &iotlb_gather);
749 struct iommu_dma_cookie *cookie = domain->iova_cookie;
750 struct iova_domain *iovad = &cookie->iovad;
765 iommu_dma_free_iova(cookie, iova, size, NULL);
839 struct iommu_dma_cookie *cookie = domain->iova_cookie;
840 struct iova_domain *iovad = &cookie->iovad;
903 iommu_dma_free_iova(cookie, iova, size, NULL);
1038 struct iommu_dma_cookie *cookie = domain->iova_cookie;
1039 struct iova_domain *iovad = &cookie->iovad;
1245 struct iommu_dma_cookie *cookie = domain->iova_cookie;
1246 struct iova_domain *iovad = &cookie->iovad;
1352 iommu_dma_free_iova(cookie, iova, iova_len, NULL);
1660 struct iommu_dma_cookie *cookie = domain->iova_cookie;
1664 size_t size = cookie_msi_granule(cookie);
1667 list_for_each_entry(msi_page, &cookie->msi_page_list, list)
1685 list_add(&msi_page->list, &cookie->msi_page_list);
1689 iommu_dma_free_iova(cookie, iova, size, NULL);