Lines Matching refs:umem
26 static void xdp_umem_unpin_pages(struct xdp_umem *umem)
28 unpin_user_pages_dirty_lock(umem->pgs, umem->npgs, true);
30 kfree(umem->pgs);
31 umem->pgs = NULL;
34 static void xdp_umem_unaccount_pages(struct xdp_umem *umem)
36 if (umem->user) {
37 atomic_long_sub(umem->npgs, &umem->user->locked_vm);
38 free_uid(umem->user);
42 static void xdp_umem_addr_unmap(struct xdp_umem *umem)
44 vunmap(umem->addrs);
45 umem->addrs = NULL;
48 static int xdp_umem_addr_map(struct xdp_umem *umem, struct page **pages,
51 umem->addrs = vmap(pages, nr_pages, VM_MAP, PAGE_KERNEL);
52 if (!umem->addrs)
57 static void xdp_umem_release(struct xdp_umem *umem)
59 umem->zc = false;
60 ida_simple_remove(&umem_ida, umem->id);
62 xdp_umem_addr_unmap(umem);
63 xdp_umem_unpin_pages(umem);
65 xdp_umem_unaccount_pages(umem);
66 kfree(umem);
71 struct xdp_umem *umem = container_of(work, struct xdp_umem, work);
73 xdp_umem_release(umem);
76 void xdp_get_umem(struct xdp_umem *umem)
78 refcount_inc(&umem->users);
81 void xdp_put_umem(struct xdp_umem *umem, bool defer_cleanup)
83 if (!umem)
86 if (refcount_dec_and_test(&umem->users)) {
88 INIT_WORK(&umem->work, xdp_umem_release_deferred);
89 schedule_work(&umem->work);
91 xdp_umem_release(umem);
96 static int xdp_umem_pin_pages(struct xdp_umem *umem, unsigned long address)
102 umem->pgs = kcalloc(umem->npgs, sizeof(*umem->pgs),
104 if (!umem->pgs)
108 npgs = pin_user_pages(address, umem->npgs,
109 gup_flags | FOLL_LONGTERM, &umem->pgs[0], NULL);
112 if (npgs != umem->npgs) {
114 umem->npgs = npgs;
124 xdp_umem_unpin_pages(umem);
126 kfree(umem->pgs);
127 umem->pgs = NULL;
131 static int xdp_umem_account_pages(struct xdp_umem *umem)
139 umem->user = get_uid(current_user());
142 old_npgs = atomic_long_read(&umem->user->locked_vm);
143 new_npgs = old_npgs + umem->npgs;
145 free_uid(umem->user);
146 umem->user = NULL;
149 } while (atomic_long_cmpxchg(&umem->user->locked_vm, old_npgs,
154 static int xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr)
205 umem->size = size;
206 umem->headroom = headroom;
207 umem->chunk_size = chunk_size;
208 umem->chunks = chunks;
209 umem->npgs = npgs;
210 umem->pgs = NULL;
211 umem->user = NULL;
212 umem->flags = mr->flags;
214 INIT_LIST_HEAD(&umem->xsk_dma_list);
215 refcount_set(&umem->users, 1);
217 err = xdp_umem_account_pages(umem);
221 err = xdp_umem_pin_pages(umem, (unsigned long)addr);
225 err = xdp_umem_addr_map(umem, umem->pgs, umem->npgs);
232 xdp_umem_unpin_pages(umem);
234 xdp_umem_unaccount_pages(umem);
240 struct xdp_umem *umem;
243 umem = kzalloc(sizeof(*umem), GFP_KERNEL);
244 if (!umem)
249 kfree(umem);
252 umem->id = err;
254 err = xdp_umem_reg(umem, mr);
256 ida_simple_remove(&umem_ida, umem->id);
257 kfree(umem);
261 return umem;