Lines Matching refs:umem
24 static void xdp_umem_unpin_pages(struct xdp_umem *umem)
26 unpin_user_pages_dirty_lock(umem->pgs, umem->npgs, true);
28 kvfree(umem->pgs);
29 umem->pgs = NULL;
32 static void xdp_umem_unaccount_pages(struct xdp_umem *umem)
34 if (umem->user) {
35 atomic_long_sub(umem->npgs, &umem->user->locked_vm);
36 free_uid(umem->user);
40 static void xdp_umem_addr_unmap(struct xdp_umem *umem)
42 vunmap(umem->addrs);
43 umem->addrs = NULL;
46 static int xdp_umem_addr_map(struct xdp_umem *umem, struct page **pages,
49 umem->addrs = vmap(pages, nr_pages, VM_MAP, PAGE_KERNEL);
50 if (!umem->addrs)
55 static void xdp_umem_release(struct xdp_umem *umem)
57 umem->zc = false;
58 ida_free(&umem_ida, umem->id);
60 xdp_umem_addr_unmap(umem);
61 xdp_umem_unpin_pages(umem);
63 xdp_umem_unaccount_pages(umem);
64 kfree(umem);
69 struct xdp_umem *umem = container_of(work, struct xdp_umem, work);
71 xdp_umem_release(umem);
74 void xdp_get_umem(struct xdp_umem *umem)
76 refcount_inc(&umem->users);
79 void xdp_put_umem(struct xdp_umem *umem, bool defer_cleanup)
81 if (!umem)
84 if (refcount_dec_and_test(&umem->users)) {
86 INIT_WORK(&umem->work, xdp_umem_release_deferred);
87 schedule_work(&umem->work);
89 xdp_umem_release(umem);
94 static int xdp_umem_pin_pages(struct xdp_umem *umem, unsigned long address)
100 umem->pgs = kvcalloc(umem->npgs, sizeof(*umem->pgs), GFP_KERNEL | __GFP_NOWARN);
101 if (!umem->pgs)
105 npgs = pin_user_pages(address, umem->npgs,
106 gup_flags | FOLL_LONGTERM, &umem->pgs[0]);
109 if (npgs != umem->npgs) {
111 umem->npgs = npgs;
121 xdp_umem_unpin_pages(umem);
123 kvfree(umem->pgs);
124 umem->pgs = NULL;
128 static int xdp_umem_account_pages(struct xdp_umem *umem)
136 umem->user = get_uid(current_user());
139 old_npgs = atomic_long_read(&umem->user->locked_vm);
140 new_npgs = old_npgs + umem->npgs;
142 free_uid(umem->user);
143 umem->user = NULL;
146 } while (atomic_long_cmpxchg(&umem->user->locked_vm, old_npgs,
151 static int xdp_umem_reg(struct xdp_umem *umem, struct xdp_umem_reg *mr)
202 umem->size = size;
203 umem->headroom = headroom;
204 umem->chunk_size = chunk_size;
205 umem->chunks = chunks;
206 umem->npgs = npgs;
207 umem->pgs = NULL;
208 umem->user = NULL;
209 umem->flags = mr->flags;
211 INIT_LIST_HEAD(&umem->xsk_dma_list);
212 refcount_set(&umem->users, 1);
214 err = xdp_umem_account_pages(umem);
218 err = xdp_umem_pin_pages(umem, (unsigned long)addr);
222 err = xdp_umem_addr_map(umem, umem->pgs, umem->npgs);
229 xdp_umem_unpin_pages(umem);
231 xdp_umem_unaccount_pages(umem);
237 struct xdp_umem *umem;
240 umem = kzalloc(sizeof(*umem), GFP_KERNEL);
241 if (!umem)
246 kfree(umem);
249 umem->id = err;
251 err = xdp_umem_reg(umem, mr);
253 ida_free(&umem_ida, umem->id);
254 kfree(umem);
258 return umem;