Lines Matching refs:pool
11 void xp_add_xsk(struct xsk_buff_pool *pool, struct xdp_sock *xs)
18 spin_lock_irqsave(&pool->xsk_tx_list_lock, flags);
19 list_add_rcu(&xs->tx_list, &pool->xsk_tx_list);
20 spin_unlock_irqrestore(&pool->xsk_tx_list_lock, flags);
23 void xp_del_xsk(struct xsk_buff_pool *pool, struct xdp_sock *xs)
30 spin_lock_irqsave(&pool->xsk_tx_list_lock, flags);
32 spin_unlock_irqrestore(&pool->xsk_tx_list_lock, flags);
35 void xp_destroy(struct xsk_buff_pool *pool)
37 if (!pool)
40 kvfree(pool->tx_descs);
41 kvfree(pool->heads);
42 kvfree(pool);
45 int xp_alloc_tx_descs(struct xsk_buff_pool *pool, struct xdp_sock *xs)
47 pool->tx_descs = kvcalloc(xs->tx->nentries, sizeof(*pool->tx_descs),
49 if (!pool->tx_descs)
59 struct xsk_buff_pool *pool;
64 pool = kvzalloc(struct_size(pool, free_heads, entries), GFP_KERNEL);
65 if (!pool)
68 pool->heads = kvcalloc(umem->chunks, sizeof(*pool->heads), GFP_KERNEL);
69 if (!pool->heads)
73 if (xp_alloc_tx_descs(pool, xs))
76 pool->chunk_mask = ~((u64)umem->chunk_size - 1);
77 pool->addrs_cnt = umem->size;
78 pool->heads_cnt = umem->chunks;
79 pool->free_heads_cnt = umem->chunks;
80 pool->headroom = umem->headroom;
81 pool->chunk_size = umem->chunk_size;
82 pool->chunk_shift = ffs(umem->chunk_size) - 1;
83 pool->unaligned = unaligned;
84 pool->frame_len = umem->chunk_size - umem->headroom -
86 pool->umem = umem;
87 pool->addrs = umem->addrs;
88 INIT_LIST_HEAD(&pool->free_list);
89 INIT_LIST_HEAD(&pool->xskb_list);
90 INIT_LIST_HEAD(&pool->xsk_tx_list);
91 spin_lock_init(&pool->xsk_tx_list_lock);
92 spin_lock_init(&pool->cq_lock);
93 refcount_set(&pool->users, 1);
95 pool->fq = xs->fq_tmp;
96 pool->cq = xs->cq_tmp;
98 for (i = 0; i < pool->free_heads_cnt; i++) {
99 xskb = &pool->heads[i];
100 xskb->pool = pool;
104 if (pool->unaligned)
105 pool->free_heads[i] = xskb;
107 xp_init_xskb_addr(xskb, pool, i * pool->chunk_size);
110 return pool;
113 xp_destroy(pool);
117 void xp_set_rxq_info(struct xsk_buff_pool *pool, struct xdp_rxq_info *rxq)
121 for (i = 0; i < pool->heads_cnt; i++)
122 pool->heads[i].xdp.rxq = rxq;
126 static void xp_disable_drv_zc(struct xsk_buff_pool *pool)
133 if (pool->umem->zc) {
135 bpf.xsk.pool = NULL;
136 bpf.xsk.queue_id = pool->queue_id;
138 err = pool->netdev->netdev_ops->ndo_bpf(pool->netdev, &bpf);
149 int xp_assign_dev(struct xsk_buff_pool *pool,
167 pool->netdev = netdev;
168 pool->queue_id = queue_id;
169 err = xsk_reg_pool_at_qid(netdev, pool, queue_id);
174 pool->umem->flags |= XDP_UMEM_SG_FLAG;
177 pool->uses_need_wakeup = true;
182 pool->cached_need_wakeup = XDP_WAKEUP_TX;
201 bpf.xsk.pool = pool;
208 if (!pool->dma_pages) {
213 pool->umem->zc = true;
217 xp_disable_drv_zc(pool);
228 int xp_assign_dev_shared(struct xsk_buff_pool *pool, struct xdp_sock *umem_xs,
235 if (!pool->fq || !pool->cq)
239 if (umem_xs->pool->uses_need_wakeup)
242 return xp_assign_dev(pool, dev, queue_id, flags);
245 void xp_clear_dev(struct xsk_buff_pool *pool)
247 if (!pool->netdev)
250 xp_disable_drv_zc(pool);
251 xsk_clear_pool_at_qid(pool->netdev, pool->queue_id);
252 dev_put(pool->netdev);
253 pool->netdev = NULL;
258 struct xsk_buff_pool *pool = container_of(work, struct xsk_buff_pool,
262 xp_clear_dev(pool);
265 if (pool->fq) {
266 xskq_destroy(pool->fq);
267 pool->fq = NULL;
270 if (pool->cq) {
271 xskq_destroy(pool->cq);
272 pool->cq = NULL;
275 xdp_put_umem(pool->umem, false);
276 xp_destroy(pool);
279 void xp_get_pool(struct xsk_buff_pool *pool)
281 refcount_inc(&pool->users);
284 bool xp_put_pool(struct xsk_buff_pool *pool)
286 if (!pool)
289 if (refcount_dec_and_test(&pool->users)) {
290 INIT_WORK(&pool->work, xp_release_deferred);
291 schedule_work(&pool->work);
298 static struct xsk_dma_map *xp_find_dma_map(struct xsk_buff_pool *pool)
302 list_for_each_entry(dma_map, &pool->umem->xsk_dma_list, list) {
303 if (dma_map->netdev == pool->netdev)
359 void xp_dma_unmap(struct xsk_buff_pool *pool, unsigned long attrs)
363 if (!pool->dma_pages)
366 dma_map = xp_find_dma_map(pool);
376 kvfree(pool->dma_pages);
377 pool->dma_pages = NULL;
378 pool->dma_pages_cnt = 0;
379 pool->dev = NULL;
395 static int xp_init_dma_info(struct xsk_buff_pool *pool, struct xsk_dma_map *dma_map)
397 if (!pool->unaligned) {
400 for (i = 0; i < pool->heads_cnt; i++) {
401 struct xdp_buff_xsk *xskb = &pool->heads[i];
403 xp_init_xskb_dma(xskb, pool, dma_map->dma_pages, xskb->orig_addr);
407 pool->dma_pages = kvcalloc(dma_map->dma_pages_cnt, sizeof(*pool->dma_pages), GFP_KERNEL);
408 if (!pool->dma_pages)
411 pool->dev = dma_map->dev;
412 pool->dma_pages_cnt = dma_map->dma_pages_cnt;
413 pool->dma_need_sync = dma_map->dma_need_sync;
414 memcpy(pool->dma_pages, dma_map->dma_pages,
415 pool->dma_pages_cnt * sizeof(*pool->dma_pages));
420 int xp_dma_map(struct xsk_buff_pool *pool, struct device *dev,
428 dma_map = xp_find_dma_map(pool);
430 err = xp_init_dma_info(pool, dma_map);
438 dma_map = xp_create_dma_map(dev, pool->netdev, nr_pages, pool->umem);
454 if (pool->unaligned)
457 err = xp_init_dma_info(pool, dma_map);
467 static bool xp_addr_crosses_non_contig_pg(struct xsk_buff_pool *pool,
470 return xp_desc_crosses_non_contig_pg(pool, addr, pool->chunk_size);
473 static bool xp_check_unaligned(struct xsk_buff_pool *pool, u64 *addr)
476 if (*addr >= pool->addrs_cnt ||
477 *addr + pool->chunk_size > pool->addrs_cnt ||
478 xp_addr_crosses_non_contig_pg(pool, *addr))
483 static bool xp_check_aligned(struct xsk_buff_pool *pool, u64 *addr)
485 *addr = xp_aligned_extract_addr(pool, *addr);
486 return *addr < pool->addrs_cnt;
489 static struct xdp_buff_xsk *__xp_alloc(struct xsk_buff_pool *pool)
495 if (pool->free_heads_cnt == 0)
499 if (!xskq_cons_peek_addr_unchecked(pool->fq, &addr)) {
500 pool->fq->queue_empty_descs++;
504 ok = pool->unaligned ? xp_check_unaligned(pool, &addr) :
505 xp_check_aligned(pool, &addr);
507 pool->fq->invalid_descs++;
508 xskq_cons_release(pool->fq);
514 if (pool->unaligned) {
515 xskb = pool->free_heads[--pool->free_heads_cnt];
516 xp_init_xskb_addr(xskb, pool, addr);
517 if (pool->dma_pages)
518 xp_init_xskb_dma(xskb, pool, pool->dma_pages, addr);
520 xskb = &pool->heads[xp_aligned_extract_idx(pool, addr)];
523 xskq_cons_release(pool->fq);
527 struct xdp_buff *xp_alloc(struct xsk_buff_pool *pool)
531 if (!pool->free_list_cnt) {
532 xskb = __xp_alloc(pool);
536 pool->free_list_cnt--;
537 xskb = list_first_entry(&pool->free_list, struct xdp_buff_xsk,
546 if (pool->dma_need_sync) {
547 dma_sync_single_range_for_device(pool->dev, xskb->dma, 0,
548 pool->frame_len,
555 static u32 xp_alloc_new_from_fq(struct xsk_buff_pool *pool, struct xdp_buff **xdp, u32 max)
559 if (max > pool->free_heads_cnt)
560 max = pool->free_heads_cnt;
561 max = xskq_cons_nb_entries(pool->fq, max);
563 cached_cons = pool->fq->cached_cons;
571 __xskq_cons_read_addr_unchecked(pool->fq, cached_cons++, &addr);
573 ok = pool->unaligned ? xp_check_unaligned(pool, &addr) :
574 xp_check_aligned(pool, &addr);
576 pool->fq->invalid_descs++;
581 if (pool->unaligned) {
582 xskb = pool->free_heads[--pool->free_heads_cnt];
583 xp_init_xskb_addr(xskb, pool, addr);
584 if (pool->dma_pages)
585 xp_init_xskb_dma(xskb, pool, pool->dma_pages, addr);
587 xskb = &pool->heads[xp_aligned_extract_idx(pool, addr)];
594 xskq_cons_release_n(pool->fq, max);
598 static u32 xp_alloc_reused(struct xsk_buff_pool *pool, struct xdp_buff **xdp, u32 nb_entries)
603 nb_entries = min_t(u32, nb_entries, pool->free_list_cnt);
607 xskb = list_first_entry(&pool->free_list, struct xdp_buff_xsk, free_list_node);
613 pool->free_list_cnt -= nb_entries;
618 u32 xp_alloc_batch(struct xsk_buff_pool *pool, struct xdp_buff **xdp, u32 max)
622 if (unlikely(pool->dma_need_sync)) {
626 buff = xp_alloc(pool);
632 if (unlikely(pool->free_list_cnt)) {
633 nb_entries1 = xp_alloc_reused(pool, xdp, max);
641 nb_entries2 = xp_alloc_new_from_fq(pool, xdp, max);
643 pool->fq->queue_empty_descs++;
649 bool xp_can_alloc(struct xsk_buff_pool *pool, u32 count)
651 if (pool->free_list_cnt >= count)
653 return xskq_cons_has_entries(pool->fq, count - pool->free_list_cnt);
662 xskb->pool->free_list_cnt++;
663 list_add(&xskb->free_list_node, &xskb->pool->free_list);
667 void *xp_raw_get_data(struct xsk_buff_pool *pool, u64 addr)
669 addr = pool->unaligned ? xp_unaligned_add_offset_to_addr(addr) : addr;
670 return pool->addrs + addr;
674 dma_addr_t xp_raw_get_dma(struct xsk_buff_pool *pool, u64 addr)
676 addr = pool->unaligned ? xp_unaligned_add_offset_to_addr(addr) : addr;
677 return (pool->dma_pages[addr >> PAGE_SHIFT] &
685 dma_sync_single_range_for_cpu(xskb->pool->dev, xskb->dma, 0,
686 xskb->pool->frame_len, DMA_BIDIRECTIONAL);
690 void xp_dma_sync_for_device_slow(struct xsk_buff_pool *pool, dma_addr_t dma,
693 dma_sync_single_range_for_device(pool->dev, dma, 0,