| /kernel/linux/linux-6.6/include/net/ |
| H A D | xsk_buff_pool.h | 116 void xp_free(struct xdp_buff_xsk *xskb);
118 static inline void xp_init_xskb_addr(struct xdp_buff_xsk *xskb, struct xsk_buff_pool *pool, in xp_init_xskb_addr() argument
121 xskb->orig_addr = addr; in xp_init_xskb_addr()
122 xskb->xdp.data_hard_start = pool->addrs + addr + pool->headroom; in xp_init_xskb_addr()
125 static inline void xp_init_xskb_dma(struct xdp_buff_xsk *xskb, struct xsk_buff_pool *pool, in xp_init_xskb_dma() argument
128 xskb->frame_dma = (dma_pages[addr >> PAGE_SHIFT] & ~XSK_NEXT_PG_CONTIG_MASK) + in xp_init_xskb_dma()
130 xskb->dma = xskb->frame_dma + pool->headroom + XDP_PACKET_HEADROOM; in xp_init_xskb_dma()
143 static inline dma_addr_t xp_get_dma(struct xdp_buff_xsk *xskb) in xp_get_dma() argument
145 return xskb in xp_get_dma()
148 xp_get_frame_dma(struct xdp_buff_xsk *xskb) xp_get_frame_dma() argument
154 xp_dma_sync_for_cpu(struct xdp_buff_xsk *xskb) xp_dma_sync_for_cpu() argument
220 xp_release(struct xdp_buff_xsk *xskb) xp_release() argument
226 xp_get_handle(struct xdp_buff_xsk *xskb) xp_get_handle() argument
[all...] |
| H A D | xdp_sock_drv.h | 75 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); in xsk_buff_xdp_get_dma() local
77 return xp_get_dma(xskb); in xsk_buff_xdp_get_dma()
82 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); in xsk_buff_xdp_get_frame_dma() local
84 return xp_get_frame_dma(xskb); in xsk_buff_xdp_get_frame_dma()
110 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); in xsk_buff_free() local
111 struct list_head *xskb_list = &xskb->pool->xskb_list; in xsk_buff_free()
124 xp_free(xskb); in xsk_buff_free()
136 struct xdp_buff_xsk *xskb = container_of(first, struct xdp_buff_xsk, xdp); in xsk_buff_get_frag() local
140 frag = list_first_entry_or_null(&xskb->pool->xskb_list, in xsk_buff_get_frag()
152 struct xdp_buff_xsk *xskb in xsk_buff_del_tail() local
159 struct xdp_buff_xsk *xskb = container_of(first, struct xdp_buff_xsk, xdp); xsk_buff_get_tail() local
188 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); xsk_buff_dma_sync_for_cpu() local
[all...] |
| /kernel/linux/linux-6.6/net/xdp/ |
| H A D | xsk_buff_pool.c | 60 struct xdp_buff_xsk *xskb; in xp_create_and_assign_umem() local
99 xskb = &pool->heads[i]; in xp_create_and_assign_umem()
100 xskb->pool = pool; in xp_create_and_assign_umem()
101 xskb->xdp.frame_sz = umem->chunk_size - umem->headroom; in xp_create_and_assign_umem()
102 INIT_LIST_HEAD(&xskb->free_list_node); in xp_create_and_assign_umem()
103 INIT_LIST_HEAD(&xskb->xskb_list_node); in xp_create_and_assign_umem()
105 pool->free_heads[i] = xskb; in xp_create_and_assign_umem()
107 xp_init_xskb_addr(xskb, pool, i * pool->chunk_size); in xp_create_and_assign_umem()
401 struct xdp_buff_xsk *xskb = &pool->heads[i]; in xp_init_dma_info() local
403 xp_init_xskb_dma(xskb, poo in xp_init_dma_info()
491 struct xdp_buff_xsk *xskb; __xp_alloc() local
529 struct xdp_buff_xsk *xskb; xp_alloc() local
567 struct xdp_buff_xsk *xskb; xp_alloc_new_from_fq() local
600 struct xdp_buff_xsk *xskb; xp_alloc_reused() local
657 xp_free(struct xdp_buff_xsk *xskb) xp_free() argument
683 xp_dma_sync_for_cpu_slow(struct xdp_buff_xsk *xskb) xp_dma_sync_for_cpu_slow() argument
[all...] |
| H A D | xsk.c | 139 static int __xsk_rcv_zc(struct xdp_sock *xs, struct xdp_buff_xsk *xskb, u32 len, in __xsk_rcv_zc() argument
145 addr = xp_get_handle(xskb); in __xsk_rcv_zc()
152 xp_release(xskb); in __xsk_rcv_zc()
158 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); in xsk_rcv_zc() local
168 err = __xsk_rcv_zc(xs, xskb, len, contd); in xsk_rcv_zc()
174 xskb_list = &xskb->pool->xskb_list; in xsk_rcv_zc()
232 struct xdp_buff_xsk *xskb; in __xsk_rcv() local
249 xskb = container_of(xsk_xdp, struct xdp_buff_xsk, xdp); in __xsk_rcv()
250 err = __xsk_rcv_zc(xs, xskb, len, 0); in __xsk_rcv()
287 xskb in __xsk_rcv()
[all...] |
| /kernel/linux/linux-5.10/net/xdp/ |
| H A D | xsk_buff_pool.c | 48 struct xdp_buff_xsk *xskb; in xp_create_and_assign_umem() local
81 xskb = &pool->heads[i]; in xp_create_and_assign_umem()
82 xskb->pool = pool; in xp_create_and_assign_umem()
83 xskb->xdp.frame_sz = umem->chunk_size - umem->headroom; in xp_create_and_assign_umem()
84 pool->free_heads[i] = xskb; in xp_create_and_assign_umem()
453 struct xdp_buff_xsk *xskb; in __xp_alloc() local
460 xskb = pool->free_heads[--pool->free_heads_cnt]; in __xp_alloc()
465 xp_release(xskb); in __xp_alloc()
480 xskb->orig_addr = addr; in __xp_alloc()
481 xskb in __xp_alloc()
494 struct xdp_buff_xsk *xskb; xp_alloc() local
527 xp_free(struct xdp_buff_xsk *xskb) xp_free() argument
550 xp_dma_sync_for_cpu_slow(struct xdp_buff_xsk *xskb) xp_dma_sync_for_cpu_slow() argument
[all...] |
| H A D | xsk.c | 136 void xp_release(struct xdp_buff_xsk *xskb) in xp_release() argument
138 xskb->pool->free_heads[xskb->pool->free_heads_cnt++] = xskb; in xp_release()
141 static u64 xp_get_handle(struct xdp_buff_xsk *xskb) in xp_get_handle() argument
143 u64 offset = xskb->xdp.data - xskb->xdp.data_hard_start; in xp_get_handle()
145 offset += xskb->pool->headroom; in xp_get_handle()
146 if (!xskb->pool->unaligned) in xp_get_handle()
147 return xskb in xp_get_handle()
153 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); __xsk_rcv_zc() local
[all...] |
| /kernel/linux/linux-5.10/include/net/ |
| H A D | xsk_buff_pool.h | 92 void xp_release(struct xdp_buff_xsk *xskb);
100 void xp_free(struct xdp_buff_xsk *xskb);
111 static inline dma_addr_t xp_get_dma(struct xdp_buff_xsk *xskb) in xp_get_dma() argument
113 return xskb->dma; in xp_get_dma()
116 static inline dma_addr_t xp_get_frame_dma(struct xdp_buff_xsk *xskb) in xp_get_frame_dma() argument
118 return xskb->frame_dma; in xp_get_frame_dma()
121 void xp_dma_sync_for_cpu_slow(struct xdp_buff_xsk *xskb);
122 static inline void xp_dma_sync_for_cpu(struct xdp_buff_xsk *xskb) in xp_dma_sync_for_cpu() argument
124 xp_dma_sync_for_cpu_slow(xskb); in xp_dma_sync_for_cpu()
|
| H A D | xdp_sock_drv.h | 62 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); in xsk_buff_xdp_get_dma() local
64 return xp_get_dma(xskb); in xsk_buff_xdp_get_dma()
69 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); in xsk_buff_xdp_get_frame_dma() local
71 return xp_get_frame_dma(xskb); in xsk_buff_xdp_get_frame_dma()
86 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); in xsk_buff_free() local
88 xp_free(xskb); in xsk_buff_free()
104 struct xdp_buff_xsk *xskb = container_of(xdp, struct xdp_buff_xsk, xdp); in xsk_buff_dma_sync_for_cpu() local
109 xp_dma_sync_for_cpu(xskb); in xsk_buff_dma_sync_for_cpu()
|