Lines Matching refs:nvq
177 static int vhost_net_buf_produce(struct vhost_net_virtqueue *nvq)
179 struct vhost_net_buf *rxq = &nvq->rxq;
182 rxq->tail = ptr_ring_consume_batched(nvq->rx_ring, rxq->queue,
187 static void vhost_net_buf_unproduce(struct vhost_net_virtqueue *nvq)
189 struct vhost_net_buf *rxq = &nvq->rxq;
191 if (nvq->rx_ring && !vhost_net_buf_is_empty(rxq)) {
192 ptr_ring_unconsume(nvq->rx_ring, rxq->queue + rxq->head,
210 static int vhost_net_buf_peek(struct vhost_net_virtqueue *nvq)
212 struct vhost_net_buf *rxq = &nvq->rxq;
217 if (!vhost_net_buf_produce(nvq))
362 struct vhost_net_virtqueue *nvq =
367 for (i = nvq->done_idx; i != nvq->upend_idx; i = (i + 1) % UIO_MAXIOV) {
377 add = min(UIO_MAXIOV - nvq->done_idx, j);
379 &vq->heads[nvq->done_idx], add);
380 nvq->done_idx = (nvq->done_idx + add) % UIO_MAXIOV;
427 struct vhost_net_virtqueue *nvq =
429 struct vhost_poll *poll = n->poll + (nvq - n->vqs);
438 struct vhost_net_virtqueue *nvq =
440 struct vhost_poll *poll = n->poll + (nvq - n->vqs);
450 static void vhost_net_signal_used(struct vhost_net_virtqueue *nvq)
452 struct vhost_virtqueue *vq = &nvq->vq;
455 if (!nvq->done_idx)
458 vhost_add_used_and_signal_n(dev, vq, vq->heads, nvq->done_idx);
459 nvq->done_idx = 0;
463 struct vhost_net_virtqueue *nvq,
469 .num = nvq->batched_xdp,
470 .ptr = nvq->xdp,
474 if (nvq->batched_xdp == 0)
481 vq_err(&nvq->vq, "Fail to batch sending packets\n");
487 for (i = 0; i < nvq->batched_xdp; ++i)
488 put_page(virt_to_head_page(nvq->xdp[i].data));
489 nvq->batched_xdp = 0;
490 nvq->done_idx = 0;
495 vhost_net_signal_used(nvq);
496 nvq->batched_xdp = 0;
602 struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];
603 struct vhost_virtqueue *vq = &nvq->vq;
605 return (nvq->upend_idx + UIO_MAXIOV - nvq->done_idx) % UIO_MAXIOV >
622 struct vhost_net_virtqueue *nvq,
627 struct vhost_virtqueue *vq = &nvq->vq;
630 ret = vhost_net_tx_get_vq_desc(net, nvq, out, in, msg, busyloop_intr);
642 *len = init_iov_iter(vq, &msg->msg_iter, nvq->vhost_hlen, *out);
645 *len, nvq->vhost_hlen);
695 static int vhost_net_build_xdp(struct vhost_net_virtqueue *nvq,
698 struct vhost_virtqueue *vq = &nvq->vq;
704 struct xdp_buff *xdp = &nvq->xdp[nvq->batched_xdp];
709 int pad = SKB_DATA_ALIGN(VHOST_NET_RX_PAD + headroom + nvq->sock_hlen);
710 int sock_hlen = nvq->sock_hlen;
714 if (unlikely(len < nvq->sock_hlen))
764 ++nvq->batched_xdp;
771 struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];
772 struct vhost_virtqueue *vq = &nvq->vq;
790 if (nvq->done_idx == VHOST_NET_BATCH)
791 vhost_tx_batch(net, nvq, sock, &msg);
793 head = get_tx_bufs(net, nvq, &msg, &out, &in, &len,
816 err = vhost_net_build_xdp(nvq, &msg.msg_iter);
820 vhost_tx_batch(net, nvq, sock, &msg);
830 vhost_tx_batch(net, nvq, sock, &msg);
851 vq->heads[nvq->done_idx].id = cpu_to_vhost32(vq, head);
852 vq->heads[nvq->done_idx].len = 0;
853 ++nvq->done_idx;
856 vhost_tx_batch(net, nvq, sock, &msg);
861 struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];
862 struct vhost_virtqueue *vq = &nvq->vq;
887 head = get_tx_bufs(net, nvq, &msg, &out, &in, &len,
909 ubuf = nvq->ubuf_info + nvq->upend_idx;
910 vq->heads[nvq->upend_idx].id = cpu_to_vhost32(vq, head);
911 vq->heads[nvq->upend_idx].len = VHOST_DMA_IN_PROGRESS;
912 ubuf->ctx = nvq->ubufs;
913 ubuf->desc = nvq->upend_idx;
921 ubufs = nvq->ubufs;
923 nvq->upend_idx = (nvq->upend_idx + 1) % UIO_MAXIOV;
944 nvq->upend_idx = ((unsigned)nvq->upend_idx - 1)
970 struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];
971 struct vhost_virtqueue *vq = &nvq->vq;
1117 struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_RX];
1118 struct vhost_virtqueue *vq = &nvq->vq;
1154 vhost_hlen = nvq->vhost_hlen;
1155 sock_hlen = nvq->sock_hlen;
1168 headcount = get_rx_bufs(vq, vq->heads + nvq->done_idx,
1189 if (nvq->rx_ring)
1190 msg.msg_control = vhost_net_buf_consume(&nvq->rxq);
1243 nvq->done_idx += headcount;
1244 if (nvq->done_idx > VHOST_NET_BATCH)
1245 vhost_net_signal_used(nvq);
1257 vhost_net_signal_used(nvq);
1366 struct vhost_net_virtqueue *nvq =
1373 vhost_net_buf_unproduce(nvq);
1374 nvq->rx_ring = NULL;
1505 struct vhost_net_virtqueue *nvq;
1519 nvq = &n->vqs[index];
1548 vhost_net_buf_unproduce(nvq);
1557 nvq->rx_ring = get_tap_ptr_ring(sock->file);
1559 nvq->rx_ring = NULL;
1562 oldubufs = nvq->ubufs;
1563 nvq->ubufs = ubufs;