Lines Matching refs:nvq
176 static int vhost_net_buf_produce(struct vhost_net_virtqueue *nvq)
178 struct vhost_net_buf *rxq = &nvq->rxq;
181 rxq->tail = ptr_ring_consume_batched(nvq->rx_ring, rxq->queue,
186 static void vhost_net_buf_unproduce(struct vhost_net_virtqueue *nvq)
188 struct vhost_net_buf *rxq = &nvq->rxq;
190 if (nvq->rx_ring && !vhost_net_buf_is_empty(rxq)) {
191 ptr_ring_unconsume(nvq->rx_ring, rxq->queue + rxq->head,
209 static int vhost_net_buf_peek(struct vhost_net_virtqueue *nvq)
211 struct vhost_net_buf *rxq = &nvq->rxq;
216 if (!vhost_net_buf_produce(nvq))
361 struct vhost_net_virtqueue *nvq =
366 for (i = nvq->done_idx; i != nvq->upend_idx; i = (i + 1) % UIO_MAXIOV) {
376 add = min(UIO_MAXIOV - nvq->done_idx, j);
378 &vq->heads[nvq->done_idx], add);
379 nvq->done_idx = (nvq->done_idx + add) % UIO_MAXIOV;
424 struct vhost_net_virtqueue *nvq =
426 struct vhost_poll *poll = n->poll + (nvq - n->vqs);
435 struct vhost_net_virtqueue *nvq =
437 struct vhost_poll *poll = n->poll + (nvq - n->vqs);
447 static void vhost_net_signal_used(struct vhost_net_virtqueue *nvq)
449 struct vhost_virtqueue *vq = &nvq->vq;
452 if (!nvq->done_idx)
455 vhost_add_used_and_signal_n(dev, vq, vq->heads, nvq->done_idx);
456 nvq->done_idx = 0;
460 struct vhost_net_virtqueue *nvq,
466 .num = nvq->batched_xdp,
467 .ptr = nvq->xdp,
471 if (nvq->batched_xdp == 0)
478 vq_err(&nvq->vq, "Fail to batch sending packets\n");
484 for (i = 0; i < nvq->batched_xdp; ++i)
485 put_page(virt_to_head_page(nvq->xdp[i].data));
486 nvq->batched_xdp = 0;
487 nvq->done_idx = 0;
492 vhost_net_signal_used(nvq);
493 nvq->batched_xdp = 0;
599 struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];
600 struct vhost_virtqueue *vq = &nvq->vq;
602 return (nvq->upend_idx + UIO_MAXIOV - nvq->done_idx) % UIO_MAXIOV >
619 struct vhost_net_virtqueue *nvq,
624 struct vhost_virtqueue *vq = &nvq->vq;
627 ret = vhost_net_tx_get_vq_desc(net, nvq, out, in, msg, busyloop_intr);
639 *len = init_iov_iter(vq, &msg->msg_iter, nvq->vhost_hlen, *out);
642 *len, nvq->vhost_hlen);
692 static int vhost_net_build_xdp(struct vhost_net_virtqueue *nvq,
695 struct vhost_virtqueue *vq = &nvq->vq;
701 struct xdp_buff *xdp = &nvq->xdp[nvq->batched_xdp];
706 int pad = SKB_DATA_ALIGN(VHOST_NET_RX_PAD + headroom + nvq->sock_hlen);
707 int sock_hlen = nvq->sock_hlen;
711 if (unlikely(len < nvq->sock_hlen))
763 ++nvq->batched_xdp;
770 struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];
771 struct vhost_virtqueue *vq = &nvq->vq;
789 if (nvq->done_idx == VHOST_NET_BATCH)
790 vhost_tx_batch(net, nvq, sock, &msg);
792 head = get_tx_bufs(net, nvq, &msg, &out, &in, &len,
815 err = vhost_net_build_xdp(nvq, &msg.msg_iter);
819 vhost_tx_batch(net, nvq, sock, &msg);
829 vhost_tx_batch(net, nvq, sock, &msg);
849 vq->heads[nvq->done_idx].id = cpu_to_vhost32(vq, head);
850 vq->heads[nvq->done_idx].len = 0;
851 ++nvq->done_idx;
854 vhost_tx_batch(net, nvq, sock, &msg);
859 struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];
860 struct vhost_virtqueue *vq = &nvq->vq;
885 head = get_tx_bufs(net, nvq, &msg, &out, &in, &len,
907 ubuf = nvq->ubuf_info + nvq->upend_idx;
908 vq->heads[nvq->upend_idx].id = cpu_to_vhost32(vq, head);
909 vq->heads[nvq->upend_idx].len = VHOST_DMA_IN_PROGRESS;
911 ubuf->ctx = nvq->ubufs;
912 ubuf->desc = nvq->upend_idx;
918 ubufs = nvq->ubufs;
920 nvq->upend_idx = (nvq->upend_idx + 1) % UIO_MAXIOV;
939 nvq->upend_idx = ((unsigned)nvq->upend_idx - 1)
961 struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_TX];
962 struct vhost_virtqueue *vq = &nvq->vq;
1108 struct vhost_net_virtqueue *nvq = &net->vqs[VHOST_NET_VQ_RX];
1109 struct vhost_virtqueue *vq = &nvq->vq;
1145 vhost_hlen = nvq->vhost_hlen;
1146 sock_hlen = nvq->sock_hlen;
1159 headcount = get_rx_bufs(vq, vq->heads + nvq->done_idx,
1180 if (nvq->rx_ring)
1181 msg.msg_control = vhost_net_buf_consume(&nvq->rxq);
1234 nvq->done_idx += headcount;
1235 if (nvq->done_idx > VHOST_NET_BATCH)
1236 vhost_net_signal_used(nvq);
1248 vhost_net_signal_used(nvq);
1355 struct vhost_net_virtqueue *nvq =
1362 vhost_net_buf_unproduce(nvq);
1363 nvq->rx_ring = NULL;
1501 struct vhost_net_virtqueue *nvq;
1515 nvq = &n->vqs[index];
1544 vhost_net_buf_unproduce(nvq);
1553 nvq->rx_ring = get_tap_ptr_ring(sock->file);
1555 nvq->rx_ring = NULL;
1558 oldubufs = nvq->ubufs;
1559 nvq->ubufs = ubufs;