Lines Matching refs:split
117 /* Available for split ring */
137 } split;
451 WARN_ON_ONCE(total_sg > vq->split.vring.num && !vq->indirect);
462 desc = vq->split.vring.desc;
518 vq->split.vring.desc[head].flags = cpu_to_virtio16(_vq->vdev,
520 vq->split.vring.desc[head].addr = cpu_to_virtio64(_vq->vdev,
523 vq->split.vring.desc[head].len = cpu_to_virtio32(_vq->vdev,
533 vq->split.vring.desc[head].next);
538 vq->split.desc_state[head].data = data;
540 vq->split.desc_state[head].indir_desc = desc;
542 vq->split.desc_state[head].indir_desc = ctx;
546 avail = vq->split.avail_idx_shadow & (vq->split.vring.num - 1);
547 vq->split.vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head);
552 vq->split.avail_idx_shadow++;
553 vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,
554 vq->split.avail_idx_shadow);
600 old = vq->split.avail_idx_shadow - vq->num_added;
601 new = vq->split.avail_idx_shadow;
609 vring_avail_event(&vq->split.vring)),
612 needs_kick = !(vq->split.vring.used->flags &
627 vq->split.desc_state[head].data = NULL;
632 while (vq->split.vring.desc[i].flags & nextflag) {
633 vring_unmap_one_split(vq, &vq->split.vring.desc[i]);
634 i = virtio16_to_cpu(vq->vq.vdev, vq->split.vring.desc[i].next);
638 vring_unmap_one_split(vq, &vq->split.vring.desc[i]);
639 vq->split.vring.desc[i].next = cpu_to_virtio16(vq->vq.vdev,
648 vq->split.desc_state[head].indir_desc;
656 vq->split.vring.desc[head].len);
658 BUG_ON(!(vq->split.vring.desc[head].flags &
666 vq->split.desc_state[head].indir_desc = NULL;
668 *ctx = vq->split.desc_state[head].indir_desc;
675 vq->split.vring.used->idx);
703 last_used = (vq->last_used_idx & (vq->split.vring.num - 1));
705 vq->split.vring.used->ring[last_used].id);
707 vq->split.vring.used->ring[last_used].len);
709 if (unlikely(i >= vq->split.vring.num)) {
713 if (unlikely(!vq->split.desc_state[i].data)) {
719 ret = vq->split.desc_state[i].data;
725 if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT))
727 &vring_used_event(&vq->split.vring),
740 if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) {
741 vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;
743 vq->split.vring.avail->flags =
745 vq->split.avail_flags_shadow);
761 if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {
762 vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;
764 vq->split.vring.avail->flags =
766 vq->split.avail_flags_shadow);
768 vring_used_event(&vq->split.vring) = cpu_to_virtio16(_vq->vdev,
779 vq->split.vring.used->idx);
794 if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {
795 vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;
797 vq->split.vring.avail->flags =
799 vq->split.avail_flags_shadow);
802 bufs = (u16)(vq->split.avail_idx_shadow - vq->last_used_idx) * 3 / 4;
805 &vring_used_event(&vq->split.vring),
808 if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->split.vring.used->idx)
826 for (i = 0; i < vq->split.vring.num; i++) {
827 if (!vq->split.desc_state[i].data)
830 buf = vq->split.desc_state[i].data;
832 vq->split.avail_idx_shadow--;
833 vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,
834 vq->split.avail_idx_shadow);
839 BUG_ON(vq->vq.num_free != vq->split.vring.num);
902 to_vvq(vq)->split.queue_dma_addr = dma_addr;
903 to_vvq(vq)->split.queue_size_in_bytes = queue_size_in_bytes;
1826 * virtqueue_kick_prepare - first half of split virtqueue_kick call.
1846 * virtqueue_notify - second half of split virtqueue_kick call.
2067 /* Only available for split ring */
2112 vq->split.queue_dma_addr = 0;
2113 vq->split.queue_size_in_bytes = 0;
2115 vq->split.vring = vring;
2116 vq->split.avail_flags_shadow = 0;
2117 vq->split.avail_idx_shadow = 0;
2121 vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;
2123 vq->split.vring.avail->flags = cpu_to_virtio16(vdev,
2124 vq->split.avail_flags_shadow);
2127 vq->split.desc_state = kmalloc_array(vring.num,
2129 if (!vq->split.desc_state) {
2137 vq->split.vring.desc[i].next = cpu_to_virtio16(vdev, i + 1);
2138 memset(vq->split.desc_state, 0, vring.num *
2172 /* Only available for split ring */
2220 vq->split.queue_size_in_bytes,
2221 vq->split.vring.desc,
2222 vq->split.queue_dma_addr);
2226 kfree(vq->split.desc_state);
2273 return vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num;
2313 return vq->split.queue_dma_addr;
2326 return vq->split.queue_dma_addr +
2327 ((char *)vq->split.vring.avail - (char *)vq->split.vring.desc);
2340 return vq->split.queue_dma_addr +
2341 ((char *)vq->split.vring.used - (char *)vq->split.vring.desc);
2345 /* Only available for split ring */
2348 return &to_vvq(vq)->split.vring;