Lines Matching defs:_vq
20 #define BAD_RING(_vq, fmt, args...) \
22 dev_err(&(_vq)->vq.vdev->dev, \
23 "%s:"fmt, (_vq)->vq.name, ##args); \
27 #define START_USE(_vq) \
29 if ((_vq)->in_use) \
31 (_vq)->vq.name, (_vq)->in_use); \
32 (_vq)->in_use = __LINE__; \
34 #define END_USE(_vq) \
35 do { BUG_ON(!(_vq)->in_use); (_vq)->in_use = 0; } while(0)
36 #define LAST_ADD_TIME_UPDATE(_vq) \
41 if ((_vq)->last_add_time_valid) \
43 (_vq)->last_add_time)) > 100); \
44 (_vq)->last_add_time = now; \
45 (_vq)->last_add_time_valid = true; \
47 #define LAST_ADD_TIME_CHECK(_vq) \
49 if ((_vq)->last_add_time_valid) { \
51 (_vq)->last_add_time)) > 100); \
54 #define LAST_ADD_TIME_INVALID(_vq) \
55 ((_vq)->last_add_time_valid = false)
57 #define BAD_RING(_vq, fmt, args...) \
59 dev_err(&_vq->vq.vdev->dev, \
60 "%s:"fmt, (_vq)->vq.name, ##args); \
61 (_vq)->broken = true; \
236 static void vring_free(struct virtqueue *_vq);
242 #define to_vvq(_vq) container_of_const(_vq, struct vring_virtqueue, vq)
492 static struct vring_desc *alloc_indirect_split(struct virtqueue *_vq,
511 desc[i].next = cpu_to_virtio16(_vq->vdev, i + 1);
544 static inline int virtqueue_add_split(struct virtqueue *_vq,
553 struct vring_virtqueue *vq = to_vvq(_vq);
577 desc = alloc_indirect_split(_vq, total_sg, gfp);
621 i = virtqueue_add_desc_split(_vq, desc, i, addr, sg->length,
637 i = virtqueue_add_desc_split(_vq, desc, i, addr,
645 desc[prev].flags &= cpu_to_virtio16(_vq->vdev, ~VRING_DESC_F_NEXT);
662 virtqueue_add_desc_split(_vq, vq->split.vring.desc,
688 vq->split.vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head);
694 vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,
704 virtqueue_kick(_vq);
721 i = virtio16_to_cpu(_vq->vdev, desc[i].next);
734 static bool virtqueue_kick_prepare_split(struct virtqueue *_vq)
736 struct vring_virtqueue *vq = to_vvq(_vq);
753 needs_kick = vring_need_event(virtio16_to_cpu(_vq->vdev,
758 cpu_to_virtio16(_vq->vdev,
823 static void *virtqueue_get_buf_ctx_split(struct virtqueue *_vq,
827 struct vring_virtqueue *vq = to_vvq(_vq);
849 i = virtio32_to_cpu(_vq->vdev,
851 *len = virtio32_to_cpu(_vq->vdev,
873 cpu_to_virtio16(_vq->vdev, vq->last_used_idx));
881 static void virtqueue_disable_cb_split(struct virtqueue *_vq)
883 struct vring_virtqueue *vq = to_vvq(_vq);
900 cpu_to_virtio16(_vq->vdev,
905 static unsigned int virtqueue_enable_cb_prepare_split(struct virtqueue *_vq)
907 struct vring_virtqueue *vq = to_vvq(_vq);
921 cpu_to_virtio16(_vq->vdev,
924 vring_used_event(&vq->split.vring) = cpu_to_virtio16(_vq->vdev,
930 static bool virtqueue_poll_split(struct virtqueue *_vq, unsigned int last_used_idx)
932 struct vring_virtqueue *vq = to_vvq(_vq);
934 return (u16)last_used_idx != virtio16_to_cpu(_vq->vdev,
938 static bool virtqueue_enable_cb_delayed_split(struct virtqueue *_vq)
940 struct vring_virtqueue *vq = to_vvq(_vq);
954 cpu_to_virtio16(_vq->vdev,
962 cpu_to_virtio16(_vq->vdev, vq->last_used_idx + bufs));
964 if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->split.vring.used->idx)
974 static void *virtqueue_detach_unused_buf_split(struct virtqueue *_vq)
976 struct vring_virtqueue *vq = to_vvq(_vq);
989 vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,
1175 static int virtqueue_resize_split(struct virtqueue *_vq, u32 num)
1178 struct vring_virtqueue *vq = to_vvq(_vq);
1179 struct virtio_device *vdev = _vq->vdev;
1401 static inline int virtqueue_add_packed(struct virtqueue *_vq,
1410 struct vring_virtqueue *vq = to_vvq(_vq);
1553 static bool virtqueue_kick_prepare_packed(struct virtqueue *_vq)
1555 struct vring_virtqueue *vq = to_vvq(_vq);
1673 static void *virtqueue_get_buf_ctx_packed(struct virtqueue *_vq,
1677 struct vring_virtqueue *vq = to_vvq(_vq);
1742 static void virtqueue_disable_cb_packed(struct virtqueue *_vq)
1744 struct vring_virtqueue *vq = to_vvq(_vq);
1761 static unsigned int virtqueue_enable_cb_prepare_packed(struct virtqueue *_vq)
1763 struct vring_virtqueue *vq = to_vvq(_vq);
1794 static bool virtqueue_poll_packed(struct virtqueue *_vq, u16 off_wrap)
1796 struct vring_virtqueue *vq = to_vvq(_vq);
1806 static bool virtqueue_enable_cb_delayed_packed(struct virtqueue *_vq)
1808 struct vring_virtqueue *vq = to_vvq(_vq);
1867 static void *virtqueue_detach_unused_buf_packed(struct virtqueue *_vq)
1869 struct vring_virtqueue *vq = to_vvq(_vq);
2125 static int virtqueue_resize_packed(struct virtqueue *_vq, u32 num)
2128 struct vring_virtqueue *vq = to_vvq(_vq);
2129 struct virtio_device *vdev = _vq->vdev;
2155 static int virtqueue_disable_and_recycle(struct virtqueue *_vq,
2158 struct vring_virtqueue *vq = to_vvq(_vq);
2172 err = vdev->config->disable_vq_and_reset(_vq);
2176 while ((buf = virtqueue_detach_unused_buf(_vq)) != NULL)
2177 recycle(_vq, buf);
2182 static int virtqueue_enable_after_reset(struct virtqueue *_vq)
2184 struct vring_virtqueue *vq = to_vvq(_vq);
2187 if (vdev->config->enable_vq_after_reset(_vq))
2197 static inline int virtqueue_add(struct virtqueue *_vq,
2206 struct vring_virtqueue *vq = to_vvq(_vq);
2208 return vq->packed_ring ? virtqueue_add_packed(_vq, sgs, total_sg,
2210 virtqueue_add_split(_vq, sgs, total_sg,
2216 * @_vq: the struct virtqueue we're talking about.
2228 int virtqueue_add_sgs(struct virtqueue *_vq,
2244 return virtqueue_add(_vq, sgs, total_sg, out_sgs, in_sgs,
2319 * @_vq: the struct virtqueue we're talking about.
2323 struct device *virtqueue_dma_dev(struct virtqueue *_vq)
2325 struct vring_virtqueue *vq = to_vvq(_vq);
2336 * @_vq: the struct virtqueue
2345 bool virtqueue_kick_prepare(struct virtqueue *_vq)
2347 struct vring_virtqueue *vq = to_vvq(_vq);
2349 return vq->packed_ring ? virtqueue_kick_prepare_packed(_vq) :
2350 virtqueue_kick_prepare_split(_vq);
2356 * @_vq: the struct virtqueue
2362 bool virtqueue_notify(struct virtqueue *_vq)
2364 struct vring_virtqueue *vq = to_vvq(_vq);
2370 if (!vq->notify(_vq)) {
2400 * @_vq: the struct virtqueue we're talking about.
2415 void *virtqueue_get_buf_ctx(struct virtqueue *_vq, unsigned int *len,
2418 struct vring_virtqueue *vq = to_vvq(_vq);
2420 return vq->packed_ring ? virtqueue_get_buf_ctx_packed(_vq, len, ctx) :
2421 virtqueue_get_buf_ctx_split(_vq, len, ctx);
2425 void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)
2427 return virtqueue_get_buf_ctx(_vq, len, NULL);
2432 * @_vq: the struct virtqueue we're talking about.
2439 void virtqueue_disable_cb(struct virtqueue *_vq)
2441 struct vring_virtqueue *vq = to_vvq(_vq);
2444 virtqueue_disable_cb_packed(_vq);
2446 virtqueue_disable_cb_split(_vq);
2452 * @_vq: the struct virtqueue we're talking about.
2462 unsigned int virtqueue_enable_cb_prepare(struct virtqueue *_vq)
2464 struct vring_virtqueue *vq = to_vvq(_vq);
2469 return vq->packed_ring ? virtqueue_enable_cb_prepare_packed(_vq) :
2470 virtqueue_enable_cb_prepare_split(_vq);
2476 * @_vq: the struct virtqueue we're talking about.
2483 bool virtqueue_poll(struct virtqueue *_vq, unsigned int last_used_idx)
2485 struct vring_virtqueue *vq = to_vvq(_vq);
2491 return vq->packed_ring ? virtqueue_poll_packed(_vq, last_used_idx) :
2492 virtqueue_poll_split(_vq, last_used_idx);
2498 * @_vq: the struct virtqueue we're talking about.
2507 bool virtqueue_enable_cb(struct virtqueue *_vq)
2509 unsigned int last_used_idx = virtqueue_enable_cb_prepare(_vq);
2511 return !virtqueue_poll(_vq, last_used_idx);
2517 * @_vq: the struct virtqueue we're talking about.
2528 bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
2530 struct vring_virtqueue *vq = to_vvq(_vq);
2535 return vq->packed_ring ? virtqueue_enable_cb_delayed_packed(_vq) :
2536 virtqueue_enable_cb_delayed_split(_vq);
2542 * @_vq: the struct virtqueue we're talking about.
2548 void *virtqueue_detach_unused_buf(struct virtqueue *_vq)
2550 struct vring_virtqueue *vq = to_vvq(_vq);
2552 return vq->packed_ring ? virtqueue_detach_unused_buf_packed(_vq) :
2553 virtqueue_detach_unused_buf_split(_vq);
2565 * @_vq: the struct virtqueue to notify
2567 * Calls the callback function of @_vq to process the virtqueue
2570 irqreturn_t vring_interrupt(int irq, void *_vq)
2572 struct vring_virtqueue *vq = to_vvq(_vq);
2716 * @_vq: the struct virtqueue we're talking about.
2738 int virtqueue_resize(struct virtqueue *_vq, u32 num,
2741 struct vring_virtqueue *vq = to_vvq(_vq);
2753 err = virtqueue_disable_and_recycle(_vq, recycle);
2758 err = virtqueue_resize_packed(_vq, num);
2760 err = virtqueue_resize_split(_vq, num);
2762 return virtqueue_enable_after_reset(_vq);
2768 * @_vq: the struct virtqueue we're talking about.
2787 int virtqueue_set_dma_premapped(struct virtqueue *_vq)
2789 struct vring_virtqueue *vq = to_vvq(_vq);
2817 * @_vq: the struct virtqueue we're talking about.
2829 int virtqueue_reset(struct virtqueue *_vq,
2832 struct vring_virtqueue *vq = to_vvq(_vq);
2835 err = virtqueue_disable_and_recycle(_vq, recycle);
2844 return virtqueue_enable_after_reset(_vq);
2872 static void vring_free(struct virtqueue *_vq)
2874 struct vring_virtqueue *vq = to_vvq(_vq);
2912 void vring_del_virtqueue(struct virtqueue *_vq)
2914 struct vring_virtqueue *vq = to_vvq(_vq);
2917 list_del(&_vq->list);
2920 vring_free(_vq);
2926 u32 vring_notification_data(struct virtqueue *_vq)
2928 struct vring_virtqueue *vq = to_vvq(_vq);
2939 return next << 16 | _vq->index;
2974 * @_vq: the struct virtqueue containing the vring of interest.
2979 unsigned int virtqueue_get_vring_size(const struct virtqueue *_vq)
2982 const struct vring_virtqueue *vq = to_vvq(_vq);
2991 void __virtqueue_break(struct virtqueue *_vq)
2993 struct vring_virtqueue *vq = to_vvq(_vq);
3003 void __virtqueue_unbreak(struct virtqueue *_vq)
3005 struct vring_virtqueue *vq = to_vvq(_vq);
3012 bool virtqueue_is_broken(const struct virtqueue *_vq)
3014 const struct vring_virtqueue *vq = to_vvq(_vq);
3026 struct virtqueue *_vq;
3029 list_for_each_entry(_vq, &dev->vqs, list) {
3030 struct vring_virtqueue *vq = to_vvq(_vq);
3048 struct virtqueue *_vq;
3051 list_for_each_entry(_vq, &dev->vqs, list) {
3052 struct vring_virtqueue *vq = to_vvq(_vq);
3061 dma_addr_t virtqueue_get_desc_addr(const struct virtqueue *_vq)
3063 const struct vring_virtqueue *vq = to_vvq(_vq);
3074 dma_addr_t virtqueue_get_avail_addr(const struct virtqueue *_vq)
3076 const struct vring_virtqueue *vq = to_vvq(_vq);
3088 dma_addr_t virtqueue_get_used_addr(const struct virtqueue *_vq)
3090 const struct vring_virtqueue *vq = to_vvq(_vq);
3110 * virtqueue_dma_map_single_attrs - map DMA for _vq
3111 * @_vq: the struct virtqueue we're talking about.
3118 * passed to this _vq when it is in pre-mapped mode.
3122 dma_addr_t virtqueue_dma_map_single_attrs(struct virtqueue *_vq, void *ptr,
3127 struct vring_virtqueue *vq = to_vvq(_vq);
3137 * virtqueue_dma_unmap_single_attrs - unmap DMA for _vq
3138 * @_vq: the struct virtqueue we're talking about.
3147 void virtqueue_dma_unmap_single_attrs(struct virtqueue *_vq, dma_addr_t addr,
3151 struct vring_virtqueue *vq = to_vvq(_vq);
3162 * @_vq: the struct virtqueue we're talking about.
3167 int virtqueue_dma_mapping_error(struct virtqueue *_vq, dma_addr_t addr)
3169 struct vring_virtqueue *vq = to_vvq(_vq);
3180 * @_vq: the struct virtqueue we're talking about.
3188 bool virtqueue_dma_need_sync(struct virtqueue *_vq, dma_addr_t addr)
3190 struct vring_virtqueue *vq = to_vvq(_vq);
3201 * @_vq: the struct virtqueue we're talking about.
3211 void virtqueue_dma_sync_single_range_for_cpu(struct virtqueue *_vq,
3216 struct vring_virtqueue *vq = to_vvq(_vq);
3228 * @_vq: the struct virtqueue we're talking about.
3237 void virtqueue_dma_sync_single_range_for_device(struct virtqueue *_vq,
3242 struct vring_virtqueue *vq = to_vvq(_vq);