Lines Matching refs:vrh
38 static inline int __vringh_get_head(const struct vringh *vrh,
39 int (*getu16)(const struct vringh *vrh,
46 err = getu16(vrh, &avail_idx, &vrh->vring.avail->idx);
49 &vrh->vring.avail->idx);
54 return vrh->vring.num;
57 virtio_rmb(vrh->weak_barriers);
59 i = *last_avail_idx & (vrh->vring.num - 1);
61 err = getu16(vrh, &head, &vrh->vring.avail->ring[i]);
64 *last_avail_idx, &vrh->vring.avail->ring[i]);
68 if (head >= vrh->vring.num) {
70 head, vrh->vring.num);
79 static inline ssize_t vringh_iov_xfer(struct vringh *vrh,
82 int (*xfer)(const struct vringh *vrh,
92 err = xfer(vrh, iov->iov[iov->i].iov_base, ptr, partlen);
116 static inline bool range_check(struct vringh *vrh, u64 addr, size_t *len,
122 if (!getrange(vrh, addr, range))
150 static inline bool no_range_check(struct vringh *vrh, u64 addr, size_t *len,
159 static int move_to_indirect(const struct vringh *vrh,
172 len = vringh32_to_cpu(vrh, desc->len);
179 if (desc->flags & cpu_to_vringh16(vrh, VRING_DESC_F_NEXT))
180 *up_next = vringh16_to_cpu(vrh, desc->next);
217 static u16 __cold return_from_indirect(const struct vringh *vrh, int *up_next,
223 *descs = vrh->vring.desc;
224 *desc_max = vrh->vring.num;
228 static int slow_copy(struct vringh *vrh, void *dst, const void *src,
229 bool (*rcheck)(struct vringh *vrh, u64 addr, size_t *len,
231 bool (*getrange)(struct vringh *vrh,
234 bool (*getrange)(struct vringh *vrh,
238 int (*copy)(const struct vringh *vrh,
250 if (!rcheck(vrh, addr, &part, range, getrange))
253 err = copy(vrh, dst, src, part);
265 __vringh_iov(struct vringh *vrh, u16 i,
268 bool (*rcheck)(struct vringh *vrh, u64 addr, size_t *len,
274 int (*copy)(const struct vringh *vrh,
283 descs = vrh->vring.desc;
284 desc_max = vrh->vring.num;
302 err = slow_copy(vrh, &desc, &descs[i], rcheck, getrange,
305 err = copy(vrh, &desc, &descs[i], sizeof(desc));
310 cpu_to_vringh16(vrh, VRING_DESC_F_INDIRECT))) {
311 u64 a = vringh64_to_cpu(vrh, desc.addr);
314 len = vringh32_to_cpu(vrh, desc.len);
315 if (!rcheck(vrh, a, &len, &range, getrange)) {
320 if (unlikely(len != vringh32_to_cpu(vrh, desc.len))) {
327 err = move_to_indirect(vrh, &up_next, &i, addr, &desc,
339 if (count > vrh->vring.num || indirect_count > desc_max) {
345 if (desc.flags & cpu_to_vringh16(vrh, VRING_DESC_F_WRITE))
366 len = vringh32_to_cpu(vrh, desc.len);
367 if (!rcheck(vrh, vringh64_to_cpu(vrh, desc.addr), &len, &range,
372 addr = (void *)(unsigned long)(vringh64_to_cpu(vrh, desc.addr) +
385 if (unlikely(len != vringh32_to_cpu(vrh, desc.len))) {
386 desc.len = cpu_to_vringh32(vrh,
387 vringh32_to_cpu(vrh, desc.len) - len);
388 desc.addr = cpu_to_vringh64(vrh,
389 vringh64_to_cpu(vrh, desc.addr) + len);
393 if (desc.flags & cpu_to_vringh16(vrh, VRING_DESC_F_NEXT)) {
394 i = vringh16_to_cpu(vrh, desc.next);
398 i = return_from_indirect(vrh, &up_next,
419 static inline int __vringh_complete(struct vringh *vrh,
422 int (*putu16)(const struct vringh *vrh,
424 int (*putused)(const struct vringh *vrh,
433 used_ring = vrh->vring.used;
434 used_idx = vrh->last_used_idx + vrh->completed;
436 off = used_idx % vrh->vring.num;
439 if (num_used > 1 && unlikely(off + num_used >= vrh->vring.num)) {
440 u16 part = vrh->vring.num - off;
441 err = putused(vrh, &used_ring->ring[off], used, part);
443 err = putused(vrh, &used_ring->ring[0], used + part,
446 err = putused(vrh, &used_ring->ring[off], used, num_used);
455 virtio_wmb(vrh->weak_barriers);
457 err = putu16(vrh, &vrh->vring.used->idx, used_idx + num_used);
460 &vrh->vring.used->idx);
464 vrh->completed += num_used;
469 static inline int __vringh_need_notify(struct vringh *vrh,
470 int (*getu16)(const struct vringh *vrh,
481 virtio_mb(vrh->weak_barriers);
484 if (!vrh->event_indices) {
486 err = getu16(vrh, &flags, &vrh->vring.avail->flags);
489 &vrh->vring.avail->flags);
496 err = getu16(vrh, &used_event, &vring_used_event(&vrh->vring));
499 &vring_used_event(&vrh->vring));
504 if (unlikely(vrh->completed > 0xffff))
508 vrh->last_used_idx + vrh->completed,
509 vrh->last_used_idx);
511 vrh->last_used_idx += vrh->completed;
512 vrh->completed = 0;
516 static inline bool __vringh_notify_enable(struct vringh *vrh,
517 int (*getu16)(const struct vringh *vrh,
519 int (*putu16)(const struct vringh *vrh,
524 if (!vrh->event_indices) {
526 if (putu16(vrh, &vrh->vring.used->flags, 0) != 0) {
528 &vrh->vring.used->flags);
532 if (putu16(vrh, &vring_avail_event(&vrh->vring),
533 vrh->last_avail_idx) != 0) {
535 &vring_avail_event(&vrh->vring));
542 virtio_mb(vrh->weak_barriers);
544 if (getu16(vrh, &avail, &vrh->vring.avail->idx) != 0) {
546 &vrh->vring.avail->idx);
553 return avail == vrh->last_avail_idx;
556 static inline void __vringh_notify_disable(struct vringh *vrh,
557 int (*putu16)(const struct vringh *vrh,
560 if (!vrh->event_indices) {
562 if (putu16(vrh, &vrh->vring.used->flags,
565 &vrh->vring.used->flags);
571 static inline int getu16_user(const struct vringh *vrh, u16 *val, const __virtio16 *p)
575 *val = vringh16_to_cpu(vrh, v);
579 static inline int putu16_user(const struct vringh *vrh, __virtio16 *p, u16 val)
581 __virtio16 v = cpu_to_vringh16(vrh, val);
585 static inline int copydesc_user(const struct vringh *vrh,
592 static inline int putused_user(const struct vringh *vrh,
601 static inline int xfer_from_user(const struct vringh *vrh, void *src,
608 static inline int xfer_to_user(const struct vringh *vrh,
617 * @vrh: the vringh to initialize.
628 int vringh_init_user(struct vringh *vrh, u64 features,
640 vrh->little_endian = (features & (1ULL << VIRTIO_F_VERSION_1));
641 vrh->event_indices = (features & (1 << VIRTIO_RING_F_EVENT_IDX));
642 vrh->weak_barriers = weak_barriers;
643 vrh->completed = 0;
644 vrh->last_avail_idx = 0;
645 vrh->last_used_idx = 0;
646 vrh->vring.num = num;
648 vrh->vring.desc = (__force struct vring_desc *)desc;
649 vrh->vring.avail = (__force struct vring_avail *)avail;
650 vrh->vring.used = (__force struct vring_used *)used;
657 * @vrh: the userspace vring.
667 * *head will be vrh->vring.num. You may be able to ignore an invalid
672 int vringh_getdesc_user(struct vringh *vrh,
675 bool (*getrange)(struct vringh *vrh,
681 *head = vrh->vring.num;
682 err = __vringh_get_head(vrh, getu16_user, &vrh->last_avail_idx);
687 if (err == vrh->vring.num)
711 err = __vringh_iov(vrh, *head, (struct vringh_kiov *)riov,
754 * @vrh: the vring.
760 void vringh_abandon_user(struct vringh *vrh, unsigned int num)
764 vrh->last_avail_idx -= num;
770 * @vrh: the vring.
777 int vringh_complete_user(struct vringh *vrh, u16 head, u32 len)
781 used.id = cpu_to_vringh32(vrh, head);
782 used.len = cpu_to_vringh32(vrh, len);
783 return __vringh_complete(vrh, &used, 1, putu16_user, putused_user);
789 * @vrh: the vring.
796 int vringh_complete_multi_user(struct vringh *vrh,
800 return __vringh_complete(vrh, used, num_used,
807 * @vrh: the vring.
812 bool vringh_notify_enable_user(struct vringh *vrh)
814 return __vringh_notify_enable(vrh, getu16_user, putu16_user);
820 * @vrh: the vring.
825 void vringh_notify_disable_user(struct vringh *vrh)
827 __vringh_notify_disable(vrh, putu16_user);
833 * @vrh: the vring we've called vringh_complete_user() on.
837 int vringh_need_notify_user(struct vringh *vrh)
839 return __vringh_need_notify(vrh, getu16_user);
844 static inline int getu16_kern(const struct vringh *vrh,
847 *val = vringh16_to_cpu(vrh, READ_ONCE(*p));
851 static inline int putu16_kern(const struct vringh *vrh, __virtio16 *p, u16 val)
853 WRITE_ONCE(*p, cpu_to_vringh16(vrh, val));
857 static inline int copydesc_kern(const struct vringh *vrh,
864 static inline int putused_kern(const struct vringh *vrh,
873 static inline int xfer_kern(const struct vringh *vrh, void *src,
880 static inline int kern_xfer(const struct vringh *vrh, void *dst,
889 * @vrh: the vringh to initialize.
899 int vringh_init_kern(struct vringh *vrh, u64 features,
911 vrh->little_endian = (features & (1ULL << VIRTIO_F_VERSION_1));
912 vrh->event_indices = (features & (1 << VIRTIO_RING_F_EVENT_IDX));
913 vrh->weak_barriers = weak_barriers;
914 vrh->completed = 0;
915 vrh->last_avail_idx = 0;
916 vrh->last_used_idx = 0;
917 vrh->vring.num = num;
918 vrh->vring.desc = desc;
919 vrh->vring.avail = avail;
920 vrh->vring.used = used;
927 * @vrh: the kernelspace vring.
937 * *head will be vrh->vring.num. You may be able to ignore an invalid
942 int vringh_getdesc_kern(struct vringh *vrh,
950 err = __vringh_get_head(vrh, getu16_kern, &vrh->last_avail_idx);
955 if (err == vrh->vring.num)
959 err = __vringh_iov(vrh, *head, riov, wiov, no_range_check, NULL,
999 * @vrh: the vring.
1005 void vringh_abandon_kern(struct vringh *vrh, unsigned int num)
1009 vrh->last_avail_idx -= num;
1015 * @vrh: the vring.
1022 int vringh_complete_kern(struct vringh *vrh, u16 head, u32 len)
1026 used.id = cpu_to_vringh32(vrh, head);
1027 used.len = cpu_to_vringh32(vrh, len);
1029 return __vringh_complete(vrh, &used, 1, putu16_kern, putused_kern);
1035 * @vrh: the vring.
1040 bool vringh_notify_enable_kern(struct vringh *vrh)
1042 return __vringh_notify_enable(vrh, getu16_kern, putu16_kern);
1048 * @vrh: the vring.
1053 void vringh_notify_disable_kern(struct vringh *vrh)
1055 __vringh_notify_disable(vrh, putu16_kern);
1061 * @vrh: the vring we've called vringh_complete_kern() on.
1065 int vringh_need_notify_kern(struct vringh *vrh)
1067 return __vringh_need_notify(vrh, getu16_kern);
1073 static int iotlb_translate(const struct vringh *vrh,
1078 struct vhost_iotlb *iotlb = vrh->iotlb;
1113 static inline int copy_from_iotlb(const struct vringh *vrh, void *dst,
1120 ret = iotlb_translate(vrh, (u64)(uintptr_t)src,
1132 static inline int copy_to_iotlb(const struct vringh *vrh, void *dst,
1139 ret = iotlb_translate(vrh, (u64)(uintptr_t)dst,
1149 static inline int getu16_iotlb(const struct vringh *vrh,
1157 ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p),
1164 *val = vringh16_to_cpu(vrh, READ_ONCE(*(__virtio16 *)from));
1170 static inline int putu16_iotlb(const struct vringh *vrh,
1178 ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p),
1185 WRITE_ONCE(*(__virtio16 *)to, cpu_to_vringh16(vrh, val));
1191 static inline int copydesc_iotlb(const struct vringh *vrh,
1196 ret = copy_from_iotlb(vrh, dst, (void *)src, len);
1203 static inline int xfer_from_iotlb(const struct vringh *vrh, void *src,
1208 ret = copy_from_iotlb(vrh, dst, src, len);
1215 static inline int xfer_to_iotlb(const struct vringh *vrh,
1220 ret = copy_to_iotlb(vrh, dst, src, len);
1227 static inline int putused_iotlb(const struct vringh *vrh,
1235 ret = copy_to_iotlb(vrh, dst, (void *)src, num * sizeof(*dst));
1244 * @vrh: the vringh to initialize.
1254 int vringh_init_iotlb(struct vringh *vrh, u64 features,
1260 return vringh_init_kern(vrh, features, num, weak_barriers,
1267 * @vrh: the vring
1270 void vringh_set_iotlb(struct vringh *vrh, struct vhost_iotlb *iotlb)
1272 vrh->iotlb = iotlb;
1279 * @vrh: the kernelspace vring.
1289 * *head will be vrh->vring.num. You may be able to ignore an invalid
1294 int vringh_getdesc_iotlb(struct vringh *vrh,
1302 err = __vringh_get_head(vrh, getu16_iotlb, &vrh->last_avail_idx);
1307 if (err == vrh->vring.num)
1311 err = __vringh_iov(vrh, *head, riov, wiov, no_range_check, NULL,
1322 * @vrh: the vring.
1329 ssize_t vringh_iov_pull_iotlb(struct vringh *vrh,
1333 return vringh_iov_xfer(vrh, riov, dst, len, xfer_from_iotlb);
1339 * @vrh: the vring.
1346 ssize_t vringh_iov_push_iotlb(struct vringh *vrh,
1350 return vringh_iov_xfer(vrh, wiov, (void *)src, len, xfer_to_iotlb);
1356 * @vrh: the vring.
1362 void vringh_abandon_iotlb(struct vringh *vrh, unsigned int num)
1367 vrh->last_avail_idx -= num;
1373 * @vrh: the vring.
1380 int vringh_complete_iotlb(struct vringh *vrh, u16 head, u32 len)
1384 used.id = cpu_to_vringh32(vrh, head);
1385 used.len = cpu_to_vringh32(vrh, len);
1387 return __vringh_complete(vrh, &used, 1, putu16_iotlb, putused_iotlb);
1393 * @vrh: the vring.
1398 bool vringh_notify_enable_iotlb(struct vringh *vrh)
1400 return __vringh_notify_enable(vrh, getu16_iotlb, putu16_iotlb);
1406 * @vrh: the vring.
1411 void vringh_notify_disable_iotlb(struct vringh *vrh)
1413 __vringh_notify_disable(vrh, putu16_iotlb);
1419 * @vrh: the vring we've called vringh_complete_iotlb() on.
1423 int vringh_need_notify_iotlb(struct vringh *vrh)
1425 return __vringh_need_notify(vrh, getu16_iotlb);