Lines Matching defs:packed
157 /* Is this a packed ring? */
190 * for packed ring:
203 /* Available for packed ring */
204 struct vring_virtqueue_packed packed;
1297 head = vq->packed.next_avail_idx;
1311 BUG_ON(id == vq->packed.vring.num);
1338 vq->packed.vring.desc[head].addr = cpu_to_le64(addr);
1339 vq->packed.vring.desc[head].len = cpu_to_le32(total_sg *
1341 vq->packed.vring.desc[head].id = cpu_to_le16(id);
1344 vq->packed.desc_extra[id].addr = addr;
1345 vq->packed.desc_extra[id].len = total_sg *
1347 vq->packed.desc_extra[id].flags = VRING_DESC_F_INDIRECT |
1348 vq->packed.avail_used_flags;
1357 vq->packed.vring.desc[head].flags = cpu_to_le16(VRING_DESC_F_INDIRECT |
1358 vq->packed.avail_used_flags);
1365 if (n >= vq->packed.vring.num) {
1367 vq->packed.avail_wrap_counter ^= 1;
1368 vq->packed.avail_used_flags ^=
1372 vq->packed.next_avail_idx = n;
1373 vq->free_head = vq->packed.desc_extra[id].next;
1376 vq->packed.desc_state[id].num = 1;
1377 vq->packed.desc_state[id].data = data;
1378 vq->packed.desc_state[id].indir_desc = desc;
1379 vq->packed.desc_state[id].last = id;
1443 head = vq->packed.next_avail_idx;
1444 avail_used_flags = vq->packed.avail_used_flags;
1446 WARN_ON_ONCE(total_sg > vq->packed.vring.num && !vq->indirect);
1448 desc = vq->packed.vring.desc;
1460 BUG_ON(id == vq->packed.vring.num);
1472 flags = cpu_to_le16(vq->packed.avail_used_flags |
1485 vq->packed.desc_extra[curr].addr = addr;
1486 vq->packed.desc_extra[curr].len = sg->length;
1487 vq->packed.desc_extra[curr].flags =
1491 curr = vq->packed.desc_extra[curr].next;
1493 if ((unlikely(++i >= vq->packed.vring.num))) {
1495 vq->packed.avail_used_flags ^=
1503 vq->packed.avail_wrap_counter ^= 1;
1509 vq->packed.next_avail_idx = i;
1513 vq->packed.desc_state[id].num = descs_used;
1514 vq->packed.desc_state[id].data = data;
1515 vq->packed.desc_state[id].indir_desc = ctx;
1516 vq->packed.desc_state[id].last = prev;
1524 vq->packed.vring.desc[head].flags = head_flags;
1537 vq->packed.avail_used_flags = avail_used_flags;
1542 vring_unmap_extra_packed(vq, &vq->packed.desc_extra[curr]);
1543 curr = vq->packed.desc_extra[curr].next;
1545 if (i >= vq->packed.vring.num)
1574 old = vq->packed.next_avail_idx - vq->num_added;
1575 new = vq->packed.next_avail_idx;
1578 snapshot.u32 = *(u32 *)vq->packed.vring.device;
1593 if (wrap_counter != vq->packed.avail_wrap_counter)
1594 event_idx -= vq->packed.vring.num;
1609 state = &vq->packed.desc_state[id];
1614 vq->packed.desc_extra[state->last].next = vq->free_head;
1622 &vq->packed.desc_extra[curr]);
1623 curr = vq->packed.desc_extra[curr].next;
1636 len = vq->packed.desc_extra[id].len;
1654 flags = le16_to_cpu(vq->packed.vring.desc[idx].flags);
1701 id = le16_to_cpu(vq->packed.vring.desc[last_used].id);
1702 *len = le32_to_cpu(vq->packed.vring.desc[last_used].len);
1704 if (unlikely(id >= vq->packed.vring.num)) {
1708 if (unlikely(!vq->packed.desc_state[id].data)) {
1714 ret = vq->packed.desc_state[id].data;
1717 last_used += vq->packed.desc_state[id].num;
1718 if (unlikely(last_used >= vq->packed.vring.num)) {
1719 last_used -= vq->packed.vring.num;
1731 if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DESC)
1733 &vq->packed.vring.driver->off_wrap,
1746 if (vq->packed.event_flags_shadow != VRING_PACKED_EVENT_FLAG_DISABLE) {
1747 vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE;
1756 vq->packed.vring.driver->flags =
1757 cpu_to_le16(vq->packed.event_flags_shadow);
1773 vq->packed.vring.driver->off_wrap =
1782 if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) {
1783 vq->packed.event_flags_shadow = vq->event ?
1786 vq->packed.vring.driver->flags =
1787 cpu_to_le16(vq->packed.event_flags_shadow);
1821 bufs = (vq->packed.vring.num - vq->vq.num_free) * 3 / 4;
1826 if (used_idx >= vq->packed.vring.num) {
1827 used_idx -= vq->packed.vring.num;
1831 vq->packed.vring.driver->off_wrap = cpu_to_le16(used_idx |
1841 if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) {
1842 vq->packed.event_flags_shadow = vq->event ?
1845 vq->packed.vring.driver->flags =
1846 cpu_to_le16(vq->packed.event_flags_shadow);
1875 for (i = 0; i < vq->packed.vring.num; i++) {
1876 if (!vq->packed.desc_state[i].data)
1879 buf = vq->packed.desc_state[i].data;
1885 BUG_ON(vq->vq.num_free != vq->packed.vring.num);
2035 vq->packed = *vring_packed;
2043 memset(vq->packed.vring.device, 0, vq->packed.event_size_in_bytes);
2044 memset(vq->packed.vring.driver, 0, vq->packed.event_size_in_bytes);
2047 memset(vq->packed.vring.desc, 0, vq->packed.ring_size_in_bytes);
2049 virtqueue_init(vq, vq->packed.vring.num);
2050 virtqueue_vring_init_packed(&vq->packed, !!vq->vq.callback);
2750 if ((vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num) == num)
2794 num = vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num;
2879 vq->packed.ring_size_in_bytes,
2880 vq->packed.vring.desc,
2881 vq->packed.ring_dma_addr,
2885 vq->packed.event_size_in_bytes,
2886 vq->packed.vring.driver,
2887 vq->packed.driver_event_dma_addr,
2891 vq->packed.event_size_in_bytes,
2892 vq->packed.vring.device,
2893 vq->packed.device_event_dma_addr,
2896 kfree(vq->packed.desc_state);
2897 kfree(vq->packed.desc_extra);
2932 next = (vq->packed.next_avail_idx &
2934 vq->packed.avail_wrap_counter <<
2984 return vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num;
3068 return vq->packed.ring_dma_addr;
3081 return vq->packed.driver_event_dma_addr;
3095 return vq->packed.device_event_dma_addr;