Lines Matching refs:packed
90 /* Is this a packed ring? */
139 /* Available for packed ring */
177 } packed;
993 head = vq->packed.next_avail_idx;
1007 BUG_ON(id == vq->packed.vring.num);
1031 vq->packed.vring.desc[head].addr = cpu_to_le64(addr);
1032 vq->packed.vring.desc[head].len = cpu_to_le32(total_sg *
1034 vq->packed.vring.desc[head].id = cpu_to_le16(id);
1037 vq->packed.desc_extra[id].addr = addr;
1038 vq->packed.desc_extra[id].len = total_sg *
1040 vq->packed.desc_extra[id].flags = VRING_DESC_F_INDIRECT |
1041 vq->packed.avail_used_flags;
1050 vq->packed.vring.desc[head].flags = cpu_to_le16(VRING_DESC_F_INDIRECT |
1051 vq->packed.avail_used_flags);
1058 if (n >= vq->packed.vring.num) {
1060 vq->packed.avail_wrap_counter ^= 1;
1061 vq->packed.avail_used_flags ^=
1065 vq->packed.next_avail_idx = n;
1066 vq->free_head = vq->packed.desc_state[id].next;
1069 vq->packed.desc_state[id].num = 1;
1070 vq->packed.desc_state[id].data = data;
1071 vq->packed.desc_state[id].indir_desc = desc;
1072 vq->packed.desc_state[id].last = id;
1135 head = vq->packed.next_avail_idx;
1136 avail_used_flags = vq->packed.avail_used_flags;
1138 WARN_ON_ONCE(total_sg > vq->packed.vring.num && !vq->indirect);
1140 desc = vq->packed.vring.desc;
1152 BUG_ON(id == vq->packed.vring.num);
1163 flags = cpu_to_le16(vq->packed.avail_used_flags |
1176 vq->packed.desc_extra[curr].addr = addr;
1177 vq->packed.desc_extra[curr].len = sg->length;
1178 vq->packed.desc_extra[curr].flags =
1182 curr = vq->packed.desc_state[curr].next;
1184 if ((unlikely(++i >= vq->packed.vring.num))) {
1186 vq->packed.avail_used_flags ^=
1194 vq->packed.avail_wrap_counter ^= 1;
1200 vq->packed.next_avail_idx = i;
1204 vq->packed.desc_state[id].num = descs_used;
1205 vq->packed.desc_state[id].data = data;
1206 vq->packed.desc_state[id].indir_desc = ctx;
1207 vq->packed.desc_state[id].last = prev;
1215 vq->packed.vring.desc[head].flags = head_flags;
1227 vq->packed.avail_used_flags = avail_used_flags;
1234 if (i >= vq->packed.vring.num)
1263 old = vq->packed.next_avail_idx - vq->num_added;
1264 new = vq->packed.next_avail_idx;
1267 snapshot.u32 = *(u32 *)vq->packed.vring.device;
1282 if (wrap_counter != vq->packed.avail_wrap_counter)
1283 event_idx -= vq->packed.vring.num;
1298 state = &vq->packed.desc_state[id];
1303 vq->packed.desc_state[state->last].next = vq->free_head;
1311 &vq->packed.desc_extra[curr]);
1312 curr = vq->packed.desc_state[curr].next;
1325 len = vq->packed.desc_extra[id].len;
1343 flags = le16_to_cpu(vq->packed.vring.desc[idx].flags);
1353 vq->packed.used_wrap_counter);
1381 id = le16_to_cpu(vq->packed.vring.desc[last_used].id);
1382 *len = le32_to_cpu(vq->packed.vring.desc[last_used].len);
1384 if (unlikely(id >= vq->packed.vring.num)) {
1388 if (unlikely(!vq->packed.desc_state[id].data)) {
1394 ret = vq->packed.desc_state[id].data;
1397 vq->last_used_idx += vq->packed.desc_state[id].num;
1398 if (unlikely(vq->last_used_idx >= vq->packed.vring.num)) {
1399 vq->last_used_idx -= vq->packed.vring.num;
1400 vq->packed.used_wrap_counter ^= 1;
1408 if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DESC)
1410 &vq->packed.vring.driver->off_wrap,
1412 (vq->packed.used_wrap_counter <<
1425 if (vq->packed.event_flags_shadow != VRING_PACKED_EVENT_FLAG_DISABLE) {
1426 vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE;
1427 vq->packed.vring.driver->flags =
1428 cpu_to_le16(vq->packed.event_flags_shadow);
1444 vq->packed.vring.driver->off_wrap =
1446 (vq->packed.used_wrap_counter <<
1455 if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) {
1456 vq->packed.event_flags_shadow = vq->event ?
1459 vq->packed.vring.driver->flags =
1460 cpu_to_le16(vq->packed.event_flags_shadow);
1464 return vq->last_used_idx | ((u16)vq->packed.used_wrap_counter <<
1495 bufs = (vq->packed.vring.num - vq->vq.num_free) * 3 / 4;
1496 wrap_counter = vq->packed.used_wrap_counter;
1499 if (used_idx >= vq->packed.vring.num) {
1500 used_idx -= vq->packed.vring.num;
1504 vq->packed.vring.driver->off_wrap = cpu_to_le16(used_idx |
1514 if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) {
1515 vq->packed.event_flags_shadow = vq->event ?
1518 vq->packed.vring.driver->flags =
1519 cpu_to_le16(vq->packed.event_flags_shadow);
1530 vq->packed.used_wrap_counter)) {
1547 for (i = 0; i < vq->packed.vring.num; i++) {
1548 if (!vq->packed.desc_state[i].data)
1551 buf = vq->packed.desc_state[i].data;
1557 BUG_ON(vq->vq.num_free != vq->packed.vring.num);
1633 vq->packed.ring_dma_addr = ring_dma_addr;
1634 vq->packed.driver_event_dma_addr = driver_event_dma_addr;
1635 vq->packed.device_event_dma_addr = device_event_dma_addr;
1637 vq->packed.ring_size_in_bytes = ring_size_in_bytes;
1638 vq->packed.event_size_in_bytes = event_size_in_bytes;
1640 vq->packed.vring.num = num;
1641 vq->packed.vring.desc = ring;
1642 vq->packed.vring.driver = driver;
1643 vq->packed.vring.device = device;
1645 vq->packed.next_avail_idx = 0;
1646 vq->packed.avail_wrap_counter = 1;
1647 vq->packed.used_wrap_counter = 1;
1648 vq->packed.event_flags_shadow = 0;
1649 vq->packed.avail_used_flags = 1 << VRING_PACKED_DESC_F_AVAIL;
1651 vq->packed.desc_state = kmalloc_array(num,
1654 if (!vq->packed.desc_state)
1657 memset(vq->packed.desc_state, 0,
1663 vq->packed.desc_state[i].next = i + 1;
1665 vq->packed.desc_extra = kmalloc_array(num,
1668 if (!vq->packed.desc_extra)
1671 memset(vq->packed.desc_extra, 0,
1676 vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE;
1677 vq->packed.vring.driver->flags =
1678 cpu_to_le16(vq->packed.event_flags_shadow);
1687 kfree(vq->packed.desc_state);
2202 vq->packed.ring_size_in_bytes,
2203 vq->packed.vring.desc,
2204 vq->packed.ring_dma_addr);
2207 vq->packed.event_size_in_bytes,
2208 vq->packed.vring.driver,
2209 vq->packed.driver_event_dma_addr);
2212 vq->packed.event_size_in_bytes,
2213 vq->packed.vring.device,
2214 vq->packed.device_event_dma_addr);
2216 kfree(vq->packed.desc_state);
2217 kfree(vq->packed.desc_extra);
2273 return vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num;
2311 return vq->packed.ring_dma_addr;
2324 return vq->packed.driver_event_dma_addr;
2338 return vq->packed.device_event_dma_addr;