Lines Matching refs:rings
186 return ctx->cached_cq_tail - READ_ONCE(ctx->rings->cq.head);
191 return READ_ONCE(ctx->rings->cq.tail) - READ_ONCE(ctx->rings->cq.head);
354 struct io_rings *r = ctx->rings;
573 if (READ_ONCE(ctx->rings->cq_flags) & IORING_CQ_EVENTFD_DISABLED)
648 /* IOPOLL rings only need to wake up if it's also SQPOLL */
708 atomic_andnot(IORING_SQ_CQ_OVERFLOW, &ctx->rings->sq_flags);
802 atomic_or(IORING_SQ_CQ_OVERFLOW, &ctx->rings->sq_flags);
831 struct io_rings *rings = ctx->rings;
856 ctx->cqe_cached = &rings->cqes[off];
1159 atomic_andnot(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);
1306 atomic_or(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);
1333 atomic_or(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);
1374 atomic_or(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);
1388 atomic_andnot(IORING_SQ_TASKRUN, &ctx->rings->sq_flags);
2334 struct io_rings *rings = ctx->rings;
2341 smp_store_release(&rings->sq.head, ctx->cached_sq_head);
2364 WRITE_ONCE(ctx->rings->sq_dropped,
2365 READ_ONCE(ctx->rings->sq_dropped) + 1);
2453 int dist = READ_ONCE(ctx->rings->cq.tail) - (int) iowq->cq_tail;
2542 struct io_rings *rings = ctx->rings;
2573 iowq.cq_tail = READ_ONCE(ctx->rings->cq.head) + min_events;
2586 int nr_wait = (int) iowq.cq_tail - READ_ONCE(ctx->rings->cq.tail);
2644 return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0;
2751 io_mem_free(ctx->rings);
2753 ctx->rings = NULL;
2777 struct io_rings *rings;
2780 off = struct_size(rings, cqes, cq_entries);
3164 if (ctx->rings)
3169 * If we failed setting up the ctx, we might not have any rings
3172 if (ctx->rings)
3180 * if we're exiting a ton of rings at the same time. It just adds
3276 if (!ctx->rings)
3283 * Cancels requests of all rings, not only @ctx, but
3423 ptr = ctx->rings;
3741 struct io_rings *rings;
3754 rings = io_mem_alloc(size);
3756 rings = io_rings_map(ctx, p->cq_off.user_addr, size);
3758 if (IS_ERR(rings))
3759 return PTR_ERR(rings);
3761 ctx->rings = rings;
3763 ctx->sq_array = (u32 *)((char *)rings + sq_array_offset);
3764 rings->sq_ring_mask = p->sq_entries - 1;
3765 rings->cq_ring_mask = p->cq_entries - 1;
3766 rings->sq_ring_entries = p->sq_entries;
3767 rings->cq_ring_entries = p->cq_entries;
3955 p->sq_off.array = (char *)ctx->sq_array - (char *)ctx->rings;
4142 /* Restrictions allowed only if rings started disabled */