Lines Matching defs:tctx
1477 struct io_uring_task *tctx = req->task->io_uring;
1482 BUG_ON(!tctx);
1483 BUG_ON(!tctx->io_wq);
1500 io_wq_enqueue(tctx->io_wq, &req->work);
1738 struct io_uring_task *tctx = task->io_uring;
1741 tctx->cached_refs += nr;
1743 percpu_counter_sub(&tctx->inflight, nr);
1744 if (unlikely(atomic_read(&tctx->in_idle)))
1745 wake_up(&tctx->wait);
1750 static void io_task_refs_refill(struct io_uring_task *tctx)
1752 unsigned int refill = -tctx->cached_refs + IO_TCTX_REFS_CACHE_NR;
1754 percpu_counter_add(&tctx->inflight, refill);
1756 tctx->cached_refs += refill;
1761 struct io_uring_task *tctx = current->io_uring;
1763 tctx->cached_refs -= nr;
1764 if (unlikely(tctx->cached_refs < 0))
1765 io_task_refs_refill(tctx);
1770 struct io_uring_task *tctx = task->io_uring;
1771 unsigned int refs = tctx->cached_refs;
1774 tctx->cached_refs = 0;
1775 percpu_counter_sub(&tctx->inflight, refs);
2185 struct io_uring_task *tctx = container_of(cb, struct io_uring_task,
2191 if (!tctx->task_list.first && locked && ctx->submit_state.compl_nr)
2194 spin_lock_irq(&tctx->task_lock);
2195 node = tctx->task_list.first;
2196 INIT_WQ_LIST(&tctx->task_list);
2198 tctx->task_running = false;
2199 spin_unlock_irq(&tctx->task_lock);
2228 if (unlikely(atomic_read(&tctx->in_idle)))
2235 struct io_uring_task *tctx = tsk->io_uring;
2241 WARN_ON_ONCE(!tctx);
2243 spin_lock_irqsave(&tctx->task_lock, flags);
2244 wq_list_add_tail(&req->io_task_work.node, &tctx->task_list);
2245 running = tctx->task_running;
2247 tctx->task_running = true;
2248 spin_unlock_irqrestore(&tctx->task_lock, flags);
2261 if (!task_work_add(tsk, &tctx->task_work, notify)) {
2266 spin_lock_irqsave(&tctx->task_lock, flags);
2267 tctx->task_running = false;
2268 node = tctx->task_list.first;
2269 INIT_WQ_LIST(&tctx->task_list);
2270 spin_unlock_irqrestore(&tctx->task_lock, flags);
6357 static int io_async_cancel_one(struct io_uring_task *tctx, u64 user_data,
6364 if (!tctx || !tctx->io_wq)
6367 cancel_ret = io_wq_cancel_cb(tctx->io_wq, io_cancel_cb, &data, false);
6436 struct io_uring_task *tctx = node->task->io_uring;
6438 ret = io_async_cancel_one(tctx, req->cancel.addr, ctx);
6735 struct io_uring_task *tctx = req->task->io_uring;
6737 atomic_dec(&tctx->inflight_tracked);
6894 /* either cancelled or io-wq is dying, so don't touch tctx->iowq */
7647 struct io_uring_task *tctx = current->io_uring;
7649 if (!tctx)
7651 return percpu_counter_read_positive(&tctx->inflight);
8670 struct io_uring_task *tctx;
8673 tctx = kzalloc(sizeof(*tctx), GFP_KERNEL);
8674 if (unlikely(!tctx))
8677 ret = percpu_counter_init(&tctx->inflight, 0, GFP_KERNEL);
8679 kfree(tctx);
8683 tctx->io_wq = io_init_wq_offload(ctx, task);
8684 if (IS_ERR(tctx->io_wq)) {
8685 ret = PTR_ERR(tctx->io_wq);
8686 percpu_counter_destroy(&tctx->inflight);
8687 kfree(tctx);
8691 xa_init(&tctx->xa);
8692 init_waitqueue_head(&tctx->wait);
8693 atomic_set(&tctx->in_idle, 0);
8694 atomic_set(&tctx->inflight_tracked, 0);
8695 task->io_uring = tctx;
8696 spin_lock_init(&tctx->task_lock);
8697 INIT_WQ_LIST(&tctx->task_list);
8698 init_task_work(&tctx->task_work, tctx_task_work);
8704 struct io_uring_task *tctx = tsk->io_uring;
8706 WARN_ON_ONCE(!xa_empty(&tctx->xa));
8707 WARN_ON_ONCE(tctx->io_wq);
8708 WARN_ON_ONCE(tctx->cached_refs);
8710 percpu_counter_destroy(&tctx->inflight);
8711 kfree(tctx);
9477 struct io_uring_task *tctx = current->io_uring;
9484 * tctx can be NULL if the queueing of this task_work raced with
9487 if (tctx && !atomic_read(&tctx->in_idle))
9696 struct io_uring_task *tctx = node->task->io_uring;
9702 if (!tctx || !tctx->io_wq)
9704 cret = io_wq_cancel_cb(tctx->io_wq, io_cancel_ctx_cb, ctx, true);
9717 struct io_uring_task *tctx = task ? task->io_uring : NULL;
9725 } else if (tctx && tctx->io_wq) {
9730 cret = io_wq_cancel_cb(tctx->io_wq, io_cancel_task_cb,
9758 struct io_uring_task *tctx = current->io_uring;
9762 if (unlikely(!tctx)) {
9767 tctx = current->io_uring;
9772 ret = io_wq_max_workers(tctx->io_wq, limits);
9777 if (!xa_load(&tctx->xa, (unsigned long)ctx)) {
9784 ret = xa_err(xa_store(&tctx->xa, (unsigned long)ctx,
9795 tctx->last = ctx;
9804 struct io_uring_task *tctx = current->io_uring;
9806 if (likely(tctx && tctx->last == ctx))
9816 struct io_uring_task *tctx = current->io_uring;
9819 if (!tctx)
9821 node = xa_erase(&tctx->xa, index);
9832 if (tctx->last == node->ctx)
9833 tctx->last = NULL;
9837 static void io_uring_clean_tctx(struct io_uring_task *tctx)
9839 struct io_wq *wq = tctx->io_wq;
9843 xa_for_each(&tctx->xa, index, node) {
9853 tctx->io_wq = NULL;
9857 static s64 tctx_inflight(struct io_uring_task *tctx, bool tracked)
9860 return atomic_read(&tctx->inflight_tracked);
9861 return percpu_counter_sum(&tctx->inflight);
9870 struct io_uring_task *tctx = current->io_uring;
9879 if (tctx->io_wq)
9880 io_wq_exit_start(tctx->io_wq);
9882 atomic_inc(&tctx->in_idle);
9886 inflight = tctx_inflight(tctx, !cancel_all);
9894 xa_for_each(&tctx->xa, index, node) {
9907 prepare_to_wait(&tctx->wait, &wait, TASK_INTERRUPTIBLE);
9916 if (inflight == tctx_inflight(tctx, !cancel_all))
9918 finish_wait(&tctx->wait, &wait);
9921 io_uring_clean_tctx(tctx);
9927 atomic_dec(&tctx->in_idle);
9928 /* for exec all current's requests should be gone, kill tctx */
10746 struct io_uring_task *tctx = current->io_uring;
10750 if (!tctx || !tctx->io_wq)
10777 ret = io_wq_cpu_affinity(tctx->io_wq, new_mask);
10784 struct io_uring_task *tctx = current->io_uring;
10786 if (!tctx || !tctx->io_wq)
10789 return io_wq_cpu_affinity(tctx->io_wq, NULL);
10797 struct io_uring_task *tctx = NULL;
10821 tctx = sqd->thread->io_uring;
10824 tctx = current->io_uring;
10835 if (tctx && tctx->io_wq) {
10836 ret = io_wq_max_workers(tctx->io_wq, new_count);
10857 struct io_uring_task *tctx = node->task->io_uring;
10859 if (WARN_ON_ONCE(!tctx->io_wq))
10865 (void)io_wq_max_workers(tctx->io_wq, new_count);