Lines Matching refs:io_ring_ctx
210 struct io_ring_ctx;
245 typedef void (rsrc_put_fn)(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc);
248 struct io_ring_ctx *ctx;
330 struct io_ring_ctx {
469 const struct io_ring_ctx *last;
872 struct io_ring_ctx *ctx;
900 struct io_ring_ctx *ctx;
1081 static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
1092 static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type,
1096 static struct file *io_file_get(struct io_ring_ctx *ctx,
1103 static void io_submit_flush_completions(struct io_ring_ctx *ctx);
1120 struct io_ring_ctx *ctx = file->private_data;
1129 static inline void io_tw_lock(struct io_ring_ctx *ctx, bool *locked)
1184 struct io_ring_ctx *ctx = req->ctx;
1248 struct io_ring_ctx *ctx = head->ctx;
1273 struct io_ring_ctx *ctx = container_of(ref, struct io_ring_ctx, refs);
1285 struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx,
1304 static struct io_ring_ctx *io_ring_ctx_alloc(struct io_uring_params *p)
1306 struct io_ring_ctx *ctx;
1370 static void io_account_cq_overflow(struct io_ring_ctx *ctx)
1381 struct io_ring_ctx *ctx = req->ctx;
1435 struct io_ring_ctx *ctx = req->ctx;
1461 struct io_ring_ctx *ctx = req->ctx;
1475 struct io_ring_ctx *ctx = req->ctx;
1522 static void io_queue_deferred(struct io_ring_ctx *ctx)
1538 static void io_flush_timeouts(struct io_ring_ctx *ctx)
1569 static void __io_commit_cqring_flush(struct io_ring_ctx *ctx)
1577 static inline bool io_commit_needs_flush(struct io_ring_ctx *ctx)
1582 static inline void __io_commit_cqring(struct io_ring_ctx *ctx)
1588 static inline void io_commit_cqring(struct io_ring_ctx *ctx)
1595 static inline bool io_sqring_full(struct io_ring_ctx *ctx)
1602 static inline unsigned int __io_cqring_events(struct io_ring_ctx *ctx)
1607 static inline struct io_uring_cqe *io_get_cqe(struct io_ring_ctx *ctx)
1624 static inline bool io_should_trigger_evfd(struct io_ring_ctx *ctx)
1640 static void io_cqring_ev_posted(struct io_ring_ctx *ctx)
1659 static void io_cqring_ev_posted_iopoll(struct io_ring_ctx *ctx)
1677 static bool __io_cqring_overflow_flush(struct io_ring_ctx *ctx, bool force)
1719 static bool io_cqring_overflow_flush(struct io_ring_ctx *ctx)
1780 static bool io_cqring_event_overflow(struct io_ring_ctx *ctx, u64 user_data,
1808 static inline bool __io_fill_cqe(struct io_ring_ctx *ctx, u64 user_data,
1835 static noinline bool io_fill_cqe_aux(struct io_ring_ctx *ctx, u64 user_data,
1845 struct io_ring_ctx *ctx = req->ctx;
1929 static void io_preinit_req(struct io_kiocb *req, struct io_ring_ctx *ctx)
1938 static void io_flush_cached_locked_reqs(struct io_ring_ctx *ctx,
1948 static bool io_flush_cached_reqs(struct io_ring_ctx *ctx)
1982 static struct io_kiocb *io_alloc_req(struct io_ring_ctx *ctx)
2040 struct io_ring_ctx *ctx = req->ctx;
2120 struct io_ring_ctx *ctx = req->ctx;
2145 struct io_ring_ctx *ctx = req->ctx;
2168 static void ctx_flush_and_put(struct io_ring_ctx *ctx, bool *locked)
2184 struct io_ring_ctx *ctx = NULL;
2283 struct io_ring_ctx *ctx = req->ctx;
2292 struct io_ring_ctx *ctx = req->ctx;
2353 static void io_req_free_batch_finish(struct io_ring_ctx *ctx,
2383 static void io_submit_flush_completions(struct io_ring_ctx *ctx)
2442 static unsigned io_cqring_events(struct io_ring_ctx *ctx)
2449 static inline unsigned int io_sqring_entries(struct io_ring_ctx *ctx)
2501 static void io_iopoll_complete(struct io_ring_ctx *ctx, unsigned int *nr_events,
2546 static int io_do_iopoll(struct io_ring_ctx *ctx, unsigned int *nr_events,
2596 static void io_iopoll_try_reap_events(struct io_ring_ctx *ctx)
2624 static int io_iopoll_check(struct io_ring_ctx *ctx, long min)
2707 struct io_ring_ctx *ctx = req->ctx;
2795 struct io_ring_ctx *ctx = req->ctx;
2851 struct io_ring_ctx *ctx = req->ctx;
2962 struct io_ring_ctx *ctx = req->ctx;
3017 struct io_ring_ctx *ctx = req->ctx;
3098 struct io_ring_ctx *ctx = req->ctx;
3178 static void io_ring_submit_unlock(struct io_ring_ctx *ctx, bool needs_lock)
3184 static void io_ring_submit_lock(struct io_ring_ctx *ctx, bool needs_lock)
4084 struct io_ring_ctx *ctx = req->ctx;
4095 struct io_ring_ctx *ctx = req->ctx;
4317 static int __io_remove_buffers(struct io_ring_ctx *ctx, struct io_buffer *buf,
4347 struct io_ring_ctx *ctx = req->ctx;
4435 struct io_ring_ctx *ctx = req->ctx;
4686 struct io_ring_ctx *ctx = req->ctx;
5413 struct io_ring_ctx *ctx = req->ctx;
5480 struct io_ring_ctx *ctx = req->ctx;
5552 struct io_ring_ctx *ctx = req->ctx;
5575 struct io_ring_ctx *ctx = req->ctx;
5726 struct io_ring_ctx *ctx = req->ctx;
5806 struct io_ring_ctx *ctx = req->ctx;
5858 static bool io_poll_remove_all(struct io_ring_ctx *ctx, struct task_struct *tsk,
5883 static struct io_kiocb *io_poll_find(struct io_ring_ctx *ctx, __u64 sqe_addr,
5911 static int io_poll_cancel(struct io_ring_ctx *ctx, __u64 sqe_addr,
6007 struct io_ring_ctx *ctx = req->ctx;
6059 struct io_ring_ctx *ctx = req->ctx;
6073 static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx,
6096 static int io_timeout_cancel(struct io_ring_ctx *ctx, __u64 user_data)
6127 static int io_linked_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,
6152 static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data,
6215 struct io_ring_ctx *ctx = req->ctx;
6296 struct io_ring_ctx *ctx = req->ctx;
6345 struct io_ring_ctx *ctx;
6358 struct io_ring_ctx *ctx)
6385 struct io_ring_ctx *ctx = req->ctx;
6423 struct io_ring_ctx *ctx = req->ctx;
6468 struct io_ring_ctx *ctx = req->ctx;
6607 struct io_ring_ctx *ctx = req->ctx;
6747 struct io_ring_ctx *ctx = req->ctx;
6932 static inline struct file *io_file_from_index(struct io_ring_ctx *ctx,
6953 static inline struct file *io_file_get_fixed(struct io_ring_ctx *ctx,
6976 static struct file *io_file_get_normal(struct io_ring_ctx *ctx,
6989 static inline struct file *io_file_get(struct io_ring_ctx *ctx,
7019 struct io_ring_ctx *ctx = req->ctx;
7046 struct io_ring_ctx *ctx = req->ctx;
7081 struct io_ring_ctx *ctx = req->ctx;
7142 static inline bool io_check_restriction(struct io_ring_ctx *ctx,
7163 static int io_init_req(struct io_ring_ctx *ctx, struct io_kiocb *req,
7226 static int io_submit_sqe(struct io_ring_ctx *ctx, struct io_kiocb *req,
7312 struct io_ring_ctx *ctx)
7334 static void io_commit_sqring(struct io_ring_ctx *ctx)
7354 static const struct io_uring_sqe *io_get_sqe(struct io_ring_ctx *ctx)
7378 static int io_submit_sqes(struct io_ring_ctx *ctx, unsigned int nr)
7431 static inline void io_ring_set_wakeup_flag(struct io_ring_ctx *ctx)
7440 static inline void io_ring_clear_wakeup_flag(struct io_ring_ctx *ctx)
7448 static int __io_sq_thread(struct io_ring_ctx *ctx, bool cap_entries)
7489 struct io_ring_ctx *ctx;
7516 struct io_ring_ctx *ctx;
7601 struct io_ring_ctx *ctx;
7608 struct io_ring_ctx *ctx = iowq->ctx;
7655 static inline int io_cqring_wait_schedule(struct io_ring_ctx *ctx,
7688 static int io_cqring_wait(struct io_ring_ctx *ctx, int min_events,
7793 struct io_ring_ctx *ctx = node->rsrc_data->ctx;
7820 static struct io_rsrc_node *io_rsrc_node_alloc(struct io_ring_ctx *ctx)
7839 static void io_rsrc_node_switch(struct io_ring_ctx *ctx,
7864 static int io_rsrc_node_switch_start(struct io_ring_ctx *ctx)
7872 static int io_rsrc_ref_quiesce(struct io_rsrc_data *data, struct io_ring_ctx *ctx)
7936 static int io_rsrc_data_alloc(struct io_ring_ctx *ctx, rsrc_put_fn *do_put,
7989 static void __io_sqe_files_unregister(struct io_ring_ctx *ctx)
8016 static int io_sqe_files_unregister(struct io_ring_ctx *ctx)
8086 static void io_sq_thread_finish(struct io_ring_ctx *ctx)
8103 struct io_ring_ctx *ctx_attach;
8167 static int __io_sqe_files_scm(struct io_ring_ctx *ctx, int nr, int offset)
8227 static int io_sqe_files_scm(struct io_ring_ctx *ctx)
8258 static int io_sqe_files_scm(struct io_ring_ctx *ctx)
8264 static void io_rsrc_file_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc)
8330 struct io_ring_ctx *ctx = rsrc_data->ctx;
8359 struct io_ring_ctx *ctx;
8362 ctx = container_of(work, struct io_ring_ctx, rsrc_put_work.work);
8375 static int io_sqe_files_register(struct io_ring_ctx *ctx, void __user *arg,
8477 struct io_ring_ctx *ctx = req->ctx;
8527 struct io_ring_ctx *ctx = req->ctx;
8562 static int __io_sqe_files_update(struct io_ring_ctx *ctx,
8635 static struct io_wq *io_init_wq_offload(struct io_ring_ctx *ctx,
8668 struct io_ring_ctx *ctx)
8715 static int io_sq_offload_create(struct io_ring_ctx *ctx,
8826 static void io_unaccount_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
8835 static int io_account_mem(struct io_ring_ctx *ctx, unsigned long nr_pages)
8899 static void io_buffer_unmap(struct io_ring_ctx *ctx, struct io_mapped_ubuf **slot)
8914 static void io_rsrc_buf_put(struct io_ring_ctx *ctx, struct io_rsrc_put *prsrc)
8920 static void __io_sqe_buffers_unregister(struct io_ring_ctx *ctx)
8933 static int io_sqe_buffers_unregister(struct io_ring_ctx *ctx)
8953 static int io_copy_iov(struct io_ring_ctx *ctx, struct iovec *dst,
8987 static bool headpage_already_acct(struct io_ring_ctx *ctx, struct page **pages,
9015 static int io_buffer_account_pin(struct io_ring_ctx *ctx, struct page **pages,
9047 static int io_sqe_buffer_register(struct io_ring_ctx *ctx, struct iovec *iov,
9150 static int io_buffers_map_alloc(struct io_ring_ctx *ctx, unsigned int nr_args)
9180 static int io_sqe_buffers_register(struct io_ring_ctx *ctx, void __user *arg,
9232 static int __io_sqe_buffers_update(struct io_ring_ctx *ctx,
9292 static int io_eventfd_register(struct io_ring_ctx *ctx, void __user *arg)
9314 static int io_eventfd_unregister(struct io_ring_ctx *ctx)
9325 static void io_destroy_buffers(struct io_ring_ctx *ctx)
9344 static void io_req_caches_free(struct io_ring_ctx *ctx)
9366 static void io_ring_ctx_free(struct io_ring_ctx *ctx)
9425 struct io_ring_ctx *ctx = file->private_data;
9456 static int io_unregister_personality(struct io_ring_ctx *ctx, unsigned id)
9472 struct io_ring_ctx *ctx;
9501 struct io_ring_ctx *ctx = container_of(work, struct io_ring_ctx, exit_work);
9584 static bool io_kill_timeouts(struct io_ring_ctx *ctx, struct task_struct *tsk,
9607 static void io_ring_ctx_wait_and_kill(struct io_ring_ctx *ctx)
9642 struct io_ring_ctx *ctx = file->private_data;
9662 static bool io_cancel_defer_files(struct io_ring_ctx *ctx,
9688 static bool io_uring_try_cancel_iowq(struct io_ring_ctx *ctx)
9712 static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
9756 static int __io_uring_add_tctx_node(struct io_ring_ctx *ctx)
9802 static inline int io_uring_add_tctx_node(struct io_ring_ctx *ctx)
9871 struct io_ring_ctx *ctx;
9941 struct io_ring_ctx *ctx = file->private_data;
10008 static int io_sqpoll_wait_sq(struct io_ring_ctx *ctx)
10062 struct io_ring_ctx *ctx;
10186 static void __io_uring_show_fdinfo(struct io_ring_ctx *ctx, struct seq_file *m)
10255 struct io_ring_ctx *ctx = f->private_data;
10277 static int io_allocate_scq_urings(struct io_ring_ctx *ctx,
10319 static int io_uring_install_fd(struct io_ring_ctx *ctx, struct file *file)
10342 static struct file *io_uring_get_file(struct io_ring_ctx *ctx)
10370 struct io_ring_ctx *ctx;
10526 static int io_probe(struct io_ring_ctx *ctx, void __user *arg, unsigned nr_args)
10565 static int io_register_personality(struct io_ring_ctx *ctx)
10582 static int io_register_restrictions(struct io_ring_ctx *ctx, void __user *arg,
10652 static int io_register_enable_rings(struct io_ring_ctx *ctx)
10666 static int __io_register_rsrc_update(struct io_ring_ctx *ctx, unsigned type,
10688 static int io_register_files_update(struct io_ring_ctx *ctx, void __user *arg,
10703 static int io_register_rsrc_update(struct io_ring_ctx *ctx, void __user *arg,
10717 static int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg,
10743 static int io_register_iowq_aff(struct io_ring_ctx *ctx, void __user *arg,
10782 static int io_unregister_iowq_aff(struct io_ring_ctx *ctx)
10792 static int io_register_iowq_max_workers(struct io_ring_ctx *ctx,
10900 static int io_ctx_quiesce(struct io_ring_ctx *ctx)
10927 static int __io_uring_register(struct io_ring_ctx *ctx, unsigned opcode,
11070 struct io_ring_ctx *ctx;