xref: /kernel/linux/linux-6.6/io_uring/cancel.c (revision 62306a36) 
1// SPDX-License-Identifier: GPL-2.0
2#include <linux/kernel.h>
3#include <linux/errno.h>
4#include <linux/fs.h>
5#include <linux/file.h>
6#include <linux/mm.h>
7#include <linux/slab.h>
8#include <linux/namei.h>
9#include <linux/nospec.h>
10#include <linux/io_uring.h>
11
12#include <uapi/linux/io_uring.h>
13
14#include "io_uring.h"
15#include "tctx.h"
16#include "poll.h"
17#include "timeout.h"
18#include "cancel.h"
19
20struct io_cancel {
21	struct file			*file;
22	u64				addr;
23	u32				flags;
24	s32				fd;
25	u8				opcode;
26};
27
28#define CANCEL_FLAGS	(IORING_ASYNC_CANCEL_ALL | IORING_ASYNC_CANCEL_FD | \
29			 IORING_ASYNC_CANCEL_ANY | IORING_ASYNC_CANCEL_FD_FIXED | \
30			 IORING_ASYNC_CANCEL_USERDATA | IORING_ASYNC_CANCEL_OP)
31
32/*
33 * Returns true if the request matches the criteria outlined by 'cd'.
34 */
35bool io_cancel_req_match(struct io_kiocb *req, struct io_cancel_data *cd)
36{
37	bool match_user_data = cd->flags & IORING_ASYNC_CANCEL_USERDATA;
38
39	if (req->ctx != cd->ctx)
40		return false;
41
42	if (!(cd->flags & (IORING_ASYNC_CANCEL_FD | IORING_ASYNC_CANCEL_OP)))
43		match_user_data = true;
44
45	if (cd->flags & IORING_ASYNC_CANCEL_ANY)
46		goto check_seq;
47	if (cd->flags & IORING_ASYNC_CANCEL_FD) {
48		if (req->file != cd->file)
49			return false;
50	}
51	if (cd->flags & IORING_ASYNC_CANCEL_OP) {
52		if (req->opcode != cd->opcode)
53			return false;
54	}
55	if (match_user_data && req->cqe.user_data != cd->data)
56		return false;
57	if (cd->flags & IORING_ASYNC_CANCEL_ALL) {
58check_seq:
59		if (cd->seq == req->work.cancel_seq)
60			return false;
61		req->work.cancel_seq = cd->seq;
62	}
63
64	return true;
65}
66
67static bool io_cancel_cb(struct io_wq_work *work, void *data)
68{
69	struct io_kiocb *req = container_of(work, struct io_kiocb, work);
70	struct io_cancel_data *cd = data;
71
72	return io_cancel_req_match(req, cd);
73}
74
75static int io_async_cancel_one(struct io_uring_task *tctx,
76			       struct io_cancel_data *cd)
77{
78	enum io_wq_cancel cancel_ret;
79	int ret = 0;
80	bool all;
81
82	if (!tctx || !tctx->io_wq)
83		return -ENOENT;
84
85	all = cd->flags & (IORING_ASYNC_CANCEL_ALL|IORING_ASYNC_CANCEL_ANY);
86	cancel_ret = io_wq_cancel_cb(tctx->io_wq, io_cancel_cb, cd, all);
87	switch (cancel_ret) {
88	case IO_WQ_CANCEL_OK:
89		ret = 0;
90		break;
91	case IO_WQ_CANCEL_RUNNING:
92		ret = -EALREADY;
93		break;
94	case IO_WQ_CANCEL_NOTFOUND:
95		ret = -ENOENT;
96		break;
97	}
98
99	return ret;
100}
101
102int io_try_cancel(struct io_uring_task *tctx, struct io_cancel_data *cd,
103		  unsigned issue_flags)
104{
105	struct io_ring_ctx *ctx = cd->ctx;
106	int ret;
107
108	WARN_ON_ONCE(!io_wq_current_is_worker() && tctx != current->io_uring);
109
110	ret = io_async_cancel_one(tctx, cd);
111	/*
112	 * Fall-through even for -EALREADY, as we may have poll armed
113	 * that need unarming.
114	 */
115	if (!ret)
116		return 0;
117
118	ret = io_poll_cancel(ctx, cd, issue_flags);
119	if (ret != -ENOENT)
120		return ret;
121
122	spin_lock(&ctx->completion_lock);
123	if (!(cd->flags & IORING_ASYNC_CANCEL_FD))
124		ret = io_timeout_cancel(ctx, cd);
125	spin_unlock(&ctx->completion_lock);
126	return ret;
127}
128
129int io_async_cancel_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
130{
131	struct io_cancel *cancel = io_kiocb_to_cmd(req, struct io_cancel);
132
133	if (unlikely(req->flags & REQ_F_BUFFER_SELECT))
134		return -EINVAL;
135	if (sqe->off || sqe->splice_fd_in)
136		return -EINVAL;
137
138	cancel->addr = READ_ONCE(sqe->addr);
139	cancel->flags = READ_ONCE(sqe->cancel_flags);
140	if (cancel->flags & ~CANCEL_FLAGS)
141		return -EINVAL;
142	if (cancel->flags & IORING_ASYNC_CANCEL_FD) {
143		if (cancel->flags & IORING_ASYNC_CANCEL_ANY)
144			return -EINVAL;
145		cancel->fd = READ_ONCE(sqe->fd);
146	}
147	if (cancel->flags & IORING_ASYNC_CANCEL_OP) {
148		if (cancel->flags & IORING_ASYNC_CANCEL_ANY)
149			return -EINVAL;
150		cancel->opcode = READ_ONCE(sqe->len);
151	}
152
153	return 0;
154}
155
156static int __io_async_cancel(struct io_cancel_data *cd,
157			     struct io_uring_task *tctx,
158			     unsigned int issue_flags)
159{
160	bool all = cd->flags & (IORING_ASYNC_CANCEL_ALL|IORING_ASYNC_CANCEL_ANY);
161	struct io_ring_ctx *ctx = cd->ctx;
162	struct io_tctx_node *node;
163	int ret, nr = 0;
164
165	do {
166		ret = io_try_cancel(tctx, cd, issue_flags);
167		if (ret == -ENOENT)
168			break;
169		if (!all)
170			return ret;
171		nr++;
172	} while (1);
173
174	/* slow path, try all io-wq's */
175	io_ring_submit_lock(ctx, issue_flags);
176	ret = -ENOENT;
177	list_for_each_entry(node, &ctx->tctx_list, ctx_node) {
178		struct io_uring_task *tctx = node->task->io_uring;
179
180		ret = io_async_cancel_one(tctx, cd);
181		if (ret != -ENOENT) {
182			if (!all)
183				break;
184			nr++;
185		}
186	}
187	io_ring_submit_unlock(ctx, issue_flags);
188	return all ? nr : ret;
189}
190
191int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags)
192{
193	struct io_cancel *cancel = io_kiocb_to_cmd(req, struct io_cancel);
194	struct io_cancel_data cd = {
195		.ctx	= req->ctx,
196		.data	= cancel->addr,
197		.flags	= cancel->flags,
198		.opcode	= cancel->opcode,
199		.seq	= atomic_inc_return(&req->ctx->cancel_seq),
200	};
201	struct io_uring_task *tctx = req->task->io_uring;
202	int ret;
203
204	if (cd.flags & IORING_ASYNC_CANCEL_FD) {
205		if (req->flags & REQ_F_FIXED_FILE ||
206		    cd.flags & IORING_ASYNC_CANCEL_FD_FIXED) {
207			req->flags |= REQ_F_FIXED_FILE;
208			req->file = io_file_get_fixed(req, cancel->fd,
209							issue_flags);
210		} else {
211			req->file = io_file_get_normal(req, cancel->fd);
212		}
213		if (!req->file) {
214			ret = -EBADF;
215			goto done;
216		}
217		cd.file = req->file;
218	}
219
220	ret = __io_async_cancel(&cd, tctx, issue_flags);
221done:
222	if (ret < 0)
223		req_set_fail(req);
224	io_req_set_res(req, ret, 0);
225	return IOU_OK;
226}
227
228void init_hash_table(struct io_hash_table *table, unsigned size)
229{
230	unsigned int i;
231
232	for (i = 0; i < size; i++) {
233		spin_lock_init(&table->hbs[i].lock);
234		INIT_HLIST_HEAD(&table->hbs[i].list);
235	}
236}
237
238static int __io_sync_cancel(struct io_uring_task *tctx,
239			    struct io_cancel_data *cd, int fd)
240{
241	struct io_ring_ctx *ctx = cd->ctx;
242
243	/* fixed must be grabbed every time since we drop the uring_lock */
244	if ((cd->flags & IORING_ASYNC_CANCEL_FD) &&
245	    (cd->flags & IORING_ASYNC_CANCEL_FD_FIXED)) {
246		if (unlikely(fd >= ctx->nr_user_files))
247			return -EBADF;
248		fd = array_index_nospec(fd, ctx->nr_user_files);
249		cd->file = io_file_from_index(&ctx->file_table, fd);
250		if (!cd->file)
251			return -EBADF;
252	}
253
254	return __io_async_cancel(cd, tctx, 0);
255}
256
257int io_sync_cancel(struct io_ring_ctx *ctx, void __user *arg)
258	__must_hold(&ctx->uring_lock)
259{
260	struct io_cancel_data cd = {
261		.ctx	= ctx,
262		.seq	= atomic_inc_return(&ctx->cancel_seq),
263	};
264	ktime_t timeout = KTIME_MAX;
265	struct io_uring_sync_cancel_reg sc;
266	struct file *file = NULL;
267	DEFINE_WAIT(wait);
268	int ret, i;
269
270	if (copy_from_user(&sc, arg, sizeof(sc)))
271		return -EFAULT;
272	if (sc.flags & ~CANCEL_FLAGS)
273		return -EINVAL;
274	for (i = 0; i < ARRAY_SIZE(sc.pad); i++)
275		if (sc.pad[i])
276			return -EINVAL;
277	for (i = 0; i < ARRAY_SIZE(sc.pad2); i++)
278		if (sc.pad2[i])
279			return -EINVAL;
280
281	cd.data = sc.addr;
282	cd.flags = sc.flags;
283	cd.opcode = sc.opcode;
284
285	/* we can grab a normal file descriptor upfront */
286	if ((cd.flags & IORING_ASYNC_CANCEL_FD) &&
287	   !(cd.flags & IORING_ASYNC_CANCEL_FD_FIXED)) {
288		file = fget(sc.fd);
289		if (!file)
290			return -EBADF;
291		cd.file = file;
292	}
293
294	ret = __io_sync_cancel(current->io_uring, &cd, sc.fd);
295
296	/* found something, done! */
297	if (ret != -EALREADY)
298		goto out;
299
300	if (sc.timeout.tv_sec != -1UL || sc.timeout.tv_nsec != -1UL) {
301		struct timespec64 ts = {
302			.tv_sec		= sc.timeout.tv_sec,
303			.tv_nsec	= sc.timeout.tv_nsec
304		};
305
306		timeout = ktime_add_ns(timespec64_to_ktime(ts), ktime_get_ns());
307	}
308
309	/*
310	 * Keep looking until we get -ENOENT. we'll get woken everytime
311	 * every time a request completes and will retry the cancelation.
312	 */
313	do {
314		cd.seq = atomic_inc_return(&ctx->cancel_seq);
315
316		prepare_to_wait(&ctx->cq_wait, &wait, TASK_INTERRUPTIBLE);
317
318		ret = __io_sync_cancel(current->io_uring, &cd, sc.fd);
319
320		mutex_unlock(&ctx->uring_lock);
321		if (ret != -EALREADY)
322			break;
323
324		ret = io_run_task_work_sig(ctx);
325		if (ret < 0)
326			break;
327		ret = schedule_hrtimeout(&timeout, HRTIMER_MODE_ABS);
328		if (!ret) {
329			ret = -ETIME;
330			break;
331		}
332		mutex_lock(&ctx->uring_lock);
333	} while (1);
334
335	finish_wait(&ctx->cq_wait, &wait);
336	mutex_lock(&ctx->uring_lock);
337
338	if (ret == -ENOENT || ret > 0)
339		ret = 0;
340out:
341	if (file)
342		fput(file);
343	return ret;
344}
345