1// SPDX-License-Identifier: GPL-2.0-or-later 2/* 3 * Copyright (c) 2020 ARM. All rights reserved. 4 * Copyright (c) 2020 Petr Vorel <pvorel@suse.cz> 5 * 6 * Mostly copied/adapted from <linux/io_uring.h> 7 */ 8 9#ifndef LAPI_IO_URING_H__ 10#define LAPI_IO_URING_H__ 11 12#include <unistd.h> 13#include <fcntl.h> 14#include <sys/types.h> 15#include <sys/uio.h> 16#include <stdlib.h> 17#include <linux/fs.h> 18 19#include "lapi/syscalls.h" 20 21#ifdef HAVE_LINUX_IO_URING_H 22#include <linux/io_uring.h> 23#endif 24 25#ifndef IOSQE_FIXED_FILE 26 27#ifndef __kernel_rwf_t 28typedef int __kernel_rwf_t; 29#endif 30 31/* 32 * IO submission data structure (Submission Queue Entry) 33 */ 34struct io_uring_sqe { 35 uint8_t opcode; /* type of operation for this sqe */ 36 uint8_t flags; /* IOSQE_ flags */ 37 uint16_t ioprio; /* ioprio for the request */ 38 int32_t fd; /* file descriptor to do IO on */ 39 union { 40 uint64_t off; /* offset into file */ 41 uint64_t addr2; 42 }; 43 uint64_t addr; /* pointer to buffer or iovecs */ 44 uint32_t len; /* buffer size or number of iovecs */ 45 union { 46 __kernel_rwf_t rw_flags; 47 uint32_t fsync_flags; 48 uint16_t poll_events; 49 uint32_t sync_range_flags; 50 uint32_t msg_flags; 51 uint32_t timeout_flags; 52 uint32_t accept_flags; 53 uint32_t cancel_flags; 54 uint32_t open_flags; 55 uint32_t statx_flags; 56 uint32_t fadvise_advice; 57 }; 58 uint64_t user_data; /* data to be passed back at completion time */ 59 union { 60 struct { 61 /* index into fixed buffers, if used */ 62 uint16_t buf_index; 63 /* personality to use, if used */ 64 uint16_t personality; 65 }; 66 uint64_t __pad2[3]; 67 }; 68}; 69 70enum { 71 IOSQE_FIXED_FILE_BIT, 72 IOSQE_IO_DRAIN_BIT, 73 IOSQE_IO_LINK_BIT, 74}; 75 76/* 77 * sqe->flags 78 */ 79/* use fixed fileset */ 80#define IOSQE_FIXED_FILE (1U << IOSQE_FIXED_FILE_BIT) 81/* issue after inflight IO */ 82#define IOSQE_IO_DRAIN (1U << IOSQE_IO_DRAIN_BIT) 83/* links next sqe */ 84#define IOSQE_IO_LINK (1U << IOSQE_IO_LINK_BIT) 85 86/* 87 * io_uring_setup() flags 88 */ 89#define IORING_SETUP_IOPOLL (1U << 0) /* io_context is polled */ 90#define IORING_SETUP_SQPOLL (1U << 1) /* SQ poll thread */ 91#define IORING_SETUP_SQ_AFF (1U << 2) /* sq_thread_cpu is valid */ 92#define IORING_SETUP_CQSIZE (1U << 3) /* app defines CQ size */ 93#define IORING_SETUP_CLAMP (1U << 4) /* clamp SQ/CQ ring sizes */ 94#define IORING_SETUP_ATTACH_WQ (1U << 5) /* attach to existing wq */ 95 96enum { 97 IORING_OP_NOP, 98 IORING_OP_READV, 99 IORING_OP_WRITEV, 100 IORING_OP_FSYNC, 101 IORING_OP_READ_FIXED, 102 IORING_OP_WRITE_FIXED, 103 IORING_OP_POLL_ADD, 104 IORING_OP_POLL_REMOVE, 105 IORING_OP_SYNC_FILE_RANGE, 106 IORING_OP_SENDMSG, 107 IORING_OP_RECVMSG, 108 IORING_OP_TIMEOUT, 109 IORING_OP_TIMEOUT_REMOVE, 110 IORING_OP_ACCEPT, 111 IORING_OP_ASYNC_CANCEL, 112 IORING_OP_LINK_TIMEOUT, 113 IORING_OP_CONNECT, 114 IORING_OP_FALLOCATE, 115 IORING_OP_OPENAT, 116 IORING_OP_CLOSE, 117 IORING_OP_FILES_UPDATE, 118 IORING_OP_STATX, 119 IORING_OP_READ, 120 IORING_OP_WRITE, 121 IORING_OP_FADVISE, 122 IORING_OP_MADVISE, 123 IORING_OP_SEND, 124 IORING_OP_RECV, 125 IORING_OP_OPENAT2, 126 IORING_OP_EPOLL_CTL, 127 128 /* this goes last, obviously */ 129 IORING_OP_LAST, 130}; 131 132/* 133 * sqe->fsync_flags 134 */ 135#define IORING_FSYNC_DATASYNC (1U << 0) 136 137/* 138 * sqe->timeout_flags 139 */ 140#define IORING_TIMEOUT_ABS (1U << 0) 141 142/* 143 * IO completion data structure (Completion Queue Entry) 144 */ 145struct io_uring_cqe { 146 uint64_t user_data; /* sqe->data submission passed back */ 147 int32_t res; /* result code for this event */ 148 uint32_t flags; 149}; 150 151/* 152 * Magic offsets for the application to mmap the data it needs 153 */ 154#define IORING_OFF_SQ_RING 0ULL 155#define IORING_OFF_CQ_RING 0x8000000ULL 156#define IORING_OFF_SQES 0x10000000ULL 157 158/* 159 * Filled with the offset for mmap(2) 160 */ 161struct io_sqring_offsets { 162 uint32_t head; 163 uint32_t tail; 164 uint32_t ring_mask; 165 uint32_t ring_entries; 166 uint32_t flags; 167 uint32_t dropped; 168 uint32_t array; 169 uint32_t resv1; 170 uint64_t resv2; 171}; 172 173/* 174 * sq_ring->flags 175 */ 176#define IORING_SQ_NEED_WAKEUP (1U << 0) /* needs io_uring_enter wakeup */ 177 178struct io_cqring_offsets { 179 uint32_t head; 180 uint32_t tail; 181 uint32_t ring_mask; 182 uint32_t ring_entries; 183 uint32_t overflow; 184 uint32_t cqes; 185 uint64_t resv[2]; 186}; 187 188/* 189 * io_uring_enter(2) flags 190 */ 191#define IORING_ENTER_GETEVENTS (1U << 0) 192#define IORING_ENTER_SQ_WAKEUP (1U << 1) 193 194/* 195 * Passed in for io_uring_setup(2). Copied back with updated info on success 196 */ 197struct io_uring_params { 198 uint32_t sq_entries; 199 uint32_t cq_entries; 200 uint32_t flags; 201 uint32_t sq_thread_cpu; 202 uint32_t sq_thread_idle; 203 uint32_t features; 204 uint32_t wq_fd; 205 uint32_t resv[3]; 206 struct io_sqring_offsets sq_off; 207 struct io_cqring_offsets cq_off; 208}; 209 210/* 211 * io_uring_params->features flags 212 */ 213#define IORING_FEAT_SINGLE_MMAP (1U << 0) 214#define IORING_FEAT_NODROP (1U << 1) 215#define IORING_FEAT_SUBMIT_STABLE (1U << 2) 216#define IORING_FEAT_RW_CUR_POS (1U << 3) 217#define IORING_FEAT_CUR_PERSONALITY (1U << 4) 218 219/* 220 * io_uring_register(2) opcodes and arguments 221 */ 222#define IORING_REGISTER_BUFFERS 0 223#define IORING_UNREGISTER_BUFFERS 1 224#define IORING_REGISTER_FILES 2 225#define IORING_UNREGISTER_FILES 3 226#define IORING_REGISTER_EVENTFD 4 227#define IORING_UNREGISTER_EVENTFD 5 228#define IORING_REGISTER_FILES_UPDATE 6 229#define IORING_REGISTER_EVENTFD_ASYNC 7 230#define IORING_REGISTER_PROBE 8 231#define IORING_REGISTER_PERSONALITY 9 232#define IORING_UNREGISTER_PERSONALITY 10 233 234struct io_uring_files_update { 235 uint32_t offset; 236 uint32_t resv; 237 uint64_t __attribute__((aligned(8))) fds; 238}; 239 240#define IO_URING_OP_SUPPORTED (1U << 0) 241 242struct io_uring_probe_op { 243 uint8_t op; 244 uint8_t resv; 245 uint16_t flags; /* IO_URING_OP_* flags */ 246 uint32_t resv2; 247}; 248 249struct io_uring_probe { 250 uint8_t last_op; /* last opcode supported */ 251 uint8_t ops_len; /* length of ops[] array below */ 252 uint16_t resv; 253 uint32_t resv2[3]; 254 struct io_uring_probe_op ops[0]; 255}; 256 257#endif /* IOSQE_FIXED_FILE */ 258 259#ifndef IOSQE_IO_HADRLINK 260/* like LINK, but stronger */ 261#define IOSQE_IO_HARDLINK_BIT 3 262#define IOSQE_IO_HARDLINK (1U << IOSQE_IO_HARDLINK_BIT) 263#endif /* IOSQE_IO_HADRLINK */ 264 265#ifndef IOSQE_ASYNC 266/* always go async */ 267#define IOSQE_ASYNC_BIT 4 268#define IOSQE_ASYNC (1U << IOSQE_ASYNC_BIT) 269#endif /* IOSQE_ASYNC */ 270 271#ifndef HAVE_IO_URING_REGISTER 272static inline int io_uring_register(int fd, unsigned int opcode, void *arg, 273 unsigned int nr_args) 274{ 275 return tst_syscall(__NR_io_uring_register, fd, opcode, arg, nr_args); 276} 277#endif /* HAVE_IO_URING_REGISTER */ 278 279 280#ifndef HAVE_IO_URING_SETUP 281static inline int io_uring_setup(unsigned int entries, 282 struct io_uring_params *p) 283{ 284 return tst_syscall(__NR_io_uring_setup, entries, p); 285} 286#endif /* HAVE_IO_URING_SETUP */ 287 288#ifndef HAVE_IO_URING_ENTER 289static inline int io_uring_enter(int fd, unsigned int to_submit, 290 unsigned int min_complete, unsigned int flags, sigset_t *sig) 291{ 292 return tst_syscall(__NR_io_uring_enter, fd, to_submit, min_complete, 293 flags, sig, _NSIG / 8); 294} 295#endif /* HAVE_IO_URING_ENTER */ 296 297static inline void io_uring_setup_supported_by_kernel(void) 298{ 299 long ret; 300 ret = syscall(__NR_io_uring_setup, NULL, 0); 301 if (ret != -1) { 302 SAFE_CLOSE(ret); 303 return; 304 } 305 306 if (errno == ENOSYS) { 307 if ((tst_kvercmp(5, 1, 0)) < 0) { 308 tst_brk(TCONF, 309 "Test not supported on kernel version < v5.1"); 310 } 311 tst_brk(TCONF, "CONFIG_IO_URING not set?"); 312 } 313} 314 315#endif /* LAPI_IO_URING_H__ */ 316