1 //! linux_raw syscalls supporting `rustix::io`.
2 //!
3 //! # Safety
4 //!
5 //! See the `rustix::backend` module documentation for details.
6 #![allow(unsafe_code)]
7 #![allow(clippy::undocumented_unsafe_blocks)]
8
9 use super::super::c;
10 #[cfg(target_pointer_width = "64")]
11 use super::super::conv::loff_t_from_u64;
12 use super::super::conv::{
13 by_ref, c_int, c_uint, opt_mut, pass_usize, raw_fd, ret, ret_c_uint, ret_discarded_fd,
14 ret_owned_fd, ret_usize, slice, slice_mut, zero,
15 };
16 #[cfg(target_pointer_width = "32")]
17 use super::super::conv::{hi, lo};
18 use crate::fd::{AsFd, BorrowedFd, OwnedFd, RawFd};
19 #[cfg(any(target_os = "android", target_os = "linux"))]
20 use crate::io::SpliceFlags;
21 use crate::io::{
22 self, epoll, DupFlags, EventfdFlags, FdFlags, IoSlice, IoSliceMut, IoSliceRaw, PipeFlags,
23 PollFd, ReadWriteFlags,
24 };
25 #[cfg(all(feature = "fs", feature = "net"))]
26 use crate::net::{RecvFlags, SendFlags};
27 use core::cmp;
28 use core::mem::MaybeUninit;
29 #[cfg(target_os = "espidf")]
30 use linux_raw_sys::general::F_DUPFD;
31 use linux_raw_sys::general::{
32 epoll_event, EPOLL_CTL_ADD, EPOLL_CTL_DEL, EPOLL_CTL_MOD, F_DUPFD_CLOEXEC, F_GETFD, F_SETFD,
33 UIO_MAXIOV,
34 };
35 use linux_raw_sys::ioctl::{BLKPBSZGET, BLKSSZGET, FIONBIO, FIONREAD, TIOCEXCL, TIOCNXCL};
36 #[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))]
37 use {
38 super::super::conv::{opt_ref, size_of},
39 linux_raw_sys::general::{__kernel_timespec, sigset_t},
40 };
41
42 #[inline]
43 pub(crate) fn read(fd: BorrowedFd<'_>, buf: &mut [u8]) -> io::Result<usize> {
44 let (buf_addr_mut, buf_len) = slice_mut(buf);
45
46 unsafe { ret_usize(syscall!(__NR_read, fd, buf_addr_mut, buf_len)) }
47 }
48
49 #[inline]
50 pub(crate) fn pread(fd: BorrowedFd<'_>, buf: &mut [u8], pos: u64) -> io::Result<usize> {
51 let (buf_addr_mut, buf_len) = slice_mut(buf);
52
53 // <https://github.com/torvalds/linux/blob/fcadab740480e0e0e9fa9bd272acd409884d431a/arch/arm64/kernel/sys32.c#L75>
54 #[cfg(all(
55 target_pointer_width = "32",
56 any(target_arch = "arm", target_arch = "mips", target_arch = "power"),
57 ))]
58 unsafe {
59 ret_usize(syscall!(
60 __NR_pread64,
61 fd,
62 buf_addr_mut,
63 buf_len,
64 zero(),
65 hi(pos),
66 lo(pos)
67 ))
68 }
69 #[cfg(all(
70 target_pointer_width = "32",
71 not(any(target_arch = "arm", target_arch = "mips", target_arch = "power")),
72 ))]
73 unsafe {
74 ret_usize(syscall!(
75 __NR_pread64,
76 fd,
77 buf_addr_mut,
78 buf_len,
79 hi(pos),
80 lo(pos)
81 ))
82 }
83 #[cfg(target_pointer_width = "64")]
84 unsafe {
85 ret_usize(syscall!(
86 __NR_pread64,
87 fd,
88 buf_addr_mut,
89 buf_len,
90 loff_t_from_u64(pos)
91 ))
92 }
93 }
94
95 #[inline]
96 pub(crate) fn readv(fd: BorrowedFd<'_>, bufs: &mut [IoSliceMut<'_>]) -> io::Result<usize> {
97 let (bufs_addr, bufs_len) = slice(&bufs[..cmp::min(bufs.len(), max_iov())]);
98
99 unsafe { ret_usize(syscall!(__NR_readv, fd, bufs_addr, bufs_len)) }
100 }
101
102 #[inline]
103 pub(crate) fn preadv(
104 fd: BorrowedFd<'_>,
105 bufs: &mut [IoSliceMut<'_>],
106 pos: u64,
107 ) -> io::Result<usize> {
108 let (bufs_addr, bufs_len) = slice(&bufs[..cmp::min(bufs.len(), max_iov())]);
109
110 #[cfg(target_pointer_width = "32")]
111 unsafe {
112 ret_usize(syscall!(
113 __NR_preadv,
114 fd,
115 bufs_addr,
116 bufs_len,
117 hi(pos),
118 lo(pos)
119 ))
120 }
121 #[cfg(target_pointer_width = "64")]
122 unsafe {
123 ret_usize(syscall!(
124 __NR_preadv,
125 fd,
126 bufs_addr,
127 bufs_len,
128 loff_t_from_u64(pos)
129 ))
130 }
131 }
132
133 #[inline]
134 pub(crate) fn preadv2(
135 fd: BorrowedFd<'_>,
136 bufs: &mut [IoSliceMut<'_>],
137 pos: u64,
138 flags: ReadWriteFlags,
139 ) -> io::Result<usize> {
140 let (bufs_addr, bufs_len) = slice(&bufs[..cmp::min(bufs.len(), max_iov())]);
141
142 #[cfg(target_pointer_width = "32")]
143 unsafe {
144 ret_usize(syscall!(
145 __NR_preadv2,
146 fd,
147 bufs_addr,
148 bufs_len,
149 hi(pos),
150 lo(pos),
151 flags
152 ))
153 }
154 #[cfg(target_pointer_width = "64")]
155 unsafe {
156 ret_usize(syscall!(
157 __NR_preadv2,
158 fd,
159 bufs_addr,
160 bufs_len,
161 loff_t_from_u64(pos),
162 flags
163 ))
164 }
165 }
166
167 #[inline]
168 pub(crate) fn write(fd: BorrowedFd<'_>, buf: &[u8]) -> io::Result<usize> {
169 let (buf_addr, buf_len) = slice(buf);
170
171 unsafe { ret_usize(syscall_readonly!(__NR_write, fd, buf_addr, buf_len)) }
172 }
173
174 #[inline]
175 pub(crate) fn pwrite(fd: BorrowedFd<'_>, buf: &[u8], pos: u64) -> io::Result<usize> {
176 let (buf_addr, buf_len) = slice(buf);
177
178 // <https://github.com/torvalds/linux/blob/fcadab740480e0e0e9fa9bd272acd409884d431a/arch/arm64/kernel/sys32.c#L81-L83>
179 #[cfg(all(
180 target_pointer_width = "32",
181 any(target_arch = "arm", target_arch = "mips", target_arch = "power"),
182 ))]
183 unsafe {
184 ret_usize(syscall_readonly!(
185 __NR_pwrite64,
186 fd,
187 buf_addr,
188 buf_len,
189 zero(),
190 hi(pos),
191 lo(pos)
192 ))
193 }
194 #[cfg(all(
195 target_pointer_width = "32",
196 not(any(target_arch = "arm", target_arch = "mips", target_arch = "power")),
197 ))]
198 unsafe {
199 ret_usize(syscall_readonly!(
200 __NR_pwrite64,
201 fd,
202 buf_addr,
203 buf_len,
204 hi(pos),
205 lo(pos)
206 ))
207 }
208 #[cfg(target_pointer_width = "64")]
209 unsafe {
210 ret_usize(syscall_readonly!(
211 __NR_pwrite64,
212 fd,
213 buf_addr,
214 buf_len,
215 loff_t_from_u64(pos)
216 ))
217 }
218 }
219
220 #[inline]
221 pub(crate) fn writev(fd: BorrowedFd<'_>, bufs: &[IoSlice<'_>]) -> io::Result<usize> {
222 let (bufs_addr, bufs_len) = slice(&bufs[..cmp::min(bufs.len(), max_iov())]);
223
224 unsafe { ret_usize(syscall_readonly!(__NR_writev, fd, bufs_addr, bufs_len)) }
225 }
226
227 #[inline]
228 pub(crate) fn pwritev(fd: BorrowedFd<'_>, bufs: &[IoSlice<'_>], pos: u64) -> io::Result<usize> {
229 let (bufs_addr, bufs_len) = slice(&bufs[..cmp::min(bufs.len(), max_iov())]);
230
231 #[cfg(target_pointer_width = "32")]
232 unsafe {
233 ret_usize(syscall_readonly!(
234 __NR_pwritev,
235 fd,
236 bufs_addr,
237 bufs_len,
238 hi(pos),
239 lo(pos)
240 ))
241 }
242 #[cfg(target_pointer_width = "64")]
243 unsafe {
244 ret_usize(syscall_readonly!(
245 __NR_pwritev,
246 fd,
247 bufs_addr,
248 bufs_len,
249 loff_t_from_u64(pos)
250 ))
251 }
252 }
253
254 #[inline]
255 pub(crate) fn pwritev2(
256 fd: BorrowedFd<'_>,
257 bufs: &[IoSlice<'_>],
258 pos: u64,
259 flags: ReadWriteFlags,
260 ) -> io::Result<usize> {
261 let (bufs_addr, bufs_len) = slice(&bufs[..cmp::min(bufs.len(), max_iov())]);
262
263 #[cfg(target_pointer_width = "32")]
264 unsafe {
265 ret_usize(syscall_readonly!(
266 __NR_pwritev2,
267 fd,
268 bufs_addr,
269 bufs_len,
270 hi(pos),
271 lo(pos),
272 flags
273 ))
274 }
275 #[cfg(target_pointer_width = "64")]
276 unsafe {
277 ret_usize(syscall_readonly!(
278 __NR_pwritev2,
279 fd,
280 bufs_addr,
281 bufs_len,
282 loff_t_from_u64(pos),
283 flags
284 ))
285 }
286 }
287
288 /// The maximum number of buffers that can be passed into a vectored I/O system
289 /// call on the current platform.
290 const fn max_iov() -> usize {
291 UIO_MAXIOV as usize
292 }
293
294 #[inline]
295 pub(crate) unsafe fn close(fd: RawFd) {
296 // See the documentation for [`io::close`] for why errors are ignored.
297 syscall_readonly!(__NR_close, raw_fd(fd)).decode_void();
298 }
299
300 #[inline]
301 pub(crate) fn eventfd(initval: u32, flags: EventfdFlags) -> io::Result<OwnedFd> {
302 unsafe { ret_owned_fd(syscall_readonly!(__NR_eventfd2, c_uint(initval), flags)) }
303 }
304
305 #[inline]
306 pub(crate) fn ioctl_fionread(fd: BorrowedFd<'_>) -> io::Result<u64> {
307 unsafe {
308 let mut result = MaybeUninit::<c::c_int>::uninit();
309 ret(syscall!(__NR_ioctl, fd, c_uint(FIONREAD), &mut result))?;
310 Ok(result.assume_init() as u64)
311 }
312 }
313
314 #[inline]
315 pub(crate) fn ioctl_fionbio(fd: BorrowedFd<'_>, value: bool) -> io::Result<()> {
316 unsafe {
317 let data = c::c_int::from(value);
318 ret(syscall_readonly!(
319 __NR_ioctl,
320 fd,
321 c_uint(FIONBIO),
322 by_ref(&data)
323 ))
324 }
325 }
326
327 #[inline]
328 pub(crate) fn ioctl_tiocexcl(fd: BorrowedFd<'_>) -> io::Result<()> {
329 unsafe { ret(syscall_readonly!(__NR_ioctl, fd, c_uint(TIOCEXCL))) }
330 }
331
332 #[inline]
333 pub(crate) fn ioctl_tiocnxcl(fd: BorrowedFd<'_>) -> io::Result<()> {
334 unsafe { ret(syscall_readonly!(__NR_ioctl, fd, c_uint(TIOCNXCL))) }
335 }
336
337 #[inline]
338 pub(crate) fn ioctl_blksszget(fd: BorrowedFd) -> io::Result<u32> {
339 let mut result = MaybeUninit::<c::c_uint>::uninit();
340 unsafe {
341 ret(syscall!(__NR_ioctl, fd, c_uint(BLKSSZGET), &mut result))?;
342 Ok(result.assume_init() as u32)
343 }
344 }
345
346 #[inline]
347 pub(crate) fn ioctl_blkpbszget(fd: BorrowedFd) -> io::Result<u32> {
348 let mut result = MaybeUninit::<c::c_uint>::uninit();
349 unsafe {
350 ret(syscall!(__NR_ioctl, fd, c_uint(BLKPBSZGET), &mut result))?;
351 Ok(result.assume_init() as u32)
352 }
353 }
354
355 #[cfg(all(feature = "fs", feature = "net"))]
356 pub(crate) fn is_read_write(fd: BorrowedFd<'_>) -> io::Result<(bool, bool)> {
357 let (mut read, mut write) = crate::fs::fd::_is_file_read_write(fd)?;
358 let mut not_socket = false;
359 if read {
360 // Do a `recv` with `PEEK` and `DONTWAIT` for 1 byte. A 0 indicates
361 // the read side is shut down; an `EWOULDBLOCK` indicates the read
362 // side is still open.
363 //
364 // TODO: This code would benefit from having a better way to read into
365 // uninitialized memory.
366 let mut buf = [0];
367 match super::super::net::syscalls::recv(fd, &mut buf, RecvFlags::PEEK | RecvFlags::DONTWAIT)
368 {
369 Ok(0) => read = false,
370 Err(err) => {
371 #[allow(unreachable_patterns)] // `EAGAIN` may equal `EWOULDBLOCK`
372 match err {
373 io::Errno::AGAIN | io::Errno::WOULDBLOCK => (),
374 io::Errno::NOTSOCK => not_socket = true,
375 _ => return Err(err),
376 }
377 }
378 Ok(_) => (),
379 }
380 }
381 if write && !not_socket {
382 // Do a `send` with `DONTWAIT` for 0 bytes. An `EPIPE` indicates
383 // the write side is shut down.
384 #[allow(unreachable_patterns)] // `EAGAIN` equals `EWOULDBLOCK`
385 match super::super::net::syscalls::send(fd, &[], SendFlags::DONTWAIT) {
386 // TODO or-patterns when we don't need 1.51
387 Err(io::Errno::AGAIN) => (),
388 Err(io::Errno::WOULDBLOCK) => (),
389 Err(io::Errno::NOTSOCK) => (),
390 Err(io::Errno::PIPE) => write = false,
391 Err(err) => return Err(err),
392 Ok(_) => (),
393 }
394 }
395 Ok((read, write))
396 }
397
398 #[inline]
399 pub(crate) fn dup(fd: BorrowedFd<'_>) -> io::Result<OwnedFd> {
400 unsafe { ret_owned_fd(syscall_readonly!(__NR_dup, fd)) }
401 }
402
403 #[inline]
404 pub(crate) fn dup2(fd: BorrowedFd<'_>, new: &mut OwnedFd) -> io::Result<()> {
405 #[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))]
406 {
407 // We don't need to worry about the difference between `dup2` and
408 // `dup3` when the file descriptors are equal because we have an
409 // `&mut OwnedFd` which means `fd` doesn't alias it.
410 dup3(fd, new, DupFlags::empty())
411 }
412
413 #[cfg(not(any(target_arch = "aarch64", target_arch = "riscv64")))]
414 unsafe {
415 ret_discarded_fd(syscall_readonly!(__NR_dup2, fd, new.as_fd()))
416 }
417 }
418
419 #[inline]
420 pub(crate) fn dup3(fd: BorrowedFd<'_>, new: &mut OwnedFd, flags: DupFlags) -> io::Result<()> {
421 unsafe { ret_discarded_fd(syscall_readonly!(__NR_dup3, fd, new.as_fd(), flags)) }
422 }
423
424 #[inline]
425 pub(crate) fn fcntl_getfd(fd: BorrowedFd<'_>) -> io::Result<FdFlags> {
426 #[cfg(target_pointer_width = "32")]
427 unsafe {
428 ret_c_uint(syscall_readonly!(__NR_fcntl64, fd, c_uint(F_GETFD)))
429 .map(FdFlags::from_bits_truncate)
430 }
431 #[cfg(target_pointer_width = "64")]
432 unsafe {
433 ret_c_uint(syscall_readonly!(__NR_fcntl, fd, c_uint(F_GETFD)))
434 .map(FdFlags::from_bits_truncate)
435 }
436 }
437
438 #[inline]
439 pub(crate) fn fcntl_setfd(fd: BorrowedFd<'_>, flags: FdFlags) -> io::Result<()> {
440 #[cfg(target_pointer_width = "32")]
441 unsafe {
442 ret(syscall_readonly!(__NR_fcntl64, fd, c_uint(F_SETFD), flags))
443 }
444 #[cfg(target_pointer_width = "64")]
445 unsafe {
446 ret(syscall_readonly!(__NR_fcntl, fd, c_uint(F_SETFD), flags))
447 }
448 }
449
450 #[cfg(target_os = "espidf")]
451 #[inline]
452 pub(crate) fn fcntl_dupfd(fd: BorrowedFd<'_>, min: RawFd) -> io::Result<OwnedFd> {
453 #[cfg(target_pointer_width = "32")]
454 unsafe {
455 ret_owned_fd(syscall_readonly!(
456 __NR_fcntl64,
457 fd,
458 c_uint(F_DUPFD),
459 raw_fd(min)
460 ))
461 }
462 #[cfg(target_pointer_width = "64")]
463 unsafe {
464 ret_owned_fd(syscall_readonly!(
465 __NR_fcntl,
466 fd,
467 c_uint(F_DUPFD),
468 raw_fd(min)
469 ))
470 }
471 }
472
473 #[inline]
474 pub(crate) fn fcntl_dupfd_cloexec(fd: BorrowedFd<'_>, min: RawFd) -> io::Result<OwnedFd> {
475 #[cfg(target_pointer_width = "32")]
476 unsafe {
477 ret_owned_fd(syscall_readonly!(
478 __NR_fcntl64,
479 fd,
480 c_uint(F_DUPFD_CLOEXEC),
481 raw_fd(min)
482 ))
483 }
484 #[cfg(target_pointer_width = "64")]
485 unsafe {
486 ret_owned_fd(syscall_readonly!(
487 __NR_fcntl,
488 fd,
489 c_uint(F_DUPFD_CLOEXEC),
490 raw_fd(min)
491 ))
492 }
493 }
494
495 #[inline]
496 pub(crate) fn pipe_with(flags: PipeFlags) -> io::Result<(OwnedFd, OwnedFd)> {
497 unsafe {
498 let mut result = MaybeUninit::<[OwnedFd; 2]>::uninit();
499 ret(syscall!(__NR_pipe2, &mut result, flags))?;
500 let [p0, p1] = result.assume_init();
501 Ok((p0, p1))
502 }
503 }
504
505 #[inline]
506 pub(crate) fn pipe() -> io::Result<(OwnedFd, OwnedFd)> {
507 // aarch64 and risc64 omit `__NR_pipe`. On mips, `__NR_pipe` uses a special
508 // calling convention, but using it is not worth complicating our syscall
509 // wrapping infrastructure at this time.
510 #[cfg(any(
511 target_arch = "aarch64",
512 target_arch = "mips",
513 target_arch = "mips64",
514 target_arch = "riscv64",
515 ))]
516 {
517 pipe_with(PipeFlags::empty())
518 }
519 #[cfg(not(any(
520 target_arch = "aarch64",
521 target_arch = "mips",
522 target_arch = "mips64",
523 target_arch = "riscv64",
524 )))]
525 unsafe {
526 let mut result = MaybeUninit::<[OwnedFd; 2]>::uninit();
527 ret(syscall!(__NR_pipe, &mut result))?;
528 let [p0, p1] = result.assume_init();
529 Ok((p0, p1))
530 }
531 }
532
533 #[inline]
534 pub(crate) fn poll(fds: &mut [PollFd<'_>], timeout: c::c_int) -> io::Result<usize> {
535 let (fds_addr_mut, fds_len) = slice_mut(fds);
536
537 #[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))]
538 unsafe {
539 let timeout = if timeout >= 0 {
540 Some(__kernel_timespec {
541 tv_sec: (timeout as i64) / 1000,
542 tv_nsec: (timeout as i64) % 1000 * 1_000_000,
543 })
544 } else {
545 None
546 };
547 ret_usize(syscall!(
548 __NR_ppoll,
549 fds_addr_mut,
550 fds_len,
551 opt_ref(timeout.as_ref()),
552 zero(),
553 size_of::<sigset_t, _>()
554 ))
555 }
556 #[cfg(not(any(target_arch = "aarch64", target_arch = "riscv64")))]
557 unsafe {
558 ret_usize(syscall!(__NR_poll, fds_addr_mut, fds_len, c_int(timeout)))
559 }
560 }
561
562 #[inline]
563 pub(crate) fn epoll_create(flags: epoll::CreateFlags) -> io::Result<OwnedFd> {
564 unsafe { ret_owned_fd(syscall_readonly!(__NR_epoll_create1, flags)) }
565 }
566
567 #[inline]
568 pub(crate) unsafe fn epoll_add(
569 epfd: BorrowedFd<'_>,
570 fd: c::c_int,
571 event: &epoll_event,
572 ) -> io::Result<()> {
573 ret(syscall_readonly!(
574 __NR_epoll_ctl,
575 epfd,
576 c_uint(EPOLL_CTL_ADD),
577 raw_fd(fd),
578 by_ref(event)
579 ))
580 }
581
582 #[inline]
583 pub(crate) unsafe fn epoll_mod(
584 epfd: BorrowedFd<'_>,
585 fd: c::c_int,
586 event: &epoll_event,
587 ) -> io::Result<()> {
588 ret(syscall_readonly!(
589 __NR_epoll_ctl,
590 epfd,
591 c_uint(EPOLL_CTL_MOD),
592 raw_fd(fd),
593 by_ref(event)
594 ))
595 }
596
597 #[inline]
598 pub(crate) unsafe fn epoll_del(epfd: BorrowedFd<'_>, fd: c::c_int) -> io::Result<()> {
599 ret(syscall_readonly!(
600 __NR_epoll_ctl,
601 epfd,
602 c_uint(EPOLL_CTL_DEL),
603 raw_fd(fd),
604 zero()
605 ))
606 }
607
608 #[inline]
609 pub(crate) fn epoll_wait(
610 epfd: BorrowedFd<'_>,
611 events: *mut epoll_event,
612 num_events: usize,
613 timeout: c::c_int,
614 ) -> io::Result<usize> {
615 #[cfg(not(any(target_arch = "aarch64", target_arch = "riscv64")))]
616 unsafe {
617 ret_usize(syscall!(
618 __NR_epoll_wait,
619 epfd,
620 events,
621 pass_usize(num_events),
622 c_int(timeout)
623 ))
624 }
625 #[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))]
626 unsafe {
627 ret_usize(syscall!(
628 __NR_epoll_pwait,
629 epfd,
630 events,
631 pass_usize(num_events),
632 c_int(timeout),
633 zero()
634 ))
635 }
636 }
637
638 #[cfg(any(target_os = "android", target_os = "linux"))]
639 #[inline]
splicenull640 pub fn splice(
641 fd_in: BorrowedFd,
642 off_in: Option<&mut u64>,
643 fd_out: BorrowedFd,
644 off_out: Option<&mut u64>,
645 len: usize,
646 flags: SpliceFlags,
647 ) -> io::Result<usize> {
648 unsafe {
649 ret_usize(syscall!(
650 __NR_splice,
651 fd_in,
652 opt_mut(off_in),
653 fd_out,
654 opt_mut(off_out),
655 pass_usize(len),
656 c_uint(flags.bits())
657 ))
658 }
659 }
660
661 #[cfg(any(target_os = "android", target_os = "linux"))]
662 #[inline]
vmsplicenull663 pub unsafe fn vmsplice(
664 fd: BorrowedFd,
665 bufs: &[IoSliceRaw],
666 flags: SpliceFlags,
667 ) -> io::Result<usize> {
668 let (bufs_addr, bufs_len) = slice(&bufs[..cmp::min(bufs.len(), max_iov())]);
669 ret_usize(syscall!(
670 __NR_vmsplice,
671 fd,
672 bufs_addr,
673 bufs_len,
674 c_uint(flags.bits())
675 ))
676 }
677