Lines Matching refs:pipe
3 * linux/fs/pipe.c
35 * New pipe buffers will be restricted to this size while the user is exceeding
36 * their pipe buffer quota. The general pipe use case needs at least two
38 * than two, then a write to a non-empty pipe may block even if the pipe is not
41 * pipe before reading tokens: https://lore.kernel.org/lkml/1628086770.5rn8p04n6j.none@localhost/.
43 * Users can reduce their pipe buffers with F_SETPIPE_SZ below this at their
44 * own risk, namely: pipe writes to non-full pipes may block until the pipe is
50 * The max size that a non-root user is allowed to grow the pipe. Can
51 * be set by root in /proc/sys/fs/pipe-max-size
78 static void pipe_lock_nested(struct pipe_inode_info *pipe, int subclass)
80 if (pipe->files)
81 mutex_lock_nested(&pipe->mutex, subclass);
84 void pipe_lock(struct pipe_inode_info *pipe)
87 * pipe_lock() nests non-pipe inode locks (for writing to a file)
89 pipe_lock_nested(pipe, I_MUTEX_PARENT);
93 void pipe_unlock(struct pipe_inode_info *pipe)
95 if (pipe->files)
96 mutex_unlock(&pipe->mutex);
100 static inline void __pipe_lock(struct pipe_inode_info *pipe)
102 mutex_lock_nested(&pipe->mutex, I_MUTEX_PARENT);
105 static inline void __pipe_unlock(struct pipe_inode_info *pipe)
107 mutex_unlock(&pipe->mutex);
124 static void anon_pipe_buf_release(struct pipe_inode_info *pipe,
134 if (page_count(page) == 1 && !pipe->tmp_page)
135 pipe->tmp_page = page;
140 static bool anon_pipe_buf_try_steal(struct pipe_inode_info *pipe,
154 * @pipe: the pipe that the buffer belongs to
164 bool generic_pipe_buf_try_steal(struct pipe_inode_info *pipe,
184 * @pipe: the pipe that the buffer belongs to
190 * pipe into another.
192 bool generic_pipe_buf_get(struct pipe_inode_info *pipe, struct pipe_buffer *buf)
200 * @pipe: the pipe that the buffer belongs to
206 void generic_pipe_buf_release(struct pipe_inode_info *pipe,
219 /* Done while waiting without holding the pipe lock - thus the READ_ONCE() */
220 static inline bool pipe_readable(const struct pipe_inode_info *pipe)
222 unsigned int head = READ_ONCE(pipe->head);
223 unsigned int tail = READ_ONCE(pipe->tail);
224 unsigned int writers = READ_ONCE(pipe->writers);
234 struct pipe_inode_info *pipe = filp->private_data;
243 __pipe_lock(pipe);
246 * We only wake up writers if the pipe was full when we started
253 was_full = pipe_full(pipe->head, pipe->tail, pipe->max_usage);
256 unsigned int head = smp_load_acquire(&pipe->head);
257 unsigned int tail = pipe->tail;
258 unsigned int mask = pipe->ring_size - 1;
261 if (pipe->note_loss) {
280 pipe->note_loss = false;
285 struct pipe_buffer *buf = &pipe->bufs[tail & mask];
299 error = pipe_buf_confirm(pipe, buf);
323 pipe_buf_release(pipe, buf);
324 spin_lock_irq(&pipe->rd_wait.lock);
327 pipe->note_loss = true;
330 pipe->tail = tail;
331 spin_unlock_irq(&pipe->rd_wait.lock);
340 if (!pipe->writers)
348 __pipe_unlock(pipe);
354 * pipe buffer, and might have made space in the buffers
357 * You can't make zero-sized pipe buffers by doing an empty
364 * _very_ unlikely case that the pipe was full, but we got
368 wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM);
369 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
377 if (wait_event_interruptible_exclusive(pipe->rd_wait, pipe_readable(pipe)) < 0)
380 __pipe_lock(pipe);
381 was_full = pipe_full(pipe->head, pipe->tail, pipe->max_usage);
384 if (pipe_empty(pipe->head, pipe->tail))
386 __pipe_unlock(pipe);
389 wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM);
391 wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM);
392 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
403 /* Done while waiting without holding the pipe lock - thus the READ_ONCE() */
404 static inline bool pipe_writable(const struct pipe_inode_info *pipe)
406 unsigned int head = READ_ONCE(pipe->head);
407 unsigned int tail = READ_ONCE(pipe->tail);
408 unsigned int max_usage = READ_ONCE(pipe->max_usage);
411 !READ_ONCE(pipe->readers);
418 struct pipe_inode_info *pipe = filp->private_data;
430 __pipe_lock(pipe);
432 if (!pipe->readers) {
438 if (pipe_has_watch_queue(pipe)) {
451 head = pipe->head;
452 was_empty = pipe_empty(head, pipe->tail);
455 unsigned int mask = pipe->ring_size - 1;
456 struct pipe_buffer *buf = &pipe->bufs[(head - 1) & mask];
461 ret = pipe_buf_confirm(pipe, buf);
478 if (!pipe->readers) {
485 head = pipe->head;
486 if (!pipe_full(head, pipe->tail, pipe->max_usage)) {
487 unsigned int mask = pipe->ring_size - 1;
488 struct pipe_buffer *buf = &pipe->bufs[head & mask];
489 struct page *page = pipe->tmp_page;
498 pipe->tmp_page = page;
506 spin_lock_irq(&pipe->rd_wait.lock);
508 head = pipe->head;
509 if (pipe_full(head, pipe->tail, pipe->max_usage)) {
510 spin_unlock_irq(&pipe->rd_wait.lock);
514 pipe->head = head + 1;
515 spin_unlock_irq(&pipe->rd_wait.lock);
518 buf = &pipe->bufs[head & mask];
527 pipe->tmp_page = NULL;
543 if (!pipe_full(head, pipe->tail, pipe->max_usage))
559 * We're going to release the pipe lock and wait for more
561 * after waiting we need to re-check whether the pipe
564 __pipe_unlock(pipe);
566 wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM);
567 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
568 wait_event_interruptible_exclusive(pipe->wr_wait, pipe_writable(pipe));
569 __pipe_lock(pipe);
570 was_empty = pipe_empty(pipe->head, pipe->tail);
574 if (pipe_full(pipe->head, pipe->tail, pipe->max_usage))
576 __pipe_unlock(pipe);
587 * Epoll nonsensically wants a wakeup whether the pipe
590 if (was_empty || pipe->poll_usage)
591 wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM);
592 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
594 wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM);
606 struct pipe_inode_info *pipe = filp->private_data;
611 __pipe_lock(pipe);
613 head = pipe->head;
614 tail = pipe->tail;
615 mask = pipe->ring_size - 1;
618 count += pipe->bufs[tail & mask].len;
621 __pipe_unlock(pipe);
628 __pipe_lock(pipe);
629 ret = watch_queue_set_size(pipe, arg);
630 __pipe_unlock(pipe);
636 pipe, (struct watch_notification_filter __user *)arg);
649 struct pipe_inode_info *pipe = filp->private_data;
653 WRITE_ONCE(pipe->poll_usage, true);
656 * Reading pipe state only -- no need for acquiring the semaphore.
662 poll_wait(filp, &pipe->rd_wait, wait);
664 poll_wait(filp, &pipe->wr_wait, wait);
671 head = READ_ONCE(pipe->head);
672 tail = READ_ONCE(pipe->tail);
678 if (!pipe->writers && filp->f_version != pipe->w_counter)
683 if (!pipe_full(head, tail, pipe->max_usage))
689 if (!pipe->readers)
696 static void put_pipe_info(struct inode *inode, struct pipe_inode_info *pipe)
701 if (!--pipe->files) {
708 free_pipe_info(pipe);
714 struct pipe_inode_info *pipe = file->private_data;
716 __pipe_lock(pipe);
718 pipe->readers--;
720 pipe->writers--;
723 if (!pipe->readers != !pipe->writers) {
724 wake_up_interruptible_all(&pipe->rd_wait);
725 wake_up_interruptible_all(&pipe->wr_wait);
726 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
727 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
729 __pipe_unlock(pipe);
731 put_pipe_info(inode, pipe);
738 struct pipe_inode_info *pipe = filp->private_data;
741 __pipe_lock(pipe);
743 retval = fasync_helper(fd, filp, on, &pipe->fasync_readers);
745 retval = fasync_helper(fd, filp, on, &pipe->fasync_writers);
748 fasync_helper(-1, filp, 0, &pipe->fasync_readers);
750 __pipe_unlock(pipe);
781 struct pipe_inode_info *pipe;
787 pipe = kzalloc(sizeof(struct pipe_inode_info), GFP_KERNEL_ACCOUNT);
788 if (pipe == NULL)
804 pipe->bufs = kcalloc(pipe_bufs, sizeof(struct pipe_buffer),
807 if (pipe->bufs) {
808 init_waitqueue_head(&pipe->rd_wait);
809 init_waitqueue_head(&pipe->wr_wait);
810 pipe->r_counter = pipe->w_counter = 1;
811 pipe->max_usage = pipe_bufs;
812 pipe->ring_size = pipe_bufs;
813 pipe->nr_accounted = pipe_bufs;
814 pipe->user = user;
815 mutex_init(&pipe->mutex);
816 return pipe;
821 kfree(pipe);
827 void free_pipe_info(struct pipe_inode_info *pipe)
832 if (pipe->watch_queue)
833 watch_queue_clear(pipe->watch_queue);
836 (void) account_pipe_buffers(pipe->user, pipe->nr_accounted, 0);
837 free_uid(pipe->user);
838 for (i = 0; i < pipe->ring_size; i++) {
839 struct pipe_buffer *buf = pipe->bufs + i;
841 pipe_buf_release(pipe, buf);
844 if (pipe->watch_queue)
845 put_watch_queue(pipe->watch_queue);
847 if (pipe->tmp_page)
848 __free_page(pipe->tmp_page);
849 kfree(pipe->bufs);
850 kfree(pipe);
860 return dynamic_dname(dentry, buffer, buflen, "pipe:[%lu]",
871 struct pipe_inode_info *pipe;
878 pipe = alloc_pipe_info();
879 if (!pipe)
882 inode->i_pipe = pipe;
883 pipe->files = 2;
884 pipe->readers = pipe->writers = 1;
999 * a pipe. It's not the way Unix traditionally does this, though.
1028 SYSCALL_DEFINE1(pipe, int __user *, fildes)
1034 * This is the stupid "wait for pipe to be readable or writable"
1042 void pipe_wait_readable(struct pipe_inode_info *pipe)
1044 pipe_unlock(pipe);
1045 wait_event_interruptible(pipe->rd_wait, pipe_readable(pipe));
1046 pipe_lock(pipe);
1049 void pipe_wait_writable(struct pipe_inode_info *pipe)
1051 pipe_unlock(pipe);
1052 wait_event_interruptible(pipe->wr_wait, pipe_writable(pipe));
1053 pipe_lock(pipe);
1058 * holding the pipe lock, so "*cnt" is stable and we know a wakeup cannot
1063 * because of the pipe lock, we can check the condition before being on
1066 * We use the 'rd_wait' waitqueue for pipe partner waiting.
1068 static int wait_for_partner(struct pipe_inode_info *pipe, unsigned int *cnt)
1074 prepare_to_wait(&pipe->rd_wait, &rdwait, TASK_INTERRUPTIBLE);
1075 pipe_unlock(pipe);
1077 finish_wait(&pipe->rd_wait, &rdwait);
1078 pipe_lock(pipe);
1085 static void wake_up_partner(struct pipe_inode_info *pipe)
1087 wake_up_interruptible_all(&pipe->rd_wait);
1092 struct pipe_inode_info *pipe;
1100 pipe = inode->i_pipe;
1101 pipe->files++;
1105 pipe = alloc_pipe_info();
1106 if (!pipe)
1108 pipe->files = 1;
1113 free_pipe_info(pipe);
1114 pipe = inode->i_pipe;
1116 inode->i_pipe = pipe;
1120 filp->private_data = pipe;
1121 /* OK, we have a pipe and it's pinned down */
1123 __pipe_lock(pipe);
1135 pipe->r_counter++;
1136 if (pipe->readers++ == 0)
1137 wake_up_partner(pipe);
1139 if (!is_pipe && !pipe->writers) {
1143 filp->f_version = pipe->w_counter;
1145 if (wait_for_partner(pipe, &pipe->w_counter))
1158 if (!is_pipe && (filp->f_flags & O_NONBLOCK) && !pipe->readers)
1161 pipe->w_counter++;
1162 if (!pipe->writers++)
1163 wake_up_partner(pipe);
1165 if (!is_pipe && !pipe->readers) {
1166 if (wait_for_partner(pipe, &pipe->r_counter))
1179 pipe->readers++;
1180 pipe->writers++;
1181 pipe->r_counter++;
1182 pipe->w_counter++;
1183 if (pipe->readers == 1 || pipe->writers == 1)
1184 wake_up_partner(pipe);
1193 __pipe_unlock(pipe);
1197 if (!--pipe->readers)
1198 wake_up_interruptible(&pipe->wr_wait);
1203 if (!--pipe->writers)
1204 wake_up_interruptible_all(&pipe->rd_wait);
1209 __pipe_unlock(pipe);
1211 put_pipe_info(inode, pipe);
1228 * Currently we rely on the pipe array holding a power-of-2 number
1236 /* Minimum pipe size, as required by POSIX */
1244 * Resize the pipe ring to a number of slots.
1246 * Note the pipe can be reduced in capacity, but only if the current
1250 int pipe_resize_ring(struct pipe_inode_info *pipe, unsigned int nr_slots)
1260 spin_lock_irq(&pipe->rd_wait.lock);
1261 mask = pipe->ring_size - 1;
1262 head = pipe->head;
1263 tail = pipe->tail;
1267 spin_unlock_irq(&pipe->rd_wait.lock);
1273 * The pipe array wraps around, so just start the new one at zero
1280 memcpy(bufs, pipe->bufs + t,
1283 unsigned int tsize = pipe->ring_size - t;
1285 memcpy(bufs + tsize, pipe->bufs,
1287 memcpy(bufs, pipe->bufs + t,
1295 kfree(pipe->bufs);
1296 pipe->bufs = bufs;
1297 pipe->ring_size = nr_slots;
1298 if (pipe->max_usage > nr_slots)
1299 pipe->max_usage = nr_slots;
1300 pipe->tail = tail;
1301 pipe->head = head;
1303 if (!pipe_has_watch_queue(pipe)) {
1304 pipe->max_usage = nr_slots;
1305 pipe->nr_accounted = nr_slots;
1308 spin_unlock_irq(&pipe->rd_wait.lock);
1311 wake_up_interruptible(&pipe->wr_wait);
1316 * Allocate a new array of pipe buffers and copy the info over. Returns the
1317 * pipe size if successful, or return -ERROR on error.
1319 static long pipe_set_size(struct pipe_inode_info *pipe, unsigned long arg)
1325 if (pipe_has_watch_queue(pipe))
1335 * If trying to increase the pipe capacity, check that an
1338 * Decreasing the pipe capacity is always permitted, even
1341 if (nr_slots > pipe->max_usage &&
1345 user_bufs = account_pipe_buffers(pipe->user, pipe->nr_accounted, nr_slots);
1347 if (nr_slots > pipe->max_usage &&
1355 ret = pipe_resize_ring(pipe, nr_slots);
1359 return pipe->max_usage * PAGE_SIZE;
1362 (void) account_pipe_buffers(pipe->user, nr_slots, pipe->nr_accounted);
1369 * pipe.
1373 struct pipe_inode_info *pipe = file->private_data;
1375 if (file->f_op != &pipefifo_fops || !pipe)
1377 if (for_splice && pipe_has_watch_queue(pipe))
1379 return pipe;
1384 struct pipe_inode_info *pipe;
1387 pipe = get_pipe_info(file, false);
1388 if (!pipe)
1391 __pipe_lock(pipe);
1395 ret = pipe_set_size(pipe, arg);
1398 ret = pipe->max_usage * PAGE_SIZE;
1405 __pipe_unlock(pipe);
1418 * d_name - pipe: will go nicely and kill the special-casing in procfs.