Lines Matching refs:pipe
3 * linux/fs/pipe.c
36 * New pipe buffers will be restricted to this size while the user is exceeding
37 * their pipe buffer quota. The general pipe use case needs at least two
39 * than two, then a write to a non-empty pipe may block even if the pipe is not
42 * pipe before reading tokens: https://lore.kernel.org/lkml/1628086770.5rn8p04n6j.none@localhost/.
44 * Users can reduce their pipe buffers with F_SETPIPE_SZ below this at their
45 * own risk, namely: pipe writes to non-full pipes may block until the pipe is
51 * The max size that a non-root user is allowed to grow the pipe. Can
52 * be set by root in /proc/sys/fs/pipe-max-size
79 static void pipe_lock_nested(struct pipe_inode_info *pipe, int subclass)
81 if (pipe->files)
82 mutex_lock_nested(&pipe->mutex, subclass);
85 void pipe_lock(struct pipe_inode_info *pipe)
88 * pipe_lock() nests non-pipe inode locks (for writing to a file)
90 pipe_lock_nested(pipe, I_MUTEX_PARENT);
94 void pipe_unlock(struct pipe_inode_info *pipe)
96 if (pipe->files)
97 mutex_unlock(&pipe->mutex);
101 static inline void __pipe_lock(struct pipe_inode_info *pipe)
103 mutex_lock_nested(&pipe->mutex, I_MUTEX_PARENT);
106 static inline void __pipe_unlock(struct pipe_inode_info *pipe)
108 mutex_unlock(&pipe->mutex);
125 static void anon_pipe_buf_release(struct pipe_inode_info *pipe,
135 if (page_count(page) == 1 && !pipe->tmp_page)
136 pipe->tmp_page = page;
141 static bool anon_pipe_buf_try_steal(struct pipe_inode_info *pipe,
155 * @pipe: the pipe that the buffer belongs to
165 bool generic_pipe_buf_try_steal(struct pipe_inode_info *pipe,
185 * @pipe: the pipe that the buffer belongs to
191 * pipe into another.
193 bool generic_pipe_buf_get(struct pipe_inode_info *pipe, struct pipe_buffer *buf)
201 * @pipe: the pipe that the buffer belongs to
207 void generic_pipe_buf_release(struct pipe_inode_info *pipe,
220 /* Done while waiting without holding the pipe lock - thus the READ_ONCE() */
221 static inline bool pipe_readable(const struct pipe_inode_info *pipe)
223 unsigned int head = READ_ONCE(pipe->head);
224 unsigned int tail = READ_ONCE(pipe->tail);
225 unsigned int writers = READ_ONCE(pipe->writers);
235 struct pipe_inode_info *pipe = filp->private_data;
244 __pipe_lock(pipe);
247 * We only wake up writers if the pipe was full when we started
254 was_full = pipe_full(pipe->head, pipe->tail, pipe->max_usage);
257 unsigned int head = smp_load_acquire(&pipe->head);
258 unsigned int tail = pipe->tail;
259 unsigned int mask = pipe->ring_size - 1;
262 if (pipe->note_loss) {
281 pipe->note_loss = false;
286 struct pipe_buffer *buf = &pipe->bufs[tail & mask];
300 error = pipe_buf_confirm(pipe, buf);
324 pipe_buf_release(pipe, buf);
325 spin_lock_irq(&pipe->rd_wait.lock);
328 pipe->note_loss = true;
331 pipe->tail = tail;
332 spin_unlock_irq(&pipe->rd_wait.lock);
341 if (!pipe->writers)
350 __pipe_unlock(pipe);
356 * pipe buffer, and might have made space in the buffers
359 * You can't make zero-sized pipe buffers by doing an empty
366 * _very_ unlikely case that the pipe was full, but we got
370 wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM);
371 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
379 if (wait_event_interruptible_exclusive(pipe->rd_wait, pipe_readable(pipe)) < 0)
382 __pipe_lock(pipe);
383 was_full = pipe_full(pipe->head, pipe->tail, pipe->max_usage);
386 if (pipe_empty(pipe->head, pipe->tail))
388 __pipe_unlock(pipe);
391 wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM);
393 wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM);
394 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
405 /* Done while waiting without holding the pipe lock - thus the READ_ONCE() */
406 static inline bool pipe_writable(const struct pipe_inode_info *pipe)
408 unsigned int head = READ_ONCE(pipe->head);
409 unsigned int tail = READ_ONCE(pipe->tail);
410 unsigned int max_usage = READ_ONCE(pipe->max_usage);
413 !READ_ONCE(pipe->readers);
420 struct pipe_inode_info *pipe = filp->private_data;
432 __pipe_lock(pipe);
434 if (!pipe->readers) {
440 if (pipe_has_watch_queue(pipe)) {
453 head = pipe->head;
454 was_empty = pipe_empty(head, pipe->tail);
457 unsigned int mask = pipe->ring_size - 1;
458 struct pipe_buffer *buf = &pipe->bufs[(head - 1) & mask];
463 ret = pipe_buf_confirm(pipe, buf);
480 if (!pipe->readers) {
487 head = pipe->head;
488 if (!pipe_full(head, pipe->tail, pipe->max_usage)) {
489 unsigned int mask = pipe->ring_size - 1;
491 struct page *page = pipe->tmp_page;
500 pipe->tmp_page = page;
508 spin_lock_irq(&pipe->rd_wait.lock);
510 head = pipe->head;
511 if (pipe_full(head, pipe->tail, pipe->max_usage)) {
512 spin_unlock_irq(&pipe->rd_wait.lock);
516 pipe->head = head + 1;
517 spin_unlock_irq(&pipe->rd_wait.lock);
520 buf = &pipe->bufs[head & mask];
529 pipe->tmp_page = NULL;
544 if (!pipe_full(head, pipe->tail, pipe->max_usage))
561 * We're going to release the pipe lock and wait for more
563 * after waiting we need to re-check whether the pipe
566 __pipe_unlock(pipe);
568 wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM);
569 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
570 wait_event_interruptible_exclusive(pipe->wr_wait, pipe_writable(pipe));
571 __pipe_lock(pipe);
572 was_empty = pipe_empty(pipe->head, pipe->tail);
576 if (pipe_full(pipe->head, pipe->tail, pipe->max_usage))
578 __pipe_unlock(pipe);
589 * Epoll nonsensically wants a wakeup whether the pipe
592 if (was_empty || pipe->poll_usage)
593 wake_up_interruptible_sync_poll(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM);
594 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
596 wake_up_interruptible_sync_poll(&pipe->wr_wait, EPOLLOUT | EPOLLWRNORM);
608 struct pipe_inode_info *pipe = filp->private_data;
613 __pipe_lock(pipe);
615 head = pipe->head;
616 tail = pipe->tail;
617 mask = pipe->ring_size - 1;
620 count += pipe->bufs[tail & mask].len;
623 __pipe_unlock(pipe);
630 __pipe_lock(pipe);
631 ret = watch_queue_set_size(pipe, arg);
632 __pipe_unlock(pipe);
638 pipe, (struct watch_notification_filter __user *)arg);
651 struct pipe_inode_info *pipe = filp->private_data;
655 WRITE_ONCE(pipe->poll_usage, true);
658 * Reading pipe state only -- no need for acquiring the semaphore.
664 poll_wait(filp, &pipe->rd_wait, wait);
666 poll_wait(filp, &pipe->wr_wait, wait);
673 head = READ_ONCE(pipe->head);
674 tail = READ_ONCE(pipe->tail);
680 if (!pipe->writers && filp->f_version != pipe->w_counter)
685 if (!pipe_full(head, tail, pipe->max_usage))
691 if (!pipe->readers)
698 static void put_pipe_info(struct inode *inode, struct pipe_inode_info *pipe)
703 if (!--pipe->files) {
710 free_pipe_info(pipe);
716 struct pipe_inode_info *pipe = file->private_data;
718 __pipe_lock(pipe);
720 pipe->readers--;
722 pipe->writers--;
725 if (!pipe->readers != !pipe->writers) {
726 wake_up_interruptible_all(&pipe->rd_wait);
727 wake_up_interruptible_all(&pipe->wr_wait);
728 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
729 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
731 __pipe_unlock(pipe);
733 put_pipe_info(inode, pipe);
740 struct pipe_inode_info *pipe = filp->private_data;
743 __pipe_lock(pipe);
745 retval = fasync_helper(fd, filp, on, &pipe->fasync_readers);
747 retval = fasync_helper(fd, filp, on, &pipe->fasync_writers);
750 fasync_helper(-1, filp, 0, &pipe->fasync_readers);
752 __pipe_unlock(pipe);
783 struct pipe_inode_info *pipe;
789 pipe = kzalloc(sizeof(struct pipe_inode_info), GFP_KERNEL_ACCOUNT);
790 if (pipe == NULL)
806 pipe->bufs = kcalloc(pipe_bufs, sizeof(struct pipe_buffer),
809 if (pipe->bufs) {
810 init_waitqueue_head(&pipe->rd_wait);
811 init_waitqueue_head(&pipe->wr_wait);
812 pipe->r_counter = pipe->w_counter = 1;
813 pipe->max_usage = pipe_bufs;
814 pipe->ring_size = pipe_bufs;
815 pipe->nr_accounted = pipe_bufs;
816 pipe->user = user;
817 mutex_init(&pipe->mutex);
818 return pipe;
823 kfree(pipe);
829 void free_pipe_info(struct pipe_inode_info *pipe)
834 if (pipe->watch_queue)
835 watch_queue_clear(pipe->watch_queue);
838 (void) account_pipe_buffers(pipe->user, pipe->nr_accounted, 0);
839 free_uid(pipe->user);
840 for (i = 0; i < pipe->ring_size; i++) {
841 struct pipe_buffer *buf = pipe->bufs + i;
843 pipe_buf_release(pipe, buf);
846 if (pipe->watch_queue)
847 put_watch_queue(pipe->watch_queue);
849 if (pipe->tmp_page)
850 __free_page(pipe->tmp_page);
851 kfree(pipe->bufs);
852 kfree(pipe);
862 return dynamic_dname(buffer, buflen, "pipe:[%lu]",
873 struct pipe_inode_info *pipe;
880 pipe = alloc_pipe_info();
881 if (!pipe)
884 inode->i_pipe = pipe;
885 pipe->files = 2;
886 pipe->readers = pipe->writers = 1;
978 /* pipe groks IOCB_NOWAIT */
1004 * a pipe. It's not the way Unix traditionally does this, though.
1033 SYSCALL_DEFINE1(pipe, int __user *, fildes)
1039 * This is the stupid "wait for pipe to be readable or writable"
1047 void pipe_wait_readable(struct pipe_inode_info *pipe)
1049 pipe_unlock(pipe);
1050 wait_event_interruptible(pipe->rd_wait, pipe_readable(pipe));
1051 pipe_lock(pipe);
1054 void pipe_wait_writable(struct pipe_inode_info *pipe)
1056 pipe_unlock(pipe);
1057 wait_event_interruptible(pipe->wr_wait, pipe_writable(pipe));
1058 pipe_lock(pipe);
1063 * holding the pipe lock, so "*cnt" is stable and we know a wakeup cannot
1068 * because of the pipe lock, we can check the condition before being on
1071 * We use the 'rd_wait' waitqueue for pipe partner waiting.
1073 static int wait_for_partner(struct pipe_inode_info *pipe, unsigned int *cnt)
1079 prepare_to_wait(&pipe->rd_wait, &rdwait, TASK_INTERRUPTIBLE);
1080 pipe_unlock(pipe);
1082 finish_wait(&pipe->rd_wait, &rdwait);
1083 pipe_lock(pipe);
1090 static void wake_up_partner(struct pipe_inode_info *pipe)
1092 wake_up_interruptible_all(&pipe->rd_wait);
1097 struct pipe_inode_info *pipe;
1105 pipe = inode->i_pipe;
1106 pipe->files++;
1110 pipe = alloc_pipe_info();
1111 if (!pipe)
1113 pipe->files = 1;
1118 free_pipe_info(pipe);
1119 pipe = inode->i_pipe;
1121 inode->i_pipe = pipe;
1125 filp->private_data = pipe;
1126 /* OK, we have a pipe and it's pinned down */
1128 __pipe_lock(pipe);
1140 pipe->r_counter++;
1141 if (pipe->readers++ == 0)
1142 wake_up_partner(pipe);
1144 if (!is_pipe && !pipe->writers) {
1148 filp->f_version = pipe->w_counter;
1150 if (wait_for_partner(pipe, &pipe->w_counter))
1163 if (!is_pipe && (filp->f_flags & O_NONBLOCK) && !pipe->readers)
1166 pipe->w_counter++;
1167 if (!pipe->writers++)
1168 wake_up_partner(pipe);
1170 if (!is_pipe && !pipe->readers) {
1171 if (wait_for_partner(pipe, &pipe->r_counter))
1184 pipe->readers++;
1185 pipe->writers++;
1186 pipe->r_counter++;
1187 pipe->w_counter++;
1188 if (pipe->readers == 1 || pipe->writers == 1)
1189 wake_up_partner(pipe);
1198 __pipe_unlock(pipe);
1202 if (!--pipe->readers)
1203 wake_up_interruptible(&pipe->wr_wait);
1208 if (!--pipe->writers)
1209 wake_up_interruptible_all(&pipe->rd_wait);
1214 __pipe_unlock(pipe);
1216 put_pipe_info(inode, pipe);
1233 * Currently we rely on the pipe array holding a power-of-2 number
1241 /* Minimum pipe size, as required by POSIX */
1249 * Resize the pipe ring to a number of slots.
1251 * Note the pipe can be reduced in capacity, but only if the current
1255 int pipe_resize_ring(struct pipe_inode_info *pipe, unsigned int nr_slots)
1265 spin_lock_irq(&pipe->rd_wait.lock);
1266 mask = pipe->ring_size - 1;
1267 head = pipe->head;
1268 tail = pipe->tail;
1272 spin_unlock_irq(&pipe->rd_wait.lock);
1278 * The pipe array wraps around, so just start the new one at zero
1285 memcpy(bufs, pipe->bufs + t,
1288 unsigned int tsize = pipe->ring_size - t;
1290 memcpy(bufs + tsize, pipe->bufs,
1292 memcpy(bufs, pipe->bufs + t,
1300 kfree(pipe->bufs);
1301 pipe->bufs = bufs;
1302 pipe->ring_size = nr_slots;
1303 if (pipe->max_usage > nr_slots)
1304 pipe->max_usage = nr_slots;
1305 pipe->tail = tail;
1306 pipe->head = head;
1308 if (!pipe_has_watch_queue(pipe)) {
1309 pipe->max_usage = nr_slots;
1310 pipe->nr_accounted = nr_slots;
1313 spin_unlock_irq(&pipe->rd_wait.lock);
1316 wake_up_interruptible(&pipe->wr_wait);
1321 * Allocate a new array of pipe buffers and copy the info over. Returns the
1322 * pipe size if successful, or return -ERROR on error.
1324 static long pipe_set_size(struct pipe_inode_info *pipe, unsigned int arg)
1330 if (pipe_has_watch_queue(pipe))
1340 * If trying to increase the pipe capacity, check that an
1343 * Decreasing the pipe capacity is always permitted, even
1346 if (nr_slots > pipe->max_usage &&
1350 user_bufs = account_pipe_buffers(pipe->user, pipe->nr_accounted, nr_slots);
1352 if (nr_slots > pipe->max_usage &&
1360 ret = pipe_resize_ring(pipe, nr_slots);
1364 return pipe->max_usage * PAGE_SIZE;
1367 (void) account_pipe_buffers(pipe->user, nr_slots, pipe->nr_accounted);
1373 * not enough to verify that this is a pipe.
1377 struct pipe_inode_info *pipe = file->private_data;
1379 if (file->f_op != &pipefifo_fops || !pipe)
1381 if (for_splice && pipe_has_watch_queue(pipe))
1383 return pipe;
1388 struct pipe_inode_info *pipe;
1391 pipe = get_pipe_info(file, false);
1392 if (!pipe)
1395 __pipe_lock(pipe);
1399 ret = pipe_set_size(pipe, arg);
1402 ret = pipe->max_usage * PAGE_SIZE;
1409 __pipe_unlock(pipe);
1422 * d_name - pipe: will go nicely and kill the special-casing in procfs.
1471 .procname = "pipe-max-size",
1478 .procname = "pipe-user-pages-hard",
1485 .procname = "pipe-user-pages-soft",