Lines Matching full:pipe
5 * This is the "extended pipe" functionality, where a pipe is used as
6 * an arbitrary in-memory buffer. Think of a pipe as a small kernel
10 * that transfers data buffers to or from a pipe buffer.
45 * here if set to avoid blocking other users of this pipe if splice is
59 * Attempt to steal a page from a pipe buffer. This should perhaps go into
64 static bool page_cache_pipe_buf_try_steal(struct pipe_inode_info *pipe, in page_cache_pipe_buf_try_steal() argument
108 static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe, in page_cache_pipe_buf_release() argument
119 static int page_cache_pipe_buf_confirm(struct pipe_inode_info *pipe, in page_cache_pipe_buf_confirm() argument
162 static bool user_page_pipe_buf_try_steal(struct pipe_inode_info *pipe, in user_page_pipe_buf_try_steal() argument
169 return generic_pipe_buf_try_steal(pipe, buf); in user_page_pipe_buf_try_steal()
178 static void wakeup_pipe_readers(struct pipe_inode_info *pipe) in wakeup_pipe_readers() argument
181 if (waitqueue_active(&pipe->rd_wait)) in wakeup_pipe_readers()
182 wake_up_interruptible(&pipe->rd_wait); in wakeup_pipe_readers()
183 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); in wakeup_pipe_readers()
187 * splice_to_pipe - fill passed data into a pipe
188 * @pipe: pipe to fill
194 * function will link that data to the pipe.
197 ssize_t splice_to_pipe(struct pipe_inode_info *pipe, in splice_to_pipe() argument
201 unsigned int tail = pipe->tail; in splice_to_pipe()
202 unsigned int head = pipe->head; in splice_to_pipe()
203 unsigned int mask = pipe->ring_size - 1; in splice_to_pipe()
210 if (unlikely(!pipe->readers)) { in splice_to_pipe()
216 while (!pipe_full(head, tail, pipe->max_usage)) { in splice_to_pipe()
217 struct pipe_buffer *buf = &pipe->bufs[head & mask]; in splice_to_pipe()
227 pipe->head = head; in splice_to_pipe()
246 ssize_t add_to_pipe(struct pipe_inode_info *pipe, struct pipe_buffer *buf) in add_to_pipe() argument
248 unsigned int head = pipe->head; in add_to_pipe()
249 unsigned int tail = pipe->tail; in add_to_pipe()
250 unsigned int mask = pipe->ring_size - 1; in add_to_pipe()
253 if (unlikely(!pipe->readers)) { in add_to_pipe()
256 } else if (pipe_full(head, tail, pipe->max_usage)) { in add_to_pipe()
259 pipe->bufs[head & mask] = *buf; in add_to_pipe()
260 pipe->head = head + 1; in add_to_pipe()
263 pipe_buf_release(pipe, buf); in add_to_pipe()
272 int splice_grow_spd(const struct pipe_inode_info *pipe, struct splice_pipe_desc *spd) in splice_grow_spd() argument
274 unsigned int max_usage = READ_ONCE(pipe->max_usage); in splice_grow_spd()
302 * copy_splice_read - Copy data from a file and splice the copy into a pipe
305 * @pipe: The pipe to splice into
310 * amount of data (but limited by the remaining pipe capacity), passes it to
312 * the pipe.
316 * to be read; -EAGAIN will be returned if the pipe had no space, and some
318 * if the pipe has insufficient space, we reach the end of the data or we hit a
322 struct pipe_inode_info *pipe, in copy_splice_read() argument
333 /* Work out how much data we can actually add into the pipe */ in copy_splice_read()
334 used = pipe_buf_usage(pipe); in copy_splice_read()
335 npages = max_t(ssize_t, pipe->max_usage - used, 0); in copy_splice_read()
383 /* Push the remaining pages into the pipe. */ in copy_splice_read()
386 struct pipe_buffer *buf = pipe_head_buf(pipe); in copy_splice_read()
395 pipe->head++; in copy_splice_read()
410 /* Pipe buffer operations for a socket and similar. */
417 static void wakeup_pipe_writers(struct pipe_inode_info *pipe) in wakeup_pipe_writers() argument
420 if (waitqueue_active(&pipe->wr_wait)) in wakeup_pipe_writers()
421 wake_up_interruptible(&pipe->wr_wait); in wakeup_pipe_writers()
422 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT); in wakeup_pipe_writers()
426 * splice_from_pipe_feed - feed available data from a pipe to a file
427 * @pipe: pipe to splice from
432 * This function loops over the pipe and calls @actor to do the
435 * the pipe or if the requested number of bytes (@sd->total_len)
437 * pipe needs to be filled with more data, zero if the required
442 * locking is required around copying the pipe buffers to the
445 static int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_desc *sd, in splice_from_pipe_feed() argument
448 unsigned int head = pipe->head; in splice_from_pipe_feed()
449 unsigned int tail = pipe->tail; in splice_from_pipe_feed()
450 unsigned int mask = pipe->ring_size - 1; in splice_from_pipe_feed()
454 struct pipe_buffer *buf = &pipe->bufs[tail & mask]; in splice_from_pipe_feed()
460 ret = pipe_buf_confirm(pipe, buf); in splice_from_pipe_feed()
467 ret = actor(pipe, buf, sd); in splice_from_pipe_feed()
480 pipe_buf_release(pipe, buf); in splice_from_pipe_feed()
482 pipe->tail = tail; in splice_from_pipe_feed()
483 if (pipe->files) in splice_from_pipe_feed()
494 /* We know we have a pipe buffer, but maybe it's empty? */
495 static inline bool eat_empty_buffer(struct pipe_inode_info *pipe) in eat_empty_buffer() argument
497 unsigned int tail = pipe->tail; in eat_empty_buffer()
498 unsigned int mask = pipe->ring_size - 1; in eat_empty_buffer()
499 struct pipe_buffer *buf = &pipe->bufs[tail & mask]; in eat_empty_buffer()
502 pipe_buf_release(pipe, buf); in eat_empty_buffer()
503 pipe->tail = tail+1; in eat_empty_buffer()
512 * @pipe: pipe to splice from
517 * value (one) if pipe buffers are available. It will return zero
520 static int splice_from_pipe_next(struct pipe_inode_info *pipe, struct splice_desc *sd) in splice_from_pipe_next() argument
530 while (pipe_is_empty(pipe)) { in splice_from_pipe_next()
531 if (!pipe->writers) in splice_from_pipe_next()
544 wakeup_pipe_writers(pipe); in splice_from_pipe_next()
548 pipe_wait_readable(pipe); in splice_from_pipe_next()
551 if (eat_empty_buffer(pipe)) in splice_from_pipe_next()
558 * splice_from_pipe_begin - start splicing from pipe
573 * splice_from_pipe_end - finish splicing from pipe
574 * @pipe: pipe to splice from
578 * This function will wake up pipe writers if necessary. It should
582 static void splice_from_pipe_end(struct pipe_inode_info *pipe, struct splice_desc *sd) in splice_from_pipe_end() argument
585 wakeup_pipe_writers(pipe); in splice_from_pipe_end()
589 * __splice_from_pipe - splice data from a pipe to given actor
590 * @pipe: pipe to splice from
595 * This function does little more than loop over the pipe and call
601 ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd, in __splice_from_pipe() argument
609 ret = splice_from_pipe_next(pipe, sd); in __splice_from_pipe()
611 ret = splice_from_pipe_feed(pipe, sd, actor); in __splice_from_pipe()
613 splice_from_pipe_end(pipe, sd); in __splice_from_pipe()
620 * splice_from_pipe - splice data from a pipe to a file
621 * @pipe: pipe to splice from
629 * See __splice_from_pipe. This function locks the pipe inode,
633 ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out, in splice_from_pipe() argument
645 pipe_lock(pipe); in splice_from_pipe()
646 ret = __splice_from_pipe(pipe, &sd, actor); in splice_from_pipe()
647 pipe_unlock(pipe); in splice_from_pipe()
653 * iter_file_splice_write - splice data from a pipe to a file
654 * @pipe: pipe info
662 * the given pipe inode to the given file.
667 iter_file_splice_write(struct pipe_inode_info *pipe, struct file *out, in iter_file_splice_write() argument
676 int nbufs = pipe->max_usage; in iter_file_splice_write()
687 pipe_lock(pipe); in iter_file_splice_write()
697 ret = splice_from_pipe_next(pipe, &sd); in iter_file_splice_write()
701 if (unlikely(nbufs < pipe->max_usage)) { in iter_file_splice_write()
703 nbufs = pipe->max_usage; in iter_file_splice_write()
712 head = pipe->head; in iter_file_splice_write()
713 tail = pipe->tail; in iter_file_splice_write()
714 mask = pipe->ring_size - 1; in iter_file_splice_write()
719 struct pipe_buffer *buf = &pipe->bufs[tail & mask]; in iter_file_splice_write()
727 ret = pipe_buf_confirm(pipe, buf); in iter_file_splice_write()
753 tail = pipe->tail; in iter_file_splice_write()
755 struct pipe_buffer *buf = &pipe->bufs[tail & mask]; in iter_file_splice_write()
759 pipe_buf_release(pipe, buf); in iter_file_splice_write()
761 pipe->tail = tail; in iter_file_splice_write()
762 if (pipe->files) in iter_file_splice_write()
773 splice_from_pipe_end(pipe, &sd); in iter_file_splice_write()
775 pipe_unlock(pipe); in iter_file_splice_write()
787 * splice_to_socket - splice data from a pipe to a socket
788 * @pipe: pipe to splice from
795 * Will send @len bytes from the pipe to a network socket. No data copying
799 ssize_t splice_to_socket(struct pipe_inode_info *pipe, struct file *out, in splice_to_socket() argument
809 pipe_lock(pipe); in splice_to_socket()
823 while (pipe_is_empty(pipe)) { in splice_to_socket()
825 if (!pipe->writers) in splice_to_socket()
840 wakeup_pipe_writers(pipe); in splice_to_socket()
844 pipe_wait_readable(pipe); in splice_to_socket()
847 head = pipe->head; in splice_to_socket()
848 tail = pipe->tail; in splice_to_socket()
849 mask = pipe->ring_size - 1; in splice_to_socket()
852 struct pipe_buffer *buf = &pipe->bufs[tail & mask]; in splice_to_socket()
862 ret = pipe_buf_confirm(pipe, buf); in splice_to_socket()
882 if (remain && pipe_occupancy(pipe->head, tail) > 0) in splice_to_socket()
895 tail = pipe->tail; in splice_to_socket()
897 struct pipe_buffer *buf = &pipe->bufs[tail & mask]; in splice_to_socket()
905 pipe_buf_release(pipe, buf); in splice_to_socket()
910 if (tail != pipe->tail) { in splice_to_socket()
911 pipe->tail = tail; in splice_to_socket()
912 if (pipe->files) in splice_to_socket()
918 pipe_unlock(pipe); in splice_to_socket()
920 wakeup_pipe_writers(pipe); in splice_to_socket()
934 * Attempt to initiate a splice from pipe to file.
936 static ssize_t do_splice_from(struct pipe_inode_info *pipe, struct file *out, in do_splice_from() argument
941 return out->f_op->splice_write(pipe, out, ppos, len, flags); in do_splice_from()
960 struct pipe_inode_info *pipe, size_t len, in do_splice_read() argument
970 /* Don't try to read more the pipe has space for. */ in do_splice_read()
971 p_space = pipe->max_usage - pipe_buf_usage(pipe); in do_splice_read()
981 * buffer, copy into it and splice that into the pipe. in do_splice_read()
984 return copy_splice_read(in, ppos, pipe, len, flags); in do_splice_read()
985 return in->f_op->splice_read(in, ppos, pipe, len, flags); in do_splice_read()
989 * vfs_splice_read - Read data from a file and splice it into a pipe
992 * @pipe: Pipe to splice to
996 * Splice the requested amount of data from the input file to the pipe. This
997 * is synchronous as the caller must hold the pipe lock across the entire
1004 struct pipe_inode_info *pipe, size_t len, in vfs_splice_read() argument
1013 return do_splice_read(in, ppos, pipe, len, flags); in vfs_splice_read()
1025 * points, without requiring an explicit pipe. Internally an allocated
1026 * pipe is cached in the process, and reused during the lifetime of
1033 struct pipe_inode_info *pipe; in splice_direct_to_actor() local
1047 * neither in nor out is a pipe, setup an internal pipe attached to in splice_direct_to_actor()
1050 pipe = current->splice_pipe; in splice_direct_to_actor()
1051 if (unlikely(!pipe)) { in splice_direct_to_actor()
1052 pipe = alloc_pipe_info(); in splice_direct_to_actor()
1053 if (!pipe) in splice_direct_to_actor()
1058 * out of the pipe right after the splice_to_pipe(). So set in splice_direct_to_actor()
1061 pipe->readers = 1; in splice_direct_to_actor()
1063 current->splice_pipe = pipe; in splice_direct_to_actor()
1072 /* Don't block on output, we have to drain the direct pipe. */ in splice_direct_to_actor()
1083 WARN_ON_ONCE(!pipe_is_empty(pipe)); in splice_direct_to_actor()
1089 ret = do_splice_read(in, &pos, pipe, len, flags); in splice_direct_to_actor()
1106 * could get stuck data in the internal pipe: in splice_direct_to_actor()
1108 ret = actor(pipe, sd); in splice_direct_to_actor()
1125 pipe->tail = pipe->head = 0; in splice_direct_to_actor()
1141 * the pipe buffers in question: in splice_direct_to_actor()
1143 for (i = 0; i < pipe->ring_size; i++) { in splice_direct_to_actor()
1144 struct pipe_buffer *buf = &pipe->bufs[i]; in splice_direct_to_actor()
1147 pipe_buf_release(pipe, buf); in splice_direct_to_actor()
1157 static int direct_splice_actor(struct pipe_inode_info *pipe, in direct_splice_actor() argument
1164 ret = do_splice_from(pipe, file, sd->opos, sd->total_len, sd->flags); in direct_splice_actor()
1169 static int splice_file_range_actor(struct pipe_inode_info *pipe, in splice_file_range_actor() argument
1174 return do_splice_from(pipe, file, sd->opos, sd->total_len, sd->flags); in splice_file_range_actor()
1226 * can splice directly through a process-private pipe.
1264 static int wait_for_space(struct pipe_inode_info *pipe, unsigned flags) in wait_for_space() argument
1267 if (unlikely(!pipe->readers)) { in wait_for_space()
1271 if (!pipe_is_full(pipe)) in wait_for_space()
1277 pipe_wait_writable(pipe); in wait_for_space()
1449 struct pipe_inode_info *pipe, in iter_to_pipe() argument
1478 ret = add_to_pipe(pipe, &buf); in iter_to_pipe()
1495 static int pipe_to_user(struct pipe_inode_info *pipe, struct pipe_buffer *buf, in pipe_to_user() argument
1509 struct pipe_inode_info *pipe = get_pipe_info(file, true); in vmsplice_to_user() local
1517 if (!pipe) in vmsplice_to_user()
1523 pipe_lock(pipe); in vmsplice_to_user()
1524 ret = __splice_from_pipe(pipe, &sd, pipe_to_user); in vmsplice_to_user()
1525 pipe_unlock(pipe); in vmsplice_to_user()
1535 * vmsplice splices a user address range into a pipe. It can be thought of
1537 * to file). In both cases the output is a pipe, naturally.
1542 struct pipe_inode_info *pipe; in vmsplice_to_pipe() local
1549 pipe = get_pipe_info(file, true); in vmsplice_to_pipe()
1550 if (!pipe) in vmsplice_to_pipe()
1555 pipe_lock(pipe); in vmsplice_to_pipe()
1556 ret = wait_for_space(pipe, flags); in vmsplice_to_pipe()
1558 ret = iter_to_pipe(iter, pipe, buf_flag); in vmsplice_to_pipe()
1559 pipe_unlock(pipe); in vmsplice_to_pipe()
1561 wakeup_pipe_readers(pipe); in vmsplice_to_pipe()
1569 * to a pipe, not the other way around. Splicing from user memory is a simple
1572 * a pipe. The reverse isn't quite as easy, though. There are two possible
1578 * has restriction limitations on both ends of the pipe).
1647 static int ipipe_prep(struct pipe_inode_info *pipe, unsigned int flags) in ipipe_prep() argument
1652 * Check the pipe occupancy without the inode lock first. This function in ipipe_prep()
1655 if (!pipe_is_empty(pipe)) in ipipe_prep()
1659 pipe_lock(pipe); in ipipe_prep()
1661 while (pipe_is_empty(pipe)) { in ipipe_prep()
1666 if (!pipe->writers) in ipipe_prep()
1672 pipe_wait_readable(pipe); in ipipe_prep()
1675 pipe_unlock(pipe); in ipipe_prep()
1683 static int opipe_prep(struct pipe_inode_info *pipe, unsigned int flags) in opipe_prep() argument
1688 * Check pipe occupancy without the inode lock first. This function in opipe_prep()
1691 if (!pipe_is_full(pipe)) in opipe_prep()
1695 pipe_lock(pipe); in opipe_prep()
1697 while (pipe_is_full(pipe)) { in opipe_prep()
1698 if (!pipe->readers) { in opipe_prep()
1711 pipe_wait_writable(pipe); in opipe_prep()
1714 pipe_unlock(pipe); in opipe_prep()
1744 * grabbing by pipe info address. Otherwise two different processes in splice_pipe_to_pipe()
1772 * pipe is empty or the output pipe is full. in splice_pipe_to_pipe()
1812 * Get a reference to this pipe buffer, in splice_pipe_to_pipe()
1844 * If we put data in the output pipe, wakeup any potential readers. in splice_pipe_to_pipe()
1870 * grabbing by pipe info address. Otherwise two different processes in link_pipe()
1903 * Get a reference to this pipe buffer, in link_pipe()
1935 * If we put data in the output pipe, wakeup any potential readers. in link_pipe()
1945 * any data, it simply references the 'in' pages on the 'out' pipe.