summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--drivers/char/virtio_console.c2
-rw-r--r--fs/fuse/dev.c2
-rw-r--r--fs/pipe.c96
-rw-r--r--fs/splice.c81
-rw-r--r--include/linux/pipe_fs_i.h40
-rw-r--r--kernel/relay.c7
-rw-r--r--kernel/trace/trace.c11
-rw-r--r--net/smc/smc_rx.c8
8 files changed, 77 insertions, 170 deletions
diff --git a/drivers/char/virtio_console.c b/drivers/char/virtio_console.c
index 3cbaec925606..00c5e3acee46 100644
--- a/drivers/char/virtio_console.c
+++ b/drivers/char/virtio_console.c
@@ -871,7 +871,7 @@ static int pipe_to_sg(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
return 0;
/* Try lock this page */
- if (pipe_buf_steal(pipe, buf) == 0) {
+ if (pipe_buf_try_steal(pipe, buf)) {
/* Get reference and unlock page for moving */
get_page(buf->page);
unlock_page(buf->page);
diff --git a/fs/fuse/dev.c b/fs/fuse/dev.c
index c7a65cf2bcca..4907e3d1f56c 100644
--- a/fs/fuse/dev.c
+++ b/fs/fuse/dev.c
@@ -805,7 +805,7 @@ static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep)
if (cs->len != PAGE_SIZE)
goto out_fallback;
- if (pipe_buf_steal(cs->pipe, buf) != 0)
+ if (!pipe_buf_try_steal(cs->pipe, buf))
goto out_fallback;
newpage = buf->page;
diff --git a/fs/pipe.c b/fs/pipe.c
index 16fb72e9abf7..c7c4fb5f345f 100644
--- a/fs/pipe.c
+++ b/fs/pipe.c
@@ -140,21 +140,20 @@ static void anon_pipe_buf_release(struct pipe_inode_info *pipe,
put_page(page);
}
-static int anon_pipe_buf_steal(struct pipe_inode_info *pipe,
- struct pipe_buffer *buf)
+static bool anon_pipe_buf_try_steal(struct pipe_inode_info *pipe,
+ struct pipe_buffer *buf)
{
struct page *page = buf->page;
- if (page_count(page) == 1) {
- memcg_kmem_uncharge_page(page, 0);
- __SetPageLocked(page);
- return 0;
- }
- return 1;
+ if (page_count(page) != 1)
+ return false;
+ memcg_kmem_uncharge_page(page, 0);
+ __SetPageLocked(page);
+ return true;
}
/**
- * generic_pipe_buf_steal - attempt to take ownership of a &pipe_buffer
+ * generic_pipe_buf_try_steal - attempt to take ownership of a &pipe_buffer
* @pipe: the pipe that the buffer belongs to
* @buf: the buffer to attempt to steal
*
@@ -165,8 +164,8 @@ static int anon_pipe_buf_steal(struct pipe_inode_info *pipe,
* he wishes; the typical use is insertion into a different file
* page cache.
*/
-int generic_pipe_buf_steal(struct pipe_inode_info *pipe,
- struct pipe_buffer *buf)
+bool generic_pipe_buf_try_steal(struct pipe_inode_info *pipe,
+ struct pipe_buffer *buf)
{
struct page *page = buf->page;
@@ -177,12 +176,11 @@ int generic_pipe_buf_steal(struct pipe_inode_info *pipe,
*/
if (page_count(page) == 1) {
lock_page(page);
- return 0;
+ return true;
}
-
- return 1;
+ return false;
}
-EXPORT_SYMBOL(generic_pipe_buf_steal);
+EXPORT_SYMBOL(generic_pipe_buf_try_steal);
/**
* generic_pipe_buf_get - get a reference to a &struct pipe_buffer
@@ -201,22 +199,6 @@ bool generic_pipe_buf_get(struct pipe_inode_info *pipe, struct pipe_buffer *buf)
EXPORT_SYMBOL(generic_pipe_buf_get);
/**
- * generic_pipe_buf_confirm - verify contents of the pipe buffer
- * @info: the pipe that the buffer belongs to
- * @buf: the buffer to confirm
- *
- * Description:
- * This function does nothing, because the generic pipe code uses
- * pages that are always good when inserted into the pipe.
- */
-int generic_pipe_buf_confirm(struct pipe_inode_info *info,
- struct pipe_buffer *buf)
-{
- return 0;
-}
-EXPORT_SYMBOL(generic_pipe_buf_confirm);
-
-/**
* generic_pipe_buf_release - put a reference to a &struct pipe_buffer
* @pipe: the pipe that the buffer belongs to
* @buf: the buffer to put a reference to
@@ -231,48 +213,12 @@ void generic_pipe_buf_release(struct pipe_inode_info *pipe,
}
EXPORT_SYMBOL(generic_pipe_buf_release);
-/* New data written to a pipe may be appended to a buffer with this type. */
static const struct pipe_buf_operations anon_pipe_buf_ops = {
- .confirm = generic_pipe_buf_confirm,
- .release = anon_pipe_buf_release,
- .steal = anon_pipe_buf_steal,
- .get = generic_pipe_buf_get,
-};
-
-static const struct pipe_buf_operations anon_pipe_buf_nomerge_ops = {
- .confirm = generic_pipe_buf_confirm,
- .release = anon_pipe_buf_release,
- .steal = anon_pipe_buf_steal,
- .get = generic_pipe_buf_get,
-};
-
-static const struct pipe_buf_operations packet_pipe_buf_ops = {
- .confirm = generic_pipe_buf_confirm,
- .release = anon_pipe_buf_release,
- .steal = anon_pipe_buf_steal,
- .get = generic_pipe_buf_get,
+ .release = anon_pipe_buf_release,
+ .try_steal = anon_pipe_buf_try_steal,
+ .get = generic_pipe_buf_get,
};
-/**
- * pipe_buf_mark_unmergeable - mark a &struct pipe_buffer as unmergeable
- * @buf: the buffer to mark
- *
- * Description:
- * This function ensures that no future writes will be merged into the
- * given &struct pipe_buffer. This is necessary when multiple pipe buffers
- * share the same backing page.
- */
-void pipe_buf_mark_unmergeable(struct pipe_buffer *buf)
-{
- if (buf->ops == &anon_pipe_buf_ops)
- buf->ops = &anon_pipe_buf_nomerge_ops;
-}
-
-static bool pipe_buf_can_merge(struct pipe_buffer *buf)
-{
- return buf->ops == &anon_pipe_buf_ops;
-}
-
/* Done while waiting without holding the pipe lock - thus the READ_ONCE() */
static inline bool pipe_readable(const struct pipe_inode_info *pipe)
{
@@ -478,7 +424,8 @@ pipe_write(struct kiocb *iocb, struct iov_iter *from)
struct pipe_buffer *buf = &pipe->bufs[(head - 1) & mask];
int offset = buf->offset + buf->len;
- if (pipe_buf_can_merge(buf) && offset + chars <= PAGE_SIZE) {
+ if ((buf->flags & PIPE_BUF_FLAG_CAN_MERGE) &&
+ offset + chars <= PAGE_SIZE) {
ret = pipe_buf_confirm(pipe, buf);
if (ret)
goto out;
@@ -541,11 +488,10 @@ pipe_write(struct kiocb *iocb, struct iov_iter *from)
buf->ops = &anon_pipe_buf_ops;
buf->offset = 0;
buf->len = 0;
- buf->flags = 0;
- if (is_packetized(filp)) {
- buf->ops = &packet_pipe_buf_ops;
+ if (is_packetized(filp))
buf->flags = PIPE_BUF_FLAG_PACKET;
- }
+ else
+ buf->flags = PIPE_BUF_FLAG_CAN_MERGE;
pipe->tmp_page = NULL;
copied = copy_page_from_iter(page, 0, PAGE_SIZE, from);
diff --git a/fs/splice.c b/fs/splice.c
index 5013565eb756..6b3c9a018a8e 100644
--- a/fs/splice.c
+++ b/fs/splice.c
@@ -44,8 +44,8 @@
* addition of remove_mapping(). If success is returned, the caller may
* attempt to reuse this page for another destination.
*/
-static int page_cache_pipe_buf_steal(struct pipe_inode_info *pipe,
- struct pipe_buffer *buf)
+static bool page_cache_pipe_buf_try_steal(struct pipe_inode_info *pipe,
+ struct pipe_buffer *buf)
{
struct page *page = buf->page;
struct address_space *mapping;
@@ -76,7 +76,7 @@ static int page_cache_pipe_buf_steal(struct pipe_inode_info *pipe,
*/
if (remove_mapping(mapping, page)) {
buf->flags |= PIPE_BUF_FLAG_LRU;
- return 0;
+ return true;
}
}
@@ -86,7 +86,7 @@ static int page_cache_pipe_buf_steal(struct pipe_inode_info *pipe,
*/
out_unlock:
unlock_page(page);
- return 1;
+ return false;
}
static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe,
@@ -139,27 +139,26 @@ error:
}
const struct pipe_buf_operations page_cache_pipe_buf_ops = {
- .confirm = page_cache_pipe_buf_confirm,
- .release = page_cache_pipe_buf_release,
- .steal = page_cache_pipe_buf_steal,
- .get = generic_pipe_buf_get,
+ .confirm = page_cache_pipe_buf_confirm,
+ .release = page_cache_pipe_buf_release,
+ .try_steal = page_cache_pipe_buf_try_steal,
+ .get = generic_pipe_buf_get,
};
-static int user_page_pipe_buf_steal(struct pipe_inode_info *pipe,
- struct pipe_buffer *buf)
+static bool user_page_pipe_buf_try_steal(struct pipe_inode_info *pipe,
+ struct pipe_buffer *buf)
{
if (!(buf->flags & PIPE_BUF_FLAG_GIFT))
- return 1;
+ return false;
buf->flags |= PIPE_BUF_FLAG_LRU;
- return generic_pipe_buf_steal(pipe, buf);
+ return generic_pipe_buf_try_steal(pipe, buf);
}
static const struct pipe_buf_operations user_page_pipe_buf_ops = {
- .confirm = generic_pipe_buf_confirm,
- .release = page_cache_pipe_buf_release,
- .steal = user_page_pipe_buf_steal,
- .get = generic_pipe_buf_get,
+ .release = page_cache_pipe_buf_release,
+ .try_steal = user_page_pipe_buf_try_steal,
+ .get = generic_pipe_buf_get,
};
static void wakeup_pipe_readers(struct pipe_inode_info *pipe)
@@ -331,24 +330,15 @@ ssize_t generic_file_splice_read(struct file *in, loff_t *ppos,
EXPORT_SYMBOL(generic_file_splice_read);
const struct pipe_buf_operations default_pipe_buf_ops = {
- .confirm = generic_pipe_buf_confirm,
- .release = generic_pipe_buf_release,
- .steal = generic_pipe_buf_steal,
- .get = generic_pipe_buf_get,
+ .release = generic_pipe_buf_release,
+ .try_steal = generic_pipe_buf_try_steal,
+ .get = generic_pipe_buf_get,
};
-int generic_pipe_buf_nosteal(struct pipe_inode_info *pipe,
- struct pipe_buffer *buf)
-{
- return 1;
-}
-
/* Pipe buffer operations for a socket and similar. */
const struct pipe_buf_operations nosteal_pipe_buf_ops = {
- .confirm = generic_pipe_buf_confirm,
- .release = generic_pipe_buf_release,
- .steal = generic_pipe_buf_nosteal,
- .get = generic_pipe_buf_get,
+ .release = generic_pipe_buf_release,
+ .get = generic_pipe_buf_get,
};
EXPORT_SYMBOL(nosteal_pipe_buf_ops);
@@ -852,15 +842,9 @@ EXPORT_SYMBOL(generic_splice_sendpage);
static long do_splice_from(struct pipe_inode_info *pipe, struct file *out,
loff_t *ppos, size_t len, unsigned int flags)
{
- ssize_t (*splice_write)(struct pipe_inode_info *, struct file *,
- loff_t *, size_t, unsigned int);
-
if (out->f_op->splice_write)
- splice_write = out->f_op->splice_write;
- else
- splice_write = default_file_splice_write;
-
- return splice_write(pipe, out, ppos, len, flags);
+ return out->f_op->splice_write(pipe, out, ppos, len, flags);
+ return default_file_splice_write(pipe, out, ppos, len, flags);
}
/*
@@ -870,8 +854,6 @@ static long do_splice_to(struct file *in, loff_t *ppos,
struct pipe_inode_info *pipe, size_t len,
unsigned int flags)
{
- ssize_t (*splice_read)(struct file *, loff_t *,
- struct pipe_inode_info *, size_t, unsigned int);
int ret;
if (unlikely(!(in->f_mode & FMODE_READ)))
@@ -885,11 +867,8 @@ static long do_splice_to(struct file *in, loff_t *ppos,
len = MAX_RW_COUNT;
if (in->f_op->splice_read)
- splice_read = in->f_op->splice_read;
- else
- splice_read = default_file_splice_read;
-
- return splice_read(in, ppos, pipe, len, flags);
+ return in->f_op->splice_read(in, ppos, pipe, len, flags);
+ return default_file_splice_read(in, ppos, pipe, len, flags);
}
/**
@@ -1626,12 +1605,11 @@ retry:
*obuf = *ibuf;
/*
- * Don't inherit the gift flag, we need to
+ * Don't inherit the gift and merge flags, we need to
* prevent multiple steals of this page.
*/
obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
-
- pipe_buf_mark_unmergeable(obuf);
+ obuf->flags &= ~PIPE_BUF_FLAG_CAN_MERGE;
obuf->len = len;
ibuf->offset += len;
@@ -1719,12 +1697,11 @@ static int link_pipe(struct pipe_inode_info *ipipe,
*obuf = *ibuf;
/*
- * Don't inherit the gift flag, we need to
- * prevent multiple steals of this page.
+ * Don't inherit the gift and merge flag, we need to prevent
+ * multiple steals of this page.
*/
obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
-
- pipe_buf_mark_unmergeable(obuf);
+ obuf->flags &= ~PIPE_BUF_FLAG_CAN_MERGE;
if (obuf->len > len)
obuf->len = len;
diff --git a/include/linux/pipe_fs_i.h b/include/linux/pipe_fs_i.h
index ae58fad7f1e0..0c31b9461262 100644
--- a/include/linux/pipe_fs_i.h
+++ b/include/linux/pipe_fs_i.h
@@ -8,6 +8,7 @@
#define PIPE_BUF_FLAG_ATOMIC 0x02 /* was atomically mapped */
#define PIPE_BUF_FLAG_GIFT 0x04 /* page is a gift */
#define PIPE_BUF_FLAG_PACKET 0x08 /* read() as a packet */
+#define PIPE_BUF_FLAG_CAN_MERGE 0x10 /* can merge buffers */
/**
* struct pipe_buffer - a linux kernel pipe buffer
@@ -69,11 +70,11 @@ struct pipe_inode_info {
* Note on the nesting of these functions:
*
* ->confirm()
- * ->steal()
+ * ->try_steal()
*
- * That is, ->steal() must be called on a confirmed buffer.
- * See below for the meaning of each operation. Also see kerneldoc
- * in fs/pipe.c for the pipe and generic variants of these hooks.
+ * That is, ->try_steal() must be called on a confirmed buffer. See below for
+ * the meaning of each operation. Also see the kerneldoc in fs/pipe.c for the
+ * pipe and generic variants of these hooks.
*/
struct pipe_buf_operations {
/*
@@ -81,7 +82,7 @@ struct pipe_buf_operations {
* and that the contents are good. If the pages in the pipe belong
* to a file system, we may need to wait for IO completion in this
* hook. Returns 0 for good, or a negative error value in case of
- * error.
+ * error. If not present all pages are considered good.
*/
int (*confirm)(struct pipe_inode_info *, struct pipe_buffer *);
@@ -93,13 +94,13 @@ struct pipe_buf_operations {
/*
* Attempt to take ownership of the pipe buffer and its contents.
- * ->steal() returns 0 for success, in which case the contents
- * of the pipe (the buf->page) is locked and now completely owned
- * by the caller. The page may then be transferred to a different
- * mapping, the most often used case is insertion into different
- * file address space cache.
+ * ->try_steal() returns %true for success, in which case the contents
+ * of the pipe (the buf->page) is locked and now completely owned by the
+ * caller. The page may then be transferred to a different mapping, the
+ * most often used case is insertion into different file address space
+ * cache.
*/
- int (*steal)(struct pipe_inode_info *, struct pipe_buffer *);
+ bool (*try_steal)(struct pipe_inode_info *, struct pipe_buffer *);
/*
* Get a reference to the pipe buffer.
@@ -194,18 +195,22 @@ static inline void pipe_buf_release(struct pipe_inode_info *pipe,
static inline int pipe_buf_confirm(struct pipe_inode_info *pipe,
struct pipe_buffer *buf)
{
+ if (!buf->ops->confirm)
+ return 0;
return buf->ops->confirm(pipe, buf);
}
/**
- * pipe_buf_steal - attempt to take ownership of a pipe_buffer
+ * pipe_buf_try_steal - attempt to take ownership of a pipe_buffer
* @pipe: the pipe that the buffer belongs to
* @buf: the buffer to attempt to steal
*/
-static inline int pipe_buf_steal(struct pipe_inode_info *pipe,
- struct pipe_buffer *buf)
+static inline bool pipe_buf_try_steal(struct pipe_inode_info *pipe,
+ struct pipe_buffer *buf)
{
- return buf->ops->steal(pipe, buf);
+ if (!buf->ops->try_steal)
+ return false;
+ return buf->ops->try_steal(pipe, buf);
}
/* Differs from PIPE_BUF in that PIPE_SIZE is the length of the actual
@@ -229,11 +234,8 @@ void free_pipe_info(struct pipe_inode_info *);
/* Generic pipe buffer ops functions */
bool generic_pipe_buf_get(struct pipe_inode_info *, struct pipe_buffer *);
-int generic_pipe_buf_confirm(struct pipe_inode_info *, struct pipe_buffer *);
-int generic_pipe_buf_steal(struct pipe_inode_info *, struct pipe_buffer *);
-int generic_pipe_buf_nosteal(struct pipe_inode_info *, struct pipe_buffer *);
+bool generic_pipe_buf_try_steal(struct pipe_inode_info *, struct pipe_buffer *);
void generic_pipe_buf_release(struct pipe_inode_info *, struct pipe_buffer *);
-void pipe_buf_mark_unmergeable(struct pipe_buffer *buf);
extern const struct pipe_buf_operations nosteal_pipe_buf_ops;
diff --git a/kernel/relay.c b/kernel/relay.c
index d0c9c287680a..90c7a002436d 100644
--- a/kernel/relay.c
+++ b/kernel/relay.c
@@ -1177,10 +1177,9 @@ static void relay_pipe_buf_release(struct pipe_inode_info *pipe,
}
static const struct pipe_buf_operations relay_pipe_buf_ops = {
- .confirm = generic_pipe_buf_confirm,
- .release = relay_pipe_buf_release,
- .steal = generic_pipe_buf_steal,
- .get = generic_pipe_buf_get,
+ .release = relay_pipe_buf_release,
+ .try_steal = generic_pipe_buf_try_steal,
+ .get = generic_pipe_buf_get,
};
static void relay_page_release(struct splice_pipe_desc *spd, unsigned int i)
diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c
index f12e99b387b2..0b144926bd73 100644
--- a/kernel/trace/trace.c
+++ b/kernel/trace/trace.c
@@ -6305,13 +6305,6 @@ static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
__free_page(spd->pages[idx]);
}
-static const struct pipe_buf_operations tracing_pipe_buf_ops = {
- .confirm = generic_pipe_buf_confirm,
- .release = generic_pipe_buf_release,
- .steal = generic_pipe_buf_steal,
- .get = generic_pipe_buf_get,
-};
-
static size_t
tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
{
@@ -6373,7 +6366,7 @@ static ssize_t tracing_splice_read_pipe(struct file *filp,
.partial = partial_def,
.nr_pages = 0, /* This gets updated below. */
.nr_pages_max = PIPE_DEF_BUFFERS,
- .ops = &tracing_pipe_buf_ops,
+ .ops = &default_pipe_buf_ops,
.spd_release = tracing_spd_release_pipe,
};
ssize_t ret;
@@ -7582,9 +7575,7 @@ static bool buffer_pipe_buf_get(struct pipe_inode_info *pipe,
/* Pipe buffer operations for a buffer. */
static const struct pipe_buf_operations buffer_pipe_buf_ops = {
- .confirm = generic_pipe_buf_confirm,
.release = buffer_pipe_buf_release,
- .steal = generic_pipe_buf_nosteal,
.get = buffer_pipe_buf_get,
};
diff --git a/net/smc/smc_rx.c b/net/smc/smc_rx.c
index 39d7b34d06d2..fcfac59f8b72 100644
--- a/net/smc/smc_rx.c
+++ b/net/smc/smc_rx.c
@@ -129,16 +129,8 @@ out:
sock_put(sk);
}
-static int smc_rx_pipe_buf_nosteal(struct pipe_inode_info *pipe,
- struct pipe_buffer *buf)
-{
- return 1;
-}
-
static const struct pipe_buf_operations smc_pipe_ops = {
- .confirm = generic_pipe_buf_confirm,
.release = smc_rx_pipe_buf_release,
- .steal = smc_rx_pipe_buf_nosteal,
.get = generic_pipe_buf_get
};