summaryrefslogtreecommitdiffstats
path: root/block/bio.c
diff options
context:
space:
mode:
authorJens Axboe <axboe@fb.com>2015-04-17 16:15:18 -0600
committerJens Axboe <axboe@fb.com>2015-05-05 13:32:47 -0600
commitc4cf5261f8bffd9de132b50660a69148e7575bd6 (patch)
treec3ff2d0d2db5f7392e8651b4f0d2958895faf8b2 /block/bio.c
parentd9cee5d4f66ef36f69b0108dedbad7f7009bb6a8 (diff)
downloadlinux-c4cf5261f8bffd9de132b50660a69148e7575bd6.tar.bz2
bio: skip atomic inc/dec of ->bi_remaining for non-chains
Struct bio has an atomic ref count for chained bio's, and we use this to know when to end IO on the bio. However, most bio's are not chained, so we don't need to always introduce this atomic operation as part of ending IO. Add a helper to elevate the bi_remaining count, and flag the bio as now actually needing the decrement at end_io time. Rename the field to __bi_remaining to catch any current users of this doing the incrementing manually. For high IOPS workloads, this reduces the overhead of bio_endio() substantially. Tested-by: Robert Elliott <elliott@hp.com> Acked-by: Kent Overstreet <kent.overstreet@gmail.com> Reviewed-by: Jan Kara <jack@suse.cz> Signed-off-by: Jens Axboe <axboe@fb.com>
Diffstat (limited to 'block/bio.c')
-rw-r--r--block/bio.c38
1 files changed, 29 insertions, 9 deletions
diff --git a/block/bio.c b/block/bio.c
index f66a4eae16ee..117da319afb6 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -270,7 +270,7 @@ void bio_init(struct bio *bio)
{
memset(bio, 0, sizeof(*bio));
bio->bi_flags = 1 << BIO_UPTODATE;
- atomic_set(&bio->bi_remaining, 1);
+ atomic_set(&bio->__bi_remaining, 1);
atomic_set(&bio->bi_cnt, 1);
}
EXPORT_SYMBOL(bio_init);
@@ -292,8 +292,8 @@ void bio_reset(struct bio *bio)
__bio_free(bio);
memset(bio, 0, BIO_RESET_BYTES);
- bio->bi_flags = flags|(1 << BIO_UPTODATE);
- atomic_set(&bio->bi_remaining, 1);
+ bio->bi_flags = flags | (1 << BIO_UPTODATE);
+ atomic_set(&bio->__bi_remaining, 1);
}
EXPORT_SYMBOL(bio_reset);
@@ -320,7 +320,7 @@ void bio_chain(struct bio *bio, struct bio *parent)
bio->bi_private = parent;
bio->bi_end_io = bio_chain_endio;
- atomic_inc(&parent->bi_remaining);
+ bio_inc_remaining(parent);
}
EXPORT_SYMBOL(bio_chain);
@@ -1741,6 +1741,23 @@ void bio_flush_dcache_pages(struct bio *bi)
EXPORT_SYMBOL(bio_flush_dcache_pages);
#endif
+static inline bool bio_remaining_done(struct bio *bio)
+{
+ /*
+ * If we're not chaining, then ->__bi_remaining is always 1 and
+ * we always end io on the first invocation.
+ */
+ if (!bio_flagged(bio, BIO_CHAIN))
+ return true;
+
+ BUG_ON(atomic_read(&bio->__bi_remaining) <= 0);
+
+ if (atomic_dec_and_test(&bio->__bi_remaining))
+ return true;
+
+ return false;
+}
+
/**
* bio_endio - end I/O on a bio
* @bio: bio
@@ -1758,15 +1775,13 @@ EXPORT_SYMBOL(bio_flush_dcache_pages);
void bio_endio(struct bio *bio, int error)
{
while (bio) {
- BUG_ON(atomic_read(&bio->bi_remaining) <= 0);
-
if (error)
clear_bit(BIO_UPTODATE, &bio->bi_flags);
else if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
error = -EIO;
- if (!atomic_dec_and_test(&bio->bi_remaining))
- return;
+ if (unlikely(!bio_remaining_done(bio)))
+ break;
/*
* Need to have a real endio function for chained bios,
@@ -1799,7 +1814,12 @@ EXPORT_SYMBOL(bio_endio);
**/
void bio_endio_nodec(struct bio *bio, int error)
{
- atomic_inc(&bio->bi_remaining);
+ /*
+ * If it's not flagged as a chain, we are not going to dec the count
+ */
+ if (bio_flagged(bio, BIO_CHAIN))
+ bio_inc_remaining(bio);
+
bio_endio(bio, error);
}
EXPORT_SYMBOL(bio_endio_nodec);