summaryrefslogtreecommitdiffstats
path: root/block/bio.c
diff options
context:
space:
mode:
authorJens Axboe <axboe@fb.com>2015-04-17 16:23:59 -0600
committerJens Axboe <axboe@fb.com>2015-05-05 13:32:49 -0600
commitdac56212e8127dbc0bff7be35c508bc280213309 (patch)
treea1a066d7f6deb7d66403137b8edafb394ce0ca6a /block/bio.c
parentc4cf5261f8bffd9de132b50660a69148e7575bd6 (diff)
downloadlinux-dac56212e8127dbc0bff7be35c508bc280213309.tar.bz2
bio: skip atomic inc/dec of ->bi_cnt for most use cases
Struct bio has a reference count that controls when it can be freed. Most uses cases is allocating the bio, which then returns with a single reference to it, doing IO, and then dropping that single reference. We can remove this atomic_dec_and_test() in the completion path, if nobody else is holding a reference to the bio. If someone does call bio_get() on the bio, then we flag the bio as now having valid count and that we must properly honor the reference count when it's being put. Tested-by: Robert Elliott <elliott@hp.com> Signed-off-by: Jens Axboe <axboe@fb.com>
Diffstat (limited to 'block/bio.c')
-rw-r--r--block/bio.c18
1 files changed, 11 insertions, 7 deletions
diff --git a/block/bio.c b/block/bio.c
index 117da319afb6..c2ff8a88aef1 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -271,7 +271,7 @@ void bio_init(struct bio *bio)
memset(bio, 0, sizeof(*bio));
bio->bi_flags = 1 << BIO_UPTODATE;
atomic_set(&bio->__bi_remaining, 1);
- atomic_set(&bio->bi_cnt, 1);
+ atomic_set(&bio->__bi_cnt, 1);
}
EXPORT_SYMBOL(bio_init);
@@ -524,13 +524,17 @@ EXPORT_SYMBOL(zero_fill_bio);
**/
void bio_put(struct bio *bio)
{
- BIO_BUG_ON(!atomic_read(&bio->bi_cnt));
-
- /*
- * last put frees it
- */
- if (atomic_dec_and_test(&bio->bi_cnt))
+ if (!bio_flagged(bio, BIO_REFFED))
bio_free(bio);
+ else {
+ BIO_BUG_ON(!atomic_read(&bio->__bi_cnt));
+
+ /*
+ * last put frees it
+ */
+ if (atomic_dec_and_test(&bio->__bi_cnt))
+ bio_free(bio);
+ }
}
EXPORT_SYMBOL(bio_put);