summaryrefslogtreecommitdiffstats
path: root/block
diff options
context:
space:
mode:
authorJens Axboe <axboe@fb.com>2014-05-22 10:40:51 -0600
committerJens Axboe <axboe@fb.com>2014-05-22 10:43:07 -0600
commit07068d5b8ed8fa6759b2826ba9197e49b69a1fc3 (patch)
tree3e13c8e1028e3722be926487acae5473c1eda2be /block
parent484b4061e6683e0e6a09c7455f80781128dc8a6b (diff)
downloadlinux-07068d5b8ed8fa6759b2826ba9197e49b69a1fc3.tar.bz2
blk-mq: split make request handler for multi and single queue
We want slightly different behavior from them: - On single queue devices, we currently use the per-process plug for deferred IO and for merging. - On multi queue devices, we don't use the per-process plug, but we want to go straight to hardware for SYNC IO. Split blk_mq_make_request() into a blk_sq_make_request() for single queue devices, and retain blk_mq_make_request() for multi queue devices. Then we don't need multiple checks for q->nr_hw_queues in the request mapping. Signed-off-by: Jens Axboe <axboe@fb.com>
Diffstat (limited to 'block')
-rw-r--r--block/blk-mq.c207
1 files changed, 157 insertions, 50 deletions
diff --git a/block/blk-mq.c b/block/blk-mq.c
index 103aa1dbc000..54e78863c083 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -1072,43 +1072,57 @@ static void blk_mq_bio_to_request(struct request *rq, struct bio *bio)
blk_account_io_start(rq, 1);
}
-static void blk_mq_make_request(struct request_queue *q, struct bio *bio)
+static inline bool blk_mq_merge_queue_io(struct blk_mq_hw_ctx *hctx,
+ struct blk_mq_ctx *ctx,
+ struct request *rq, struct bio *bio)
{
- struct blk_mq_hw_ctx *hctx;
- struct blk_mq_ctx *ctx;
- const int is_sync = rw_is_sync(bio->bi_rw);
- const int is_flush_fua = bio->bi_rw & (REQ_FLUSH | REQ_FUA);
- int rw = bio_data_dir(bio);
- struct request *rq;
- unsigned int use_plug, request_count = 0;
-
- /*
- * If we have multiple hardware queues, just go directly to
- * one of those for sync IO.
- */
- use_plug = !is_flush_fua && ((q->nr_hw_queues == 1) || !is_sync);
+ struct request_queue *q = hctx->queue;
- blk_queue_bounce(q, &bio);
+ if (!(hctx->flags & BLK_MQ_F_SHOULD_MERGE)) {
+ blk_mq_bio_to_request(rq, bio);
+ spin_lock(&ctx->lock);
+insert_rq:
+ __blk_mq_insert_request(hctx, rq, false);
+ spin_unlock(&ctx->lock);
+ return false;
+ } else {
+ spin_lock(&ctx->lock);
+ if (!blk_mq_attempt_merge(q, ctx, bio)) {
+ blk_mq_bio_to_request(rq, bio);
+ goto insert_rq;
+ }
- if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
- bio_endio(bio, -EIO);
- return;
+ spin_unlock(&ctx->lock);
+ __blk_mq_free_request(hctx, ctx, rq);
+ return true;
}
+}
- if (use_plug && !blk_queue_nomerges(q) &&
- blk_attempt_plug_merge(q, bio, &request_count))
- return;
+struct blk_map_ctx {
+ struct blk_mq_hw_ctx *hctx;
+ struct blk_mq_ctx *ctx;
+};
+
+static struct request *blk_mq_map_request(struct request_queue *q,
+ struct bio *bio,
+ struct blk_map_ctx *data)
+{
+ struct blk_mq_hw_ctx *hctx;
+ struct blk_mq_ctx *ctx;
+ struct request *rq;
+ int rw = bio_data_dir(bio);
- if (blk_mq_queue_enter(q)) {
+ if (unlikely(blk_mq_queue_enter(q))) {
bio_endio(bio, -EIO);
- return;
+ return NULL;
}
ctx = blk_mq_get_ctx(q);
hctx = q->mq_ops->map_queue(q, ctx->cpu);
- if (is_sync)
+ if (rw_is_sync(bio->bi_rw))
rw |= REQ_SYNC;
+
trace_block_getrq(q, bio, rw);
rq = __blk_mq_alloc_request(hctx, ctx, GFP_ATOMIC, false);
if (likely(rq))
@@ -1123,6 +1137,109 @@ static void blk_mq_make_request(struct request_queue *q, struct bio *bio)
}
hctx->queued++;
+ data->hctx = hctx;
+ data->ctx = ctx;
+ return rq;
+}
+
+/*
+ * Multiple hardware queue variant. This will not use per-process plugs,
+ * but will attempt to bypass the hctx queueing if we can go straight to
+ * hardware for SYNC IO.
+ */
+static void blk_mq_make_request(struct request_queue *q, struct bio *bio)
+{
+ const int is_sync = rw_is_sync(bio->bi_rw);
+ const int is_flush_fua = bio->bi_rw & (REQ_FLUSH | REQ_FUA);
+ struct blk_map_ctx data;
+ struct request *rq;
+
+ blk_queue_bounce(q, &bio);
+
+ if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
+ bio_endio(bio, -EIO);
+ return;
+ }
+
+ rq = blk_mq_map_request(q, bio, &data);
+ if (unlikely(!rq))
+ return;
+
+ if (unlikely(is_flush_fua)) {
+ blk_mq_bio_to_request(rq, bio);
+ blk_insert_flush(rq);
+ goto run_queue;
+ }
+
+ if (is_sync) {
+ int ret;
+
+ blk_mq_bio_to_request(rq, bio);
+ blk_mq_start_request(rq, true);
+
+ /*
+ * For OK queue, we are done. For error, kill it. Any other
+ * error (busy), just add it to our list as we previously
+ * would have done
+ */
+ ret = q->mq_ops->queue_rq(data.hctx, rq);
+ if (ret == BLK_MQ_RQ_QUEUE_OK)
+ goto done;
+ else {
+ __blk_mq_requeue_request(rq);
+
+ if (ret == BLK_MQ_RQ_QUEUE_ERROR) {
+ rq->errors = -EIO;
+ blk_mq_end_io(rq, rq->errors);
+ goto done;
+ }
+ }
+ }
+
+ if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
+ /*
+ * For a SYNC request, send it to the hardware immediately. For
+ * an ASYNC request, just ensure that we run it later on. The
+ * latter allows for merging opportunities and more efficient
+ * dispatching.
+ */
+run_queue:
+ blk_mq_run_hw_queue(data.hctx, !is_sync || is_flush_fua);
+ }
+done:
+ blk_mq_put_ctx(data.ctx);
+}
+
+/*
+ * Single hardware queue variant. This will attempt to use any per-process
+ * plug for merging and IO deferral.
+ */
+static void blk_sq_make_request(struct request_queue *q, struct bio *bio)
+{
+ const int is_sync = rw_is_sync(bio->bi_rw);
+ const int is_flush_fua = bio->bi_rw & (REQ_FLUSH | REQ_FUA);
+ unsigned int use_plug, request_count = 0;
+ struct blk_map_ctx data;
+ struct request *rq;
+
+ /*
+ * If we have multiple hardware queues, just go directly to
+ * one of those for sync IO.
+ */
+ use_plug = !is_flush_fua && !is_sync;
+
+ blk_queue_bounce(q, &bio);
+
+ if (bio_integrity_enabled(bio) && bio_integrity_prep(bio)) {
+ bio_endio(bio, -EIO);
+ return;
+ }
+
+ if (use_plug && !blk_queue_nomerges(q) &&
+ blk_attempt_plug_merge(q, bio, &request_count))
+ return;
+
+ rq = blk_mq_map_request(q, bio, &data);
if (unlikely(is_flush_fua)) {
blk_mq_bio_to_request(rq, bio);
@@ -1147,37 +1264,23 @@ static void blk_mq_make_request(struct request_queue *q, struct bio *bio)
trace_block_plug(q);
}
list_add_tail(&rq->queuelist, &plug->mq_list);
- blk_mq_put_ctx(ctx);
+ blk_mq_put_ctx(data.ctx);
return;
}
}
- if (!(hctx->flags & BLK_MQ_F_SHOULD_MERGE)) {
- blk_mq_bio_to_request(rq, bio);
- spin_lock(&ctx->lock);
-insert_rq:
- __blk_mq_insert_request(hctx, rq, false);
- spin_unlock(&ctx->lock);
- } else {
- spin_lock(&ctx->lock);
- if (!blk_mq_attempt_merge(q, ctx, bio)) {
- blk_mq_bio_to_request(rq, bio);
- goto insert_rq;
- }
-
- spin_unlock(&ctx->lock);
- __blk_mq_free_request(hctx, ctx, rq);
+ if (!blk_mq_merge_queue_io(data.hctx, data.ctx, rq, bio)) {
+ /*
+ * For a SYNC request, send it to the hardware immediately. For
+ * an ASYNC request, just ensure that we run it later on. The
+ * latter allows for merging opportunities and more efficient
+ * dispatching.
+ */
+run_queue:
+ blk_mq_run_hw_queue(data.hctx, !is_sync || is_flush_fua);
}
-
- /*
- * For a SYNC request, send it to the hardware immediately. For an
- * ASYNC request, just ensure that we run it later on. The latter
- * allows for merging opportunities and more efficient dispatching.
- */
-run_queue:
- blk_mq_run_hw_queue(hctx, !is_sync || is_flush_fua);
- blk_mq_put_ctx(ctx);
+ blk_mq_put_ctx(data.ctx);
}
/*
@@ -1670,7 +1773,11 @@ struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set)
q->sg_reserved_size = INT_MAX;
- blk_queue_make_request(q, blk_mq_make_request);
+ if (q->nr_hw_queues > 1)
+ blk_queue_make_request(q, blk_mq_make_request);
+ else
+ blk_queue_make_request(q, blk_sq_make_request);
+
blk_queue_rq_timed_out(q, blk_mq_rq_timed_out);
if (set->timeout)
blk_queue_rq_timeout(q, set->timeout);