From cb96a42cc1f50ba1c7b1e9b2343bec80b926107f Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Sun, 1 Jun 2014 00:43:37 +0800 Subject: blk-mq: fix schedule from atomic context blk_mq_put_ctx() has to be called before io_schedule() in bt_get(). This patch fixes the problem by taking similar approach from percpu_ida allocation for the situation. Signed-off-by: Ming Lei Signed-off-by: Jens Axboe --- block/blk-mq-tag.c | 48 +++++++++++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 17 deletions(-) (limited to 'block/blk-mq-tag.c') diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c index d90c4aeb7dd3..1aab39f71d95 100644 --- a/block/blk-mq-tag.c +++ b/block/blk-mq-tag.c @@ -221,8 +221,10 @@ static struct bt_wait_state *bt_wait_ptr(struct blk_mq_bitmap_tags *bt, return bs; } -static int bt_get(struct blk_mq_bitmap_tags *bt, struct blk_mq_hw_ctx *hctx, - unsigned int *last_tag, gfp_t gfp) +static int bt_get(struct blk_mq_alloc_data *data, + struct blk_mq_bitmap_tags *bt, + struct blk_mq_hw_ctx *hctx, + unsigned int *last_tag) { struct bt_wait_state *bs; DEFINE_WAIT(wait); @@ -232,7 +234,7 @@ static int bt_get(struct blk_mq_bitmap_tags *bt, struct blk_mq_hw_ctx *hctx, if (tag != -1) return tag; - if (!(gfp & __GFP_WAIT)) + if (!(data->gfp & __GFP_WAIT)) return -1; bs = bt_wait_ptr(bt, hctx); @@ -249,50 +251,62 @@ static int bt_get(struct blk_mq_bitmap_tags *bt, struct blk_mq_hw_ctx *hctx, if (was_empty) atomic_set(&bs->wait_cnt, bt->wake_cnt); + blk_mq_put_ctx(data->ctx); + io_schedule(); + + data->ctx = blk_mq_get_ctx(data->q); + data->hctx = data->q->mq_ops->map_queue(data->q, + data->ctx->cpu); + if (data->reserved) { + bt = &data->hctx->tags->breserved_tags; + } else { + last_tag = &data->ctx->last_tag; + hctx = data->hctx; + bt = &hctx->tags->bitmap_tags; + } + finish_wait(&bs->wait, &wait); + bs = bt_wait_ptr(bt, hctx); } while (1); finish_wait(&bs->wait, &wait); return tag; } -static unsigned int __blk_mq_get_tag(struct blk_mq_tags *tags, - struct blk_mq_hw_ctx *hctx, - unsigned int *last_tag, gfp_t gfp) +static unsigned int __blk_mq_get_tag(struct blk_mq_alloc_data *data) { int tag; - tag = bt_get(&tags->bitmap_tags, hctx, last_tag, gfp); + tag = bt_get(data, &data->hctx->tags->bitmap_tags, data->hctx, + &data->ctx->last_tag); if (tag >= 0) - return tag + tags->nr_reserved_tags; + return tag + data->hctx->tags->nr_reserved_tags; return BLK_MQ_TAG_FAIL; } -static unsigned int __blk_mq_get_reserved_tag(struct blk_mq_tags *tags, - gfp_t gfp) +static unsigned int __blk_mq_get_reserved_tag(struct blk_mq_alloc_data *data) { int tag, zero = 0; - if (unlikely(!tags->nr_reserved_tags)) { + if (unlikely(!data->hctx->tags->nr_reserved_tags)) { WARN_ON_ONCE(1); return BLK_MQ_TAG_FAIL; } - tag = bt_get(&tags->breserved_tags, NULL, &zero, gfp); + tag = bt_get(data, &data->hctx->tags->breserved_tags, NULL, &zero); if (tag < 0) return BLK_MQ_TAG_FAIL; return tag; } -unsigned int blk_mq_get_tag(struct blk_mq_hw_ctx *hctx, unsigned int *last_tag, - gfp_t gfp, bool reserved) +unsigned int blk_mq_get_tag(struct blk_mq_alloc_data *data) { - if (!reserved) - return __blk_mq_get_tag(hctx->tags, hctx, last_tag, gfp); + if (!data->reserved) + return __blk_mq_get_tag(data); - return __blk_mq_get_reserved_tag(hctx->tags, gfp); + return __blk_mq_get_reserved_tag(data); } static struct bt_wait_state *bt_wake_ptr(struct blk_mq_bitmap_tags *bt) -- cgit v1.2.3