From c76541a932113faa5b3be65a3f0a3d4ebbc86aeb Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Fri, 19 Dec 2014 17:54:13 -0700 Subject: blk-mq: Exit queue on alloc failure Fixes usage counter when a request could not be allocated. Signed-off-by: Keith Busch Signed-off-by: Jens Axboe --- block/blk-mq.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'block') diff --git a/block/blk-mq.c b/block/blk-mq.c index da1ab5641227..bfe0f1f9cfa0 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -258,8 +258,10 @@ struct request *blk_mq_alloc_request(struct request_queue *q, int rw, gfp_t gfp, ctx = alloc_data.ctx; } blk_mq_put_ctx(ctx); - if (!rq) + if (!rq) { + blk_mq_queue_exit(q); return ERR_PTR(-EWOULDBLOCK); + } return rq; } EXPORT_SYMBOL(blk_mq_alloc_request); -- cgit v1.2.3 From b4c6a028774bcf3f20ed1e66c27a05aa51a8cf55 Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Fri, 19 Dec 2014 17:54:14 -0700 Subject: blk-mq: Export freeze/unfreeze functions Let drivers prevent entering a queue that isn't available. Signed-off-by: Keith Busch Signed-off-by: Jens Axboe --- block/blk-mq.c | 6 ++++-- include/linux/blk-mq.h | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) (limited to 'block') diff --git a/block/blk-mq.c b/block/blk-mq.c index bfe0f1f9cfa0..97ebb84b5633 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -107,7 +107,7 @@ static void blk_mq_usage_counter_release(struct percpu_ref *ref) wake_up_all(&q->mq_freeze_wq); } -static void blk_mq_freeze_queue_start(struct request_queue *q) +void blk_mq_freeze_queue_start(struct request_queue *q) { bool freeze; @@ -120,6 +120,7 @@ static void blk_mq_freeze_queue_start(struct request_queue *q) blk_mq_run_queues(q, false); } } +EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_start); static void blk_mq_freeze_queue_wait(struct request_queue *q) { @@ -136,7 +137,7 @@ void blk_mq_freeze_queue(struct request_queue *q) blk_mq_freeze_queue_wait(q); } -static void blk_mq_unfreeze_queue(struct request_queue *q) +void blk_mq_unfreeze_queue(struct request_queue *q) { bool wake; @@ -149,6 +150,7 @@ static void blk_mq_unfreeze_queue(struct request_queue *q) wake_up_all(&q->mq_freeze_wq); } } +EXPORT_SYMBOL_GPL(blk_mq_unfreeze_queue); bool blk_mq_can_queue(struct blk_mq_hw_ctx *hctx) { diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index 8aded9ab2e4e..3b43f509432c 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -212,6 +212,8 @@ void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async); void blk_mq_delay_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs); void blk_mq_tag_busy_iter(struct blk_mq_hw_ctx *hctx, busy_iter_fn *fn, void *priv); +void blk_mq_unfreeze_queue(struct request_queue *q); +void blk_mq_freeze_queue_start(struct request_queue *q); /* * Driver command data is immediately after the request. So subtract request -- cgit v1.2.3 From aed3ea94bdd2ac0a21ed0103d34097e202ee77f6 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Mon, 22 Dec 2014 14:04:42 -0700 Subject: block: wake up waiters when a queue is marked dying If it's dying, we can't expect new request to complete and come in an wake up other tasks waiting for requests. So after we have marked it as dying, wake up everybody currently waiting for a request. Once they wake, they will retry their allocation and fail appropriately due to the state of the queue. Tested-by: Keith Busch Signed-off-by: Jens Axboe --- block/blk-core.c | 21 ++++++++++++++++++++- block/blk-mq-tag.c | 14 ++++++++++---- block/blk-mq-tag.h | 1 + block/blk-mq.c | 10 ++++++++++ block/blk-mq.h | 1 + 5 files changed, 42 insertions(+), 5 deletions(-) (limited to 'block') diff --git a/block/blk-core.c b/block/blk-core.c index 30f6153a40c2..3ad405571dcc 100644 --- a/block/blk-core.c +++ b/block/blk-core.c @@ -473,6 +473,25 @@ void blk_queue_bypass_end(struct request_queue *q) } EXPORT_SYMBOL_GPL(blk_queue_bypass_end); +void blk_set_queue_dying(struct request_queue *q) +{ + queue_flag_set_unlocked(QUEUE_FLAG_DYING, q); + + if (q->mq_ops) + blk_mq_wake_waiters(q); + else { + struct request_list *rl; + + blk_queue_for_each_rl(rl, q) { + if (rl->rq_pool) { + wake_up(&rl->wait[BLK_RW_SYNC]); + wake_up(&rl->wait[BLK_RW_ASYNC]); + } + } + } +} +EXPORT_SYMBOL_GPL(blk_set_queue_dying); + /** * blk_cleanup_queue - shutdown a request queue * @q: request queue to shutdown @@ -486,7 +505,7 @@ void blk_cleanup_queue(struct request_queue *q) /* mark @q DYING, no new request or merges will be allowed afterwards */ mutex_lock(&q->sysfs_lock); - queue_flag_set_unlocked(QUEUE_FLAG_DYING, q); + blk_set_queue_dying(q); spin_lock_irq(lock); /* diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c index 32e8dbb9ad1c..60c9d4a93fe4 100644 --- a/block/blk-mq-tag.c +++ b/block/blk-mq-tag.c @@ -68,9 +68,9 @@ bool __blk_mq_tag_busy(struct blk_mq_hw_ctx *hctx) } /* - * Wakeup all potentially sleeping on normal (non-reserved) tags + * Wakeup all potentially sleeping on tags */ -static void blk_mq_tag_wakeup_all(struct blk_mq_tags *tags) +void blk_mq_tag_wakeup_all(struct blk_mq_tags *tags, bool include_reserve) { struct blk_mq_bitmap_tags *bt; int i, wake_index; @@ -85,6 +85,12 @@ static void blk_mq_tag_wakeup_all(struct blk_mq_tags *tags) wake_index = bt_index_inc(wake_index); } + + if (include_reserve) { + bt = &tags->breserved_tags; + if (waitqueue_active(&bt->bs[0].wait)) + wake_up(&bt->bs[0].wait); + } } /* @@ -100,7 +106,7 @@ void __blk_mq_tag_idle(struct blk_mq_hw_ctx *hctx) atomic_dec(&tags->active_queues); - blk_mq_tag_wakeup_all(tags); + blk_mq_tag_wakeup_all(tags, false); } /* @@ -584,7 +590,7 @@ int blk_mq_tag_update_depth(struct blk_mq_tags *tags, unsigned int tdepth) * static and should never need resizing. */ bt_update_count(&tags->bitmap_tags, tdepth); - blk_mq_tag_wakeup_all(tags); + blk_mq_tag_wakeup_all(tags, false); return 0; } diff --git a/block/blk-mq-tag.h b/block/blk-mq-tag.h index 6206ed17ef76..a6fa0fc9d41a 100644 --- a/block/blk-mq-tag.h +++ b/block/blk-mq-tag.h @@ -54,6 +54,7 @@ extern bool blk_mq_has_free_tags(struct blk_mq_tags *tags); extern ssize_t blk_mq_tag_sysfs_show(struct blk_mq_tags *tags, char *page); extern void blk_mq_tag_init_last_tag(struct blk_mq_tags *tags, unsigned int *last_tag); extern int blk_mq_tag_update_depth(struct blk_mq_tags *tags, unsigned int depth); +extern void blk_mq_tag_wakeup_all(struct blk_mq_tags *tags, bool); enum { BLK_MQ_TAG_CACHE_MIN = 1, diff --git a/block/blk-mq.c b/block/blk-mq.c index 97ebb84b5633..1a41d7aefbd5 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -152,6 +152,16 @@ void blk_mq_unfreeze_queue(struct request_queue *q) } EXPORT_SYMBOL_GPL(blk_mq_unfreeze_queue); +void blk_mq_wake_waiters(struct request_queue *q) +{ + struct blk_mq_hw_ctx *hctx; + unsigned int i; + + queue_for_each_hw_ctx(q, hctx, i) + if (blk_mq_hw_queue_mapped(hctx)) + blk_mq_tag_wakeup_all(hctx->tags, true); +} + bool blk_mq_can_queue(struct blk_mq_hw_ctx *hctx) { return blk_mq_has_free_tags(hctx->tags); diff --git a/block/blk-mq.h b/block/blk-mq.h index 206230e64f79..4f4f943c22c3 100644 --- a/block/blk-mq.h +++ b/block/blk-mq.h @@ -32,6 +32,7 @@ void blk_mq_free_queue(struct request_queue *q); void blk_mq_clone_flush_request(struct request *flush_rq, struct request *orig_rq); int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr); +void blk_mq_wake_waiters(struct request_queue *q); /* * CPU hotplug helpers -- cgit v1.2.3 From 17ded320706c6316376059cfbe9dccab32c62b42 Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 7 Jan 2015 10:44:04 -0700 Subject: blk-mq: get rid of ->cmd_size in the hardware queue We store it in the tag set, we don't need it in the hardware queue. While removing cmd_size, place ->queue_num further down to avoid a hole on 64-bit archs. It's not used in any fast paths, so we can safely move it. Signed-off-by: Jens Axboe --- block/blk-mq.c | 1 - include/linux/blk-mq.h | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) (limited to 'block') diff --git a/block/blk-mq.c b/block/blk-mq.c index 1a41d7aefbd5..cbd2a55d69b8 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -1615,7 +1615,6 @@ static int blk_mq_init_hctx(struct request_queue *q, hctx->queue = q; hctx->queue_num = hctx_idx; hctx->flags = set->flags; - hctx->cmd_size = set->cmd_size; blk_mq_init_cpu_notifier(&hctx->cpu_notifier, blk_mq_hctx_notify, hctx); diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index 3b43f509432c..d0de259e92b2 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -34,7 +34,6 @@ struct blk_mq_hw_ctx { unsigned long flags; /* BLK_MQ_F_* flags */ struct request_queue *queue; - unsigned int queue_num; struct blk_flush_queue *fq; void *driver_data; @@ -54,7 +53,7 @@ struct blk_mq_hw_ctx { unsigned long dispatched[BLK_MQ_MAX_DISPATCH_ORDER]; unsigned int numa_node; - unsigned int cmd_size; /* per-request extra data */ + unsigned int queue_num; atomic_t nr_active; -- cgit v1.2.3 From 3fd5940cb2e47a61dcff4108f39be4c72cb4dace Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Thu, 8 Jan 2015 08:53:56 -0700 Subject: blk-mq: Wake tasks entering queue on dying When the queue is set to dying, wake up tasks that are waiting on frozen queue so they realize it is dying and abandon their request. Signed-off-by: Keith Busch Modified by me to add a code comment on the need for the wakeup. Signed-off-by: Jens Axboe --- block/blk-mq.c | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'block') diff --git a/block/blk-mq.c b/block/blk-mq.c index cbd2a55d69b8..aefed96369dd 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -160,6 +160,13 @@ void blk_mq_wake_waiters(struct request_queue *q) queue_for_each_hw_ctx(q, hctx, i) if (blk_mq_hw_queue_mapped(hctx)) blk_mq_tag_wakeup_all(hctx->tags, true); + + /* + * If we are called because the queue has now been marked as + * dying, we need to ensure that processes currently waiting on + * the queue are notified as well. + */ + wake_up_all(&q->mq_freeze_wq); } bool blk_mq_can_queue(struct blk_mq_hw_ctx *hctx) -- cgit v1.2.3 From 973c01919bce7e3559b62a856b29211ec5ac325c Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Wed, 7 Jan 2015 18:55:43 -0700 Subject: blk-mq: Export if requests were started Drivers can iterate over all allocated request tags, but their callback needs a way to know if the driver started the request in the first place. Signed-off-by: Keith Busch Signed-off-by: Jens Axboe --- block/blk-mq.c | 6 ++++++ include/linux/blk-mq.h | 1 + 2 files changed, 7 insertions(+) (limited to 'block') diff --git a/block/blk-mq.c b/block/blk-mq.c index aefed96369dd..ce4811667d36 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -404,6 +404,12 @@ void blk_mq_complete_request(struct request *rq) } EXPORT_SYMBOL(blk_mq_complete_request); +int blk_mq_request_started(struct request *rq) +{ + return test_bit(REQ_ATOM_STARTED, &rq->atomic_flags); +} +EXPORT_SYMBOL_GPL(blk_mq_request_started); + void blk_mq_start_request(struct request *rq) { struct request_queue *q = rq->q; diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index d0de259e92b2..addf8a19d806 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -194,6 +194,7 @@ static inline u16 blk_mq_unique_tag_to_tag(u32 unique_tag) struct blk_mq_hw_ctx *blk_mq_map_queue(struct request_queue *, const int ctx_index); struct blk_mq_hw_ctx *blk_mq_alloc_single_hw_queue(struct blk_mq_tag_set *, unsigned int, int); +int blk_mq_request_started(struct request *rq); void blk_mq_start_request(struct request *rq); void blk_mq_end_request(struct request *rq, int error); void __blk_mq_end_request(struct request *rq, int error); -- cgit v1.2.3 From c68ed59f534c318716c6189050af3c5ea03b8071 Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Wed, 7 Jan 2015 18:55:44 -0700 Subject: blk-mq: Let drivers cancel requeue_work Kicking requeued requests will start h/w queues in a work_queue, which may alter the driver's requested state to temporarily stop them. This patch exports a method to cancel the q->requeue_work so a driver can be assured stopped h/w queues won't be started up before it is ready. Signed-off-by: Keith Busch Signed-off-by: Jens Axboe --- block/blk-mq.c | 6 ++++++ include/linux/blk-mq.h | 1 + 2 files changed, 7 insertions(+) (limited to 'block') diff --git a/block/blk-mq.c b/block/blk-mq.c index ce4811667d36..e73a5dd89fc4 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -527,6 +527,12 @@ void blk_mq_add_to_requeue_list(struct request *rq, bool at_head) } EXPORT_SYMBOL(blk_mq_add_to_requeue_list); +void blk_mq_cancel_requeue_work(struct request_queue *q) +{ + cancel_work_sync(&q->requeue_work); +} +EXPORT_SYMBOL_GPL(blk_mq_cancel_requeue_work); + void blk_mq_kick_requeue_list(struct request_queue *q) { kblockd_schedule_work(&q->requeue_work); diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index addf8a19d806..18684e0bdb8a 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -201,6 +201,7 @@ void __blk_mq_end_request(struct request *rq, int error); void blk_mq_requeue_request(struct request *rq); void blk_mq_add_to_requeue_list(struct request *rq, bool at_head); +void blk_mq_cancel_requeue_work(struct request_queue *q); void blk_mq_kick_requeue_list(struct request_queue *q); void blk_mq_complete_request(struct request *rq); -- cgit v1.2.3 From 1885b24d23716e09b9c952822b05fd7f68099cdb Mon Sep 17 00:00:00 2001 From: Jens Axboe Date: Wed, 7 Jan 2015 18:55:45 -0700 Subject: blk-mq: Add helper to abort requeued requests Adds a helper function a driver can use to abort requeued requests in case any are pending when h/w queues are being removed. Signed-off-by: Jens Axboe --- block/blk-mq.c | 20 ++++++++++++++++++++ include/linux/blk-mq.h | 1 + 2 files changed, 21 insertions(+) (limited to 'block') diff --git a/block/blk-mq.c b/block/blk-mq.c index e73a5dd89fc4..261ccd89e15d 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -539,6 +539,26 @@ void blk_mq_kick_requeue_list(struct request_queue *q) } EXPORT_SYMBOL(blk_mq_kick_requeue_list); +void blk_mq_abort_requeue_list(struct request_queue *q) +{ + unsigned long flags; + LIST_HEAD(rq_list); + + spin_lock_irqsave(&q->requeue_lock, flags); + list_splice_init(&q->requeue_list, &rq_list); + spin_unlock_irqrestore(&q->requeue_lock, flags); + + while (!list_empty(&rq_list)) { + struct request *rq; + + rq = list_first_entry(&rq_list, struct request, queuelist); + list_del_init(&rq->queuelist); + rq->errors = -EIO; + blk_mq_end_request(rq, rq->errors); + } +} +EXPORT_SYMBOL(blk_mq_abort_requeue_list); + static inline bool is_flush_request(struct request *rq, struct blk_flush_queue *fq, unsigned int tag) { diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index 18684e0bdb8a..5735e7130d63 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -203,6 +203,7 @@ void blk_mq_requeue_request(struct request *rq); void blk_mq_add_to_requeue_list(struct request *rq, bool at_head); void blk_mq_cancel_requeue_work(struct request_queue *q); void blk_mq_kick_requeue_list(struct request_queue *q); +void blk_mq_abort_requeue_list(struct request_queue *q); void blk_mq_complete_request(struct request *rq); void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx); -- cgit v1.2.3 From 5b3f25fc343690cafd3e27431a69a7bdaf9df001 Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Wed, 7 Jan 2015 18:55:46 -0700 Subject: blk-mq: Allow requests to never expire Some types of requests may be started that are not gauranteed to ever complete. This adds a request flag that a driver can use so mark the request as such. Signed-off-by: Keith Busch Signed-off-by: Jens Axboe --- block/blk-mq.c | 4 +++- block/blk-timeout.c | 3 +++ include/linux/blk_types.h | 2 ++ 3 files changed, 8 insertions(+), 1 deletion(-) (limited to 'block') diff --git a/block/blk-mq.c b/block/blk-mq.c index 261ccd89e15d..78324a2f0e10 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -619,7 +619,7 @@ void blk_mq_rq_timed_out(struct request *req, bool reserved) break; } } - + static void blk_mq_check_expired(struct blk_mq_hw_ctx *hctx, struct request *rq, void *priv, bool reserved) { @@ -627,6 +627,8 @@ static void blk_mq_check_expired(struct blk_mq_hw_ctx *hctx, if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) return; + if (rq->cmd_flags & REQ_NO_TIMEOUT) + return; if (time_after_eq(jiffies, rq->deadline)) { if (!blk_mark_rq_complete(rq)) diff --git a/block/blk-timeout.c b/block/blk-timeout.c index 56c025894cdf..246dfb16c3d9 100644 --- a/block/blk-timeout.c +++ b/block/blk-timeout.c @@ -190,6 +190,9 @@ void blk_add_timer(struct request *req) struct request_queue *q = req->q; unsigned long expiry; + if (req->cmd_flags & REQ_NO_TIMEOUT) + return; + /* blk-mq has its own handler, so we don't need ->rq_timed_out_fn */ if (!q->mq_ops && !q->rq_timed_out_fn) return; diff --git a/include/linux/blk_types.h b/include/linux/blk_types.h index 445d59231bc4..c294e3e25e37 100644 --- a/include/linux/blk_types.h +++ b/include/linux/blk_types.h @@ -190,6 +190,7 @@ enum rq_flag_bits { __REQ_PM, /* runtime pm request */ __REQ_HASHED, /* on IO scheduler merge hash */ __REQ_MQ_INFLIGHT, /* track inflight for MQ */ + __REQ_NO_TIMEOUT, /* requests may never expire */ __REQ_NR_BITS, /* stops here */ }; @@ -243,5 +244,6 @@ enum rq_flag_bits { #define REQ_PM (1ULL << __REQ_PM) #define REQ_HASHED (1ULL << __REQ_HASHED) #define REQ_MQ_INFLIGHT (1ULL << __REQ_MQ_INFLIGHT) +#define REQ_NO_TIMEOUT (1ULL << __REQ_NO_TIMEOUT) #endif /* __LINUX_BLK_TYPES_H */ -- cgit v1.2.3 From eb130dbfc40eabcd4e10797310bda6b9f6dd7e76 Mon Sep 17 00:00:00 2001 From: Keith Busch Date: Thu, 8 Jan 2015 08:59:53 -0700 Subject: blk-mq: End unstarted requests on a dying queue Requests that haven't been started prior to a queue dying can be ended in error without waiting for them to start and time out. Signed-off-by: Keith Busch Added code comment to explain why this is done. Signed-off-by: Jens Axboe --- block/blk-mq.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'block') diff --git a/block/blk-mq.c b/block/blk-mq.c index 78324a2f0e10..2f95747c287e 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -625,8 +625,17 @@ static void blk_mq_check_expired(struct blk_mq_hw_ctx *hctx, { struct blk_mq_timeout_data *data = priv; - if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) + if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) { + /* + * If a request wasn't started before the queue was + * marked dying, kill it here or it'll go unnoticed. + */ + if (unlikely(blk_queue_dying(rq->q))) { + rq->errors = -EIO; + blk_mq_complete_request(rq); + } return; + } if (rq->cmd_flags & REQ_NO_TIMEOUT) return; -- cgit v1.2.3