From e806402130c9c494e22c73ae9ead4e79d2a5811c Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Thu, 20 Oct 2016 15:12:13 +0200 Subject: block: split out request-only flags into a new namespace A lot of the REQ_* flags are only used on struct requests, and only of use to the block layer and a few drivers that dig into struct request internals. This patch adds a new req_flags_t rq_flags field to struct request for them, and thus dramatically shrinks the number of common requests. It also removes the unfortunate situation where we have to fit the fields from the same enum into 32 bits for struct bio and 64 bits for struct request. Signed-off-by: Christoph Hellwig Reviewed-by: Shaun Tancheff Signed-off-by: Jens Axboe --- drivers/md/dm-rq.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'drivers/md/dm-rq.c') diff --git a/drivers/md/dm-rq.c b/drivers/md/dm-rq.c index dc75bea0d541..f76cc36b8546 100644 --- a/drivers/md/dm-rq.c +++ b/drivers/md/dm-rq.c @@ -313,7 +313,7 @@ static void dm_unprep_request(struct request *rq) if (!rq->q->mq_ops) { rq->special = NULL; - rq->cmd_flags &= ~REQ_DONTPREP; + rq->rq_flags &= ~RQF_DONTPREP; } if (clone) @@ -431,7 +431,7 @@ static void dm_softirq_done(struct request *rq) return; } - if (rq->cmd_flags & REQ_FAILED) + if (rq->rq_flags & RQF_FAILED) mapped = false; dm_done(clone, tio->error, mapped); @@ -460,7 +460,7 @@ static void dm_complete_request(struct request *rq, int error) */ static void dm_kill_unmapped_request(struct request *rq, int error) { - rq->cmd_flags |= REQ_FAILED; + rq->rq_flags |= RQF_FAILED; dm_complete_request(rq, error); } @@ -476,7 +476,7 @@ static void end_clone_request(struct request *clone, int error) * For just cleaning up the information of the queue in which * the clone was dispatched. * The clone is *NOT* freed actually here because it is alloced - * from dm own mempool (REQ_ALLOCED isn't set). + * from dm own mempool (RQF_ALLOCED isn't set). */ __blk_put_request(clone->q, clone); } @@ -497,7 +497,7 @@ static void dm_dispatch_clone_request(struct request *clone, struct request *rq) int r; if (blk_queue_io_stat(clone->q)) - clone->cmd_flags |= REQ_IO_STAT; + clone->rq_flags |= RQF_IO_STAT; clone->start_time = jiffies; r = blk_insert_cloned_request(clone->q, clone); @@ -633,7 +633,7 @@ static int dm_old_prep_fn(struct request_queue *q, struct request *rq) return BLKPREP_DEFER; rq->special = tio; - rq->cmd_flags |= REQ_DONTPREP; + rq->rq_flags |= RQF_DONTPREP; return BLKPREP_OK; } -- cgit v1.2.3 From 52d7f1b5c2f33b5d34dc2b6af5175fb6a44999f6 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Fri, 28 Oct 2016 17:20:32 -0700 Subject: blk-mq: Avoid that requeueing starts stopped queues MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since blk_mq_requeue_work() starts stopped queues and since execution of this function can be scheduled after a queue has been stopped it is not possible to stop queues without using an additional state variable to track whether or not the queue has been stopped. Hence modify blk_mq_requeue_work() such that it does not start stopped queues. My conclusion after a review of the blk_mq_stop_hw_queues() and blk_mq_{delay_,}kick_requeue_list() callers is as follows: * In the dm driver starting and stopping queues should only happen if __dm_suspend() or __dm_resume() is called and not if the requeue list is processed. * In the SCSI core queue stopping and starting should only be performed by the scsi_internal_device_block() and scsi_internal_device_unblock() functions but not by any other function. Although the blk_mq_stop_hw_queue() call in scsi_queue_rq() may help to reduce CPU load if a LLD queue is full, figuring out whether or not a queue should be restarted when requeueing a command would require to introduce additional locking in scsi_mq_requeue_cmd() to avoid a race with scsi_internal_device_block(). Avoid this complexity by removing the blk_mq_stop_hw_queue() call from scsi_queue_rq(). * In the NVMe core only the functions that call blk_mq_start_stopped_hw_queues() explicitly should start stopped queues. * A blk_mq_start_stopped_hwqueues() call must be added in the xen-blkfront driver in its blkif_recover() function. Signed-off-by: Bart Van Assche Cc: Konrad Rzeszutek Wilk Cc: Roger Pau Monné Cc: Mike Snitzer Cc: James Bottomley Cc: Martin K. Petersen Reviewed-by: Sagi Grimberg Reviewed-by: Christoph Hellwig Signed-off-by: Jens Axboe --- block/blk-mq.c | 6 +----- drivers/block/xen-blkfront.c | 1 + drivers/md/dm-rq.c | 7 +------ drivers/scsi/scsi_lib.c | 1 - 4 files changed, 3 insertions(+), 12 deletions(-) (limited to 'drivers/md/dm-rq.c') diff --git a/block/blk-mq.c b/block/blk-mq.c index 447c37f39e32..d95034ae64f6 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -501,11 +501,7 @@ static void blk_mq_requeue_work(struct work_struct *work) blk_mq_insert_request(rq, false, false, false); } - /* - * Use the start variant of queue running here, so that running - * the requeue work will kick stopped queues. - */ - blk_mq_start_hw_queues(q); + blk_mq_run_hw_queues(q, false); } void blk_mq_add_to_requeue_list(struct request *rq, bool at_head) diff --git a/drivers/block/xen-blkfront.c b/drivers/block/xen-blkfront.c index 9908597c5209..71ca36eab558 100644 --- a/drivers/block/xen-blkfront.c +++ b/drivers/block/xen-blkfront.c @@ -2045,6 +2045,7 @@ static int blkif_recover(struct blkfront_info *info) BUG_ON(req->nr_phys_segments > segs); blk_mq_requeue_request(req); } + blk_mq_start_stopped_hw_queues(info->rq, true); blk_mq_kick_requeue_list(info->rq); while ((bio = bio_list_pop(&info->bio_list)) != NULL) { diff --git a/drivers/md/dm-rq.c b/drivers/md/dm-rq.c index f76cc36b8546..a9e9e781bb77 100644 --- a/drivers/md/dm-rq.c +++ b/drivers/md/dm-rq.c @@ -338,12 +338,7 @@ static void dm_old_requeue_request(struct request *rq) static void __dm_mq_kick_requeue_list(struct request_queue *q, unsigned long msecs) { - unsigned long flags; - - spin_lock_irqsave(q->queue_lock, flags); - if (!blk_queue_stopped(q)) - blk_mq_delay_kick_requeue_list(q, msecs); - spin_unlock_irqrestore(q->queue_lock, flags); + blk_mq_delay_kick_requeue_list(q, msecs); } void dm_mq_kick_requeue_list(struct mapped_device *md) diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c index 8c52622ac257..2b78ff12bf3c 100644 --- a/drivers/scsi/scsi_lib.c +++ b/drivers/scsi/scsi_lib.c @@ -1952,7 +1952,6 @@ out_put_device: out: switch (ret) { case BLK_MQ_RQ_QUEUE_BUSY: - blk_mq_stop_hw_queue(hctx); if (atomic_read(&sdev->device_busy) == 0 && !scsi_device_blocked(sdev)) blk_mq_delay_queue(hctx, SCSI_QUEUE_DELAY); -- cgit v1.2.3 From 9b7dd572cc439fa92e120290eb74d0295567c5a0 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Fri, 28 Oct 2016 17:20:49 -0700 Subject: blk-mq: Remove blk_mq_cancel_requeue_work() Since blk_mq_requeue_work() no longer restarts stopped queues canceling requeue work is no longer needed to prevent that a stopped queue would be restarted. Hence remove this function. Signed-off-by: Bart Van Assche Cc: Mike Snitzer Cc: Keith Busch Cc: Hannes Reinecke Cc: Johannes Thumshirn Reviewed-by: Sagi Grimberg Reviewed-by: Christoph Hellwig Signed-off-by: Jens Axboe --- block/blk-mq.c | 6 ------ drivers/md/dm-rq.c | 2 -- drivers/nvme/host/core.c | 1 - include/linux/blk-mq.h | 1 - 4 files changed, 10 deletions(-) (limited to 'drivers/md/dm-rq.c') diff --git a/block/blk-mq.c b/block/blk-mq.c index d95034ae64f6..a461823644fb 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -526,12 +526,6 @@ void blk_mq_add_to_requeue_list(struct request *rq, bool at_head) } EXPORT_SYMBOL(blk_mq_add_to_requeue_list); -void blk_mq_cancel_requeue_work(struct request_queue *q) -{ - cancel_delayed_work_sync(&q->requeue_work); -} -EXPORT_SYMBOL_GPL(blk_mq_cancel_requeue_work); - void blk_mq_kick_requeue_list(struct request_queue *q) { kblockd_schedule_delayed_work(&q->requeue_work, 0); diff --git a/drivers/md/dm-rq.c b/drivers/md/dm-rq.c index a9e9e781bb77..060ccc5a4b1c 100644 --- a/drivers/md/dm-rq.c +++ b/drivers/md/dm-rq.c @@ -116,8 +116,6 @@ static void dm_mq_stop_queue(struct request_queue *q) queue_flag_set(QUEUE_FLAG_STOPPED, q); spin_unlock_irqrestore(q->queue_lock, flags); - /* Avoid that requeuing could restart the queue. */ - blk_mq_cancel_requeue_work(q); blk_mq_stop_hw_queues(q); } diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 329381a28edf..a764c2aa00a1 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -2081,7 +2081,6 @@ void nvme_stop_queues(struct nvme_ctrl *ctrl) queue_flag_set(QUEUE_FLAG_STOPPED, ns->queue); spin_unlock_irq(ns->queue->queue_lock); - blk_mq_cancel_requeue_work(ns->queue); blk_mq_stop_hw_queues(ns->queue); } mutex_unlock(&ctrl->namespaces_mutex); diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index aa930009fcd3..a85a20f80aaa 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -217,7 +217,6 @@ void __blk_mq_end_request(struct request *rq, int error); void blk_mq_requeue_request(struct request *rq); void blk_mq_add_to_requeue_list(struct request *rq, bool at_head); -void blk_mq_cancel_requeue_work(struct request_queue *q); void blk_mq_kick_requeue_list(struct request_queue *q); void blk_mq_delay_kick_requeue_list(struct request_queue *q, unsigned long msecs); void blk_mq_abort_requeue_list(struct request_queue *q); -- cgit v1.2.3 From 2b053aca76b48e681be57b34ca3a8c2c10b275c5 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Fri, 28 Oct 2016 17:21:41 -0700 Subject: blk-mq: Add a kick_requeue_list argument to blk_mq_requeue_request() Most blk_mq_requeue_request() and blk_mq_add_to_requeue_list() calls are followed by kicking the requeue list. Hence add an argument to these two functions that allows to kick the requeue list. This was proposed by Christoph Hellwig. Signed-off-by: Bart Van Assche Reviewed-by: Johannes Thumshirn Reviewed-by: Christoph Hellwig Cc: Hannes Reinecke Reviewed-by: Sagi Grimberg Signed-off-by: Jens Axboe --- block/blk-flush.c | 5 +---- block/blk-mq.c | 10 +++++++--- drivers/block/xen-blkfront.c | 2 +- drivers/md/dm-rq.c | 2 +- drivers/nvme/host/core.c | 2 +- drivers/scsi/scsi_lib.c | 4 +--- include/linux/blk-mq.h | 5 +++-- 7 files changed, 15 insertions(+), 15 deletions(-) (limited to 'drivers/md/dm-rq.c') diff --git a/block/blk-flush.c b/block/blk-flush.c index d35beca18481..c486b7aa62ee 100644 --- a/block/blk-flush.c +++ b/block/blk-flush.c @@ -134,10 +134,7 @@ static void blk_flush_restore_request(struct request *rq) static bool blk_flush_queue_rq(struct request *rq, bool add_front) { if (rq->q->mq_ops) { - struct request_queue *q = rq->q; - - blk_mq_add_to_requeue_list(rq, add_front); - blk_mq_kick_requeue_list(q); + blk_mq_add_to_requeue_list(rq, add_front, true); return false; } else { if (add_front) diff --git a/block/blk-mq.c b/block/blk-mq.c index 3dc323543293..8d3de5bd4d6f 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -492,12 +492,12 @@ static void __blk_mq_requeue_request(struct request *rq) } } -void blk_mq_requeue_request(struct request *rq) +void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list) { __blk_mq_requeue_request(rq); BUG_ON(blk_queued_rq(rq)); - blk_mq_add_to_requeue_list(rq, true); + blk_mq_add_to_requeue_list(rq, true, kick_requeue_list); } EXPORT_SYMBOL(blk_mq_requeue_request); @@ -531,7 +531,8 @@ static void blk_mq_requeue_work(struct work_struct *work) blk_mq_run_hw_queues(q, false); } -void blk_mq_add_to_requeue_list(struct request *rq, bool at_head) +void blk_mq_add_to_requeue_list(struct request *rq, bool at_head, + bool kick_requeue_list) { struct request_queue *q = rq->q; unsigned long flags; @@ -550,6 +551,9 @@ void blk_mq_add_to_requeue_list(struct request *rq, bool at_head) list_add_tail(&rq->queuelist, &q->requeue_list); } spin_unlock_irqrestore(&q->requeue_lock, flags); + + if (kick_requeue_list) + blk_mq_kick_requeue_list(q); } EXPORT_SYMBOL(blk_mq_add_to_requeue_list); diff --git a/drivers/block/xen-blkfront.c b/drivers/block/xen-blkfront.c index 71ca36eab558..c000fdf048b2 100644 --- a/drivers/block/xen-blkfront.c +++ b/drivers/block/xen-blkfront.c @@ -2043,7 +2043,7 @@ static int blkif_recover(struct blkfront_info *info) /* Requeue pending requests (flush or discard) */ list_del_init(&req->queuelist); BUG_ON(req->nr_phys_segments > segs); - blk_mq_requeue_request(req); + blk_mq_requeue_request(req, false); } blk_mq_start_stopped_hw_queues(info->rq, true); blk_mq_kick_requeue_list(info->rq); diff --git a/drivers/md/dm-rq.c b/drivers/md/dm-rq.c index 060ccc5a4b1c..315257959fc0 100644 --- a/drivers/md/dm-rq.c +++ b/drivers/md/dm-rq.c @@ -347,7 +347,7 @@ EXPORT_SYMBOL(dm_mq_kick_requeue_list); static void dm_mq_delay_requeue_request(struct request *rq, unsigned long msecs) { - blk_mq_requeue_request(rq); + blk_mq_requeue_request(rq, false); __dm_mq_kick_requeue_list(rq->q, msecs); } diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index a764c2aa00a1..e8070e3cc7c7 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -203,7 +203,7 @@ void nvme_requeue_req(struct request *req) { unsigned long flags; - blk_mq_requeue_request(req); + blk_mq_requeue_request(req, false); spin_lock_irqsave(req->q->queue_lock, flags); if (!blk_queue_stopped(req->q)) blk_mq_kick_requeue_list(req->q); diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c index 2b78ff12bf3c..2e35132f8be1 100644 --- a/drivers/scsi/scsi_lib.c +++ b/drivers/scsi/scsi_lib.c @@ -86,10 +86,8 @@ scsi_set_blocked(struct scsi_cmnd *cmd, int reason) static void scsi_mq_requeue_cmd(struct scsi_cmnd *cmd) { struct scsi_device *sdev = cmd->device; - struct request_queue *q = cmd->request->q; - blk_mq_requeue_request(cmd->request); - blk_mq_kick_requeue_list(q); + blk_mq_requeue_request(cmd->request, true); put_device(&sdev->sdev_gendev); } diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index ed20ac74c62a..35a0af5ede6d 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -218,8 +218,9 @@ void blk_mq_start_request(struct request *rq); void blk_mq_end_request(struct request *rq, int error); void __blk_mq_end_request(struct request *rq, int error); -void blk_mq_requeue_request(struct request *rq); -void blk_mq_add_to_requeue_list(struct request *rq, bool at_head); +void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list); +void blk_mq_add_to_requeue_list(struct request *rq, bool at_head, + bool kick_requeue_list); void blk_mq_kick_requeue_list(struct request_queue *q); void blk_mq_delay_kick_requeue_list(struct request_queue *q, unsigned long msecs); void blk_mq_abort_requeue_list(struct request_queue *q); -- cgit v1.2.3 From f0d33ab76cfcd696328c1330e2462b5d62e74f00 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Fri, 28 Oct 2016 17:22:00 -0700 Subject: dm: Use BLK_MQ_S_STOPPED instead of QUEUE_FLAG_STOPPED in blk-mq code Instead of manipulating both QUEUE_FLAG_STOPPED and BLK_MQ_S_STOPPED in the dm start and stop queue functions, only manipulate the latter flag. Change blk_queue_stopped() tests into blk_mq_queue_stopped(). Signed-off-by: Bart Van Assche Acked-by: Mike Snitzer Reviewed-by: Christoph Hellwig Reviewed-by: Hannes Reinecke Reviewed-by: Johannes Thumshirn Signed-off-by: Jens Axboe --- drivers/md/dm-rq.c | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) (limited to 'drivers/md/dm-rq.c') diff --git a/drivers/md/dm-rq.c b/drivers/md/dm-rq.c index 315257959fc0..09c958b6f038 100644 --- a/drivers/md/dm-rq.c +++ b/drivers/md/dm-rq.c @@ -75,12 +75,6 @@ static void dm_old_start_queue(struct request_queue *q) static void dm_mq_start_queue(struct request_queue *q) { - unsigned long flags; - - spin_lock_irqsave(q->queue_lock, flags); - queue_flag_clear(QUEUE_FLAG_STOPPED, q); - spin_unlock_irqrestore(q->queue_lock, flags); - blk_mq_start_stopped_hw_queues(q, true); blk_mq_kick_requeue_list(q); } @@ -105,16 +99,8 @@ static void dm_old_stop_queue(struct request_queue *q) static void dm_mq_stop_queue(struct request_queue *q) { - unsigned long flags; - - spin_lock_irqsave(q->queue_lock, flags); - if (blk_queue_stopped(q)) { - spin_unlock_irqrestore(q->queue_lock, flags); + if (blk_mq_queue_stopped(q)) return; - } - - queue_flag_set(QUEUE_FLAG_STOPPED, q); - spin_unlock_irqrestore(q->queue_lock, flags); blk_mq_stop_hw_queues(q); } -- cgit v1.2.3 From 7b17c2f7292ba1f3f98dae3f7077f9e569653276 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Fri, 28 Oct 2016 17:22:16 -0700 Subject: dm: Fix a race condition related to stopping and starting queues Ensure that all ongoing dm_mq_queue_rq() and dm_mq_requeue_request() calls have stopped before setting the "queue stopped" flag. This allows to remove the "queue stopped" test from dm_mq_queue_rq() and dm_mq_requeue_request(). This patch fixes a race condition because dm_mq_queue_rq() is called without holding the queue lock and hence BLK_MQ_S_STOPPED can be set at any time while dm_mq_queue_rq() is in progress. This patch prevents that the following hang occurs sporadically when using dm-mq: INFO: task systemd-udevd:10111 blocked for more than 480 seconds. Call Trace: [] schedule+0x37/0x90 [] schedule_timeout+0x27f/0x470 [] io_schedule_timeout+0x9f/0x110 [] bit_wait_io+0x16/0x60 [] __wait_on_bit_lock+0x49/0xa0 [] __lock_page+0xb9/0xc0 [] truncate_inode_pages_range+0x3e0/0x760 [] truncate_inode_pages+0x10/0x20 [] kill_bdev+0x30/0x40 [] __blkdev_put+0x71/0x360 [] blkdev_put+0x49/0x170 [] blkdev_close+0x20/0x30 [] __fput+0xe8/0x1f0 [] ____fput+0x9/0x10 [] task_work_run+0x83/0xb0 [] do_exit+0x3ee/0xc40 [] do_group_exit+0x4b/0xc0 [] get_signal+0x2ca/0x940 [] do_signal+0x23/0x660 [] exit_to_usermode_loop+0x73/0xb0 [] syscall_return_slowpath+0xb0/0xc0 [] entry_SYSCALL_64_fastpath+0xa6/0xa8 Signed-off-by: Bart Van Assche Acked-by: Mike Snitzer Reviewed-by: Hannes Reinecke Reviewed-by: Johannes Thumshirn Reviewed-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/md/dm-rq.c | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) (limited to 'drivers/md/dm-rq.c') diff --git a/drivers/md/dm-rq.c b/drivers/md/dm-rq.c index 09c958b6f038..8b92e066bb69 100644 --- a/drivers/md/dm-rq.c +++ b/drivers/md/dm-rq.c @@ -102,7 +102,7 @@ static void dm_mq_stop_queue(struct request_queue *q) if (blk_mq_queue_stopped(q)) return; - blk_mq_stop_hw_queues(q); + blk_mq_quiesce_queue(q); } void dm_stop_queue(struct request_queue *q) @@ -880,17 +880,6 @@ static int dm_mq_queue_rq(struct blk_mq_hw_ctx *hctx, dm_put_live_table(md, srcu_idx); } - /* - * On suspend dm_stop_queue() handles stopping the blk-mq - * request_queue BUT: even though the hw_queues are marked - * BLK_MQ_S_STOPPED at that point there is still a race that - * is allowing block/blk-mq.c to call ->queue_rq against a - * hctx that it really shouldn't. The following check guards - * against this rarity (albeit _not_ race-free). - */ - if (unlikely(test_bit(BLK_MQ_S_STOPPED, &hctx->state))) - return BLK_MQ_RQ_QUEUE_BUSY; - if (ti->type->busy && ti->type->busy(ti)) return BLK_MQ_RQ_QUEUE_BUSY; -- cgit v1.2.3