summaryrefslogtreecommitdiffstats
path: root/block/blk-mq.c
diff options
context:
space:
mode:
authorLinus Torvalds <torvalds@linux-foundation.org>2021-04-28 17:22:10 -0700
committerLinus Torvalds <torvalds@linux-foundation.org>2021-04-28 17:22:10 -0700
commitd72cd4ad4174cfd2257c426ad51e4f53bcfde9c9 (patch)
treeb291d1c28bbf6ce61edc3bdf022ea857414230f6 /block/blk-mq.c
parent238da4d004856ac5f832899f6f3fa27c0102381f (diff)
parent7a3beeae289385f7be9f61a33a6e4f6c7e2400d3 (diff)
downloadlinux-d72cd4ad4174cfd2257c426ad51e4f53bcfde9c9.tar.bz2
Merge tag 'scsi-misc' of git://git.kernel.org/pub/scm/linux/kernel/git/jejb/scsi
Pull SCSI updates from James Bottomley: "This consists of the usual driver updates (ufs, target, tcmu, smartpqi, lpfc, zfcp, qla2xxx, mpt3sas, pm80xx). The major core change is using a sbitmap instead of an atomic for queue tracking" * tag 'scsi-misc' of git://git.kernel.org/pub/scm/linux/kernel/git/jejb/scsi: (412 commits) scsi: target: tcm_fc: Fix a kernel-doc header scsi: target: Shorten ALUA error messages scsi: target: Fix two format specifiers scsi: target: Compare explicitly with SAM_STAT_GOOD scsi: sd: Introduce a new local variable in sd_check_events() scsi: dc395x: Open-code status_byte(u8) calls scsi: 53c700: Open-code status_byte(u8) calls scsi: smartpqi: Remove unused functions scsi: qla4xxx: Remove an unused function scsi: myrs: Remove unused functions scsi: myrb: Remove unused functions scsi: mpt3sas: Fix two kernel-doc headers scsi: fcoe: Suppress a compiler warning scsi: libfc: Fix a format specifier scsi: aacraid: Remove an unused function scsi: core: Introduce enum scsi_disposition scsi: core: Modify the scsi_send_eh_cmnd() return value for the SDEV_BLOCK case scsi: core: Rename scsi_softirq_done() into scsi_complete() scsi: core: Remove an incorrect comment scsi: core: Make the scsi_alloc_sgtables() documentation more accurate ...
Diffstat (limited to 'block/blk-mq.c')
-rw-r--r--block/blk-mq.c38
1 files changed, 26 insertions, 12 deletions
diff --git a/block/blk-mq.c b/block/blk-mq.c
index cd4ad58c88a5..466676bc2f0b 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -1278,10 +1278,15 @@ static enum prep_dispatch blk_mq_prep_dispatch_rq(struct request *rq,
bool need_budget)
{
struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
+ int budget_token = -1;
- if (need_budget && !blk_mq_get_dispatch_budget(rq->q)) {
- blk_mq_put_driver_tag(rq);
- return PREP_DISPATCH_NO_BUDGET;
+ if (need_budget) {
+ budget_token = blk_mq_get_dispatch_budget(rq->q);
+ if (budget_token < 0) {
+ blk_mq_put_driver_tag(rq);
+ return PREP_DISPATCH_NO_BUDGET;
+ }
+ blk_mq_set_rq_budget_token(rq, budget_token);
}
if (!blk_mq_get_driver_tag(rq)) {
@@ -1298,7 +1303,7 @@ static enum prep_dispatch blk_mq_prep_dispatch_rq(struct request *rq,
* together during handling partial dispatch
*/
if (need_budget)
- blk_mq_put_dispatch_budget(rq->q);
+ blk_mq_put_dispatch_budget(rq->q, budget_token);
return PREP_DISPATCH_NO_TAG;
}
}
@@ -1308,12 +1313,16 @@ static enum prep_dispatch blk_mq_prep_dispatch_rq(struct request *rq,
/* release all allocated budgets before calling to blk_mq_dispatch_rq_list */
static void blk_mq_release_budgets(struct request_queue *q,
- unsigned int nr_budgets)
+ struct list_head *list)
{
- int i;
+ struct request *rq;
- for (i = 0; i < nr_budgets; i++)
- blk_mq_put_dispatch_budget(q);
+ list_for_each_entry(rq, list, queuelist) {
+ int budget_token = blk_mq_get_rq_budget_token(rq);
+
+ if (budget_token >= 0)
+ blk_mq_put_dispatch_budget(q, budget_token);
+ }
}
/*
@@ -1411,7 +1420,8 @@ out:
(hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED);
bool no_budget_avail = prep == PREP_DISPATCH_NO_BUDGET;
- blk_mq_release_budgets(q, nr_budgets);
+ if (nr_budgets)
+ blk_mq_release_budgets(q, list);
spin_lock(&hctx->lock);
list_splice_tail_init(list, &hctx->dispatch);
@@ -2011,6 +2021,7 @@ static blk_status_t __blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,
{
struct request_queue *q = rq->q;
bool run_queue = true;
+ int budget_token;
/*
* RCU or SRCU read lock is needed before checking quiesced flag.
@@ -2028,11 +2039,14 @@ static blk_status_t __blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,
if (q->elevator && !bypass_insert)
goto insert;
- if (!blk_mq_get_dispatch_budget(q))
+ budget_token = blk_mq_get_dispatch_budget(q);
+ if (budget_token < 0)
goto insert;
+ blk_mq_set_rq_budget_token(rq, budget_token);
+
if (!blk_mq_get_driver_tag(rq)) {
- blk_mq_put_dispatch_budget(q);
+ blk_mq_put_dispatch_budget(q, budget_token);
goto insert;
}
@@ -2704,7 +2718,7 @@ blk_mq_alloc_hctx(struct request_queue *q, struct blk_mq_tag_set *set,
goto free_cpumask;
if (sbitmap_init_node(&hctx->ctx_map, nr_cpu_ids, ilog2(8),
- gfp, node))
+ gfp, node, false, false))
goto free_ctxs;
hctx->nr_ctx = 0;