summaryrefslogtreecommitdiffstats
path: root/block
diff options
context:
space:
mode:
Diffstat (limited to 'block')
-rw-r--r--block/bio.c20
-rw-r--r--block/blk-core.c17
-rw-r--r--block/blk-crypto-internal.h21
-rw-r--r--block/blk-crypto.c33
-rw-r--r--block/blk-integrity.c2
-rw-r--r--block/blk-iocost.c4
-rw-r--r--block/blk-iolatency.c2
-rw-r--r--block/blk-merge.c62
-rw-r--r--block/blk-mq-sched.c38
-rw-r--r--block/blk-mq-sysfs.c2
-rw-r--r--block/blk-mq.c29
-rw-r--r--block/blk-settings.c46
-rw-r--r--block/blk-sysfs.c14
-rw-r--r--block/blk-throttle.c69
-rw-r--r--block/blk.h34
-rw-r--r--block/bounce.c19
-rw-r--r--block/elevator.c23
-rw-r--r--block/genhd.c2
-rw-r--r--block/partitions/core.c2
-rw-r--r--block/partitions/ibm.c7
-rw-r--r--block/scsi_ioctl.c19
21 files changed, 257 insertions, 208 deletions
diff --git a/block/bio.c b/block/bio.c
index e865ea55b9f9..640d0fb74a8b 100644
--- a/block/bio.c
+++ b/block/bio.c
@@ -713,20 +713,18 @@ struct bio *bio_clone_fast(struct bio *bio, gfp_t gfp_mask, struct bio_set *bs)
__bio_clone_fast(b, bio);
- bio_crypt_clone(b, bio, gfp_mask);
+ if (bio_crypt_clone(b, bio, gfp_mask) < 0)
+ goto err_put;
- if (bio_integrity(bio)) {
- int ret;
-
- ret = bio_integrity_clone(b, bio, gfp_mask);
-
- if (ret < 0) {
- bio_put(b);
- return NULL;
- }
- }
+ if (bio_integrity(bio) &&
+ bio_integrity_clone(b, bio, gfp_mask) < 0)
+ goto err_put;
return b;
+
+err_put:
+ bio_put(b);
+ return NULL;
}
EXPORT_SYMBOL(bio_clone_fast);
diff --git a/block/blk-core.c b/block/blk-core.c
index 4884f1e7451b..ac00d2fa4eb4 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -646,11 +646,10 @@ static void handle_bad_sector(struct bio *bio, sector_t maxsector)
{
char b[BDEVNAME_SIZE];
- printk(KERN_INFO "attempt to access beyond end of device\n");
- printk(KERN_INFO "%s: rw=%d, want=%Lu, limit=%Lu\n",
- bio_devname(bio, b), bio->bi_opf,
- (unsigned long long)bio_end_sector(bio),
- (long long)maxsector);
+ pr_info_ratelimited("attempt to access beyond end of device\n"
+ "%s: rw=%d, want=%llu, limit=%llu\n",
+ bio_devname(bio, b), bio->bi_opf,
+ bio_end_sector(bio), maxsector);
}
#ifdef CONFIG_FAIL_MAKE_REQUEST
@@ -1617,8 +1616,10 @@ int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
if (rq->bio) {
rq->biotail->bi_next = bio;
rq->biotail = bio;
- } else
+ } else {
rq->bio = rq->biotail = bio;
+ }
+ bio = NULL;
}
/* Copy attributes of the original request to the clone request. */
@@ -1631,8 +1632,8 @@ int blk_rq_prep_clone(struct request *rq, struct request *rq_src,
rq->nr_phys_segments = rq_src->nr_phys_segments;
rq->ioprio = rq_src->ioprio;
- if (rq->bio)
- blk_crypto_rq_bio_prep(rq, rq->bio, gfp_mask);
+ if (rq->bio && blk_crypto_rq_bio_prep(rq, rq->bio, gfp_mask) < 0)
+ goto free_and_out;
return 0;
diff --git a/block/blk-crypto-internal.h b/block/blk-crypto-internal.h
index d2b0f565d83c..0d36aae538d7 100644
--- a/block/blk-crypto-internal.h
+++ b/block/blk-crypto-internal.h
@@ -142,13 +142,24 @@ static inline void blk_crypto_free_request(struct request *rq)
__blk_crypto_free_request(rq);
}
-void __blk_crypto_rq_bio_prep(struct request *rq, struct bio *bio,
- gfp_t gfp_mask);
-static inline void blk_crypto_rq_bio_prep(struct request *rq, struct bio *bio,
- gfp_t gfp_mask)
+int __blk_crypto_rq_bio_prep(struct request *rq, struct bio *bio,
+ gfp_t gfp_mask);
+/**
+ * blk_crypto_rq_bio_prep - Prepare a request's crypt_ctx when its first bio
+ * is inserted
+ * @rq: The request to prepare
+ * @bio: The first bio being inserted into the request
+ * @gfp_mask: Memory allocation flags
+ *
+ * Return: 0 on success, -ENOMEM if out of memory. -ENOMEM is only possible if
+ * @gfp_mask doesn't include %__GFP_DIRECT_RECLAIM.
+ */
+static inline int blk_crypto_rq_bio_prep(struct request *rq, struct bio *bio,
+ gfp_t gfp_mask)
{
if (bio_has_crypt_ctx(bio))
- __blk_crypto_rq_bio_prep(rq, bio, gfp_mask);
+ return __blk_crypto_rq_bio_prep(rq, bio, gfp_mask);
+ return 0;
}
/**
diff --git a/block/blk-crypto.c b/block/blk-crypto.c
index 2d5e60023b08..5da43f0973b4 100644
--- a/block/blk-crypto.c
+++ b/block/blk-crypto.c
@@ -81,7 +81,15 @@ subsys_initcall(bio_crypt_ctx_init);
void bio_crypt_set_ctx(struct bio *bio, const struct blk_crypto_key *key,
const u64 dun[BLK_CRYPTO_DUN_ARRAY_SIZE], gfp_t gfp_mask)
{
- struct bio_crypt_ctx *bc = mempool_alloc(bio_crypt_ctx_pool, gfp_mask);
+ struct bio_crypt_ctx *bc;
+
+ /*
+ * The caller must use a gfp_mask that contains __GFP_DIRECT_RECLAIM so
+ * that the mempool_alloc() can't fail.
+ */
+ WARN_ON_ONCE(!(gfp_mask & __GFP_DIRECT_RECLAIM));
+
+ bc = mempool_alloc(bio_crypt_ctx_pool, gfp_mask);
bc->bc_key = key;
memcpy(bc->bc_dun, dun, sizeof(bc->bc_dun));
@@ -95,10 +103,13 @@ void __bio_crypt_free_ctx(struct bio *bio)
bio->bi_crypt_context = NULL;
}
-void __bio_crypt_clone(struct bio *dst, struct bio *src, gfp_t gfp_mask)
+int __bio_crypt_clone(struct bio *dst, struct bio *src, gfp_t gfp_mask)
{
dst->bi_crypt_context = mempool_alloc(bio_crypt_ctx_pool, gfp_mask);
+ if (!dst->bi_crypt_context)
+ return -ENOMEM;
*dst->bi_crypt_context = *src->bi_crypt_context;
+ return 0;
}
EXPORT_SYMBOL_GPL(__bio_crypt_clone);
@@ -280,20 +291,16 @@ fail:
return false;
}
-/**
- * __blk_crypto_rq_bio_prep - Prepare a request's crypt_ctx when its first bio
- * is inserted
- *
- * @rq: The request to prepare
- * @bio: The first bio being inserted into the request
- * @gfp_mask: gfp mask
- */
-void __blk_crypto_rq_bio_prep(struct request *rq, struct bio *bio,
- gfp_t gfp_mask)
+int __blk_crypto_rq_bio_prep(struct request *rq, struct bio *bio,
+ gfp_t gfp_mask)
{
- if (!rq->crypt_ctx)
+ if (!rq->crypt_ctx) {
rq->crypt_ctx = mempool_alloc(bio_crypt_ctx_pool, gfp_mask);
+ if (!rq->crypt_ctx)
+ return -ENOMEM;
+ }
*rq->crypt_ctx = *bio->bi_crypt_context;
+ return 0;
}
/**
diff --git a/block/blk-integrity.c b/block/blk-integrity.c
index 2b36a8f9b813..410da060d1f5 100644
--- a/block/blk-integrity.c
+++ b/block/blk-integrity.c
@@ -183,7 +183,6 @@ bool blk_integrity_merge_rq(struct request_queue *q, struct request *req,
return true;
}
-EXPORT_SYMBOL(blk_integrity_merge_rq);
bool blk_integrity_merge_bio(struct request_queue *q, struct request *req,
struct bio *bio)
@@ -212,7 +211,6 @@ bool blk_integrity_merge_bio(struct request_queue *q, struct request *req,
return true;
}
-EXPORT_SYMBOL(blk_integrity_merge_bio);
struct integrity_sysfs_entry {
struct attribute attr;
diff --git a/block/blk-iocost.c b/block/blk-iocost.c
index b82649c1440b..bbe86d1199dc 100644
--- a/block/blk-iocost.c
+++ b/block/blk-iocost.c
@@ -669,7 +669,7 @@ static struct ioc *q_to_ioc(struct request_queue *q)
static const char *q_name(struct request_queue *q)
{
- if (test_bit(QUEUE_FLAG_REGISTERED, &q->queue_flags))
+ if (blk_queue_registered(q))
return kobject_name(q->kobj.parent);
else
return "<unknown>";
@@ -3405,7 +3405,7 @@ static int __init ioc_init(void)
static void __exit ioc_exit(void)
{
- return blkcg_policy_unregister(&blkcg_policy_iocost);
+ blkcg_policy_unregister(&blkcg_policy_iocost);
}
module_init(ioc_init);
diff --git a/block/blk-iolatency.c b/block/blk-iolatency.c
index f90429cf4edf..81be0096411d 100644
--- a/block/blk-iolatency.c
+++ b/block/blk-iolatency.c
@@ -1046,7 +1046,7 @@ static int __init iolatency_init(void)
static void __exit iolatency_exit(void)
{
- return blkcg_policy_unregister(&blkcg_policy_iolatency);
+ blkcg_policy_unregister(&blkcg_policy_iolatency);
}
module_init(iolatency_init);
diff --git a/block/blk-merge.c b/block/blk-merge.c
index 6ed715835d45..bcf5e4580603 100644
--- a/block/blk-merge.c
+++ b/block/blk-merge.c
@@ -580,7 +580,8 @@ int ll_back_merge_fn(struct request *req, struct bio *bio, unsigned int nr_segs)
return ll_new_hw_segment(req, bio, nr_segs);
}
-int ll_front_merge_fn(struct request *req, struct bio *bio, unsigned int nr_segs)
+static int ll_front_merge_fn(struct request *req, struct bio *bio,
+ unsigned int nr_segs)
{
if (req_gap_front_merge(req, bio))
return 0;
@@ -810,7 +811,8 @@ static struct request *attempt_merge(struct request_queue *q,
return next;
}
-struct request *attempt_back_merge(struct request_queue *q, struct request *rq)
+static struct request *attempt_back_merge(struct request_queue *q,
+ struct request *rq)
{
struct request *next = elv_latter_request(q, rq);
@@ -820,7 +822,8 @@ struct request *attempt_back_merge(struct request_queue *q, struct request *rq)
return NULL;
}
-struct request *attempt_front_merge(struct request_queue *q, struct request *rq)
+static struct request *attempt_front_merge(struct request_queue *q,
+ struct request *rq)
{
struct request *prev = elv_former_request(q, rq);
@@ -907,9 +910,14 @@ static void blk_account_io_merge_bio(struct request *req)
part_stat_unlock();
}
-enum bio_merge_status bio_attempt_back_merge(struct request *req,
- struct bio *bio,
- unsigned int nr_segs)
+enum bio_merge_status {
+ BIO_MERGE_OK,
+ BIO_MERGE_NONE,
+ BIO_MERGE_FAILED,
+};
+
+static enum bio_merge_status bio_attempt_back_merge(struct request *req,
+ struct bio *bio, unsigned int nr_segs)
{
const int ff = bio->bi_opf & REQ_FAILFAST_MASK;
@@ -932,9 +940,8 @@ enum bio_merge_status bio_attempt_back_merge(struct request *req,
return BIO_MERGE_OK;
}
-enum bio_merge_status bio_attempt_front_merge(struct request *req,
- struct bio *bio,
- unsigned int nr_segs)
+static enum bio_merge_status bio_attempt_front_merge(struct request *req,
+ struct bio *bio, unsigned int nr_segs)
{
const int ff = bio->bi_opf & REQ_FAILFAST_MASK;
@@ -959,9 +966,8 @@ enum bio_merge_status bio_attempt_front_merge(struct request *req,
return BIO_MERGE_OK;
}
-enum bio_merge_status bio_attempt_discard_merge(struct request_queue *q,
- struct request *req,
- struct bio *bio)
+static enum bio_merge_status bio_attempt_discard_merge(struct request_queue *q,
+ struct request *req, struct bio *bio)
{
unsigned short segments = blk_rq_nr_discard_segments(req);
@@ -1096,3 +1102,35 @@ bool blk_bio_list_merge(struct request_queue *q, struct list_head *list,
return false;
}
EXPORT_SYMBOL_GPL(blk_bio_list_merge);
+
+bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio,
+ unsigned int nr_segs, struct request **merged_request)
+{
+ struct request *rq;
+
+ switch (elv_merge(q, &rq, bio)) {
+ case ELEVATOR_BACK_MERGE:
+ if (!blk_mq_sched_allow_merge(q, rq, bio))
+ return false;
+ if (bio_attempt_back_merge(rq, bio, nr_segs) != BIO_MERGE_OK)
+ return false;
+ *merged_request = attempt_back_merge(q, rq);
+ if (!*merged_request)
+ elv_merged_request(q, rq, ELEVATOR_BACK_MERGE);
+ return true;
+ case ELEVATOR_FRONT_MERGE:
+ if (!blk_mq_sched_allow_merge(q, rq, bio))
+ return false;
+ if (bio_attempt_front_merge(rq, bio, nr_segs) != BIO_MERGE_OK)
+ return false;
+ *merged_request = attempt_front_merge(q, rq);
+ if (!*merged_request)
+ elv_merged_request(q, rq, ELEVATOR_FRONT_MERGE);
+ return true;
+ case ELEVATOR_DISCARD_MERGE:
+ return bio_attempt_discard_merge(q, rq, bio) == BIO_MERGE_OK;
+ default:
+ return false;
+ }
+}
+EXPORT_SYMBOL_GPL(blk_mq_sched_try_merge);
diff --git a/block/blk-mq-sched.c b/block/blk-mq-sched.c
index 3e9596738852..d1eafe2c045c 100644
--- a/block/blk-mq-sched.c
+++ b/block/blk-mq-sched.c
@@ -344,38 +344,6 @@ void blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx)
}
}
-bool blk_mq_sched_try_merge(struct request_queue *q, struct bio *bio,
- unsigned int nr_segs, struct request **merged_request)
-{
- struct request *rq;
-
- switch (elv_merge(q, &rq, bio)) {
- case ELEVATOR_BACK_MERGE:
- if (!blk_mq_sched_allow_merge(q, rq, bio))
- return false;
- if (bio_attempt_back_merge(rq, bio, nr_segs) != BIO_MERGE_OK)
- return false;
- *merged_request = attempt_back_merge(q, rq);
- if (!*merged_request)
- elv_merged_request(q, rq, ELEVATOR_BACK_MERGE);
- return true;
- case ELEVATOR_FRONT_MERGE:
- if (!blk_mq_sched_allow_merge(q, rq, bio))
- return false;
- if (bio_attempt_front_merge(rq, bio, nr_segs) != BIO_MERGE_OK)
- return false;
- *merged_request = attempt_front_merge(q, rq);
- if (!*merged_request)
- elv_merged_request(q, rq, ELEVATOR_FRONT_MERGE);
- return true;
- case ELEVATOR_DISCARD_MERGE:
- return bio_attempt_discard_merge(q, rq, bio) == BIO_MERGE_OK;
- default:
- return false;
- }
-}
-EXPORT_SYMBOL_GPL(blk_mq_sched_try_merge);
-
bool __blk_mq_sched_bio_merge(struct request_queue *q, struct bio *bio,
unsigned int nr_segs)
{
@@ -454,12 +422,6 @@ void blk_mq_sched_insert_request(struct request *rq, bool at_head,
struct blk_mq_ctx *ctx = rq->mq_ctx;
struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
- /* flush rq in flush machinery need to be dispatched directly */
- if (!(rq->rq_flags & RQF_FLUSH_SEQ) && op_is_flush(rq->cmd_flags)) {
- blk_insert_flush(rq);
- goto run;
- }
-
WARN_ON(e && (rq->tag != BLK_MQ_NO_TAG));
if (blk_mq_sched_bypass_insert(hctx, !!e, rq)) {
diff --git a/block/blk-mq-sysfs.c b/block/blk-mq-sysfs.c
index 062229395a50..7b52e7657b2d 100644
--- a/block/blk-mq-sysfs.c
+++ b/block/blk-mq-sysfs.c
@@ -36,8 +36,6 @@ static void blk_mq_hw_sysfs_release(struct kobject *kobj)
struct blk_mq_hw_ctx *hctx = container_of(kobj, struct blk_mq_hw_ctx,
kobj);
- cancel_delayed_work_sync(&hctx->run_work);
-
if (hctx->flags & BLK_MQ_F_BLOCKING)
cleanup_srcu_struct(hctx->srcu);
blk_free_flush_queue(hctx->fq);
diff --git a/block/blk-mq.c b/block/blk-mq.c
index 0191fc0447f7..696450257ac1 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -105,7 +105,7 @@ static bool blk_mq_check_inflight(struct blk_mq_hw_ctx *hctx,
{
struct mq_inflight *mi = priv;
- if (rq->part == mi->part)
+ if (rq->part == mi->part && blk_mq_rq_state(rq) == MQ_RQ_IN_FLIGHT)
mi->inflight[rq_data_dir(rq)]++;
return true;
@@ -1413,6 +1413,11 @@ out:
hctx->dispatched[queued_to_index(queued)]++;
+ /* If we didn't flush the entire list, we could have told the driver
+ * there was more coming, but that turned out to be a lie.
+ */
+ if ((!list_empty(list) || errors) && q->mq_ops->commit_rqs && queued)
+ q->mq_ops->commit_rqs(hctx);
/*
* Any items that need requeuing? Stuff them into hctx->dispatch,
* that is where we will continue on next queue run.
@@ -1426,14 +1431,6 @@ out:
blk_mq_release_budgets(q, nr_budgets);
- /*
- * If we didn't flush the entire list, we could have told
- * the driver there was more coming, but that turned out to
- * be a lie.
- */
- if (q->mq_ops->commit_rqs && queued)
- q->mq_ops->commit_rqs(hctx);
-
spin_lock(&hctx->lock);
list_splice_tail_init(list, &hctx->dispatch);
spin_unlock(&hctx->lock);
@@ -1807,7 +1804,7 @@ static void blk_mq_run_work_fn(struct work_struct *work)
/*
* If we are stopped, don't run the queue.
*/
- if (test_bit(BLK_MQ_S_STOPPED, &hctx->state))
+ if (blk_mq_hctx_stopped(hctx))
return;
__blk_mq_run_hw_queue(hctx);
@@ -1940,13 +1937,18 @@ void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
static void blk_mq_bio_to_request(struct request *rq, struct bio *bio,
unsigned int nr_segs)
{
+ int err;
+
if (bio->bi_opf & REQ_RAHEAD)
rq->cmd_flags |= REQ_FAILFAST_MASK;
rq->__sector = bio->bi_iter.bi_sector;
rq->write_hint = bio->bi_write_hint;
blk_rq_bio_prep(rq, bio, nr_segs);
- blk_crypto_rq_bio_prep(rq, bio, GFP_NOIO);
+
+ /* This can't fail, since GFP_NOIO includes __GFP_DIRECT_RECLAIM. */
+ err = blk_crypto_rq_bio_prep(rq, bio, GFP_NOIO);
+ WARN_ON_ONCE(err);
blk_account_io_start(rq);
}
@@ -2080,6 +2082,7 @@ void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx,
struct list_head *list)
{
int queued = 0;
+ int errors = 0;
while (!list_empty(list)) {
blk_status_t ret;
@@ -2096,6 +2099,7 @@ void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx,
break;
}
blk_mq_end_request(rq, ret);
+ errors++;
} else
queued++;
}
@@ -2105,7 +2109,8 @@ void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx,
* the driver there was more coming, but that turned out to
* be a lie.
*/
- if (!list_empty(list) && hctx->queue->mq_ops->commit_rqs && queued)
+ if ((!list_empty(list) || errors) &&
+ hctx->queue->mq_ops->commit_rqs && queued)
hctx->queue->mq_ops->commit_rqs(hctx);
}
diff --git a/block/blk-settings.c b/block/blk-settings.c
index 4f6eb4bb1723..9741d1d83e98 100644
--- a/block/blk-settings.c
+++ b/block/blk-settings.c
@@ -817,6 +817,52 @@ bool blk_queue_can_use_dma_map_merging(struct request_queue *q,
}
EXPORT_SYMBOL_GPL(blk_queue_can_use_dma_map_merging);
+/**
+ * blk_queue_set_zoned - configure a disk queue zoned model.
+ * @disk: the gendisk of the queue to configure
+ * @model: the zoned model to set
+ *
+ * Set the zoned model of the request queue of @disk according to @model.
+ * When @model is BLK_ZONED_HM (host managed), this should be called only
+ * if zoned block device support is enabled (CONFIG_BLK_DEV_ZONED option).
+ * If @model specifies BLK_ZONED_HA (host aware), the effective model used
+ * depends on CONFIG_BLK_DEV_ZONED settings and on the existence of partitions
+ * on the disk.
+ */
+void blk_queue_set_zoned(struct gendisk *disk, enum blk_zoned_model model)
+{
+ switch (model) {
+ case BLK_ZONED_HM:
+ /*
+ * Host managed devices are supported only if
+ * CONFIG_BLK_DEV_ZONED is enabled.
+ */
+ WARN_ON_ONCE(!IS_ENABLED(CONFIG_BLK_DEV_ZONED));
+ break;
+ case BLK_ZONED_HA:
+ /*
+ * Host aware devices can be treated either as regular block
+ * devices (similar to drive managed devices) or as zoned block
+ * devices to take advantage of the zone command set, similarly
+ * to host managed devices. We try the latter if there are no
+ * partitions and zoned block device support is enabled, else
+ * we do nothing special as far as the block layer is concerned.
+ */
+ if (!IS_ENABLED(CONFIG_BLK_DEV_ZONED) ||
+ disk_has_partitions(disk))
+ model = BLK_ZONED_NONE;
+ break;
+ case BLK_ZONED_NONE:
+ default:
+ if (WARN_ON_ONCE(model != BLK_ZONED_NONE))
+ model = BLK_ZONED_NONE;
+ break;
+ }
+
+ disk->queue->limits.zoned = model;
+}
+EXPORT_SYMBOL_GPL(blk_queue_set_zoned);
+
static int __init blk_settings_init(void)
{
blk_max_low_pfn = max_low_pfn - 1;
diff --git a/block/blk-sysfs.c b/block/blk-sysfs.c
index 76b54c7750b0..b513f1683af0 100644
--- a/block/blk-sysfs.c
+++ b/block/blk-sysfs.c
@@ -740,7 +740,6 @@ static void blk_exit_queue(struct request_queue *q)
if (q->elevator) {
ioc_clear_queue(q);
__elevator_exit(q, q->elevator);
- q->elevator = NULL;
}
/*
@@ -791,9 +790,16 @@ static void blk_release_queue(struct kobject *kobj)
blk_free_queue_stats(q->stats);
- if (queue_is_mq(q))
+ if (queue_is_mq(q)) {
+ struct blk_mq_hw_ctx *hctx;
+ int i;
+
cancel_delayed_work_sync(&q->requeue_work);
+ queue_for_each_hw_ctx(q, hctx, i)
+ cancel_delayed_work_sync(&hctx->run_work);
+ }
+
blk_exit_queue(q);
blk_queue_free_zone_bitmaps(q);
@@ -834,7 +840,6 @@ int blk_register_queue(struct gendisk *disk)
int ret;
struct device *dev = disk_to_dev(disk);
struct request_queue *q = disk->queue;
- bool has_elevator = false;
if (WARN_ON(!q))
return -ENXIO;
@@ -900,7 +905,6 @@ int blk_register_queue(struct gendisk *disk)
kobject_put(&dev->kobj);
return ret;
}
- has_elevator = true;
}
blk_queue_flag_set(QUEUE_FLAG_REGISTERED, q);
@@ -909,7 +913,7 @@ int blk_register_queue(struct gendisk *disk)
/* Now everything is ready and send out KOBJ_ADD uevent */
kobject_uevent(&q->kobj, KOBJ_ADD);
- if (has_elevator)
+ if (q->elevator)
kobject_uevent(&q->elevator->kobj, KOBJ_ADD);
mutex_unlock(&q->sysfs_lock);
diff --git a/block/blk-throttle.c b/block/blk-throttle.c
index 36ba61c5cdbd..b771c4299982 100644
--- a/block/blk-throttle.c
+++ b/block/blk-throttle.c
@@ -423,12 +423,13 @@ static void throtl_qnode_add_bio(struct bio *bio, struct throtl_qnode *qn,
*/
static struct bio *throtl_peek_queued(struct list_head *queued)
{
- struct throtl_qnode *qn = list_first_entry(queued, struct throtl_qnode, node);
+ struct throtl_qnode *qn;
struct bio *bio;
if (list_empty(queued))
return NULL;
+ qn = list_first_entry(queued, struct throtl_qnode, node);
bio = bio_list_peek(&qn->bios);
WARN_ON_ONCE(!bio);
return bio;
@@ -451,12 +452,13 @@ static struct bio *throtl_peek_queued(struct list_head *queued)
static struct bio *throtl_pop_queued(struct list_head *queued,
struct throtl_grp **tg_to_put)
{
- struct throtl_qnode *qn = list_first_entry(queued, struct throtl_qnode, node);
+ struct throtl_qnode *qn;
struct bio *bio;
if (list_empty(queued))
return NULL;
+ qn = list_first_entry(queued, struct throtl_qnode, node);
bio = bio_list_pop(&qn->bios);
WARN_ON_ONCE(!bio);
@@ -636,9 +638,6 @@ static struct throtl_grp *
throtl_rb_first(struct throtl_service_queue *parent_sq)
{
struct rb_node *n;
- /* Service tree is empty */
- if (!parent_sq->nr_pending)
- return NULL;
n = rb_first_cached(&parent_sq->pending_tree);
WARN_ON_ONCE(!n);
@@ -692,29 +691,21 @@ static void tg_service_queue_add(struct throtl_grp *tg)
leftmost);
}
-static void __throtl_enqueue_tg(struct throtl_grp *tg)
-{
- tg_service_queue_add(tg);
- tg->flags |= THROTL_TG_PENDING;
- tg->service_queue.parent_sq->nr_pending++;
-}
-
static void throtl_enqueue_tg(struct throtl_grp *tg)
{
- if (!(tg->flags & THROTL_TG_PENDING))
- __throtl_enqueue_tg(tg);
-}
-
-static void __throtl_dequeue_tg(struct throtl_grp *tg)
-{
- throtl_rb_erase(&tg->rb_node, tg->service_queue.parent_sq);
- tg->flags &= ~THROTL_TG_PENDING;
+ if (!(tg->flags & THROTL_TG_PENDING)) {
+ tg_service_queue_add(tg);
+ tg->flags |= THROTL_TG_PENDING;
+ tg->service_queue.parent_sq->nr_pending++;
+ }
}
static void throtl_dequeue_tg(struct throtl_grp *tg)
{
- if (tg->flags & THROTL_TG_PENDING)
- __throtl_dequeue_tg(tg);
+ if (tg->flags & THROTL_TG_PENDING) {
+ throtl_rb_erase(&tg->rb_node, tg->service_queue.parent_sq);
+ tg->flags &= ~THROTL_TG_PENDING;
+ }
}
/* Call with queue lock held */
@@ -817,7 +808,7 @@ static inline void throtl_set_slice_end(struct throtl_grp *tg, bool rw,
static inline void throtl_extend_slice(struct throtl_grp *tg, bool rw,
unsigned long jiffy_end)
{
- tg->slice_end[rw] = roundup(jiffy_end, tg->td->throtl_slice);
+ throtl_set_slice_end(tg, rw, jiffy_end);
throtl_log(&tg->service_queue,
"[%c] extend slice start=%lu end=%lu jiffies=%lu",
rw == READ ? 'R' : 'W', tg->slice_start[rw],
@@ -1222,9 +1213,13 @@ static int throtl_select_dispatch(struct throtl_service_queue *parent_sq)
unsigned int nr_disp = 0;
while (1) {
- struct throtl_grp *tg = throtl_rb_first(parent_sq);
+ struct throtl_grp *tg;
struct throtl_service_queue *sq;
+ if (!parent_sq->nr_pending)
+ break;
+
+ tg = throtl_rb_first(parent_sq);
if (!tg)
break;
@@ -1687,13 +1682,13 @@ static ssize_t tg_set_limit(struct kernfs_open_file *of,
goto out_finish;
ret = -EINVAL;
- if (!strcmp(tok, "rbps"))
+ if (!strcmp(tok, "rbps") && val > 1)
v[0] = val;
- else if (!strcmp(tok, "wbps"))
+ else if (!strcmp(tok, "wbps") && val > 1)
v[1] = val;
- else if (!strcmp(tok, "riops"))
+ else if (!strcmp(tok, "riops") && val > 1)
v[2] = min_t(u64, val, UINT_MAX);
- else if (!strcmp(tok, "wiops"))
+ else if (!strcmp(tok, "wiops") && val > 1)
v[3] = min_t(u64, val, UINT_MAX);
else if (off == LIMIT_LOW && !strcmp(tok, "idle"))
idle_time = val;
@@ -1970,7 +1965,7 @@ static void throtl_upgrade_state(struct throtl_data *td)
queue_work(kthrotld_workqueue, &td->dispatch_work);
}
-static void throtl_downgrade_state(struct throtl_data *td, int new)
+static void throtl_downgrade_state(struct throtl_data *td)
{
td->scale /= 2;
@@ -1980,7 +1975,7 @@ static void throtl_downgrade_state(struct throtl_data *td, int new)
return;
}
- td->limit_index = new;
+ td->limit_index = LIMIT_LOW;
td->low_downgrade_time = jiffies;
}
@@ -2067,7 +2062,7 @@ static void throtl_downgrade_check(struct throtl_grp *tg)
* cgroups
*/
if (throtl_hierarchy_can_downgrade(tg))
- throtl_downgrade_state(tg->td, LIMIT_LOW);
+ throtl_downgrade_state(tg->td);
tg->last_bytes_disp[READ] = 0;
tg->last_bytes_disp[WRITE] = 0;
@@ -2077,10 +2072,14 @@ static void throtl_downgrade_check(struct throtl_grp *tg)
static void blk_throtl_update_idletime(struct throtl_grp *tg)
{
- unsigned long now = ktime_get_ns() >> 10;
+ unsigned long now;
unsigned long last_finish_time = tg->last_finish_time;
- if (now <= last_finish_time || last_finish_time == 0 ||
+ if (last_finish_time == 0)
+ return;
+
+ now = ktime_get_ns() >> 10;
+ if (now <= last_finish_time ||
last_finish_time == tg->checked_last_finish_time)
return;
@@ -2096,7 +2095,7 @@ static void throtl_update_latency_buckets(struct throtl_data *td)
unsigned long last_latency[2] = { 0 };
unsigned long latency[2];
- if (!blk_queue_nonrot(td->queue))
+ if (!blk_queue_nonrot(td->queue) || !td->limit_valid[LIMIT_LOW])
return;
if (time_before(jiffies, td->last_calculate_time + HZ))
return;
@@ -2334,6 +2333,8 @@ void blk_throtl_bio_endio(struct bio *bio)
if (!blkg)
return;
tg = blkg_to_tg(blkg);
+ if (!tg->td->limit_valid[LIMIT_LOW])
+ return;
finish_time_ns = ktime_get_ns();
tg->last_finish_time = finish_time_ns >> 10;
diff --git a/block/blk.h b/block/blk.h
index c08762e10b04..dfab98465db9 100644
--- a/block/blk.h
+++ b/block/blk.h
@@ -29,12 +29,6 @@ struct blk_flush_queue {
spinlock_t mq_flush_lock;
};
-enum bio_merge_status {
- BIO_MERGE_OK,
- BIO_MERGE_NONE,
- BIO_MERGE_FAILED,
-};
-
extern struct kmem_cache *blk_requestq_cachep;
extern struct kobj_type blk_queue_ktype;
extern struct ida blk_queue_ida;
@@ -120,6 +114,11 @@ static inline bool bio_integrity_endio(struct bio *bio)
return true;
}
+bool blk_integrity_merge_rq(struct request_queue *, struct request *,
+ struct request *);
+bool blk_integrity_merge_bio(struct request_queue *, struct request *,
+ struct bio *);
+
static inline bool integrity_req_gap_back_merge(struct request *req,
struct bio *next)
{
@@ -143,6 +142,16 @@ static inline bool integrity_req_gap_front_merge(struct request *req,
void blk_integrity_add(struct gendisk *);
void blk_integrity_del(struct gendisk *);
#else /* CONFIG_BLK_DEV_INTEGRITY */
+static inline bool blk_integrity_merge_rq(struct request_queue *rq,
+ struct request *r1, struct request *r2)
+{
+ return true;
+}
+static inline bool blk_integrity_merge_bio(struct request_queue *rq,
+ struct request *r, struct bio *b)
+{
+ return true;
+}
static inline bool integrity_req_gap_back_merge(struct request *req,
struct bio *next)
{
@@ -175,15 +184,6 @@ static inline void blk_integrity_del(struct gendisk *disk)
unsigned long blk_rq_timeout(unsigned long timeout);
void blk_add_timer(struct request *req);
-enum bio_merge_status bio_attempt_front_merge(struct request *req,
- struct bio *bio,
- unsigned int nr_segs);
-enum bio_merge_status bio_attempt_back_merge(struct request *req,
- struct bio *bio,
- unsigned int nr_segs);
-enum bio_merge_status bio_attempt_discard_merge(struct request_queue *q,
- struct request *req,
- struct bio *bio);
bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
unsigned int nr_segs, struct request **same_queue_rq);
bool blk_bio_list_merge(struct request_queue *q, struct list_head *list,
@@ -234,10 +234,6 @@ ssize_t part_timeout_store(struct device *, struct device_attribute *,
void __blk_queue_split(struct bio **bio, unsigned int *nr_segs);
int ll_back_merge_fn(struct request *req, struct bio *bio,
unsigned int nr_segs);
-int ll_front_merge_fn(struct request *req, struct bio *bio,
- unsigned int nr_segs);
-struct request *attempt_back_merge(struct request_queue *q, struct request *rq);
-struct request *attempt_front_merge(struct request_queue *q, struct request *rq);
int blk_attempt_req_merge(struct request_queue *q, struct request *rq,
struct request *next);
unsigned int blk_recalc_rq_segments(struct request *rq);
diff --git a/block/bounce.c b/block/bounce.c
index 431be88a0240..162a6eee8999 100644
--- a/block/bounce.c
+++ b/block/bounce.c
@@ -267,22 +267,21 @@ static struct bio *bounce_clone_bio(struct bio *bio_src, gfp_t gfp_mask,
break;
}
- bio_crypt_clone(bio, bio_src, gfp_mask);
+ if (bio_crypt_clone(bio, bio_src, gfp_mask) < 0)
+ goto err_put;
- if (bio_integrity(bio_src)) {
- int ret;
-
- ret = bio_integrity_clone(bio, bio_src, gfp_mask);
- if (ret < 0) {
- bio_put(bio);
- return NULL;
- }
- }
+ if (bio_integrity(bio_src) &&
+ bio_integrity_clone(bio, bio_src, gfp_mask) < 0)
+ goto err_put;
bio_clone_blkg_association(bio, bio_src);
blkcg_bio_issue_init(bio);
return bio;
+
+err_put:
+ bio_put(bio);
+ return NULL;
}
static void __blk_queue_bounce(struct request_queue *q, struct bio **bio_orig,
diff --git a/block/elevator.c b/block/elevator.c
index 90ed7a28c21d..293c5c81397a 100644
--- a/block/elevator.c
+++ b/block/elevator.c
@@ -191,8 +191,7 @@ static void elevator_release(struct kobject *kobj)
void __elevator_exit(struct request_queue *q, struct elevator_queue *e)
{
mutex_lock(&e->sysfs_lock);
- if (e->type->ops.exit_sched)
- blk_mq_exit_sched(q, e);
+ blk_mq_exit_sched(q, e);
mutex_unlock(&e->sysfs_lock);
kobject_put(&e->kobj);
@@ -480,16 +479,13 @@ static struct kobj_type elv_ktype = {
.release = elevator_release,
};
-/*
- * elv_register_queue is called from either blk_register_queue or
- * elevator_switch, elevator switch is prevented from being happen
- * in the two paths, so it is safe to not hold q->sysfs_lock.
- */
int elv_register_queue(struct request_queue *q, bool uevent)
{
struct elevator_queue *e = q->elevator;
int error;
+ lockdep_assert_held(&q->sysfs_lock);
+
error = kobject_add(&e->kobj, &q->kobj, "%s", "iosched");
if (!error) {
struct elv_fs_entry *attr = e->type->elevator_attrs;
@@ -508,13 +504,10 @@ int elv_register_queue(struct request_queue *q, bool uevent)
return error;
}
-/*
- * elv_unregister_queue is called from either blk_unregister_queue or
- * elevator_switch, elevator switch is prevented from being happen
- * in the two paths, so it is safe to not hold q->sysfs_lock.
- */
void elv_unregister_queue(struct request_queue *q)
{
+ lockdep_assert_held(&q->sysfs_lock);
+
if (q) {
struct elevator_queue *e = q->elevator;
@@ -616,7 +609,7 @@ out:
static inline bool elv_support_iosched(struct request_queue *q)
{
- if (!q->mq_ops ||
+ if (!queue_is_mq(q) ||
(q->tag_set && (q->tag_set->flags & BLK_MQ_F_NO_SCHED)))
return false;
return true;
@@ -673,7 +666,7 @@ void elevator_init_mq(struct request_queue *q)
if (!elv_support_iosched(q))
return;
- WARN_ON_ONCE(test_bit(QUEUE_FLAG_REGISTERED, &q->queue_flags));
+ WARN_ON_ONCE(blk_queue_registered(q));
if (unlikely(q->elevator))
return;
@@ -764,7 +757,7 @@ ssize_t elv_iosched_store(struct request_queue *q, const char *name,
{
int ret;
- if (!queue_is_mq(q) || !elv_support_iosched(q))
+ if (!elv_support_iosched(q))
return count;
ret = __elevator_change(q, name);
diff --git a/block/genhd.c b/block/genhd.c
index e5f17f022ec7..0a273211fec2 100644
--- a/block/genhd.c
+++ b/block/genhd.c
@@ -1048,7 +1048,7 @@ struct block_device *bdget_disk(struct gendisk *disk, int partno)
part = disk_get_part(disk, partno);
if (part)
- bdev = bdget(part_devt(part));
+ bdev = bdget_part(part);
disk_put_part(part);
return bdev;
diff --git a/block/partitions/core.c b/block/partitions/core.c
index 5cacbac30107..a02e22411594 100644
--- a/block/partitions/core.c
+++ b/block/partitions/core.c
@@ -580,7 +580,7 @@ int bdev_resize_partition(struct block_device *bdev, int partno,
return -ENXIO;
ret = -ENOMEM;
- bdevp = bdget(part_devt(part));
+ bdevp = bdget_part(part);
if (!bdevp)
goto out_put_part;
diff --git a/block/partitions/ibm.c b/block/partitions/ibm.c
index d6e18df9c53c..4b044e620d35 100644
--- a/block/partitions/ibm.c
+++ b/block/partitions/ibm.c
@@ -305,8 +305,6 @@ int ibm_partition(struct parsed_partitions *state)
if (!disk->fops->getgeo)
goto out_exit;
fn = symbol_get(dasd_biodasdinfo);
- if (!fn)
- goto out_exit;
blocksize = bdev_logical_block_size(bdev);
if (blocksize <= 0)
goto out_symbol;
@@ -326,7 +324,7 @@ int ibm_partition(struct parsed_partitions *state)
geo->start = get_start_sect(bdev);
if (disk->fops->getgeo(bdev, geo))
goto out_freeall;
- if (fn(disk, info)) {
+ if (!fn || fn(disk, info)) {
kfree(info);
info = NULL;
}
@@ -370,7 +368,8 @@ out_nolab:
out_nogeo:
kfree(info);
out_symbol:
- symbol_put(dasd_biodasdinfo);
+ if (fn)
+ symbol_put(dasd_biodasdinfo);
out_exit:
return res;
}
diff --git a/block/scsi_ioctl.c b/block/scsi_ioctl.c
index d4abd1ed5a2d..c9f009cc0446 100644
--- a/block/scsi_ioctl.c
+++ b/block/scsi_ioctl.c
@@ -331,16 +331,8 @@ static int sg_io(struct request_queue *q, struct gendisk *bd_disk,
struct iov_iter i;
struct iovec *iov = NULL;
-#ifdef CONFIG_COMPAT
- if (in_compat_syscall())
- ret = compat_import_iovec(rq_data_dir(rq),
- hdr->dxferp, hdr->iovec_count,
- 0, &iov, &i);
- else
-#endif
- ret = import_iovec(rq_data_dir(rq),
- hdr->dxferp, hdr->iovec_count,
- 0, &iov, &i);
+ ret = import_iovec(rq_data_dir(rq), hdr->dxferp,
+ hdr->iovec_count, 0, &iov, &i);
if (ret < 0)
goto out_free_cdb;
@@ -649,9 +641,10 @@ struct compat_cdrom_generic_command {
compat_int_t stat;
compat_caddr_t sense;
unsigned char data_direction;
+ unsigned char pad[3];
compat_int_t quiet;
compat_int_t timeout;
- compat_caddr_t reserved[1];
+ compat_caddr_t unused;
};
#endif
@@ -673,7 +666,7 @@ static int scsi_get_cdrom_generic_arg(struct cdrom_generic_command *cgc,
.data_direction = cgc32.data_direction,
.quiet = cgc32.quiet,
.timeout = cgc32.timeout,
- .reserved[0] = compat_ptr(cgc32.reserved[0]),
+ .unused = compat_ptr(cgc32.unused),
};
memcpy(&cgc->cmd, &cgc32.cmd, CDROM_PACKET_SIZE);
return 0;
@@ -698,7 +691,7 @@ static int scsi_put_cdrom_generic_arg(const struct cdrom_generic_command *cgc,
.data_direction = cgc->data_direction,
.quiet = cgc->quiet,
.timeout = cgc->timeout,
- .reserved[0] = (uintptr_t)(cgc->reserved[0]),
+ .unused = (uintptr_t)(cgc->unused),
};
memcpy(&cgc32.cmd, &cgc->cmd, CDROM_PACKET_SIZE);
generated by cgit v1.2.3 (git 2.46.0) at 2025-10-08 15:00:59 +0000