diff options
author | Jens Axboe <axboe@kernel.dk> | 2020-07-20 15:38:23 -0600 |
---|---|---|
committer | Jens Axboe <axboe@kernel.dk> | 2020-07-20 15:38:23 -0600 |
commit | 9caaa66c918c020fd16e84d1c6ebcce9960df1b2 (patch) | |
tree | 578d455a53453a1aead77da399c5134e73224a66 /drivers/nvme | |
parent | ba47d845d715a010f7b51f6f89bae32845e6acb7 (diff) | |
parent | ef45fe470e1e5410db4af87abc5d5055427945ac (diff) | |
download | linux-9caaa66c918c020fd16e84d1c6ebcce9960df1b2.tar.bz2 |
Merge branch 'for-5.9/block' into for-5.9/block-merge
* for-5.9/block: (124 commits)
blk-cgroup: show global disk stats in root cgroup io.stat
blk-cgroup: make iostat functions visible to stat printing
block: improve discard bio alignment in __blkdev_issue_discard()
block: change REQ_OP_ZONE_RESET and REQ_OP_ZONE_RESET_ALL to be odd numbers
block: defer flush request no matter whether we have elevator
block: make blk_timeout_init() static
block: remove retry loop in ioc_release_fn()
block: remove unnecessary ioc nested locking
block: integrate bd_start_claiming into __blkdev_get
block: use bd_prepare_to_claim directly in the loop driver
block: refactor bd_start_claiming
block: simplify the restart case in __blkdev_get
Revert "blk-rq-qos: remove redundant finish_wait to rq_qos_wait."
block: always remove partitions from blk_drop_partitions()
block: relax jiffies rounding for timeouts
blk-mq: remove redundant validation in __blk_mq_end_request()
blk-mq: Remove unnecessary local variable
writeback: remove bdi->congested_fn
writeback: remove struct bdi_writeback_congested
writeback: remove {set,clear}_wb_congested
...
Diffstat (limited to 'drivers/nvme')
-rw-r--r-- | drivers/nvme/host/core.c | 3 | ||||
-rw-r--r-- | drivers/nvme/host/fc.c | 4 | ||||
-rw-r--r-- | drivers/nvme/host/multipath.c | 18 | ||||
-rw-r--r-- | drivers/nvme/host/nvme.h | 7 | ||||
-rw-r--r-- | drivers/nvme/host/pci.c | 3 | ||||
-rw-r--r-- | drivers/nvme/host/rdma.c | 35 | ||||
-rw-r--r-- | drivers/nvme/host/tcp.c | 6 | ||||
-rw-r--r-- | drivers/nvme/target/core.c | 2 | ||||
-rw-r--r-- | drivers/nvme/target/loop.c | 3 |
9 files changed, 45 insertions, 36 deletions
diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index add040168e67..71c2c1bf3cc1 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -304,7 +304,7 @@ bool nvme_cancel_request(struct request *req, void *data, bool reserved) return true; nvme_req(req)->status = NVME_SC_HOST_ABORTED_CMD; - blk_mq_force_complete_rq(req); + blk_mq_complete_request(req); return true; } EXPORT_SYMBOL_GPL(nvme_cancel_request); @@ -2184,6 +2184,7 @@ static void nvme_ns_head_release(struct gendisk *disk, fmode_t mode) const struct block_device_operations nvme_ns_head_ops = { .owner = THIS_MODULE, + .submit_bio = nvme_ns_head_submit_bio, .open = nvme_ns_head_open, .release = nvme_ns_head_release, .ioctl = nvme_ioctl, diff --git a/drivers/nvme/host/fc.c b/drivers/nvme/host/fc.c index e999a8c4b7e8..6aa30bb5a762 100644 --- a/drivers/nvme/host/fc.c +++ b/drivers/nvme/host/fc.c @@ -227,6 +227,7 @@ static DECLARE_COMPLETION(nvme_fc_unload_proceed); */ static struct device *fc_udev_device; +static void nvme_fc_complete_rq(struct request *rq); /* *********************** FC-NVME Port Management ************************ */ @@ -2033,7 +2034,8 @@ done: } __nvme_fc_fcpop_chk_teardowns(ctrl, op, opstate); - nvme_end_request(rq, status, result); + if (!nvme_end_request(rq, status, result)) + nvme_fc_complete_rq(rq); check_error: if (terminate_assoc) diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c index 66509472fe06..5a37a595411e 100644 --- a/drivers/nvme/host/multipath.c +++ b/drivers/nvme/host/multipath.c @@ -291,8 +291,7 @@ static bool nvme_available_path(struct nvme_ns_head *head) return false; } -static blk_qc_t nvme_ns_head_make_request(struct request_queue *q, - struct bio *bio) +blk_qc_t nvme_ns_head_submit_bio(struct bio *bio) { struct nvme_ns_head *head = bio->bi_disk->private_data; struct device *dev = disk_to_dev(head->disk); @@ -301,12 +300,11 @@ static blk_qc_t nvme_ns_head_make_request(struct request_queue *q, int srcu_idx; /* - * The namespace might be going away and the bio might - * be moved to a different queue via blk_steal_bios(), - * so we need to use the bio_split pool from the original - * queue to allocate the bvecs from. + * The namespace might be going away and the bio might be moved to a + * different queue via blk_steal_bios(), so we need to use the bio_split + * pool from the original queue to allocate the bvecs from. */ - blk_queue_split(q, &bio); + blk_queue_split(&bio); srcu_idx = srcu_read_lock(&head->srcu); ns = nvme_find_path(head); @@ -316,7 +314,7 @@ static blk_qc_t nvme_ns_head_make_request(struct request_queue *q, trace_block_bio_remap(bio->bi_disk->queue, bio, disk_devt(ns->head->disk), bio->bi_iter.bi_sector); - ret = direct_make_request(bio); + ret = submit_bio_noacct(bio); } else if (nvme_available_path(head)) { dev_warn_ratelimited(dev, "no usable path - requeuing I/O\n"); @@ -353,7 +351,7 @@ static void nvme_requeue_work(struct work_struct *work) * path. */ bio->bi_disk = head->disk; - generic_make_request(bio); + submit_bio_noacct(bio); } } @@ -375,7 +373,7 @@ int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl, struct nvme_ns_head *head) if (!(ctrl->subsys->cmic & NVME_CTRL_CMIC_MULTI_CTRL) || !multipath) return 0; - q = blk_alloc_queue(nvme_ns_head_make_request, ctrl->numa_node); + q = blk_alloc_queue(ctrl->numa_node); if (!q) goto out; blk_queue_flag_set(QUEUE_FLAG_NONROT, q); diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index 1de3f9b827aa..26099eaf1b1c 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -474,7 +474,7 @@ static inline u32 nvme_bytes_to_numd(size_t len) return (len >> 2) - 1; } -static inline void nvme_end_request(struct request *req, __le16 status, +static inline bool nvme_end_request(struct request *req, __le16 status, union nvme_result result) { struct nvme_request *rq = nvme_req(req); @@ -483,7 +483,9 @@ static inline void nvme_end_request(struct request *req, __le16 status, rq->result = result; /* inject error when permitted by fault injection framework */ nvme_should_fail(req); - blk_mq_complete_request(req); + if (unlikely(blk_should_fake_timeout(req->q))) + return true; + return blk_mq_complete_request_remote(req); } static inline void nvme_get_ctrl(struct nvme_ctrl *ctrl) @@ -586,6 +588,7 @@ void nvme_mpath_stop(struct nvme_ctrl *ctrl); bool nvme_mpath_clear_current_path(struct nvme_ns *ns); void nvme_mpath_clear_ctrl_paths(struct nvme_ctrl *ctrl); struct nvme_ns *nvme_find_path(struct nvme_ns_head *head); +blk_qc_t nvme_ns_head_submit_bio(struct bio *bio); static inline void nvme_mpath_check_last_path(struct nvme_ns *ns) { diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index b1d18f0633c7..74a2e2e00794 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -963,7 +963,8 @@ static inline void nvme_handle_cqe(struct nvme_queue *nvmeq, u16 idx) req = blk_mq_tag_to_rq(nvme_queue_tagset(nvmeq), cqe->command_id); trace_nvme_sq(req, cqe->sq_head, nvmeq->sq_tail); - nvme_end_request(req, cqe->status, cqe->result); + if (!nvme_end_request(req, cqe->status, cqe->result)) + nvme_pci_complete_rq(req); } static inline void nvme_update_cq_head(struct nvme_queue *nvmeq) diff --git a/drivers/nvme/host/rdma.c b/drivers/nvme/host/rdma.c index 13506a87a444..e881f879ac63 100644 --- a/drivers/nvme/host/rdma.c +++ b/drivers/nvme/host/rdma.c @@ -149,6 +149,7 @@ MODULE_PARM_DESC(register_always, static int nvme_rdma_cm_handler(struct rdma_cm_id *cm_id, struct rdma_cm_event *event); static void nvme_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc); +static void nvme_rdma_complete_rq(struct request *rq); static const struct blk_mq_ops nvme_rdma_mq_ops; static const struct blk_mq_ops nvme_rdma_admin_mq_ops; @@ -1149,6 +1150,16 @@ static void nvme_rdma_error_recovery(struct nvme_rdma_ctrl *ctrl) queue_work(nvme_reset_wq, &ctrl->err_work); } +static void nvme_rdma_end_request(struct nvme_rdma_request *req) +{ + struct request *rq = blk_mq_rq_from_pdu(req); + + if (!refcount_dec_and_test(&req->ref)) + return; + if (!nvme_end_request(rq, req->status, req->result)) + nvme_rdma_complete_rq(rq); +} + static void nvme_rdma_wr_error(struct ib_cq *cq, struct ib_wc *wc, const char *op) { @@ -1173,16 +1184,11 @@ static void nvme_rdma_inv_rkey_done(struct ib_cq *cq, struct ib_wc *wc) { struct nvme_rdma_request *req = container_of(wc->wr_cqe, struct nvme_rdma_request, reg_cqe); - struct request *rq = blk_mq_rq_from_pdu(req); - if (unlikely(wc->status != IB_WC_SUCCESS)) { + if (unlikely(wc->status != IB_WC_SUCCESS)) nvme_rdma_wr_error(cq, wc, "LOCAL_INV"); - return; - } - - if (refcount_dec_and_test(&req->ref)) - nvme_end_request(rq, req->status, req->result); - + else + nvme_rdma_end_request(req); } static int nvme_rdma_inv_rkey(struct nvme_rdma_queue *queue, @@ -1547,15 +1553,11 @@ static void nvme_rdma_send_done(struct ib_cq *cq, struct ib_wc *wc) container_of(wc->wr_cqe, struct nvme_rdma_qe, cqe); struct nvme_rdma_request *req = container_of(qe, struct nvme_rdma_request, sqe); - struct request *rq = blk_mq_rq_from_pdu(req); - if (unlikely(wc->status != IB_WC_SUCCESS)) { + if (unlikely(wc->status != IB_WC_SUCCESS)) nvme_rdma_wr_error(cq, wc, "SEND"); - return; - } - - if (refcount_dec_and_test(&req->ref)) - nvme_end_request(rq, req->status, req->result); + else + nvme_rdma_end_request(req); } static int nvme_rdma_post_send(struct nvme_rdma_queue *queue, @@ -1697,8 +1699,7 @@ static void nvme_rdma_process_nvme_rsp(struct nvme_rdma_queue *queue, return; } - if (refcount_dec_and_test(&req->ref)) - nvme_end_request(rq, req->status, req->result); + nvme_rdma_end_request(req); } static void nvme_rdma_recv_done(struct ib_cq *cq, struct ib_wc *wc) diff --git a/drivers/nvme/host/tcp.c b/drivers/nvme/host/tcp.c index 79ef2b8e2b3c..7006aca89456 100644 --- a/drivers/nvme/host/tcp.c +++ b/drivers/nvme/host/tcp.c @@ -464,7 +464,8 @@ static int nvme_tcp_process_nvme_cqe(struct nvme_tcp_queue *queue, return -EINVAL; } - nvme_end_request(rq, cqe->status, cqe->result); + if (!nvme_end_request(rq, cqe->status, cqe->result)) + nvme_complete_rq(rq); queue->nr_cqe++; return 0; @@ -654,7 +655,8 @@ static inline void nvme_tcp_end_request(struct request *rq, u16 status) { union nvme_result res = {}; - nvme_end_request(rq, cpu_to_le16(status << 1), res); + if (!nvme_end_request(rq, cpu_to_le16(status << 1), res)) + nvme_complete_rq(rq); } static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue, struct sk_buff *skb, diff --git a/drivers/nvme/target/core.c b/drivers/nvme/target/core.c index 6e2f623e472e..6816507fba58 100644 --- a/drivers/nvme/target/core.c +++ b/drivers/nvme/target/core.c @@ -467,7 +467,7 @@ static int nvmet_p2pmem_ns_enable(struct nvmet_ns *ns) return -EINVAL; } - if (!blk_queue_pci_p2pdma(ns->bdev->bd_queue)) { + if (!blk_queue_pci_p2pdma(ns->bdev->bd_disk->queue)) { pr_err("peer-to-peer DMA is not supported by the driver of %s\n", ns->device_path); return -EINVAL; diff --git a/drivers/nvme/target/loop.c b/drivers/nvme/target/loop.c index 6344e73c9354..8a0d4fe7bc18 100644 --- a/drivers/nvme/target/loop.c +++ b/drivers/nvme/target/loop.c @@ -116,7 +116,8 @@ static void nvme_loop_queue_response(struct nvmet_req *req) return; } - nvme_end_request(rq, cqe->status, cqe->result); + if (!nvme_end_request(rq, cqe->status, cqe->result)) + nvme_loop_complete_rq(rq); } } |