On 12/10/2021 19:17, Jens Axboe wrote:
Signed-off-by: Jens Axboe<axboe@xxxxxxxxx>
---
drivers/nvme/host/pci.c | 69 +++++++++++++++++++++++++++++++++++++----
1 file changed, 63 insertions(+), 6 deletions(-)
diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c
index 4ad63bb9f415..4713da708cd4 100644
--- a/drivers/nvme/host/pci.c
+++ b/drivers/nvme/host/pci.c
@@ -959,7 +959,7 @@ static blk_status_t nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
return ret;
}
-static void nvme_pci_complete_rq(struct request *req)
+static void nvme_pci_unmap_rq(struct request *req)
{
struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
struct nvme_dev *dev = iod->nvmeq->dev;
@@ -969,9 +969,34 @@ static void nvme_pci_complete_rq(struct request *req)
rq_integrity_vec(req)->bv_len, rq_data_dir(req));
if (blk_rq_nr_phys_segments(req))
nvme_unmap_data(dev, req);
+}
+
+static void nvme_pci_complete_rq(struct request *req)
+{
+ nvme_pci_unmap_rq(req);
nvme_complete_rq(req);
}
+static void nvme_pci_complete_batch(struct io_batch *ib)
+{
+ struct request *req;
+
+ req = ib->req_list;
+ while (req) {
+ nvme_pci_unmap_rq(req);
This will do the DMA SG unmap per request. Often this is a performance
bottle neck when we have an IOMMU enabled in strict mode. So since we
complete in batches, could we combine all the SGs in the batch to do one
big DMA unmap SG, and not one-by-one?
Just a thought.
+ if (req->rq_flags & RQF_SPECIAL_PAYLOAD)
+ nvme_cleanup_cmd(req);
+ if (IS_ENABLED(CONFIG_BLK_DEV_ZONED) &&
+ req_op(req) == REQ_OP_ZONE_APPEND)
+ req->__sector = nvme_lba_to_sect(req->q->queuedata,
+ le64_to_cpu(nvme_req(req)->result.u64));
+ req->status = nvme_error_status(nvme_req(req)->status);
+ req = req->rq_next;
+ }
+
+ blk_mq_end_request_batch(ib);
+}
