If we have a list of requests in our plug list, send it to the driver in
one go, if possible. The driver must set mq_ops->queue_rqs() to support
this, if not the usual one-by-one path is used.
Signed-off-by: Jens Axboe <axboe@xxxxxxxxx>
---
block/blk-mq.c | 17 +++++++++++++++++
include/linux/blk-mq.h | 8 ++++++++
2 files changed, 25 insertions(+)
diff --git a/block/blk-mq.c b/block/blk-mq.c
index 9b4e79e2ac1e..005715206b16 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -2208,6 +2208,19 @@ static void blk_mq_plug_issue_direct(struct blk_plug *plug, bool from_schedule)
int queued = 0;
int errors = 0;
+ /*
+ * Peek first request and see if we have a ->queue_rqs() hook. If we
+ * do, we can dispatch the whole plug list in one go. We already know
+ * at this point that all requests belong to the same queue, caller
+ * must ensure that's the case.
+ */
+ rq = rq_list_peek(&plug->mq_list);
+ if (rq->q->mq_ops->queue_rqs) {
+ rq->q->mq_ops->queue_rqs(&plug->mq_list);
+ if (rq_list_empty(plug->mq_list))
+ return;
+ }
+
while ((rq = rq_list_pop(&plug->mq_list))) {
bool last = rq_list_empty(plug->mq_list);
blk_status_t ret;
@@ -2256,6 +2269,10 @@ void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
if (!plug->multiple_queues && !plug->has_elevator && !from_schedule) {
blk_mq_plug_issue_direct(plug, false);
+ /*
+ * Expected case, all requests got dispatched. If not, fall
+ * through to individual dispatch of the remainder.
+ */
if (rq_list_empty(plug->mq_list))
return;
}
diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h
index 3ba1e750067b..897cf475e7eb 100644
--- a/include/linux/blk-mq.h
+++ b/include/linux/blk-mq.h
@@ -503,6 +503,14 @@ struct blk_mq_ops {
*/
void (*commit_rqs)(struct blk_mq_hw_ctx *);
+ /**
+ * @queue_rqs: Queue a list of new requests. Driver is guaranteed
+ * that each request belongs to the same queue. If the driver doesn't
+ * empty the @rqlist completely, then the rest will be queued
+ * individually by the block layer upon return.
+ */
+ void (*queue_rqs)(struct request **rqlist);
+
/**
* @get_budget: Reserve budget before queue request, once .queue_rq is
* run, it is driver's responsibility to release the
--
2.33.1
