In legacy path, when one request is executed via blk_execute_rq(),
the request is added to the front of q->queue_head directly,
and I/O scheduler's queue is bypassed because either merging
or sorting isn't needed.
When SCSI device is put into quiece state, such as during
system suspend, we need to add the RQF_PM request into
front of the queue.
This patch fixes I/O hang after system resume by taking
the similar implementation of legacy path.
Tested-by: Oleksandr Natalenko <oleksandr@xxxxxxxxxxxxxx>
Reported-by: Oleksandr Natalenko <oleksandr@xxxxxxxxxxxxxx>
Signed-off-by: Ming Lei <ming.lei@xxxxxxxxxx>
---
block/blk-core.c | 2 +-
block/blk-exec.c | 2 +-
block/blk-flush.c | 2 +-
block/blk-mq-sched.c | 58 ++++++++++++++++++++++++++++++++++++++++++++++++++++
block/blk-mq-sched.h | 2 ++
5 files changed, 63 insertions(+), 3 deletions(-)
diff --git a/block/blk-core.c b/block/blk-core.c
index dbecbf4a64e0..fb75bc646ebc 100644
--- a/block/blk-core.c
+++ b/block/blk-core.c
@@ -2330,7 +2330,7 @@ blk_status_t blk_insert_cloned_request(struct request_queue *q, struct request *
if (q->mq_ops) {
if (blk_queue_io_stat(q))
blk_account_io_start(rq, true);
- blk_mq_sched_insert_request(rq, false, true, false, false);
+ blk_mq_sched_insert_request_bypass(rq, false, true, false, false);
return BLK_STS_OK;
}
diff --git a/block/blk-exec.c b/block/blk-exec.c
index 5c0f3dc446dc..4565aa6bb624 100644
--- a/block/blk-exec.c
+++ b/block/blk-exec.c
@@ -61,7 +61,7 @@ void blk_execute_rq_nowait(struct request_queue *q, struct gendisk *bd_disk,
* be reused after dying flag is set
*/
if (q->mq_ops) {
- blk_mq_sched_insert_request(rq, at_head, true, false, false);
+ blk_mq_sched_insert_request_bypass(rq, at_head, true, false, false);
return;
}
diff --git a/block/blk-flush.c b/block/blk-flush.c
index ed5fe322abba..51e89e5c525a 100644
--- a/block/blk-flush.c
+++ b/block/blk-flush.c
@@ -463,7 +463,7 @@ void blk_insert_flush(struct request *rq)
if ((policy & REQ_FSEQ_DATA) &&
!(policy & (REQ_FSEQ_PREFLUSH | REQ_FSEQ_POSTFLUSH))) {
if (q->mq_ops)
- blk_mq_sched_insert_request(rq, false, true, false, false);
+ blk_mq_sched_insert_request_bypass(rq, false, true, false, false);
else
list_add_tail(&rq->queuelist, &q->queue_head);
return;
diff --git a/block/blk-mq-sched.c b/block/blk-mq-sched.c
index 8d97df40fc28..b40dd063d61f 100644
--- a/block/blk-mq-sched.c
+++ b/block/blk-mq-sched.c
@@ -354,6 +354,64 @@ static void blk_mq_sched_insert_flush(struct blk_mq_hw_ctx *hctx,
blk_mq_add_to_requeue_list(rq, false, true);
}
+static void blk_mq_flush_hctx(struct blk_mq_hw_ctx *hctx,
+ struct elevator_queue *e,
+ const bool has_sched_dispatch,
+ struct list_head *rqs)
+{
+ LIST_HEAD(list);
+
+ if (!has_sched_dispatch)
+ blk_mq_flush_busy_ctxs(hctx, &list);
+ else {
+ while (true) {
+ struct request *rq;
+
+ rq = e->type->ops.mq.dispatch_request(hctx);
+ if (!rq)
+ break;
+ list_add_tail(&rq->queuelist, &list);
+ }
+ }
+
+ list_splice_tail(&list, rqs);
+}
+
+void blk_mq_sched_insert_request_bypass(struct request *rq, bool at_head,
+ bool run_queue, bool async,
+ bool can_block)
+{
+ struct request_queue *q = rq->q;
+ struct elevator_queue *e = q->elevator;
+ struct blk_mq_ctx *ctx = rq->mq_ctx;
+ struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, ctx->cpu);
+ LIST_HEAD(list);
+ const bool has_sched_dispatch = e && e->type->ops.mq.dispatch_request;
+
+ if (rq->tag == -1 && op_is_flush(rq->cmd_flags)) {
+ blk_mq_sched_insert_flush(hctx, rq, can_block);
+ return;
+ }
+
+ if (at_head)
+ list_add_tail(&rq->queuelist, &list);
+ else {
+ blk_mq_flush_hctx(hctx, e, has_sched_dispatch, &list);
+ list_add_tail(&rq->queuelist, &list);
+ run_queue = true;
+ }
+
+ spin_lock(&hctx->lock);
+ if (at_head)
+ list_splice(&list, &hctx->dispatch);
+ else
+ list_splice_tail(&list, &hctx->dispatch);
+ spin_unlock(&hctx->lock);
+
+ if (run_queue)
+ blk_mq_run_hw_queue(hctx, async);
+}
+
void blk_mq_sched_insert_request(struct request *rq, bool at_head,
bool run_queue, bool async, bool can_block)
{
diff --git a/block/blk-mq-sched.h b/block/blk-mq-sched.h
index 9267d0b7c197..4d01697a627f 100644
--- a/block/blk-mq-sched.h
+++ b/block/blk-mq-sched.h
@@ -18,6 +18,8 @@ void blk_mq_sched_restart(struct blk_mq_hw_ctx *hctx);
void blk_mq_sched_insert_request(struct request *rq, bool at_head,
bool run_queue, bool async, bool can_block);
+void blk_mq_sched_insert_request_bypass(struct request *rq, bool at_head,
+ bool run_queue, bool async, bool can_block);
void blk_mq_sched_insert_requests(struct request_queue *q,
struct blk_mq_ctx *ctx,
struct list_head *list, bool run_queue_async);
--
2.9.5
