Supposing the following scenario with a virtio_blk driver.
CPU0 CPU1
/*
* Add request to dispatch list or set bitmap of
* software queue. 1) store virtblk_done()
*/
blk_mq_run_hw_queues()/blk_mq_delay_run_hw_queues() blk_mq_start_stopped_hw_queues()
if (blk_mq_hctx_stopped()) 2) load blk_mq_start_stopped_hw_queue()
continue clear_bit(BLK_MQ_S_STOPPED) 3) store
blk_mq_run_hw_queue()/blk_mq_delay_run_hw_queue() blk_mq_run_hw_queue()
if (!blk_mq_hctx_has_pending()) 4) load
return
blk_mq_sched_dispatch_requests()
The full memory barrier should be inserted between 1) and 2), as well as between
3) and 4) to make sure that either CPU0 sees BLK_MQ_S_STOPPED is cleared or CPU1
sees dispatch list or setting of bitmap of software queue. Otherwise, either CPU
will not re-run the hardware queue causing starvation.
Signed-off-by: Muchun Song <songmuchun@xxxxxxxxxxxxx>
---
block/blk-mq.c | 24 ++++++++++++++++++++++++
1 file changed, 24 insertions(+)
diff --git a/block/blk-mq.c b/block/blk-mq.c
index 6f18993b8f454..385a74e566874 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -2299,6 +2299,18 @@ void blk_mq_run_hw_queues(struct request_queue *q, bool async)
sq_hctx = NULL;
if (blk_queue_sq_sched(q))
sq_hctx = blk_mq_get_sq_hctx(q);
+
+ /*
+ * This barrier is used to order adding of dispatch list or setting
+ * of bitmap of any software queue outside of this function and the
+ * test of BLK_MQ_S_STOPPED in the following routine. Pairs with the
+ * barrier in blk_mq_start_stopped_hw_queue(). So dispatch code could
+ * either see BLK_MQ_S_STOPPED is cleared or dispatch list or setting
+ * of bitmap of any software queue to avoid missing dispatching
+ * requests.
+ */
+ smp_mb();
+
queue_for_each_hw_ctx(q, hctx, i) {
if (blk_mq_hctx_stopped(hctx))
continue;
@@ -2327,6 +2339,18 @@ void blk_mq_delay_run_hw_queues(struct request_queue *q, unsigned long msecs)
sq_hctx = NULL;
if (blk_queue_sq_sched(q))
sq_hctx = blk_mq_get_sq_hctx(q);
+
+ /*
+ * This barrier is used to order adding of dispatch list or setting
+ * of bitmap of any software queue outside of this function and the
+ * test of BLK_MQ_S_STOPPED in the following routine. Pairs with the
+ * barrier in blk_mq_start_stopped_hw_queue(). So dispatch code could
+ * either see BLK_MQ_S_STOPPED is cleared or dispatch list or setting
+ * of bitmap of any software queue to avoid missing dispatching
+ * requests.
+ */
+ smp_mb();
+
queue_for_each_hw_ctx(q, hctx, i) {
if (blk_mq_hctx_stopped(hctx))
continue;
--
2.20.1
