Drivers that have shared tagsets may need to quiesce potentially a lot
of request queues that all share a single tagset (e.g. nvme). Add an
interface to quiesce all the queues on a given tagset. This interface is
useful because it can speedup the quiesce by doing it in parallel.
For tagsets that have BLK_MQ_F_BLOCKING set, we first call
start_poll_synchronize_srcu for all queues of the tagset, and then call
poll_state_synchronize_srcu such that all of them wait for the same srcu
elapsed period. For tagsets that don't have BLK_MQ_F_BLOCKING set,
we simply call a single synchronize_rcu as this is sufficient.
Because some queues should not need to be quiesced(e.g. nvme connect_q)
when quiesce the tagset. So introduce QUEUE_FLAG_SKIP_TAGSET_QUIESCE to
tagset quiesce interface to skip the queue.
Signed-off-by: Sagi Grimberg <sagi@xxxxxxxxxxx>
Signed-off-by: Chao Leng <lengchao@xxxxxxxxxx>
---
block/blk-mq.c | 76 ++++++++++++++++++++++++++++++++++++++++++++++++++
include/linux/blk-mq.h | 2 ++
include/linux/blkdev.h | 3 ++
3 files changed, 81 insertions(+)
diff --git a/block/blk-mq.c b/block/blk-mq.c
index 8070b6c10e8d..f064ecda425b 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -311,6 +311,82 @@ void blk_mq_unquiesce_queue(struct request_queue *q)
}
EXPORT_SYMBOL_GPL(blk_mq_unquiesce_queue);
+static void blk_mq_quiesce_blocking_tagset(struct blk_mq_tag_set *set)
+{
+ int i, count = 0;
+ struct request_queue *q;
+ unsigned long *rcu;
+
+ list_for_each_entry(q, &set->tag_list, tag_set_list) {
+ if (blk_queue_skip_tagset_quiesce(q))
+ continue;
+
+ blk_mq_quiesce_queue_nowait(q);
+ count++;
+ }
+
+ rcu = kvmalloc(count * sizeof(*rcu), GFP_KERNEL);
+ if (rcu) {
+ i = 0;
+ list_for_each_entry(q, &set->tag_list, tag_set_list) {
+ if (blk_queue_skip_tagset_quiesce(q))
+ continue;
+
+ rcu[i++] = start_poll_synchronize_srcu(q->srcu);
+ }
+
+ i = 0;
+ list_for_each_entry(q, &set->tag_list, tag_set_list) {
+ if (blk_queue_skip_tagset_quiesce(q))
+ continue;
+
+ if (!poll_state_synchronize_srcu(q->srcu, rcu[i++]))
+ synchronize_srcu(q->srcu);
+ }
+
+ kvfree(rcu);
+ } else {
+ list_for_each_entry(q, &set->tag_list, tag_set_list)
+ synchronize_srcu(q->srcu);
+ }
+}
+
+static void blk_mq_quiesce_nonblocking_tagset(struct blk_mq_tag_set *set)
+{
+ struct request_queue *q;
+
+ list_for_each_entry(q, &set->tag_list, tag_set_list) {
+ if (blk_queue_skip_tagset_quiesce(q))
+ continue;
+
+ blk_mq_quiesce_queue_nowait(q);
+ }
+ synchronize_rcu();
+}
+
+void blk_mq_quiesce_tagset(struct blk_mq_tag_set *set)
+{
+ mutex_lock(&set->tag_list_lock);
+ if (set->flags & BLK_MQ_F_BLOCKING)
+ blk_mq_quiesce_blocking_tagset(set);
+ else
+ blk_mq_quiesce_nonblocking_tagset(set);
+
+ mutex_unlock(&set->tag_list_lock);
+}
+EXPORT_SYMBOL_GPL(blk_mq_quiesce_tagset);
+
+void blk_mq_unquiesce_tagset(struct blk_mq_tag_set *set)
+{
+ struct request_queue *q;
+
+ mutex_lock(&set->tag_list_lock);
+ list_for_each_entry(q, &set->tag_list, tag_set_list)
+ blk_mq_unquiesce_queue(q);
+ mutex_unlock(&set->tag_list_lock);
+}
+EXPORT_SYMBOL_GPL(blk_mq_unquiesce_tagset);
+
void blk_mq_wake_waiters(struct request_queue *q)
{
struct blk_mq_hw_ctx *hctx;
diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h
index ba18e9bdb799..1df47606d0a7 100644
--- a/include/linux/blk-mq.h
+++ b/include/linux/blk-mq.h
@@ -877,6 +877,8 @@ void blk_mq_start_hw_queues(struct request_queue *q);
void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async);
void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async);
void blk_mq_quiesce_queue(struct request_queue *q);
+void blk_mq_quiesce_tagset(struct blk_mq_tag_set *set);
+void blk_mq_unquiesce_tagset(struct blk_mq_tag_set *set);
void blk_mq_wait_quiesce_done(struct request_queue *q);
void blk_mq_unquiesce_queue(struct request_queue *q);
void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs);
diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h
index 50e358a19d98..efa3fa771dce 100644
--- a/include/linux/blkdev.h
+++ b/include/linux/blkdev.h
@@ -579,6 +579,7 @@ struct request_queue {
#define QUEUE_FLAG_HCTX_ACTIVE 28 /* at least one blk-mq hctx is active */
#define QUEUE_FLAG_NOWAIT 29 /* device supports NOWAIT */
#define QUEUE_FLAG_SQ_SCHED 30 /* single queue style io dispatch */
+#define QUEUE_FLAG_SKIP_TAGSET_QUIESCE 31 /* quiesce_tagset skip the queue*/
#define QUEUE_FLAG_MQ_DEFAULT ((1UL << QUEUE_FLAG_IO_STAT) | \
(1UL << QUEUE_FLAG_SAME_COMP) | \
@@ -619,6 +620,8 @@ bool blk_queue_flag_test_and_set(unsigned int flag, struct request_queue *q);
#define blk_queue_pm_only(q) atomic_read(&(q)->pm_only)
#define blk_queue_registered(q) test_bit(QUEUE_FLAG_REGISTERED, &(q)->queue_flags)
#define blk_queue_sq_sched(q) test_bit(QUEUE_FLAG_SQ_SCHED, &(q)->queue_flags)
+#define blk_queue_skip_tagset_quiesce(q) \
+ test_bit(QUEUE_FLAG_SKIP_TAGSET_QUIESCE, &(q)->queue_flags)
extern void blk_set_pm_only(struct request_queue *q);
extern void blk_clear_pm_only(struct request_queue *q);
--
2.16.4
