Protect sw-ctx busy bit manipulations via the sw queue lock.
This allows to convert the atomic bit operations into slightly
faster non-atomic operations. Document that blk_mq_run_hw_queues()
tests these bits without holding the sw queue lock.
Signed-off-by: Bart Van Assche <bart.vanassche@xxxxxxxxxxx>
Cc: Christoph Hellwig <hch@xxxxxx>
Cc: Sagi Grimberg <sagi@xxxxxxxxxxx>
---
block/blk-mq.c | 16 +++++++++++++---
1 file changed, 13 insertions(+), 3 deletions(-)
diff --git a/block/blk-mq.c b/block/blk-mq.c
index 13f5a6c..0dcafa6 100644
--- a/block/blk-mq.c
+++ b/block/blk-mq.c
@@ -66,8 +66,9 @@ static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx,
{
struct blk_align_bitmap *bm = get_bm(hctx, ctx);
+ lockdep_assert_held(&ctx->lock);
if (!test_bit(CTX_TO_BIT(hctx, ctx), &bm->word))
- set_bit(CTX_TO_BIT(hctx, ctx), &bm->word);
+ __set_bit(CTX_TO_BIT(hctx, ctx), &bm->word);
}
static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx,
@@ -75,7 +76,8 @@ static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx,
{
struct blk_align_bitmap *bm = get_bm(hctx, ctx);
- clear_bit(CTX_TO_BIT(hctx, ctx), &bm->word);
+ lockdep_assert_held(&ctx->lock);
+ __clear_bit(CTX_TO_BIT(hctx, ctx), &bm->word);
}
void blk_mq_freeze_queue_start(struct request_queue *q)
@@ -768,8 +770,8 @@ static void flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list)
break;
ctx = hctx->ctxs[bit + off];
- clear_bit(bit, &bm->word);
spin_lock(&ctx->lock);
+ __clear_bit(bit, &bm->word);
list_splice_tail_init(&ctx->rq_list, list);
spin_unlock(&ctx->lock);
@@ -940,6 +942,13 @@ void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
&hctx->run_work, 0);
}
+/**
+ * blk_mq_run_hw_queues - run all hardware queues
+ *
+ * Note: this function checks the SW and HW busy status without serialization
+ * against the functions that modify that status information. The caller is
+ * responsible for realizing that serialization.
+ */
void blk_mq_run_hw_queues(struct request_queue *q, bool async)
{
struct blk_mq_hw_ctx *hctx;
@@ -1055,6 +1064,7 @@ static void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx,
{
struct blk_mq_ctx *ctx = rq->mq_ctx;
+ lockdep_assert_held(&ctx->lock);
__blk_mq_insert_req_list(hctx, rq, at_head);
blk_mq_hctx_mark_pending(hctx, ctx);
}
--
2.9.3
--
To unsubscribe from this list: send the line "unsubscribe linux-block" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
