Signed-off-by: Ming Lei <ming.lei@xxxxxxxxxx> --- block/blk-mq-sched.c | 13 +++---------- block/blk-mq.c | 24 +++++++++++------------- block/blk-mq.h | 40 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 23 deletions(-) diff --git a/block/blk-mq-sched.c b/block/blk-mq-sched.c index 9fae76275acf..5d435f01ecc8 100644 --- a/block/blk-mq-sched.c +++ b/block/blk-mq-sched.c @@ -152,12 +152,8 @@ void blk_mq_sched_dispatch_requests(struct blk_mq_hw_ctx *hctx) * If we have previous entries on our dispatch list, grab them first for * more fair dispatch. */ - if (!list_empty_careful(&hctx->dispatch)) { - spin_lock(&hctx->lock); - if (!list_empty(&hctx->dispatch)) - list_splice_init(&hctx->dispatch, &rq_list); - spin_unlock(&hctx->lock); - } + if (blk_mq_has_dispatch_rqs(hctx)) + blk_mq_take_list_from_dispatch(hctx, &rq_list); /* * Only ask the scheduler for requests, if we didn't have residual @@ -341,10 +337,7 @@ static bool blk_mq_sched_bypass_insert(struct blk_mq_hw_ctx *hctx, * If we already have a real request tag, send directly to * the dispatch list. */ - spin_lock(&hctx->lock); - blk_mq_hctx_set_dispatch_busy(hctx); - list_add(&rq->queuelist, &hctx->dispatch); - spin_unlock(&hctx->lock); + blk_mq_add_rq_to_dispatch(hctx, rq); return true; } diff --git a/block/blk-mq.c b/block/blk-mq.c index 11042aa0501d..2392a813f5ee 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -63,7 +63,7 @@ static int blk_mq_poll_stats_bkt(const struct request *rq) bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx) { return sbitmap_any_bit_set(&hctx->ctx_map) || - !list_empty_careful(&hctx->dispatch) || + blk_mq_has_dispatch_rqs(hctx) || blk_mq_sched_has_work(hctx); } @@ -1100,14 +1100,7 @@ bool blk_mq_dispatch_rq_list(struct request_queue *q, struct list_head *list) rq = list_first_entry(list, struct request, queuelist); blk_mq_put_driver_tag(rq); - spin_lock(&hctx->lock); - /* - * DISPATCH_BUSY won't be cleared until all requests - * in hctx->dispatch are dispatched successfully - */ - blk_mq_hctx_set_dispatch_busy(hctx); - list_splice_init(list, &hctx->dispatch); - spin_unlock(&hctx->lock); + blk_mq_add_list_to_dispatch(hctx, list); /* * If SCHED_RESTART was set by the caller of this function and @@ -1882,10 +1875,7 @@ static int blk_mq_hctx_notify_dead(unsigned int cpu, struct hlist_node *node) if (list_empty(&tmp)) return 0; - spin_lock(&hctx->lock); - blk_mq_hctx_set_dispatch_busy(hctx); - list_splice_tail_init(&tmp, &hctx->dispatch); - spin_unlock(&hctx->lock); + blk_mq_add_list_to_dispatch_tail(hctx, &tmp); blk_mq_run_hw_queue(hctx, true); return 0; @@ -1935,6 +1925,13 @@ static void blk_mq_exit_hw_queues(struct request_queue *q, } } +static void blk_mq_init_dispatch(struct request_queue *q, + struct blk_mq_hw_ctx *hctx) +{ + spin_lock_init(&hctx->lock); + INIT_LIST_HEAD(&hctx->dispatch); +} + static int blk_mq_init_hctx(struct request_queue *q, struct blk_mq_tag_set *set, struct blk_mq_hw_ctx *hctx, unsigned hctx_idx) @@ -1948,6 +1945,7 @@ static int blk_mq_init_hctx(struct request_queue *q, INIT_DELAYED_WORK(&hctx->run_work, blk_mq_run_work_fn); spin_lock_init(&hctx->lock); INIT_LIST_HEAD(&hctx->dispatch); + blk_mq_init_dispatch(q, hctx); hctx->queue = q; hctx->flags = set->flags & ~BLK_MQ_F_TAG_SHARED; diff --git a/block/blk-mq.h b/block/blk-mq.h index cadc0c83a140..7f0d35ca5fea 100644 --- a/block/blk-mq.h +++ b/block/blk-mq.h @@ -151,4 +151,44 @@ static inline void blk_mq_hctx_clear_dispatch_busy(struct blk_mq_hw_ctx *hctx) clear_bit(BLK_MQ_S_DISPATCH_BUSY, &hctx->state); } +static inline bool blk_mq_has_dispatch_rqs(struct blk_mq_hw_ctx *hctx) +{ + return !list_empty_careful(&hctx->dispatch); +} + +static inline void blk_mq_add_rq_to_dispatch(struct blk_mq_hw_ctx *hctx, + struct request *rq) +{ + spin_lock(&hctx->lock); + list_add(&rq->queuelist, &hctx->dispatch); + blk_mq_hctx_set_dispatch_busy(hctx); + spin_unlock(&hctx->lock); +} + +static inline void blk_mq_add_list_to_dispatch(struct blk_mq_hw_ctx *hctx, + struct list_head *list) +{ + spin_lock(&hctx->lock); + list_splice_init(list, &hctx->dispatch); + blk_mq_hctx_set_dispatch_busy(hctx); + spin_unlock(&hctx->lock); +} + +static inline void blk_mq_add_list_to_dispatch_tail(struct blk_mq_hw_ctx *hctx, + struct list_head *list) +{ + spin_lock(&hctx->lock); + list_splice_tail_init(list, &hctx->dispatch); + blk_mq_hctx_set_dispatch_busy(hctx); + spin_unlock(&hctx->lock); +} + +static inline void blk_mq_take_list_from_dispatch(struct blk_mq_hw_ctx *hctx, + struct list_head *list) +{ + spin_lock(&hctx->lock); + list_splice_init(&hctx->dispatch, list); + spin_unlock(&hctx->lock); +} + #endif -- 2.9.4