On Sat, Oct 16, 2021 at 08:06:20PM -0600, Jens Axboe wrote:
> Instead of calling blk_mq_end_request() on a single request, add a helper
> that takes the new struct io_comp_batch and completes any request stored
> in there.
>
> Signed-off-by: Jens Axboe <axboe@xxxxxxxxx>
> ---
> block/blk-mq-tag.c | 6 ++++
> block/blk-mq-tag.h | 1 +
> block/blk-mq.c | 81 ++++++++++++++++++++++++++++++++----------
> include/linux/blk-mq.h | 29 +++++++++++++++
> 4 files changed, 98 insertions(+), 19 deletions(-)
>
> diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c
> index c43b97201161..b94c3e8ef392 100644
> --- a/block/blk-mq-tag.c
> +++ b/block/blk-mq-tag.c
> @@ -207,6 +207,12 @@ void blk_mq_put_tag(struct blk_mq_tags *tags, struct blk_mq_ctx *ctx,
> }
> }
>
> +void blk_mq_put_tags(struct blk_mq_tags *tags, int *tag_array, int nr_tags)
> +{
> + sbitmap_queue_clear_batch(&tags->bitmap_tags, tags->nr_reserved_tags,
> + tag_array, nr_tags);
> +}
> +
> struct bt_iter_data {
> struct blk_mq_hw_ctx *hctx;
> busy_iter_fn *fn;
> diff --git a/block/blk-mq-tag.h b/block/blk-mq-tag.h
> index e617c7220626..df787b5a23bd 100644
> --- a/block/blk-mq-tag.h
> +++ b/block/blk-mq-tag.h
> @@ -19,6 +19,7 @@ unsigned long blk_mq_get_tags(struct blk_mq_alloc_data *data, int nr_tags,
> unsigned int *offset);
> extern void blk_mq_put_tag(struct blk_mq_tags *tags, struct blk_mq_ctx *ctx,
> unsigned int tag);
> +void blk_mq_put_tags(struct blk_mq_tags *tags, int *tag_array, int nr_tags);
> extern int blk_mq_tag_update_depth(struct blk_mq_hw_ctx *hctx,
> struct blk_mq_tags **tags,
> unsigned int depth, bool can_grow);
> diff --git a/block/blk-mq.c b/block/blk-mq.c
> index 8eb80e70e8ea..58dc0c0c24ac 100644
> --- a/block/blk-mq.c
> +++ b/block/blk-mq.c
> @@ -292,15 +292,6 @@ void blk_mq_wake_waiters(struct request_queue *q)
> blk_mq_tag_wakeup_all(hctx->tags, true);
> }
>
> -/*
> - * Only need start/end time stamping if we have iostat or
> - * blk stats enabled, or using an IO scheduler.
> - */
> -static inline bool blk_mq_need_time_stamp(struct request *rq)
> -{
> - return (rq->rq_flags & (RQF_IO_STAT | RQF_STATS | RQF_ELV));
> -}
> -
> static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data,
> unsigned int tag, u64 alloc_time_ns)
> {
> @@ -754,19 +745,21 @@ bool blk_update_request(struct request *req, blk_status_t error,
> }
> EXPORT_SYMBOL_GPL(blk_update_request);
>
> -inline void __blk_mq_end_request(struct request *rq, blk_status_t error)
> +static inline void __blk_mq_end_request_acct(struct request *rq, u64 now)
> {
> - if (blk_mq_need_time_stamp(rq)) {
> - u64 now = ktime_get_ns();
> + if (rq->rq_flags & RQF_STATS) {
> + blk_mq_poll_stats_start(rq->q);
> + blk_stat_add(rq, now);
> + }
>
> + blk_mq_sched_completed_request(rq, now);
> + blk_account_io_done(rq, now);
> +}
>
> - blk_mq_sched_completed_request(rq, now);
> - blk_account_io_done(rq, now);
> - }
> +inline void __blk_mq_end_request(struct request *rq, blk_status_t error)
> +{
> + if (blk_mq_need_time_stamp(rq))
> + __blk_mq_end_request_acct(rq, ktime_get_ns());
>
> if (rq->end_io) {
> rq_qos_done(rq->q, rq);
> @@ -785,6 +778,56 @@ void blk_mq_end_request(struct request *rq, blk_status_t error)
> }
> EXPORT_SYMBOL(blk_mq_end_request);
>
> +#define TAG_COMP_BATCH 32
> +
> +static inline void blk_mq_flush_tag_batch(struct blk_mq_hw_ctx *hctx,
> + int *tag_array, int nr_tags)
> +{
> + struct request_queue *q = hctx->queue;
> +
> + blk_mq_put_tags(hctx->tags, tag_array, nr_tags);
> + percpu_ref_put_many(&q->q_usage_counter, nr_tags);
> +}
> +
> +void blk_mq_end_request_batch(struct io_comp_batch *iob)
> +{
> + int tags[TAG_COMP_BATCH], nr_tags = 0;
> + struct blk_mq_hw_ctx *last_hctx = NULL;
> + struct request *rq;
> + u64 now = 0;
> +
> + if (iob->need_ts)
> + now = ktime_get_ns();
> +
> + while ((rq = rq_list_pop(&iob->req_list)) != NULL) {
> + prefetch(rq->bio);
> + prefetch(rq->rq_next);
> +
> + blk_update_request(rq, BLK_STS_OK, blk_rq_bytes(rq));
> + __blk_mq_end_request_acct(rq, now);
If iob->need_ts is not set we don't need to call
__blk_mq_end_request_acct, do we?
Otherwise looks good:
Reviewed-by: Christoph Hellwig <hch@xxxxxx>
