On 10/18/21 4:18 AM, Christoph Hellwig wrote:
> On Sat, Oct 16, 2021 at 08:06:20PM -0600, Jens Axboe wrote:
>> Instead of calling blk_mq_end_request() on a single request, add a helper
>> that takes the new struct io_comp_batch and completes any request stored
>> in there.
>>
>> Signed-off-by: Jens Axboe <axboe@xxxxxxxxx>
>> ---
>> block/blk-mq-tag.c | 6 ++++
>> block/blk-mq-tag.h | 1 +
>> block/blk-mq.c | 81 ++++++++++++++++++++++++++++++++----------
>> include/linux/blk-mq.h | 29 +++++++++++++++
>> 4 files changed, 98 insertions(+), 19 deletions(-)
>>
>> diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c
>> index c43b97201161..b94c3e8ef392 100644
>> --- a/block/blk-mq-tag.c
>> +++ b/block/blk-mq-tag.c
>> @@ -207,6 +207,12 @@ void blk_mq_put_tag(struct blk_mq_tags *tags, struct blk_mq_ctx *ctx,
>> }
>> }
>>
>> +void blk_mq_put_tags(struct blk_mq_tags *tags, int *tag_array, int nr_tags)
>> +{
>> + sbitmap_queue_clear_batch(&tags->bitmap_tags, tags->nr_reserved_tags,
>> + tag_array, nr_tags);
>> +}
>> +
>> struct bt_iter_data {
>> struct blk_mq_hw_ctx *hctx;
>> busy_iter_fn *fn;
>> diff --git a/block/blk-mq-tag.h b/block/blk-mq-tag.h
>> index e617c7220626..df787b5a23bd 100644
>> --- a/block/blk-mq-tag.h
>> +++ b/block/blk-mq-tag.h
>> @@ -19,6 +19,7 @@ unsigned long blk_mq_get_tags(struct blk_mq_alloc_data *data, int nr_tags,
>> unsigned int *offset);
>> extern void blk_mq_put_tag(struct blk_mq_tags *tags, struct blk_mq_ctx *ctx,
>> unsigned int tag);
>> +void blk_mq_put_tags(struct blk_mq_tags *tags, int *tag_array, int nr_tags);
>> extern int blk_mq_tag_update_depth(struct blk_mq_hw_ctx *hctx,
>> struct blk_mq_tags **tags,
>> unsigned int depth, bool can_grow);
>> diff --git a/block/blk-mq.c b/block/blk-mq.c
>> index 8eb80e70e8ea..58dc0c0c24ac 100644
>> --- a/block/blk-mq.c
>> +++ b/block/blk-mq.c
>> @@ -292,15 +292,6 @@ void blk_mq_wake_waiters(struct request_queue *q)
>> blk_mq_tag_wakeup_all(hctx->tags, true);
>> }
>>
>> -/*
>> - * Only need start/end time stamping if we have iostat or
>> - * blk stats enabled, or using an IO scheduler.
>> - */
>> -static inline bool blk_mq_need_time_stamp(struct request *rq)
>> -{
>> - return (rq->rq_flags & (RQF_IO_STAT | RQF_STATS | RQF_ELV));
>> -}
>> -
>> static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data,
>> unsigned int tag, u64 alloc_time_ns)
>> {
>> @@ -754,19 +745,21 @@ bool blk_update_request(struct request *req, blk_status_t error,
>> }
>> EXPORT_SYMBOL_GPL(blk_update_request);
>>
>> -inline void __blk_mq_end_request(struct request *rq, blk_status_t error)
>> +static inline void __blk_mq_end_request_acct(struct request *rq, u64 now)
>> {
>> - if (blk_mq_need_time_stamp(rq)) {
>> - u64 now = ktime_get_ns();
>> + if (rq->rq_flags & RQF_STATS) {
>> + blk_mq_poll_stats_start(rq->q);
>> + blk_stat_add(rq, now);
>> + }
>>
>> + blk_mq_sched_completed_request(rq, now);
>> + blk_account_io_done(rq, now);
>> +}
>>
>> - blk_mq_sched_completed_request(rq, now);
>> - blk_account_io_done(rq, now);
>> - }
>> +inline void __blk_mq_end_request(struct request *rq, blk_status_t error)
>> +{
>> + if (blk_mq_need_time_stamp(rq))
>> + __blk_mq_end_request_acct(rq, ktime_get_ns());
>>
>> if (rq->end_io) {
>> rq_qos_done(rq->q, rq);
>> @@ -785,6 +778,56 @@ void blk_mq_end_request(struct request *rq, blk_status_t error)
>> }
>> EXPORT_SYMBOL(blk_mq_end_request);
>>
>> +#define TAG_COMP_BATCH 32
>> +
>> +static inline void blk_mq_flush_tag_batch(struct blk_mq_hw_ctx *hctx,
>> + int *tag_array, int nr_tags)
>> +{
>> + struct request_queue *q = hctx->queue;
>> +
>> + blk_mq_put_tags(hctx->tags, tag_array, nr_tags);
>> + percpu_ref_put_many(&q->q_usage_counter, nr_tags);
>> +}
>> +
>> +void blk_mq_end_request_batch(struct io_comp_batch *iob)
>> +{
>> + int tags[TAG_COMP_BATCH], nr_tags = 0;
>> + struct blk_mq_hw_ctx *last_hctx = NULL;
>> + struct request *rq;
>> + u64 now = 0;
>> +
>> + if (iob->need_ts)
>> + now = ktime_get_ns();
>> +
>> + while ((rq = rq_list_pop(&iob->req_list)) != NULL) {
>> + prefetch(rq->bio);
>> + prefetch(rq->rq_next);
>> +
>> + blk_update_request(rq, BLK_STS_OK, blk_rq_bytes(rq));
>> + __blk_mq_end_request_acct(rq, now);
>
> If iob->need_ts is not set we don't need to call
> __blk_mq_end_request_acct, do we?
We don't strictly need to, I'll make that change.
--
Jens Axboe
