The tags allocation is limited by the fair sharing algorithm. It hurts the performance for UFS devices, because the queue depth of general I/O is reduced by half once the UFS send a control command. Add a new queue flag QUEUE_FLAG_FAIR_TAG_SHARING to make the fair tag sharing configurable. See also https://lore.kernel.org/all/20230103195337.158625-1-bvanassche@xxxxxxx Signed-off-by: Ed Tsai <ed.tsai@xxxxxxxxxxxx> --- block/blk-mq-debugfs.c | 1 + block/blk-mq-tag.c | 1 + block/blk-mq.c | 3 ++- drivers/ufs/core/ufshcd.c | 3 +++ include/linux/blkdev.h | 6 +++++- 5 files changed, 12 insertions(+), 2 deletions(-) diff --git a/block/blk-mq-debugfs.c b/block/blk-mq-debugfs.c index d23a8554ec4a..f03b8bfe63be 100644 --- a/block/blk-mq-debugfs.c +++ b/block/blk-mq-debugfs.c @@ -103,6 +103,7 @@ static const char *const blk_queue_flag_name[] = { QUEUE_FLAG_NAME(RQ_ALLOC_TIME), QUEUE_FLAG_NAME(HCTX_ACTIVE), QUEUE_FLAG_NAME(NOWAIT), + QUEUE_FLAG_NAME(FAIR_TAG_SHARING), }; #undef QUEUE_FLAG_NAME diff --git a/block/blk-mq-tag.c b/block/blk-mq-tag.c index d6af9d431dc6..b8b36823f5f5 100644 --- a/block/blk-mq-tag.c +++ b/block/blk-mq-tag.c @@ -97,6 +97,7 @@ static int __blk_mq_get_tag(struct blk_mq_alloc_data *data, struct sbitmap_queue *bt) { if (!data->q->elevator && !(data->flags & BLK_MQ_REQ_RESERVED) && + blk_queue_fair_tag_sharing(data->q) && !hctx_may_queue(data->hctx, bt)) return BLK_MQ_NO_TAG; diff --git a/block/blk-mq.c b/block/blk-mq.c index f6dad0886a2f..41f7e2b500bd 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -1746,7 +1746,8 @@ static bool __blk_mq_alloc_driver_tag(struct request *rq) bt = &rq->mq_hctx->tags->breserved_tags; tag_offset = 0; } else { - if (!hctx_may_queue(rq->mq_hctx, bt)) + if (blk_queue_fair_tag_sharing(rq->q) && + !hctx_may_queue(rq->mq_hctx, bt)) return false; } diff --git a/drivers/ufs/core/ufshcd.c b/drivers/ufs/core/ufshcd.c index 17d7bb875fee..e96a50265285 100644 --- a/drivers/ufs/core/ufshcd.c +++ b/drivers/ufs/core/ufshcd.c @@ -5149,6 +5149,9 @@ static int ufshcd_slave_configure(struct scsi_device *sdev) blk_queue_update_dma_pad(q, PRDT_DATA_BYTE_COUNT_PAD - 1); if (hba->quirks & UFSHCD_QUIRK_4KB_DMA_ALIGNMENT) blk_queue_update_dma_alignment(q, 4096 - 1); + + blk_queue_flag_clear(QUEUE_FLAG_FAIR_TAG_SHARING, q); + /* * Block runtime-pm until all consumers are added. * Refer ufshcd_setup_links(). diff --git a/include/linux/blkdev.h b/include/linux/blkdev.h index b441e633f4dd..7fcb2356860d 100644 --- a/include/linux/blkdev.h +++ b/include/linux/blkdev.h @@ -561,10 +561,12 @@ struct request_queue { #define QUEUE_FLAG_NOWAIT 29 /* device supports NOWAIT */ #define QUEUE_FLAG_SQ_SCHED 30 /* single queue style io dispatch */ #define QUEUE_FLAG_SKIP_TAGSET_QUIESCE 31 /* quiesce_tagset skip the queue*/ +#define QUEUE_FLAG_FAIR_TAG_SHARING 32 /* fair allocation of shared tags */ #define QUEUE_FLAG_MQ_DEFAULT ((1UL << QUEUE_FLAG_IO_STAT) | \ (1UL << QUEUE_FLAG_SAME_COMP) | \ - (1UL << QUEUE_FLAG_NOWAIT)) + (1UL << QUEUE_FLAG_NOWAIT) | \ + (1UL << QUEUE_FLAG_FAIR_TAG_SHARING)) void blk_queue_flag_set(unsigned int flag, struct request_queue *q); void blk_queue_flag_clear(unsigned int flag, struct request_queue *q); @@ -602,6 +604,8 @@ bool blk_queue_flag_test_and_set(unsigned int flag, struct request_queue *q); #define blk_queue_sq_sched(q) test_bit(QUEUE_FLAG_SQ_SCHED, &(q)->queue_flags) #define blk_queue_skip_tagset_quiesce(q) \ test_bit(QUEUE_FLAG_SKIP_TAGSET_QUIESCE, &(q)->queue_flags) +#define blk_queue_fair_tag_sharing(q) \ + test_bit(QUEUE_FLAG_FAIR_TAG_SHARING, &(q)->queue_flags) extern void blk_set_pm_only(struct request_queue *q); extern void blk_clear_pm_only(struct request_queue *q); -- 2.18.0
