SCSI core uses the atomic variable of sdev->device_busy to track
in-flight IO requests dispatched to this scsi device. IO request may be
submitted from any CPU, so the cost for maintaining the shared atomic
counter can be very big on big NUMA machine with lots of CPU cores.
sdev->queue_depth is usually used for two purposes: 1) improve IO merge;
2) fair IO request scattered among all LUNs.
blk-mq already provides fair request allocation among all active shared
request queues(LUNs), see hctx_may_queue().
NVMe doesn't have such per-request-queue(namespace) queue depth, so it
is reasonable to ignore the limit for SCSI SSD too. Also IO merge won't
play big role for reaching top SSD performance.
With this patch, big cost for tracking in-flight per-LUN requests via
atomic variable can be saved.
Cc: Jens Axboe <axboe@xxxxxxxxx>
Cc: Ewan D. Milne <emilne@xxxxxxxxxx>
Cc: Omar Sandoval <osandov@xxxxxx>,
Cc: "Martin K. Petersen" <martin.petersen@xxxxxxxxxx>,
Cc: James Bottomley <james.bottomley@xxxxxxxxxxxxxxxxxxxxx>,
Cc: Christoph Hellwig <hch@xxxxxx>,
Cc: Kashyap Desai <kashyap.desai@xxxxxxxxxxxx>
Cc: Hannes Reinecke <hare@xxxxxxx>
Cc: Laurence Oberman <loberman@xxxxxxxxxx>
Cc: Bart Van Assche <bart.vanassche@xxxxxxx>
Signed-off-by: Ming Lei <ming.lei@xxxxxxxxxx>
---
drivers/scsi/scsi_lib.c | 24 ++++++++++++++++++------
1 file changed, 18 insertions(+), 6 deletions(-)
diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c
index b6f66dcb15a5..b8f0898a15e4 100644
--- a/drivers/scsi/scsi_lib.c
+++ b/drivers/scsi/scsi_lib.c
@@ -354,7 +354,8 @@ void scsi_device_unbusy(struct scsi_device *sdev, struct scsi_cmnd *cmd)
if (starget->can_queue > 0)
atomic_dec(&starget->target_busy);
- atomic_dec(&sdev->device_busy);
+ if (!blk_queue_nonrot(sdev->request_queue))
+ atomic_dec(&sdev->device_busy);
}
static void scsi_kick_queue(struct request_queue *q)
@@ -410,7 +411,8 @@ static void scsi_single_lun_run(struct scsi_device *current_sdev)
static inline bool scsi_device_is_busy(struct scsi_device *sdev)
{
- if (atomic_read(&sdev->device_busy) >= sdev->queue_depth)
+ if (!blk_queue_nonrot(sdev->request_queue) &&
+ atomic_read(&sdev->device_busy) >= sdev->queue_depth)
return true;
if (atomic_read(&sdev->device_blocked) > 0)
return true;
@@ -1283,8 +1285,12 @@ static inline int scsi_dev_queue_ready(struct request_queue *q,
struct scsi_device *sdev)
{
unsigned int busy;
+ bool bypass = blk_queue_nonrot(sdev->request_queue);
- busy = atomic_inc_return(&sdev->device_busy) - 1;
+ if (!bypass)
+ busy = atomic_inc_return(&sdev->device_busy) - 1;
+ else
+ busy = 0;
if (atomic_read(&sdev->device_blocked)) {
if (busy)
goto out_dec;
@@ -1298,12 +1304,16 @@ static inline int scsi_dev_queue_ready(struct request_queue *q,
"unblocking device at zero depth\n"));
}
+ if (bypass)
+ return 1;
+
if (busy >= sdev->queue_depth)
goto out_dec;
return 1;
out_dec:
- atomic_dec(&sdev->device_busy);
+ if (!bypass)
+ atomic_dec(&sdev->device_busy);
return 0;
}
@@ -1624,7 +1634,8 @@ static void scsi_mq_put_budget(struct blk_mq_hw_ctx *hctx)
struct request_queue *q = hctx->queue;
struct scsi_device *sdev = q->queuedata;
- atomic_dec(&sdev->device_busy);
+ if (!blk_queue_nonrot(sdev->request_queue))
+ atomic_dec(&sdev->device_busy);
}
static bool scsi_mq_get_budget(struct blk_mq_hw_ctx *hctx)
@@ -1706,7 +1717,8 @@ static blk_status_t scsi_queue_rq(struct blk_mq_hw_ctx *hctx,
case BLK_STS_OK:
break;
case BLK_STS_RESOURCE:
- if (atomic_read(&sdev->device_busy) ||
+ if ((!blk_queue_nonrot(sdev->request_queue) &&
+ atomic_read(&sdev->device_busy)) ||
scsi_device_blocked(sdev))
ret = BLK_STS_DEV_RESOURCE;
break;
--
2.20.1
