SCSI core uses the atomic variable of sdev->device_busy to track
in-flight IO requests dispatched to this scsi device. IO request may be
submitted from any CPU, so the cost for maintaining the shared atomic
counter can be very big on big NUMA machine with lots of CPU cores.
sdev->queue_depth is usually used for two purposes: 1) improve IO merge;
2) fair IO request scattered among all LUNs.
blk-mq already provides fair request allocation among all active shared
request queues(LUNs), see hctx_may_queue().
NVMe doesn't have such per-request-queue(namespace) queue depth, so it
is reasonable to ignore the limit for SCSI SSD too. Also IO merge won't
play big role for reaching top SSD performance.
With this patch, big cost for tracking in-flight per-LUN requests via
atomic variable can be saved for some high end controller with SSD.
Cc: Sathya Prakash <sathya.prakash@xxxxxxxxxxxx>
Cc: Chaitra P B <chaitra.basappa@xxxxxxxxxxxx>
Cc: Suganath Prabu Subramani <suganath-prabu.subramani@xxxxxxxxxxxx>
Cc: Kashyap Desai <kashyap.desai@xxxxxxxxxxxx>
Cc: Sumit Saxena <sumit.saxena@xxxxxxxxxxxx>
Cc: Shivasharan S <shivasharan.srikanteshwara@xxxxxxxxxxxx>
Cc: Ewan D. Milne <emilne@xxxxxxxxxx>
Cc: Christoph Hellwig <hch@xxxxxx>,
Cc: Hannes Reinecke <hare@xxxxxxx>
Cc: Bart Van Assche <bart.vanassche@xxxxxxx>
Signed-off-by: Ming Lei <ming.lei@xxxxxxxxxx>
---
drivers/scsi/scsi_lib.c | 34 ++++++++++++++++++++++++++++------
include/scsi/scsi_host.h | 3 +++
2 files changed, 31 insertions(+), 6 deletions(-)
diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c
index 610ee41fa54c..0903697dd843 100644
--- a/drivers/scsi/scsi_lib.c
+++ b/drivers/scsi/scsi_lib.c
@@ -344,6 +344,16 @@ static void scsi_dec_host_busy(struct Scsi_Host *shost, struct scsi_cmnd *cmd)
rcu_read_unlock();
}
+static inline bool scsi_bypass_device_busy(struct scsi_device *sdev)
+{
+ struct Scsi_Host *shost = sdev->host;
+
+ if (!shost->hostt->no_device_queue_for_ssd)
+ return false;
+
+ return blk_queue_nonrot(sdev->request_queue);
+}
+
void scsi_device_unbusy(struct scsi_device *sdev, struct scsi_cmnd *cmd)
{
struct Scsi_Host *shost = sdev->host;
@@ -354,7 +364,8 @@ void scsi_device_unbusy(struct scsi_device *sdev, struct scsi_cmnd *cmd)
if (starget->can_queue > 0)
atomic_dec(&starget->target_busy);
- atomic_dec(&sdev->device_busy);
+ if (!scsi_bypass_device_busy(sdev))
+ atomic_dec(&sdev->device_busy);
}
static void scsi_kick_queue(struct request_queue *q)
@@ -410,7 +421,8 @@ static void scsi_single_lun_run(struct scsi_device *current_sdev)
static inline bool scsi_device_is_busy(struct scsi_device *sdev)
{
- if (atomic_read(&sdev->device_busy) >= sdev->queue_depth)
+ if (!scsi_bypass_device_busy(sdev) &&
+ atomic_read(&sdev->device_busy) >= sdev->queue_depth)
return true;
if (atomic_read(&sdev->device_blocked) > 0)
return true;
@@ -1283,8 +1295,12 @@ static inline int scsi_dev_queue_ready(struct request_queue *q,
struct scsi_device *sdev)
{
unsigned int busy;
+ bool bypass = scsi_bypass_device_busy(sdev);
- busy = atomic_inc_return(&sdev->device_busy) - 1;
+ if (!bypass)
+ busy = atomic_inc_return(&sdev->device_busy) - 1;
+ else
+ busy = 0;
if (atomic_read(&sdev->device_blocked)) {
if (busy)
goto out_dec;
@@ -1298,12 +1314,16 @@ static inline int scsi_dev_queue_ready(struct request_queue *q,
"unblocking device at zero depth\n"));
}
+ if (bypass)
+ return 1;
+
if (busy >= sdev->queue_depth)
goto out_dec;
return 1;
out_dec:
- atomic_dec(&sdev->device_busy);
+ if (!bypass)
+ atomic_dec(&sdev->device_busy);
return 0;
}
@@ -1624,7 +1644,8 @@ static void scsi_mq_put_budget(struct blk_mq_hw_ctx *hctx)
struct request_queue *q = hctx->queue;
struct scsi_device *sdev = q->queuedata;
- atomic_dec(&sdev->device_busy);
+ if (!scsi_bypass_device_busy(sdev))
+ atomic_dec(&sdev->device_busy);
}
static bool scsi_mq_get_budget(struct blk_mq_hw_ctx *hctx)
@@ -1706,7 +1727,8 @@ static blk_status_t scsi_queue_rq(struct blk_mq_hw_ctx *hctx,
case BLK_STS_OK:
break;
case BLK_STS_RESOURCE:
- if (atomic_read(&sdev->device_busy) ||
+ if ((!scsi_bypass_device_busy(sdev) &&
+ atomic_read(&sdev->device_busy)) ||
scsi_device_blocked(sdev))
ret = BLK_STS_DEV_RESOURCE;
break;
diff --git a/include/scsi/scsi_host.h b/include/scsi/scsi_host.h
index 7a97fb8104cf..8e80edfd5a6d 100644
--- a/include/scsi/scsi_host.h
+++ b/include/scsi/scsi_host.h
@@ -426,6 +426,9 @@ struct scsi_host_template {
/* True if the controller does not support WRITE SAME */
unsigned no_write_same:1;
+ /* True if the controller needn't to maintain device queue for SSD */
+ unsigned no_device_queue_for_ssd:1;
+
/*
* Countdown for host blocking with no commands outstanding.
*/
--
2.20.1
