Add likely() and unlikely() hints to conditionals on or near the fastpath.
Signed-off-by: Douglas Gilbert <dgilbert@xxxxxxxxxxxx>
Reviewed-by: Johannes Thumshirn <jthumshirn@xxxxxxx>
---
A reviewer wanted any performance improvement (or otherwise) quantified.
The improvement was so small, that ftrace ignored it. Inline timing code
suggests the improvement from this whole patchset is around 7 nanoseconds
per invocation (tested on a Lenovo X270 (i5-7200U)). Not exactly huge.
Another win might be the smaller size of scsi_io_completion() after the
refactoring; this should allow more other code to fit in the instruction
cache.
drivers/scsi/scsi_lib.c | 24 +++++++++++-------------
1 file changed, 11 insertions(+), 13 deletions(-)
diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c
index 0e2dcdb63fcd..63f79b0dbbf3 100644
--- a/drivers/scsi/scsi_lib.c
+++ b/drivers/scsi/scsi_lib.c
@@ -1020,7 +1020,7 @@ void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes)
struct request *req = cmd->request;
blk_status_t blk_stat = BLK_STS_OK; /* u8: BLK_STS_OK is only 0 */
- if (result) { /* does not necessarily mean there is an error */
+ if (unlikely(result)) { /* a nz result may or may not be an error */
blk_stat = scsi_io_completion_nz_result(cmd, result);
if (blk_stat == BLK_STS_OK)
result = 0; /* suppress error processing now */
@@ -1033,14 +1033,14 @@ void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes)
}
- if (blk_rq_is_passthrough(req)) {
+ if (unlikely(blk_rq_is_passthrough(req))) {
/*
* __scsi_error_from_host_byte may have reset the host_byte
*/
scsi_req(req)->result = cmd->result;
scsi_req(req)->resid_len = scsi_get_resid(cmd);
- if (scsi_bidi_cmnd(cmd)) {
+ if (unlikely(scsi_bidi_cmnd(cmd))) {
/*
* Bidi commands Must be complete as a whole,
* both sides at once.
@@ -1053,7 +1053,7 @@ void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes)
}
}
- /* no bidi support for !blk_rq_is_passthrough yet */
+ /* no bidi support yet, other than in pass-through */
BUG_ON(blk_bidi_rq(req));
/*
@@ -1069,15 +1069,13 @@ void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes)
* handle. Failed, zero length commands always need to drop down
* to retry code. Fast path should return in this block.
*/
- if (blk_rq_bytes(req) > 0 || blk_stat == BLK_STS_OK) {
- if (!scsi_end_request(req, blk_stat, good_bytes, 0))
- return; /* no bytes remaining */
+ if (likely(blk_rq_bytes(req) > 0 || blk_stat == BLK_STS_OK)) {
+ if (likely(!scsi_end_request(req, blk_stat, good_bytes, 0)))
+ return; /* no bytes remaining */
}
- /*
- * Kill remainder if no retrys.
- */
- if (blk_stat && scsi_noretry_cmd(cmd)) {
+ /* Kill remainder if no retrys */
+ if (unlikely(blk_stat && scsi_noretry_cmd(cmd))) {
if (scsi_end_request(req, blk_stat, blk_rq_bytes(req), 0))
BUG();
return;
@@ -1087,14 +1085,14 @@ void scsi_io_completion(struct scsi_cmnd *cmd, unsigned int good_bytes)
* If there had been no error, but we have leftover bytes in the
* requeues just queue the command up again.
*/
- if (result == 0) {
+ if (likely(result == 0)) {
/*
* Unprep the request and put it back at the head of the
* queue. A new command will be prepared and issued.
* This block is the same as case ACTION_REPREP in
* scsi_io_completion_action() above.
*/
- if (q->mq_ops)
+ if (likely(q->mq_ops))
scsi_mq_requeue_cmd(cmd);
else {
scsi_release_buffers(cmd);
--
2.14.1
