Try to handle loop aio command via NOWAIT IO first, then we can avoid to queue the aio command into workqueue. This is usually one big win in case that FS block mapping is stable, Mikulas verified [1] that this way improves IO perf by close to 5X in 12jobs sequential read/write test. Fallback to workqueue in case of -EAGAIN. This way may bring a little cost from the 1st retry, but when running the following write test over loop/sparse_file, the actual effect on randwrite is obvious: ``` truncate -s 4G 1.img #1.img is created on XFS/virtio-scsi losetup -f 1.img --direct-io=on fio --direct=1 --bs=4k --runtime=40 --time_based --numjobs=1 --ioengine=libaio \ --iodepth=16 --group_reporting=1 --filename=/dev/loop0 -name=job --rw=$RW ``` - RW=randwrite: obvious IOPS drop observed - RW=write: a little drop(%5 - 10%) This perf drop on randwrite over sparse file will be addressed in the following patch. BLK_MQ_F_BLOCKING has to be set for calling into .read_iter() or .write_iter() which might sleep even though it is NOWAIT, and the only effect is that rcu read lock is replaced with srcu read lock. Link: https://lore.kernel.org/linux-block/a8e5c76a-231f-07d1-a394-847de930f638@xxxxxxxxxx/ [1] Signed-off-by: Ming Lei <ming.lei@xxxxxxxxxx> --- drivers/block/loop.c | 51 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 47 insertions(+), 4 deletions(-) diff --git a/drivers/block/loop.c b/drivers/block/loop.c index 81f6ba9bc911..542b1fe938a7 100644 --- a/drivers/block/loop.c +++ b/drivers/block/loop.c @@ -92,6 +92,8 @@ struct loop_cmd { #define LOOP_IDLE_WORKER_TIMEOUT (60 * HZ) #define LOOP_DEFAULT_HW_Q_DEPTH 128 +static void loop_queue_work(struct loop_device *lo, struct loop_cmd *cmd); + static DEFINE_IDR(loop_index_idr); static DEFINE_MUTEX(loop_ctl_mutex); static DEFINE_MUTEX(loop_validate_mutex); @@ -393,6 +395,15 @@ static void lo_rw_aio_do_completion(struct loop_cmd *cmd) if (!atomic_dec_and_test(&cmd->ref)) return; + + /* -EAGAIN could be returned from bdev's ->ki_complete */ + if (cmd->ret == -EAGAIN) { + struct loop_device *lo = rq->q->queuedata; + + loop_queue_work(lo, cmd); + return; + } + kfree(cmd->bvec); if (likely(!blk_should_fake_timeout(rq->q))) blk_mq_complete_request(rq); @@ -498,15 +509,38 @@ static int lo_submit_rw_aio(struct loop_device *lo, struct loop_cmd *cmd, static int lo_rw_aio(struct loop_device *lo, struct loop_cmd *cmd) { unsigned int nr_bvec = lo_cmd_nr_bvec(cmd); - int ret = lo_rw_aio_prep(lo, cmd, nr_bvec); + int ret; - if (ret >= 0) - ret = lo_submit_rw_aio(lo, cmd, nr_bvec); + /* + * This command is prepared, and we have tried IOCB_NOWAIT, but got + * -EAGAIN, so clear it now + */ + cmd->iocb.ki_flags &= ~IOCB_NOWAIT; + ret = lo_submit_rw_aio(lo, cmd, nr_bvec); if (ret != -EIOCBQUEUED) lo_rw_aio_complete(&cmd->iocb, ret); return 0; } +static blk_status_t lo_rw_aio_nowait(struct loop_device *lo, + struct loop_cmd *cmd) +{ + unsigned int nr_bvec = lo_cmd_nr_bvec(cmd); + int ret = lo_rw_aio_prep(lo, cmd, nr_bvec); + + if (unlikely(ret < 0)) + return BLK_STS_IOERR; + + cmd->iocb.ki_flags |= IOCB_NOWAIT; + ret = lo_submit_rw_aio(lo, cmd, nr_bvec); + if (ret == -EAGAIN) + return BLK_STS_AGAIN; + + if (ret != -EIOCBQUEUED) + lo_rw_aio_complete(&cmd->iocb, ret); + return BLK_STS_OK; +} + static int do_req_filebacked(struct loop_device *lo, struct request *rq) { struct loop_cmd *cmd = blk_mq_rq_to_pdu(rq); @@ -1937,6 +1971,14 @@ static blk_status_t loop_queue_rq(struct blk_mq_hw_ctx *hctx, break; } + if (cmd->use_aio) { + blk_status_t res = lo_rw_aio_nowait(lo, cmd); + + if (res != BLK_STS_AGAIN) + return res; + /* fallback to workqueue for handling aio */ + } + loop_queue_work(lo, cmd); return BLK_STS_OK; @@ -2087,7 +2129,8 @@ static int loop_add(int i) lo->tag_set.queue_depth = hw_queue_depth; lo->tag_set.numa_node = NUMA_NO_NODE; lo->tag_set.cmd_size = sizeof(struct loop_cmd); - lo->tag_set.flags = BLK_MQ_F_STACKING | BLK_MQ_F_NO_SCHED_BY_DEFAULT; + lo->tag_set.flags = BLK_MQ_F_STACKING | BLK_MQ_F_NO_SCHED_BY_DEFAULT | + BLK_MQ_F_BLOCKING; lo->tag_set.driver_data = lo; err = blk_mq_alloc_tag_set(&lo->tag_set); -- 2.47.0
