>
> Bart,
>
> We look forward to the support of UFSHCD polling, I did a review and test.
> comments as below:
>
>
> On Fri, 2021-11-19 at 11:57 -0800, Bart Van Assche wrote:
> > The time spent in io_schedule() and also the interrupt latency are
> > significant when submitting direct I/O to a UFS device. Hence this
> > patch that implements polling support. User space software can enable
> > polling by passing the RWF_HIPRI flag to the preadv2() system call or
> > the IORING_SETUP_IOPOLL flag to the io_uring interface.
> >
> > Although the block layer supports to partition the tag space for
> > interrupt-based completions (HCTX_TYPE_DEFAULT) purposes and polling
> > (HCTX_TYPE_POLL), the choice has been made to use the same hardware
> > queue for both hctx types because partitioning the tag space would
> > negatively affect performance.
> >
> > On my test setup this patch increases IOPS from 2736 to 22000 (8x) for
> > the following test:
> >
> > for hipri in 0 1; do
> > fio --ioengine=io_uring --iodepth=1 --rw=randread \
> > --runtime=60 --time_based=1 --direct=1 --name=qd1 \
> > --filename=/dev/block/sda --ioscheduler=none --gtod_reduce=1 \
> > --norandommap --hipri=$hipri
> > done
>
> For 4KB random read, direct IO, and iodepth=1, we did not see an
> improvement in IOPS due to this patch. Maybe the test case above is not
> sufficient.
Does your setup contains commit af1830956dc3 (scsi: core: Add mq_poll support to SCSI layer) ?
Thanks,
Avri
>
>
> >
> > Signed-off-by: Bart Van Assche <bvanassche@xxxxxxx>
> > ---
> > drivers/scsi/ufs/ufshcd.c | 85 ++++++++++++++++++++++++++++++-------
> > --
> > 1 file changed, 65 insertions(+), 20 deletions(-)
> >
> > diff --git a/drivers/scsi/ufs/ufshcd.c b/drivers/scsi/ufs/ufshcd.c
> > index 9cf4a22f1950..14043d74389d 100644
> > --- a/drivers/scsi/ufs/ufshcd.c
> > +++ b/drivers/scsi/ufs/ufshcd.c
> > @@ -2629,6 +2629,36 @@ static inline bool is_device_wlun(struct
> > scsi_device *sdev)
> > ufshcd_upiu_wlun_to_scsi_wlun(UFS_UPIU_UFS_DEVICE_WLUN)
> > ;
> > }
> >
> > +/*
> > + * Associate the UFS controller queue with the default and poll HCTX
> > types.
> > + * Initialize the mq_map[] arrays.
> > + */
> > +static int ufshcd_map_queues(struct Scsi_Host *shost) {
> > + int i, ret;
> > +
> > + for (i = 0; i < shost->nr_maps; i++) {
> > + struct blk_mq_queue_map *map = &shost->tag_set.map[i];
> > +
> > + switch (i) {
> > + case HCTX_TYPE_DEFAULT:
> > + case HCTX_TYPE_POLL:
> > + map->nr_queues = 1;
> > + break;
> > + case HCTX_TYPE_READ:
> > + map->nr_queues = 0;
> > + break;
> > + default:
> > + WARN_ON_ONCE(true);
> > + }
> > + map->queue_offset = 0;
> > + ret = blk_mq_map_queues(map);
> > + WARN_ON_ONCE(ret);
> > + }
> > +
> > + return 0;
> > +}
> > +
> > static void ufshcd_init_lrb(struct ufs_hba *hba, struct ufshcd_lrb
> > *lrb, int i) {
> > struct utp_transfer_cmd_desc *cmd_descp = hba->ucdl_base_addr;
> > @@ -2664,7 +2694,7 @@ static int ufshcd_queuecommand(struct
> Scsi_Host
> > *host, struct scsi_cmnd *cmd)
> > struct ufshcd_lrb *lrbp;
> > int err = 0;
> >
> > - WARN_ONCE(tag < 0, "Invalid tag %d\n", tag);
> > + WARN_ONCE(tag < 0 || tag >= hba->nutrs, "Invalid tag %d\n",
> > tag);
> >
> > /*
> > * Allows the UFS error handler to wait for prior
> > ufshcd_queuecommand()
> > @@ -2925,7 +2955,7 @@ static int ufshcd_exec_dev_cmd(struct ufs_hba
> > *hba,
> > }
> > req = scsi_cmd_to_rq(scmd);
> > tag = req->tag;
> > - WARN_ONCE(tag < 0, "Invalid tag %d\n", tag);
> > + WARN_ONCE(tag < 0 || tag >= hba->nutrs, "Invalid tag %d\n",
> > tag)
> probably this change should not in this patch?
>
> > /*
> > * Start the request such that blk_mq_tag_idle() is called when
> > the
> > * device management request finishes.
> > @@ -5272,6 +5302,31 @@ static void __ufshcd_transfer_req_compl(struct
> > ufs_hba *hba,
> > }
> > }
> >
> > +/*
> > + * Returns > 0 if one or more commands have been completed or 0 if
> > no
> > + * requests have been completed.
> > + */
> > +static int ufshcd_poll(struct Scsi_Host *shost, unsigned int
> > queue_num)
> > +{
> > + struct ufs_hba *hba = shost_priv(shost);
> > + unsigned long completed_reqs, flags;
> > + u32 tr_doorbell;
> > +
> > + spin_lock_irqsave(&hba->outstanding_lock, flags);
> > + tr_doorbell = ufshcd_readl(hba,
> > REG_UTP_TRANSFER_REQ_DOOR_BELL);
> > + completed_reqs = ~tr_doorbell & hba->outstanding_reqs;
> > + WARN_ONCE(completed_reqs & ~hba->outstanding_reqs,
> > + "completed: %#lx; outstanding: %#lx\n",
> > completed_reqs,
> > + hba->outstanding_reqs);
> > + hba->outstanding_reqs &= ~completed_reqs;
> > + spin_unlock_irqrestore(&hba->outstanding_lock, flags);
> > +
> > + if (completed_reqs)
> > + __ufshcd_transfer_req_compl(hba, completed_reqs);
> > +
> > + return completed_reqs;
> > +}
> > +
> > /**
> > * ufshcd_transfer_req_compl - handle SCSI and query command
> > completion
> > * @hba: per adapter instance
> > @@ -5282,9 +5337,6 @@ static void __ufshcd_transfer_req_compl(struct
> > ufs_hba *hba,
> > */
> > static irqreturn_t ufshcd_transfer_req_compl(struct ufs_hba *hba) {
> > - unsigned long completed_reqs, flags;
> > - u32 tr_doorbell;
> > -
> > /* Resetting interrupt aggregation counters first and reading
> > the
> > * DOOR_BELL afterward allows us to handle all the completed
> > requests.
> > * In order to prevent other interrupts starvation the DB is
> > read once @@ -5299,21 +5351,9 @@ static irqreturn_t
> > ufshcd_transfer_req_compl(struct ufs_hba *hba)
> > if (ufs_fail_completion())
> > return IRQ_HANDLED;
> >
> > - spin_lock_irqsave(&hba->outstanding_lock, flags);
> > - tr_doorbell = ufshcd_readl(hba,
> > REG_UTP_TRANSFER_REQ_DOOR_BELL);
> > - completed_reqs = ~tr_doorbell & hba->outstanding_reqs;
> > - WARN_ONCE(completed_reqs & ~hba->outstanding_reqs,
> > - "completed: %#lx; outstanding: %#lx\n",
> > completed_reqs,
> > - hba->outstanding_reqs);
> > - hba->outstanding_reqs &= ~completed_reqs;
> > - spin_unlock_irqrestore(&hba->outstanding_lock, flags);
> > + ufshcd_poll(hba->host, 0);
> >
> > - if (completed_reqs) {
> > - __ufshcd_transfer_req_compl(hba, completed_reqs);
> > - return IRQ_HANDLED;
> > - } else {
> > - return IRQ_NONE;
> > - }
> > + return IRQ_HANDLED;
> > }
> >
>
> ufshcd_transfer_req_compl() will never return IRQ_NONE, but ufshcd_intr()
> needs this return to check the fake interrupt.
>
>
> > int __ufshcd_write_ee_control(struct ufs_hba *hba, u32 ee_ctrl_mask)
> > @@ -6564,6 +6604,8 @@ static int __ufshcd_issue_tm_cmd(struct ufs_hba
> > *hba,
> > spin_lock_irqsave(host->host_lock, flags);
> >
> > task_tag = req->tag;
> > + WARN_ONCE(task_tag < 0 || task_tag >= hba->nutmrs, "Invalid tag
> > %d\n",
> > + task_tag);
> > hba->tmf_rqs[req->tag] = req;
> > treq->upiu_req.req_header.dword_0 |= cpu_to_be32(task_tag);
> >
> > @@ -6705,7 +6747,7 @@ static int ufshcd_issue_devman_upiu_cmd(struct
> > ufs_hba *hba,
> > }
> > req = scsi_cmd_to_rq(scmd);
> > tag = req->tag;
> > - WARN_ONCE(tag < 0, "Invalid tag %d\n", tag);
> > + WARN_ONCE(tag < 0 || tag >= hba->nutrs, "Invalid tag %d\n",
> > tag);
> > /*
> > * Start the request such that blk_mq_tag_idle() is called when
> > the
> > * device management request finishes.
> > @@ -8147,7 +8189,9 @@ static struct scsi_host_template
> > ufshcd_driver_template = {
> > .module = THIS_MODULE,
> > .name = UFSHCD,
> > .proc_name = UFSHCD,
> > + .map_queues = ufshcd_map_queues,
> > .queuecommand = ufshcd_queuecommand,
> > + .mq_poll = ufshcd_poll,
> > .slave_alloc = ufshcd_slave_alloc,
> > .slave_configure = ufshcd_slave_configure,
> > .slave_destroy = ufshcd_slave_destroy,
> > @@ -9437,6 +9481,7 @@ int ufshcd_alloc_host(struct device *dev, struct
> > ufs_hba **hba_handle)
> > err = -ENOMEM;
> > goto out_error;
> > }
> > + host->nr_maps = 3;
>
> I don't understand why not directly uses HCTX_MAX_TYPES, 3 is already
> maximum.
>
> > hba = shost_priv(host);
> > hba->host = host;
> > hba->dev = dev;
