On 05/02/2018 16:20, Ming Lei wrote:
> Now 84676c1f21e8ff5(genirq/affinity: assign vectors to all possible CPUs)
> has been merged to V4.16-rc, and it is easy to allocate all offline CPUs
> for some irq vectors, this can't be avoided even though the allocation
> is improved.
>
> For example, on a 8cores VM, 4~7 are not-present/offline, 4 queues of
> virtio-scsi, the irq affinity assigned can become the following shape:
>
> irq 36, cpu list 0-7
> irq 37, cpu list 0-7
> irq 38, cpu list 0-7
> irq 39, cpu list 0-1
> irq 40, cpu list 4,6
> irq 41, cpu list 2-3
> irq 42, cpu list 5,7
>
> Then IO hang is triggered in case of non-SCSI_MQ.
>
> Given storage IO is always C/S model, there isn't such issue with SCSI_MQ(blk-mq),
> because no IO can be submitted to one hw queue if the hw queue hasn't online
> CPUs.
>
> Fix this issue by forcing to use blk_mq.
>
> BTW, I have been used virtio-scsi(scsi_mq) for several years, and it has
> been quite stable, so it shouldn't cause extra risk.
I think that's ok now that we have I/O schedulers for blk-mq.
Acked-by: Paolo Bonzini <pbonzini@xxxxxxxxxx>
Paolo
> Cc: Arun Easi <arun.easi@xxxxxxxxxx>
> Cc: Omar Sandoval <osandov@xxxxxx>,
> Cc: "Martin K. Petersen" <martin.petersen@xxxxxxxxxx>,
> Cc: James Bottomley <james.bottomley@xxxxxxxxxxxxxxxxxxxxx>,
> Cc: Christoph Hellwig <hch@xxxxxx>,
> Cc: Don Brace <don.brace@xxxxxxxxxxxxx>
> Cc: Kashyap Desai <kashyap.desai@xxxxxxxxxxxx>
> Cc: Peter Rivera <peter.rivera@xxxxxxxxxxxx>
> Cc: Paolo Bonzini <pbonzini@xxxxxxxxxx>
> Cc: Mike Snitzer <snitzer@xxxxxxxxxx>
> Reviewed-by: Hannes Reinecke <hare@xxxxxxx>
> Tested-by: Laurence Oberman <loberman@xxxxxxxxxx>
> Signed-off-by: Ming Lei <ming.lei@xxxxxxxxxx>
> ---
> drivers/scsi/virtio_scsi.c | 59 +++-------------------------------------------
> 1 file changed, 3 insertions(+), 56 deletions(-)
>
> diff --git a/drivers/scsi/virtio_scsi.c b/drivers/scsi/virtio_scsi.c
> index 7c28e8d4955a..54e3a0f6844c 100644
> --- a/drivers/scsi/virtio_scsi.c
> +++ b/drivers/scsi/virtio_scsi.c
> @@ -91,9 +91,6 @@ struct virtio_scsi_vq {
> struct virtio_scsi_target_state {
> seqcount_t tgt_seq;
>
> - /* Count of outstanding requests. */
> - atomic_t reqs;
> -
> /* Currently active virtqueue for requests sent to this target. */
> struct virtio_scsi_vq *req_vq;
> };
> @@ -152,8 +149,6 @@ static void virtscsi_complete_cmd(struct virtio_scsi *vscsi, void *buf)
> struct virtio_scsi_cmd *cmd = buf;
> struct scsi_cmnd *sc = cmd->sc;
> struct virtio_scsi_cmd_resp *resp = &cmd->resp.cmd;
> - struct virtio_scsi_target_state *tgt =
> - scsi_target(sc->device)->hostdata;
>
> dev_dbg(&sc->device->sdev_gendev,
> "cmd %p response %u status %#02x sense_len %u\n",
> @@ -210,8 +205,6 @@ static void virtscsi_complete_cmd(struct virtio_scsi *vscsi, void *buf)
> }
>
> sc->scsi_done(sc);
> -
> - atomic_dec(&tgt->reqs);
> }
>
> static void virtscsi_vq_done(struct virtio_scsi *vscsi,
> @@ -580,10 +573,7 @@ static int virtscsi_queuecommand_single(struct Scsi_Host *sh,
> struct scsi_cmnd *sc)
> {
> struct virtio_scsi *vscsi = shost_priv(sh);
> - struct virtio_scsi_target_state *tgt =
> - scsi_target(sc->device)->hostdata;
>
> - atomic_inc(&tgt->reqs);
> return virtscsi_queuecommand(vscsi, &vscsi->req_vqs[0], sc);
> }
>
> @@ -596,55 +586,11 @@ static struct virtio_scsi_vq *virtscsi_pick_vq_mq(struct virtio_scsi *vscsi,
> return &vscsi->req_vqs[hwq];
> }
>
> -static struct virtio_scsi_vq *virtscsi_pick_vq(struct virtio_scsi *vscsi,
> - struct virtio_scsi_target_state *tgt)
> -{
> - struct virtio_scsi_vq *vq;
> - unsigned long flags;
> - u32 queue_num;
> -
> - local_irq_save(flags);
> - if (atomic_inc_return(&tgt->reqs) > 1) {
> - unsigned long seq;
> -
> - do {
> - seq = read_seqcount_begin(&tgt->tgt_seq);
> - vq = tgt->req_vq;
> - } while (read_seqcount_retry(&tgt->tgt_seq, seq));
> - } else {
> - /* no writes can be concurrent because of atomic_t */
> - write_seqcount_begin(&tgt->tgt_seq);
> -
> - /* keep previous req_vq if a reader just arrived */
> - if (unlikely(atomic_read(&tgt->reqs) > 1)) {
> - vq = tgt->req_vq;
> - goto unlock;
> - }
> -
> - queue_num = smp_processor_id();
> - while (unlikely(queue_num >= vscsi->num_queues))
> - queue_num -= vscsi->num_queues;
> - tgt->req_vq = vq = &vscsi->req_vqs[queue_num];
> - unlock:
> - write_seqcount_end(&tgt->tgt_seq);
> - }
> - local_irq_restore(flags);
> -
> - return vq;
> -}
> -
> static int virtscsi_queuecommand_multi(struct Scsi_Host *sh,
> struct scsi_cmnd *sc)
> {
> struct virtio_scsi *vscsi = shost_priv(sh);
> - struct virtio_scsi_target_state *tgt =
> - scsi_target(sc->device)->hostdata;
> - struct virtio_scsi_vq *req_vq;
> -
> - if (shost_use_blk_mq(sh))
> - req_vq = virtscsi_pick_vq_mq(vscsi, sc);
> - else
> - req_vq = virtscsi_pick_vq(vscsi, tgt);
> + struct virtio_scsi_vq *req_vq = virtscsi_pick_vq_mq(vscsi, sc);
>
> return virtscsi_queuecommand(vscsi, req_vq, sc);
> }
> @@ -775,7 +721,6 @@ static int virtscsi_target_alloc(struct scsi_target *starget)
> return -ENOMEM;
>
> seqcount_init(&tgt->tgt_seq);
> - atomic_set(&tgt->reqs, 0);
> tgt->req_vq = &vscsi->req_vqs[0];
>
> starget->hostdata = tgt;
> @@ -823,6 +768,7 @@ static struct scsi_host_template virtscsi_host_template_single = {
> .target_alloc = virtscsi_target_alloc,
> .target_destroy = virtscsi_target_destroy,
> .track_queue_depth = 1,
> + .force_blk_mq = 1,
> };
>
> static struct scsi_host_template virtscsi_host_template_multi = {
> @@ -844,6 +790,7 @@ static struct scsi_host_template virtscsi_host_template_multi = {
> .target_destroy = virtscsi_target_destroy,
> .map_queues = virtscsi_map_queues,
> .track_queue_depth = 1,
> + .force_blk_mq = 1,
> };
>
> #define virtscsi_config_get(vdev, fld) \
>
