On Fri, Apr 16 2021 at 8:02pm -0400,
Mike Snitzer <snitzer@xxxxxxxxxx> wrote:
> On Fri, Apr 16 2021 at 7:53pm -0400,
> Mike Snitzer <snitzer@xxxxxxxxxx> wrote:
>
> > Hi,
> >
> > This patchset reflects changes needed to make NVMe error handling and
> > ANA state updates work well with dm-multipath (which always sets
> > REQ_FAILFAST_TRANSPORT).
> >
> > RHEL8 has been carrying an older ~5.9 based version of this patchset
> > (since RHEL8.3, August 2020).
> >
> > RHEL9 is coming, would really prefer that these changes land upstream
> > rather than carry them within RHEL.
> >
> > All review/feedback welcome.
> >
> > Thanks,
> > Mike
> >
> > v3 -> v4, less is more:
> > - folded REQ_FAILFAST_TRANSPORT local retry and FAILUP patches
> > - simplified nvme_failup_req(), removes needless blk_path_error() et al
> > - removed comment block in nvme_decide_disposition()
> >
> > v2 -> v3:
> > - Added Reviewed-by tags to BLK_STS_DO_NOT_RETRY patch.
> > - Eliminated __nvme_end_req() and added code comment to
> > nvme_failup_req() in FAILUP handling patch.
> >
> > Mike Snitzer (3):
> > nvme: return BLK_STS_DO_NOT_RETRY if the DNR bit is set
> > nvme: allow local retry and proper failover for REQ_FAILFAST_TRANSPORT
> > nvme: decouple basic ANA log page re-read support from native
> > multipathing
> >
> > drivers/nvme/host/core.c | 22 +++++++++++++++++++---
> > drivers/nvme/host/multipath.c | 16 +++++++++++-----
> > drivers/nvme/host/nvme.h | 4 ++++
> > include/linux/blk_types.h | 8 ++++++++
> > 4 files changed, 42 insertions(+), 8 deletions(-)
>
> Sorry for all the noise, but I had a cut-and-paste issue with this cover
> letter; should've said "[PATCH v4 0/4] ..."
>
> While I'm replying, I _think_ there is consensus that patch 1 is
> worthwile and acceptable. Here is a combined diff of patches 2+3 to
> illustrate just how minimalist these proposed changes are:
>
> diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c
> index 540d6fd8ffef..83ca96292157 100644
> --- a/drivers/nvme/host/core.c
> +++ b/drivers/nvme/host/core.c
> @@ -299,6 +299,7 @@ enum nvme_disposition {
> COMPLETE,
> RETRY,
> FAILOVER,
> + FAILUP,
> };
>
> static inline enum nvme_disposition nvme_decide_disposition(struct request *req)
> @@ -306,15 +307,16 @@ static inline enum nvme_disposition nvme_decide_disposition(struct request *req)
> if (likely(nvme_req(req)->status == 0))
> return COMPLETE;
>
> - if (blk_noretry_request(req) ||
> + if ((req->cmd_flags & (REQ_FAILFAST_DEV | REQ_FAILFAST_DRIVER)) ||
> (nvme_req(req)->status & NVME_SC_DNR) ||
> nvme_req(req)->retries >= nvme_max_retries)
> return COMPLETE;
>
> - if (req->cmd_flags & REQ_NVME_MPATH) {
> + if (req->cmd_flags & (REQ_NVME_MPATH | REQ_FAILFAST_TRANSPORT)) {
> if (nvme_is_path_error(nvme_req(req)->status) ||
> blk_queue_dying(req->q))
> - return FAILOVER;
> + return (req->cmd_flags & REQ_NVME_MPATH) ?
> + FAILOVER : FAILUP;
> } else {
> if (blk_queue_dying(req->q))
> return COMPLETE;
> @@ -336,6 +338,14 @@ static inline void nvme_end_req(struct request *req)
> blk_mq_end_request(req, status);
> }
>
> +static inline void nvme_failup_req(struct request *req)
> +{
> + nvme_update_ana(req);
> +
> + nvme_req(req)->status = NVME_SC_HOST_PATH_ERROR;
> + nvme_end_req(req);
> +}
> +
> void nvme_complete_rq(struct request *req)
> {
> trace_nvme_complete_rq(req);
> @@ -354,6 +364,9 @@ void nvme_complete_rq(struct request *req)
> case FAILOVER:
> nvme_failover_req(req);
> return;
> + case FAILUP:
> + nvme_failup_req(req);
> + return;
> }
> }
> EXPORT_SYMBOL_GPL(nvme_complete_rq);
> diff --git a/drivers/nvme/host/multipath.c b/drivers/nvme/host/multipath.c
> index a1d476e1ac02..7d94250264aa 100644
> --- a/drivers/nvme/host/multipath.c
> +++ b/drivers/nvme/host/multipath.c
> @@ -65,23 +65,29 @@ void nvme_set_disk_name(char *disk_name, struct nvme_ns *ns,
> }
> }
>
> -void nvme_failover_req(struct request *req)
> +void nvme_update_ana(struct request *req)
> {
> struct nvme_ns *ns = req->q->queuedata;
> u16 status = nvme_req(req)->status & 0x7ff;
> - unsigned long flags;
> -
> - nvme_mpath_clear_current_path(ns);
>
> /*
> * If we got back an ANA error, we know the controller is alive but not
> - * ready to serve this namespace. Kick of a re-read of the ANA
> + * ready to serve this namespace. Kick off a re-read of the ANA
> * information page, and just try any other available path for now.
> */
> if (nvme_is_ana_error(status) && ns->ctrl->ana_log_buf) {
> set_bit(NVME_NS_ANA_PENDING, &ns->flags);
> queue_work(nvme_wq, &ns->ctrl->ana_work);
> }
> +}
> +
> +void nvme_failover_req(struct request *req)
> +{
> + struct nvme_ns *ns = req->q->queuedata;
> + unsigned long flags;
> +
> + nvme_mpath_clear_current_path(ns);
> + nvme_update_ana(req);
>
> spin_lock_irqsave(&ns->head->requeue_lock, flags);
> blk_steal_bios(&ns->head->requeue_list, req);
> diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h
> index 07b34175c6ce..4eed8536625c 100644
> --- a/drivers/nvme/host/nvme.h
> +++ b/drivers/nvme/host/nvme.h
> @@ -664,6 +664,7 @@ void nvme_mpath_start_freeze(struct nvme_subsystem *subsys);
> void nvme_set_disk_name(char *disk_name, struct nvme_ns *ns,
> struct nvme_ctrl *ctrl, int *flags);
> void nvme_failover_req(struct request *req);
> +void nvme_update_ana(struct request *req);
> void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl);
> int nvme_mpath_alloc_disk(struct nvme_ctrl *ctrl,struct nvme_ns_head *head);
> void nvme_mpath_add_disk(struct nvme_ns *ns, struct nvme_id_ns *id);
> @@ -714,6 +715,9 @@ static inline void nvme_set_disk_name(char *disk_name, struct nvme_ns *ns,
> static inline void nvme_failover_req(struct request *req)
> {
> }
> +static inline void nvme_update_ana(struct request *req)
> +{
> +}
> static inline void nvme_kick_requeue_lists(struct nvme_ctrl *ctrl)
> {
> }
Since I sent these changes late on Friday I figured I'd try to get them
on your radar.
I would really appreciate timely review of this series for 5.13:
https://patchwork.kernel.org/project/dm-devel/list/?series=468905
Thanks,
Mike
--
dm-devel mailing list
dm-devel@xxxxxxxxxx
https://listman.redhat.com/mailman/listinfo/dm-devel
