On Thu, Dec 06, 2018 at 02:11:44PM +0800, liyangyang (M) wrote:
> On 2018/12/6 1:06, Leon Romanovsky wrote:
> > On Wed, Dec 05, 2018 at 05:41:43PM +0800, Lijun Ou wrote:
> >> From: Yangyang Li <liyangyang20@xxxxxxxxxx>
> >>
> >> This patch adds SCC context clear support for DCQCN
> >> in kernel space driver.
> >>
> >> Signed-off-by: Yangyang Li <liyangyang20@xxxxxxxxxx>
> >> ---
> >> drivers/infiniband/hw/hns/hns_roce_device.h | 3 ++
> >> drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 59 ++++++++++++++++++++++++++++-
> >> drivers/infiniband/hw/hns/hns_roce_hw_v2.h | 17 +++++++++
> >> drivers/infiniband/hw/hns/hns_roce_qp.c | 9 +++++
> >> 4 files changed, 87 insertions(+), 1 deletion(-)
> >>
> >> diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h
> >> index 718b415..6fe1871 100644
> >> --- a/drivers/infiniband/hw/hns/hns_roce_device.h
> >> +++ b/drivers/infiniband/hw/hns/hns_roce_device.h
> >> @@ -202,6 +202,7 @@ enum {
> >> HNS_ROCE_CAP_FLAG_SRQ = BIT(5),
> >> HNS_ROCE_CAP_FLAG_MW = BIT(7),
> >> HNS_ROCE_CAP_FLAG_FRMR = BIT(8),
> >> + HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL = BIT(9),
> >> HNS_ROCE_CAP_FLAG_ATOMIC = BIT(10),
> >> };
> >>
> >> @@ -867,6 +868,8 @@ struct hns_roce_hw {
> >> int attr_mask, enum ib_qp_state cur_state,
> >> enum ib_qp_state new_state);
> >> int (*destroy_qp)(struct ib_qp *ibqp);
> >> + int (*qp_flow_control_init)(struct hns_roce_dev *hr_dev,
> >> + struct hns_roce_qp *hr_qp);
> >> int (*post_send)(struct ib_qp *ibqp, const struct ib_send_wr *wr,
> >> const struct ib_send_wr **bad_wr);
> >> int (*post_recv)(struct ib_qp *qp, const struct ib_recv_wr *recv_wr,
> >> diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
> >> index 77cfd9b..4520061 100644
> >> --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
> >> +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
> >> @@ -1392,7 +1392,8 @@ static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev)
> >>
> >> if (hr_dev->pci_dev->revision == 0x21) {
> >> caps->flags |= HNS_ROCE_CAP_FLAG_ATOMIC |
> >> - HNS_ROCE_CAP_FLAG_SRQ;
> >> + HNS_ROCE_CAP_FLAG_SRQ |
> >> + HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL;
> >> caps->scc_ctx_entry_sz = HNS_ROCE_V2_SCC_CTX_ENTRY_SZ;
> >> caps->scc_ctx_ba_pg_sz = 0;
> >> caps->scc_ctx_buf_pg_sz = 0;
> >> @@ -4210,6 +4211,61 @@ static int hns_roce_v2_destroy_qp(struct ib_qp *ibqp)
> >> return 0;
> >> }
> >>
> >> +static int hns_roce_v2_qp_flow_control_init(struct hns_roce_dev *hr_dev,
> >> + struct hns_roce_qp *hr_qp)
> >> +{
> >> + struct hns_roce_scc_ctx_clr *scc_cxt_clr;
> >> + struct hns_roce_scc_ctx_clr_done *resp;
> >> + struct hns_roce_scc_ctx_clr_done *rst;
> >> + struct hns_roce_cmq_desc desc;
> >> + int ret;
> >> + int i;
> >> +
> >> + /* set scc ctx clear done flag */
> >> + hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_RESET_SCC_CTX,
> >> + false);
> >> +
> >> + rst = (struct hns_roce_scc_ctx_clr_done *)desc.data;
> >> + memset(rst, 0, sizeof(*rst));
> >> + roce_set_bit(rst->rocee_scc_ctx_clr_done,
> >> + HNS_ROCE_V2_SCC_CTX_DONE_S,
> >> + 0);
> >> +
> >> + ret = hns_roce_cmq_send(hr_dev, &desc, 1);
> >> + if (ret)
> >> + return ret;
> >> +
> >> + /* clear scc context */
> >> + hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_SCC_CTX_CLR,
> >> + false);
> >> +
> >> + scc_cxt_clr = (struct hns_roce_scc_ctx_clr *)desc.data;
> >> + memset(scc_cxt_clr, 0, sizeof(*scc_cxt_clr));
> >> + scc_cxt_clr->rocee_scc_ctx_clr_qpn = hr_qp->qpn;
> >> +
> >> + ret = hns_roce_cmq_send(hr_dev, &desc, 1);
> >> + if (ret)
> >> + return ret;
> >> +
> >> + /* query scc context clear is done or not */
> >> + for (i = 0; i <= HNS_ROCE_CMQ_SCC_CLR_DONE_CNT; i++) {
> >> + hns_roce_cmq_setup_basic_desc(&desc,
> >> + HNS_ROCE_OPC_QUERY_SCC_CTX, true);
> >> + resp = (struct hns_roce_scc_ctx_clr_done *)desc.data;
> >> + memset(resp, 0, sizeof(*resp));
> >> +
> >> + ret = hns_roce_cmq_send(hr_dev, &desc, 1);
> >> + if (ret)
> >> + return ret;
> >> +
> >> + if (resp->rocee_scc_ctx_clr_done == 1)
> >> + return 0;
> >
> > Sorry for my question, but I'm a little bit lost here.
> >
> > You took pointer "resp", cleared the data and the resp->rocee_scc_ctx_clr_done.
> > Are you still expecting to see here resp->rocee_scc_ctx_clr_done = 1 ?
> "resp" point to "desc.data" which will be sent to hardware through
> function "hns_roce_cmq_send".
> resp->rocee_scc_ctx_clr_done will be set to 1 by hardware if
> scc context clear successful. If not, continue the query until the
> maximum number of queries is reached HNS_ROCE_CMQ_SCC_CLR_DONE_CNT.
I'm not expert in this area, but it looks like you need some sort of
memory fence and ensure that memory in-sync between device and CPU.
Thanks
Attachment:
signature.asc
Description: PGP signature
