Hi, jason & leon
Any other comments for this?
thanks
Lijun Ou
在 2019/2/28 22:42, Lijun Ou 写道:
> This patch adds support of resource track for hip08 and take
> dumping cq context state used for debugging as an example.
> More resources track supports for hns driver will be added in future.
>
> The output should be as follows.
> $ ./rdma res show cq dev hnseth0 -d
> dev hnseth0 cqe 1023 users 2 poll-ctx WORKQUEUE pid 0 comm [ib_core] drv_state 2 drv_ceq
> n 0 drv_cqn 0 drv_hopnum 1 drv_pi 0 drv_ci 0 drv_coalesce 0 drv_period 0 drv_cnt 0
>
> Signed-off-by: Tao Tian <tiantao6@xxxxxxxxxx>
> Signed-off-by: Yangyang Li <liyangyang20@xxxxxxxxxx>
> Signed-off-by: chenglang <chenglang@xxxxxxxxxx>
> Signed-off-by: Lijun Ou <oulijun@xxxxxxxxxx>
> ---
> v7->v8:
> 1. Remove the prefix for all fields of cq context in the driver because
> the prefix had been added in rdmatool..
> 2. resolve the merge conflict becasuse the other patches received.
>
> v6->v7:
> 1. Add prefix for all fields of cq context.
> 2. Remove unnecessary commit information.
>
> v5->v6:
> 1. resolve the merge conflict.
> 2. add prefix for some common fields of cq context.
>
> v4->v5:
> 1.Change the cqe and maxcnt in the driver output to a proper name.
>
> v3->v4:
> 1.print values in the granularity of parameterable fields.
>
> v2->v3:
> 1.Modify the output message.
>
> v1->v2:
> 1.Fixed the warning about the SPDX tag.
> 2.Add the output of rdmatool to the commit message.
> 3.Modify the function pointer to register res.
> ---
> drivers/infiniband/hw/hns/Makefile | 4 +-
> drivers/infiniband/hw/hns/hns_roce_cmd.h | 1 +
> drivers/infiniband/hw/hns/hns_roce_device.h | 8 ++
> drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 11 ++-
> drivers/infiniband/hw/hns/hns_roce_hw_v2.h | 8 ++
> drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c | 36 +++++++
> drivers/infiniband/hw/hns/hns_roce_restrack.c | 124 +++++++++++++++++++++++++
> 7 files changed, 187 insertions(+), 5 deletions(-)
> create mode 100644 drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c
> create mode 100644 drivers/infiniband/hw/hns/hns_roce_restrack.c
>
> diff --git a/drivers/infiniband/hw/hns/Makefile b/drivers/infiniband/hw/hns/Makefile
> index e2a7f14..eee5205 100644
> --- a/drivers/infiniband/hw/hns/Makefile
> +++ b/drivers/infiniband/hw/hns/Makefile
> @@ -7,8 +7,8 @@ ccflags-y := -I $(srctree)/drivers/net/ethernet/hisilicon/hns3
> obj-$(CONFIG_INFINIBAND_HNS) += hns-roce.o
> hns-roce-objs := hns_roce_main.o hns_roce_cmd.o hns_roce_pd.o \
> hns_roce_ah.o hns_roce_hem.o hns_roce_mr.o hns_roce_qp.o \
> - hns_roce_cq.o hns_roce_alloc.o hns_roce_db.o hns_roce_srq.o
> + hns_roce_cq.o hns_roce_alloc.o hns_roce_db.o hns_roce_srq.o hns_roce_restrack.o
> obj-$(CONFIG_INFINIBAND_HNS_HIP06) += hns-roce-hw-v1.o
> hns-roce-hw-v1-objs := hns_roce_hw_v1.o
> obj-$(CONFIG_INFINIBAND_HNS_HIP08) += hns-roce-hw-v2.o
> -hns-roce-hw-v2-objs := hns_roce_hw_v2.o
> +hns-roce-hw-v2-objs := hns_roce_hw_v2.o hns_roce_hw_v2_dfx.o
> diff --git a/drivers/infiniband/hw/hns/hns_roce_cmd.h b/drivers/infiniband/hw/hns/hns_roce_cmd.h
> index 059fd1d..2b6ac64 100644
> --- a/drivers/infiniband/hw/hns/hns_roce_cmd.h
> +++ b/drivers/infiniband/hw/hns/hns_roce_cmd.h
> @@ -53,6 +53,7 @@ enum {
> HNS_ROCE_CMD_QUERY_QPC = 0x42,
>
> HNS_ROCE_CMD_MODIFY_CQC = 0x52,
> + HNS_ROCE_CMD_QUERY_CQC = 0x53,
> /* CQC BT commands */
> HNS_ROCE_CMD_WRITE_CQC_BT0 = 0x10,
> HNS_ROCE_CMD_WRITE_CQC_BT1 = 0x11,
> diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h
> index 9ee86da..d7872ef 100644
> --- a/drivers/infiniband/hw/hns/hns_roce_device.h
> +++ b/drivers/infiniband/hw/hns/hns_roce_device.h
> @@ -869,6 +869,11 @@ struct hns_roce_work {
> int sub_type;
> };
>
> +struct hns_roce_dfx_hw {
> + int (*query_cqc_info)(struct hns_roce_dev *hr_dev, u32 cqn,
> + int *buffer);
> +};
> +
> struct hns_roce_hw {
> int (*reset)(struct hns_roce_dev *hr_dev, bool enable);
> int (*cmq_init)(struct hns_roce_dev *hr_dev);
> @@ -985,6 +990,7 @@ struct hns_roce_dev {
> const struct hns_roce_hw *hw;
> void *priv;
> struct workqueue_struct *irq_workq;
> + const struct hns_roce_dfx_hw *dfx;
> };
>
> static inline struct hns_roce_dev *to_hr_dev(struct ib_device *ib_dev)
> @@ -1202,4 +1208,6 @@ int hns_roce_alloc_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db,
> int hns_roce_init(struct hns_roce_dev *hr_dev);
> void hns_roce_exit(struct hns_roce_dev *hr_dev);
>
> +int hns_roce_fill_res_entry(struct sk_buff *msg,
> + struct rdma_restrack_entry *res);
> #endif /* _HNS_ROCE_DEVICE_H */
> diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
> index 8795266..817a4f5 100644
> --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
> +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
> @@ -953,9 +953,9 @@ static void hns_roce_v2_cmq_exit(struct hns_roce_dev *hr_dev)
> hns_roce_free_cmq_desc(hr_dev, &priv->cmq.crq);
> }
>
> -static void hns_roce_cmq_setup_basic_desc(struct hns_roce_cmq_desc *desc,
> - enum hns_roce_opcode_type opcode,
> - bool is_read)
> +void hns_roce_cmq_setup_basic_desc(struct hns_roce_cmq_desc *desc,
> + enum hns_roce_opcode_type opcode,
> + bool is_read)
> {
> memset((void *)desc, 0, sizeof(struct hns_roce_cmq_desc));
> desc->opcode = cpu_to_le16(opcode);
> @@ -6044,6 +6044,10 @@ static int hns_roce_v2_post_srq_recv(struct ib_srq *ibsrq,
> return ret;
> }
>
> +static const struct hns_roce_dfx_hw hns_roce_dfx_hw_v2 = {
> + .query_cqc_info = hns_roce_v2_query_cqc_info,
> +};
> +
> static const struct ib_device_ops hns_roce_v2_dev_ops = {
> .destroy_qp = hns_roce_v2_destroy_qp,
> .modify_cq = hns_roce_v2_modify_cq,
> @@ -6123,6 +6127,7 @@ static int hns_roce_hw_v2_get_cfg(struct hns_roce_dev *hr_dev,
> }
>
> hr_dev->hw = &hns_roce_hw_v2;
> + hr_dev->dfx = &hns_roce_dfx_hw_v2;
> hr_dev->sdb_offset = ROCEE_DB_SQ_L_0_REG;
> hr_dev->odb_offset = hr_dev->sdb_offset;
>
> diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
> index f1f1b75..15e4d53 100644
> --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
> +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
> @@ -1799,6 +1799,14 @@ struct hns_roce_sccc_clr_done {
> __le32 rsv[5];
> };
>
> +int hns_roce_v2_query_cqc_info(struct hns_roce_dev *hr_dev, u32 cqn,
> + int *buffer);
> +void hns_roce_cmq_setup_basic_desc(struct hns_roce_cmq_desc *desc,
> + enum hns_roce_opcode_type opcode,
> + bool is_read);
> +int hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
> + struct hns_roce_cmq_desc *desc, int num);
> +
> static inline void hns_roce_write64(struct hns_roce_dev *hr_dev, __le32 val[2],
> void __iomem *dest)
> {
> diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c
> new file mode 100644
> index 0000000..76cdbc9
> --- /dev/null
> +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c
> @@ -0,0 +1,36 @@
> +// SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
> +// Copyright (c) 2019 Hisilicon Limited.
> +
> +#include "hnae3.h"
> +#include "hns_roce_device.h"
> +#include "hns_roce_cmd.h"
> +#include "hns_roce_hw_v2.h"
> +
> +int hns_roce_v2_query_cqc_info(struct hns_roce_dev *hr_dev, u32 cqn,
> + int *buffer)
> +{
> + struct hns_roce_v2_cq_context *cq_context;
> + struct hns_roce_cmd_mailbox *mailbox;
> + int ret;
> +
> + mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
> + if (IS_ERR(mailbox))
> + return PTR_ERR(mailbox);
> +
> + cq_context = mailbox->buf;
> + ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma,
> + cqn, 0,
> + HNS_ROCE_CMD_QUERY_CQC,
> + HNS_ROCE_CMD_TIMEOUT_MSECS);
> + if (ret) {
> + dev_err(hr_dev->dev, "QUERY cqc cmd process error\n");
> + goto err_mailbox;
> + }
> +
> + memcpy(buffer, cq_context, sizeof(*cq_context));
> +
> +err_mailbox:
> + hns_roce_free_cmd_mailbox(hr_dev, mailbox);
> +
> + return ret;
> +}
> diff --git a/drivers/infiniband/hw/hns/hns_roce_restrack.c b/drivers/infiniband/hw/hns/hns_roce_restrack.c
> new file mode 100644
> index 0000000..5436f5f
> --- /dev/null
> +++ b/drivers/infiniband/hw/hns/hns_roce_restrack.c
> @@ -0,0 +1,124 @@
> +// SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
> +// Copyright (c) 2019 Hisilicon Limited.
> +
> +#include <rdma/rdma_cm.h>
> +#include <rdma/restrack.h>
> +#include <uapi/rdma/rdma_netlink.h>
> +#include "hnae3.h"
> +#include "hns_roce_common.h"
> +#include "hns_roce_device.h"
> +#include "hns_roce_hw_v2.h"
> +
> +static int hns_roce_fill_cq(struct sk_buff *msg,
> + struct hns_roce_v2_cq_context *context)
> +{
> + if (rdma_nl_put_driver_u32(msg, "state",
> + roce_get_field(context->byte_4_pg_ceqn,
> + V2_CQC_BYTE_4_ARM_ST_M,
> + V2_CQC_BYTE_4_ARM_ST_S)))
> + goto err;
> +
> + if (rdma_nl_put_driver_u32(msg, "ceqn",
> + roce_get_field(context->byte_4_pg_ceqn,
> + V2_CQC_BYTE_4_CEQN_M,
> + V2_CQC_BYTE_4_CEQN_S)))
> + goto err;
> +
> + if (rdma_nl_put_driver_u32(msg, "cqn",
> + roce_get_field(context->byte_8_cqn,
> + V2_CQC_BYTE_8_CQN_M,
> + V2_CQC_BYTE_8_CQN_S)))
> + goto err;
> +
> + if (rdma_nl_put_driver_u32(msg, "hopnum",
> + roce_get_field(context->byte_16_hop_addr,
> + V2_CQC_BYTE_16_CQE_HOP_NUM_M,
> + V2_CQC_BYTE_16_CQE_HOP_NUM_S)))
> + goto err;
> +
> + if (rdma_nl_put_driver_u32(msg, "pi",
> + roce_get_field(context->byte_28_cq_pi,
> + V2_CQC_BYTE_28_CQ_PRODUCER_IDX_M,
> + V2_CQC_BYTE_28_CQ_PRODUCER_IDX_S)))
> + goto err;
> +
> + if (rdma_nl_put_driver_u32(msg, "ci",
> + roce_get_field(context->byte_32_cq_ci,
> + V2_CQC_BYTE_16_CQE_HOP_NUM_M,
> + V2_CQC_BYTE_16_CQE_HOP_NUM_S)))
> + goto err;
> +
> + if (rdma_nl_put_driver_u32(msg, "coalesce",
> + roce_get_field(
> + context->byte_56_cqe_period_maxcnt,
> + V2_CQC_BYTE_56_CQ_MAX_CNT_M,
> + V2_CQC_BYTE_56_CQ_MAX_CNT_S)))
> + goto err;
> +
> + if (rdma_nl_put_driver_u32(msg, "period",
> + roce_get_field(
> + context->byte_56_cqe_period_maxcnt,
> + V2_CQC_BYTE_56_CQ_PERIOD_M,
> + V2_CQC_BYTE_56_CQ_PERIOD_S)))
> + goto err;
> +
> + if (rdma_nl_put_driver_u32(msg, "cnt",
> + roce_get_field(context->byte_52_cqe_cnt,
> + V2_CQC_BYTE_52_CQE_CNT_M,
> + V2_CQC_BYTE_52_CQE_CNT_S)))
> + goto err;
> +
> + return 0;
> +
> +err:
> + return -EMSGSIZE;
> +}
> +
> +static int hns_roce_fill_res_cq_entry(struct sk_buff *msg,
> + struct rdma_restrack_entry *res)
> +{
> + struct ib_cq *ib_cq = container_of(res, struct ib_cq, res);
> + struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq->device);
> + struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq);
> + struct hns_roce_v2_cq_context *context;
> + struct nlattr *table_attr;
> + int ret;
> +
> + if (!hr_dev->dfx->query_cqc_info)
> + return -EINVAL;
> +
> + context = kzalloc(sizeof(struct hns_roce_v2_cq_context), GFP_KERNEL);
> + if (!context)
> + return -ENOMEM;
> +
> + ret = hr_dev->dfx->query_cqc_info(hr_dev, hr_cq->cqn, (int *)context);
> + if (ret)
> + goto err;
> +
> + table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_DRIVER);
> + if (!table_attr)
> + goto err;
> +
> + if (hns_roce_fill_cq(msg, context))
> + goto err_cancel_table;
> +
> + nla_nest_end(msg, table_attr);
> + kfree(context);
> +
> + return 0;
> +
> +err_cancel_table:
> + nla_nest_cancel(msg, table_attr);
> +err:
> + kfree(context);
> + return -EMSGSIZE;
> +}
> +
> +int hns_roce_fill_res_entry(struct sk_buff *msg,
> + struct rdma_restrack_entry *res)
> +{
> + if (res->type == RDMA_RESTRACK_CQ)
> + return hns_roce_fill_res_cq_entry(msg, res);
> +
> + return 0;
> +}
>
