On Mon, Nov 19, 2018 at 06:56:56PM +0800, Tao Tian wrote:
> This patch realizes the resoures track for hip08. It allows
> dumping important cq context state which is useful for
> debugging. we will support more resource track later.
>
> the output should be as follows.
> $ ./rdma res show cq dev hnseth0 -d
> dev hnseth0 cqe 1024 users 2 poll-ctx WORKQUEUE pid 0 comm [ib_core] cq_st 0x2a8
> 1 cqn 0x0 cur_cqe 0xf7b97 cqe_hop_num 0x40000000 nxt_cqe 0xf7b96 cqe_sz 0x0 cq_p
> roducer 0x0 cq_consumer 0x0 cqe_ba 0x1ef71e00 nxt_ba 0x0 db_record_en 0xf7b98001
> db_record 0x0 cqe_cnt 0x0 cqe_period 0x0 cqe_timer 0x38cf9bf se_cqe 0x0
Except the fact that it looks pretty awful and you didn't really think
about your users who will see it, some parameters are useful for everyone
and some parameters look like addresses. The implementation looks good to me.
>
> Signed-off-by: Tao Tian <tiantao6@xxxxxxxxxx>
> ---
> v1->v2:
> 1.fixed the warning about the SPDX tag.
> 2.Add the output of rdmatool to the commit message.
> 3.Modify the function pointer to register res.
> ---
> drivers/infiniband/hw/hns/Makefile | 4 +-
> drivers/infiniband/hw/hns/hns_roce_cmd.h | 1 +
> drivers/infiniband/hw/hns/hns_roce_device.h | 8 ++
> drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 15 ++--
> drivers/infiniband/hw/hns/hns_roce_hw_v2.h | 8 ++
> drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c | 36 +++++++++
> drivers/infiniband/hw/hns/hns_roce_main.c | 1 +
> drivers/infiniband/hw/hns/hns_roce_restrack.c | 100 +++++++++++++++++++++++++
> 8 files changed, 166 insertions(+), 7 deletions(-)
> create mode 100644 drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c
> create mode 100644 drivers/infiniband/hw/hns/hns_roce_restrack.c
>
> diff --git a/drivers/infiniband/hw/hns/Makefile b/drivers/infiniband/hw/hns/Makefile
> index cf03404..5b99229 100644
> --- a/drivers/infiniband/hw/hns/Makefile
> +++ b/drivers/infiniband/hw/hns/Makefile
> @@ -7,8 +7,8 @@ ccflags-y := -Idrivers/net/ethernet/hisilicon/hns3
> obj-$(CONFIG_INFINIBAND_HNS) += hns-roce.o
> hns-roce-objs := hns_roce_main.o hns_roce_cmd.o hns_roce_pd.o \
> hns_roce_ah.o hns_roce_hem.o hns_roce_mr.o hns_roce_qp.o \
> - hns_roce_cq.o hns_roce_alloc.o hns_roce_db.o
> + hns_roce_cq.o hns_roce_alloc.o hns_roce_db.o hns_roce_restrack.o
> obj-$(CONFIG_INFINIBAND_HNS_HIP06) += hns-roce-hw-v1.o
> hns-roce-hw-v1-objs := hns_roce_hw_v1.o
> obj-$(CONFIG_INFINIBAND_HNS_HIP08) += hns-roce-hw-v2.o
> -hns-roce-hw-v2-objs := hns_roce_hw_v2.o
> +hns-roce-hw-v2-objs := hns_roce_hw_v2.o hns_roce_hw_v2_dfx.o
> diff --git a/drivers/infiniband/hw/hns/hns_roce_cmd.h b/drivers/infiniband/hw/hns/hns_roce_cmd.h
> index 9549ae5..e93b6fb 100644
> --- a/drivers/infiniband/hw/hns/hns_roce_cmd.h
> +++ b/drivers/infiniband/hw/hns/hns_roce_cmd.h
> @@ -53,6 +53,7 @@ enum {
> HNS_ROCE_CMD_QUERY_QPC = 0x42,
>
> HNS_ROCE_CMD_MODIFY_CQC = 0x52,
> + HNS_ROCE_CMD_QUERY_CQC = 0x53,
> /* CQC BT commands */
> HNS_ROCE_CMD_WRITE_CQC_BT0 = 0x10,
> HNS_ROCE_CMD_WRITE_CQC_BT1 = 0x11,
> diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h
> index d39bdfd..5f2c85e 100644
> --- a/drivers/infiniband/hw/hns/hns_roce_device.h
> +++ b/drivers/infiniband/hw/hns/hns_roce_device.h
> @@ -756,6 +756,11 @@ struct hns_roce_work {
> int sub_type;
> };
>
> +struct hns_roce_dfx_hw {
> + int (*query_cqc_info)(struct hns_roce_dev *hr_dev, u32 cqn,
> + int *buffer);
> +};
> +
> struct hns_roce_hw {
> int (*reset)(struct hns_roce_dev *hr_dev, bool enable);
> int (*cmq_init)(struct hns_roce_dev *hr_dev);
> @@ -851,6 +856,7 @@ struct hns_roce_dev {
> const struct hns_roce_hw *hw;
> void *priv;
> struct workqueue_struct *irq_workq;
> + const struct hns_roce_dfx_hw *dfx;
> };
>
> static inline struct hns_roce_dev *to_hr_dev(struct ib_device *ib_dev)
> @@ -1056,4 +1062,6 @@ int hns_get_gid_index(struct hns_roce_dev *hr_dev, u8 port, int gid_index);
> int hns_roce_init(struct hns_roce_dev *hr_dev);
> void hns_roce_exit(struct hns_roce_dev *hr_dev);
>
> +int hns_roce_fill_res_entry(struct sk_buff *msg,
> + struct rdma_restrack_entry *res);
> #endif /* _HNS_ROCE_DEVICE_H */
> diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
> index a4c62ae..dbc1931 100644
> --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
> +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
> @@ -846,9 +846,9 @@ static void hns_roce_v2_cmq_exit(struct hns_roce_dev *hr_dev)
> hns_roce_free_cmq_desc(hr_dev, &priv->cmq.crq);
> }
>
> -static void hns_roce_cmq_setup_basic_desc(struct hns_roce_cmq_desc *desc,
> - enum hns_roce_opcode_type opcode,
> - bool is_read)
> +void hns_roce_cmq_setup_basic_desc(struct hns_roce_cmq_desc *desc,
> + enum hns_roce_opcode_type opcode,
> + bool is_read)
> {
> memset((void *)desc, 0, sizeof(struct hns_roce_cmq_desc));
> desc->opcode = cpu_to_le16(opcode);
> @@ -892,8 +892,8 @@ static int hns_roce_cmq_csq_clean(struct hns_roce_dev *hr_dev)
> return clean;
> }
>
> -static int hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
> - struct hns_roce_cmq_desc *desc, int num)
> +int hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
> + struct hns_roce_cmq_desc *desc, int num)
> {
> struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
> struct hns_roce_v2_cmq_ring *csq = &priv->cmq.csq;
> @@ -5340,6 +5340,10 @@ static void hns_roce_v2_cleanup_eq_table(struct hns_roce_dev *hr_dev)
> destroy_workqueue(hr_dev->irq_workq);
> }
>
> +static const struct hns_roce_dfx_hw hns_roce_dfx_hw_v2 = {
> + .query_cqc_info = hns_roce_v2_query_cqc_info,
> +};
> +
> static const struct hns_roce_hw hns_roce_hw_v2 = {
> .cmq_init = hns_roce_v2_cmq_init,
> .cmq_exit = hns_roce_v2_cmq_exit,
> @@ -5394,6 +5398,7 @@ static int hns_roce_hw_v2_get_cfg(struct hns_roce_dev *hr_dev,
> }
>
> hr_dev->hw = &hns_roce_hw_v2;
> + hr_dev->dfx = &hns_roce_dfx_hw_v2;
> hr_dev->sdb_offset = ROCEE_DB_SQ_L_0_REG;
> hr_dev->odb_offset = hr_dev->sdb_offset;
>
> diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
> index 8bc8206..929b89e 100644
> --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
> +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
> @@ -1612,4 +1612,12 @@ struct hns_roce_wqe_atomic_seg {
> __le64 cmp_data;
> };
>
> +int hns_roce_v2_query_cqc_info(struct hns_roce_dev *hr_dev, u32 cqn,
> + int *buffer);
> +void hns_roce_cmq_setup_basic_desc(struct hns_roce_cmq_desc *desc,
> + enum hns_roce_opcode_type opcode,
> + bool is_read);
> +int hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
> + struct hns_roce_cmq_desc *desc, int num);
> +
> #endif
> diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c
> new file mode 100644
> index 0000000..ef6412b
> --- /dev/null
> +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c
> @@ -0,0 +1,36 @@
> +// SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
> +// Copyright (c) 2018 Hisilicon Limited.
> +
> +#include "hns_roce_device.h"
> +#include "hns_roce_cmd.h"
> +#include "hns_roce_hw_v2.h"
> +
> +int hns_roce_v2_query_cqc_info(struct hns_roce_dev *hr_dev, u32 cqn,
> + int *buffer)
> +{
> + struct hns_roce_v2_cq_context *cq_context;
> + struct hns_roce_cmd_mailbox *mailbox;
> + int ret;
> +
> + mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
> + if (IS_ERR(mailbox))
> + return PTR_ERR(mailbox);
> +
> + cq_context = mailbox->buf;
> + ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma,
> + cqn, 0,
> + HNS_ROCE_CMD_QUERY_CQC,
> + HNS_ROCE_CMD_TIMEOUT_MSECS);
> + if (ret) {
> + dev_err(hr_dev->dev, "QUERY cqc cmd process error\n");
> + goto err_mailbox;
> + }
> +
> + memcpy(buffer, cq_context, sizeof(*cq_context));
> +
> +err_mailbox:
> + hns_roce_free_cmd_mailbox(hr_dev, mailbox);
> +
> + return ret;
> +}
> +
> diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c
> index 1b3ee51..0aaf1ef 100644
> --- a/drivers/infiniband/hw/hns/hns_roce_main.c
> +++ b/drivers/infiniband/hw/hns/hns_roce_main.c
> @@ -544,6 +544,7 @@ static int hns_roce_register_device(struct hns_roce_dev *hr_dev)
> /* OTHERS */
> ib_dev->get_port_immutable = hns_roce_port_immutable;
> ib_dev->disassociate_ucontext = hns_roce_disassociate_ucontext;
> + ib_dev->res.fill_res_entry = hns_roce_fill_res_entry;
>
> ib_dev->driver_id = RDMA_DRIVER_HNS;
> ret = ib_register_device(ib_dev, "hns_%d", NULL);
> diff --git a/drivers/infiniband/hw/hns/hns_roce_restrack.c b/drivers/infiniband/hw/hns/hns_roce_restrack.c
> new file mode 100644
> index 0000000..a84238d
> --- /dev/null
> +++ b/drivers/infiniband/hw/hns/hns_roce_restrack.c
> @@ -0,0 +1,100 @@
> +// SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause)
> +// Copyright (c) 2018 Hisilicon Limited.
> +#include "hns_roce_device.h"
> +#include <rdma/rdma_cm.h>
> +#include <rdma/restrack.h>
> +#include <uapi/rdma/rdma_netlink.h>
> +
> +enum cq_context_index {
> + CQ_ST,
> + CQN,
> + CUR_CQE,
> + CQE_HOP_NUM,
> + NXT_CQE,
> + CQE_SZ,
> + CQ_PRODUCER,
> + CQ_CONSUMER,
> + CQE_BA,
> + NXT_BA,
> + DB_RECORD_EN,
> + DB_RECORD,
> + CQE_CNT,
> + CQE_PERIOD,
> + CQE_TIMER,
> + SE_CQE,
> + CQC_CONTEXT_MAX
> +};
> +
> +static const char * const cq_names[CQC_CONTEXT_MAX] = {
> + [CQ_ST] = "cq_st",
> + [CQN] = "cqn",
> + [CUR_CQE] = "cur_cqe",
> + [CQE_HOP_NUM] = "cqe_hop_num",
> + [NXT_CQE] = "nxt_cqe",
> + [CQE_SZ] = "cqe_sz",
> + [CQ_PRODUCER] = "cq_producer",
> + [CQ_CONSUMER] = "cq_consumer",
> + [CQE_BA] = "cqe_ba",
> + [NXT_BA] = "nxt_ba",
> + [DB_RECORD_EN] = "db_record_en",
> + [DB_RECORD] = "db_record",
> + [CQE_CNT] = "cqe_cnt",
> + [CQE_PERIOD] = "cqe_period",
> + [CQE_TIMER] = "cqe_timer",
> + [SE_CQE] = "se_cqe",
> +};
> +
> +static int hns_roce_fill_cq(struct sk_buff *msg, int *cqc)
> +{
> + int i;
> +
> + for (i = 0; i < CQC_CONTEXT_MAX; i++) {
> + if (rdma_nl_put_driver_u32_hex(msg, cq_names[i], *(cqc + i)))
> + return -EMSGSIZE;
> + }
> +
> + return 0;
> +}
> +
> +static int hns_roce_fill_res_cq_entry(struct sk_buff *msg,
> + struct rdma_restrack_entry *res)
> +{
> + struct ib_cq *ib_cq = container_of(res, struct ib_cq, res);
> + struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq->device);
> + struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq);
> + int buffer[hr_dev->caps.cqc_entry_sz];
> + struct nlattr *table_attr;
> + int ret;
> +
> + if (!hr_dev->dfx->query_cqc_info)
> + return -EINVAL;
> +
> + ret = hr_dev->dfx->query_cqc_info(hr_dev, hr_cq->cqn, buffer);
> + if (ret)
> + return ret;
> +
> + table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_DRIVER);
> + if (!table_attr)
> + goto err;
> +
> + if (hns_roce_fill_cq(msg, buffer))
> + goto err_cancel_table;
> +
> + nla_nest_end(msg, table_attr);
> +
> + return 0;
> +
> +err_cancel_table:
> + nla_nest_cancel(msg, table_attr);
> +err:
> + return -EMSGSIZE;
> +}
> +
> +int hns_roce_fill_res_entry(struct sk_buff *msg,
> + struct rdma_restrack_entry *res)
> +{
> + if (res->type == RDMA_RESTRACK_CQ)
> + return hns_roce_fill_res_cq_entry(msg, res);
> +
> + return 0;
> +}
> --
> 2.7.4
>
Attachment:
signature.asc
Description: PGP signature
