On Sat, Nov 17, 2018 at 04:01:03PM +0800, Tao Tian wrote:
> This patch realizes the resoures track for hip08. It allows
> dumping important cq context state which is useful for
> debugging. we will support more resource track later.
Please provide output of rdmatool.
>
> Signed-off-by: Tao Tian <tiantao6@xxxxxxxxxx>
> ---
> drivers/infiniband/hw/hns/Makefile | 4 +-
> drivers/infiniband/hw/hns/hns_roce_cmd.h | 1 +
> drivers/infiniband/hw/hns/hns_roce_device.h | 10 ++
> drivers/infiniband/hw/hns/hns_roce_hw_v2.c | 9 +-
> drivers/infiniband/hw/hns/hns_roce_hw_v2.h | 8 ++
> drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c | 65 +++++++++++++
> drivers/infiniband/hw/hns/hns_roce_main.c | 8 ++
> drivers/infiniband/hw/hns/hns_roce_restrack.c | 126 +++++++++++++++++++++++++
> 8 files changed, 227 insertions(+), 4 deletions(-)
> create mode 100644 drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c
> create mode 100644 drivers/infiniband/hw/hns/hns_roce_restrack.c
>
> diff --git a/drivers/infiniband/hw/hns/Makefile b/drivers/infiniband/hw/hns/Makefile
> index cf03404..5b99229 100644
> --- a/drivers/infiniband/hw/hns/Makefile
> +++ b/drivers/infiniband/hw/hns/Makefile
> @@ -7,8 +7,8 @@ ccflags-y := -Idrivers/net/ethernet/hisilicon/hns3
> obj-$(CONFIG_INFINIBAND_HNS) += hns-roce.o
> hns-roce-objs := hns_roce_main.o hns_roce_cmd.o hns_roce_pd.o \
> hns_roce_ah.o hns_roce_hem.o hns_roce_mr.o hns_roce_qp.o \
> - hns_roce_cq.o hns_roce_alloc.o hns_roce_db.o
> + hns_roce_cq.o hns_roce_alloc.o hns_roce_db.o hns_roce_restrack.o
> obj-$(CONFIG_INFINIBAND_HNS_HIP06) += hns-roce-hw-v1.o
> hns-roce-hw-v1-objs := hns_roce_hw_v1.o
> obj-$(CONFIG_INFINIBAND_HNS_HIP08) += hns-roce-hw-v2.o
> -hns-roce-hw-v2-objs := hns_roce_hw_v2.o
> +hns-roce-hw-v2-objs := hns_roce_hw_v2.o hns_roce_hw_v2_dfx.o
> diff --git a/drivers/infiniband/hw/hns/hns_roce_cmd.h b/drivers/infiniband/hw/hns/hns_roce_cmd.h
> index 9549ae5..e93b6fb 100644
> --- a/drivers/infiniband/hw/hns/hns_roce_cmd.h
> +++ b/drivers/infiniband/hw/hns/hns_roce_cmd.h
> @@ -53,6 +53,7 @@ enum {
> HNS_ROCE_CMD_QUERY_QPC = 0x42,
>
> HNS_ROCE_CMD_MODIFY_CQC = 0x52,
> + HNS_ROCE_CMD_QUERY_CQC = 0x53,
> /* CQC BT commands */
> HNS_ROCE_CMD_WRITE_CQC_BT0 = 0x10,
> HNS_ROCE_CMD_WRITE_CQC_BT1 = 0x11,
> diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h
> index d39bdfd..151cb59 100644
> --- a/drivers/infiniband/hw/hns/hns_roce_device.h
> +++ b/drivers/infiniband/hw/hns/hns_roce_device.h
> @@ -756,6 +756,11 @@ struct hns_roce_work {
> int sub_type;
> };
>
> +struct hns_roce_dfx_hw {
> + int (*query_cqc_info)(struct hns_roce_dev *hr_dev, u32 cqn,
> + int *buffer);
> +};
> +
> struct hns_roce_hw {
> int (*reset)(struct hns_roce_dev *hr_dev, bool enable);
> int (*cmq_init)(struct hns_roce_dev *hr_dev);
> @@ -851,6 +856,7 @@ struct hns_roce_dev {
> const struct hns_roce_hw *hw;
> void *priv;
> struct workqueue_struct *irq_workq;
> + const struct hns_roce_dfx_hw *dfx;
> };
>
> static inline struct hns_roce_dev *to_hr_dev(struct ib_device *ib_dev)
> @@ -1056,4 +1062,8 @@ int hns_get_gid_index(struct hns_roce_dev *hr_dev, u8 port, int gid_index);
> int hns_roce_init(struct hns_roce_dev *hr_dev);
> void hns_roce_exit(struct hns_roce_dev *hr_dev);
>
> +typedef int hns_roce_restrack(struct sk_buff *msg,
> + struct rdma_restrack_entry *res);
> +extern hns_roce_restrack *hns_roce_restrack_funcs[RDMA_RESTRACK_MAX];
> +
I see that cxgb4 uses the same construction. Please do and use general
variant of this function signature in in include/rdma/restrack.h because
you are proposing second implementation.
> #endif /* _HNS_ROCE_DEVICE_H */
> diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
> index a4c62ae..868c7ee 100644
> --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
> +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
> @@ -846,7 +846,7 @@ static void hns_roce_v2_cmq_exit(struct hns_roce_dev *hr_dev)
> hns_roce_free_cmq_desc(hr_dev, &priv->cmq.crq);
> }
>
> -static void hns_roce_cmq_setup_basic_desc(struct hns_roce_cmq_desc *desc,
> +void hns_roce_cmq_setup_basic_desc(struct hns_roce_cmq_desc *desc,
> enum hns_roce_opcode_type opcode,
> bool is_read)
> {
> @@ -892,7 +892,7 @@ static int hns_roce_cmq_csq_clean(struct hns_roce_dev *hr_dev)
> return clean;
> }
>
> -static int hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
> +int hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
> struct hns_roce_cmq_desc *desc, int num)
> {
> struct hns_roce_v2_priv *priv = (struct hns_roce_v2_priv *)hr_dev->priv;
> @@ -5340,6 +5340,10 @@ static void hns_roce_v2_cleanup_eq_table(struct hns_roce_dev *hr_dev)
> destroy_workqueue(hr_dev->irq_workq);
> }
>
> +static const struct hns_roce_dfx_hw hns_roce_dfx_hw_v2 = {
> + .query_cqc_info = hns_roce_v2_query_cqc_info,
> +};
> +
> static const struct hns_roce_hw hns_roce_hw_v2 = {
> .cmq_init = hns_roce_v2_cmq_init,
> .cmq_exit = hns_roce_v2_cmq_exit,
> @@ -5394,6 +5398,7 @@ static int hns_roce_hw_v2_get_cfg(struct hns_roce_dev *hr_dev,
> }
>
> hr_dev->hw = &hns_roce_hw_v2;
> + hr_dev->dfx = &hns_roce_dfx_hw_v2;
> hr_dev->sdb_offset = ROCEE_DB_SQ_L_0_REG;
> hr_dev->odb_offset = hr_dev->sdb_offset;
>
> diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
> index 8bc8206..262198d 100644
> --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
> +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
> @@ -1612,4 +1612,12 @@ struct hns_roce_wqe_atomic_seg {
> __le64 cmp_data;
> };
>
> +int hns_roce_v2_query_cqc_info(struct hns_roce_dev *hr_dev, u32 cqn,
> + int *buffer);
> +void hns_roce_cmq_setup_basic_desc(struct hns_roce_cmq_desc *desc,
> + enum hns_roce_opcode_type opcode,
> + bool is_read);
> +int hns_roce_cmq_send(struct hns_roce_dev *hr_dev,
> + struct hns_roce_cmq_desc *desc, int num);
> +
> #endif
> diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c
> new file mode 100644
> index 0000000..259410e
> --- /dev/null
> +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2_dfx.c
> @@ -0,0 +1,65 @@
> +/*
> + * Copyright (c) 2018 Hisilicon Limited.
> + *
> + * This software is available to you under a choice of one of two
> + * licenses. You may choose to be licensed under the terms of the GNU
> + * General Public License (GPL) Version 2, available from the file
> + * COPYING in the main directory of this source tree, or the
> + * OpenIB.org BSD license below:
> + *
> + * Redistribution and use in source and binary forms, with or
> + * without modification, are permitted provided that the following
> + * conditions are met:
> + *
> + * - Redistributions of source code must retain the above
> + * copyright notice, this list of conditions and the following
> + * disclaimer.
> + *
> + * - Redistributions in binary form must reproduce the above
> + * copyright notice, this list of conditions and the following
> + * disclaimer in the documentation and/or other materials
> + * provided with the distribution.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
> + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
> + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
> + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
> + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
> + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
> + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
> + * SOFTWARE.
> + */
Please change to SPDX tag instead of license agreement.
See drivers/infiniband/hw/mlx5/devx.c as an example.
> +
> +#include "hns_roce_device.h"
> +#include "hns_roce_cmd.h"
> +#include "hns_roce_hw_v2.h"
> +
> +int hns_roce_v2_query_cqc_info(struct hns_roce_dev *hr_dev, u32 cqn,
> + int *buffer)
> +{
> + struct hns_roce_v2_cq_context *cq_context;
> + struct hns_roce_cmd_mailbox *mailbox;
> + int ret;
> +
> + mailbox = hns_roce_alloc_cmd_mailbox(hr_dev);
> + if (IS_ERR(mailbox))
> + return PTR_ERR(mailbox);
> +
> + cq_context = mailbox->buf;
> + ret = hns_roce_cmd_mbox(hr_dev, 0, mailbox->dma,
> + cqn, 0,
> + HNS_ROCE_CMD_QUERY_CQC,
> + HNS_ROCE_CMD_TIMEOUT_MSECS);
> + if (ret) {
> + dev_err(hr_dev->dev, "QUERY cqc cmd process error\n");
> + goto err_mailbox;
> + }
> +
> + memcpy(buffer, cq_context, sizeof(*cq_context));
> +
> +err_mailbox:
> + hns_roce_free_cmd_mailbox(hr_dev, mailbox);
> +
> + return ret;
> +}
> +
> diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c
> index 1b3ee51..68af8b7 100644
> --- a/drivers/infiniband/hw/hns/hns_roce_main.c
> +++ b/drivers/infiniband/hw/hns/hns_roce_main.c
> @@ -53,6 +53,13 @@
> * GID[0][0], GID[1][0],.....GID[N - 1][0],
> * And so on
> */
> +static int hns_roce_fill_res_entry(struct sk_buff *msg, struct rdma_restrack_entry *res)
> +{
> + return (res->type < ARRAY_SIZE(hns_roce_restrack_funcs) &&
> + hns_roce_restrack_funcs[res->type]) ?
> + hns_roce_restrack_funcs[res->type](msg, res) : 0;
> +}
> +
> int hns_get_gid_index(struct hns_roce_dev *hr_dev, u8 port, int gid_index)
> {
> return gid_index * hr_dev->caps.num_ports + port;
> @@ -544,6 +551,7 @@ static int hns_roce_register_device(struct hns_roce_dev *hr_dev)
> /* OTHERS */
> ib_dev->get_port_immutable = hns_roce_port_immutable;
> ib_dev->disassociate_ucontext = hns_roce_disassociate_ucontext;
> + ib_dev->res.fill_res_entry = hns_roce_fill_res_entry;
>
> ib_dev->driver_id = RDMA_DRIVER_HNS;
> ret = ib_register_device(ib_dev, "hns_%d", NULL);
> diff --git a/drivers/infiniband/hw/hns/hns_roce_restrack.c b/drivers/infiniband/hw/hns/hns_roce_restrack.c
> new file mode 100644
> index 0000000..fe174cd
> --- /dev/null
> +++ b/drivers/infiniband/hw/hns/hns_roce_restrack.c
> @@ -0,0 +1,126 @@
> +/*
> + * Copyright (c) 2018 Hisilicon Limited.
> + *
> + * This software is available to you under a choice of one of two
> + * licenses. You may choose to be licensed under the terms of the GNU
> + * General Public License (GPL) Version 2, available from the file
> + * COPYING in the main directory of this source tree, or the
> + * OpenIB.org BSD license below:
> + *
> + * Redistribution and use in source and binary forms, with or
> + * without modification, are permitted provided that the following
> + * conditions are met:
> + *
> + * - Redistributions of source code must retain the above
> + * copyright notice, this list of conditions and the following
> + * disclaimer.
> + *
> + * - Redistributions in binary form must reproduce the above
> + * copyright notice, this list of conditions and the following
> + * disclaimer in the documentation and/or other materials
> + * provided with the distribution.
> + *
> + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
> + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
> + * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
> + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
> + * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
> + * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
> + * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
> + * SOFTWARE.
> + */
Again, SPDX tag.
> +
> +#include "hns_roce_device.h"
> +#include <rdma/rdma_cm.h>
> +#include <rdma/restrack.h>
> +#include <uapi/rdma/rdma_netlink.h>
> +
> +enum cq_context_index {
> + BYTE_4_PG_CEQN,
> + BYTE_8_CQN,
> + CQE_CUR_BLK_ADDR,
> + BYTE_16_HOP_ADDR,
> + CQE_NXT_BLK_ADDR,
> + BYTE_24_PGSZ_ADDR,
> + BYTE_28_CQ_PI,
> + BYTE_32_CQ_CI,
> + CQE_BA,
> + BYTE_40_CQE_BA,
> + BYTE_44_DB_RECORD,
> + DB_RECORD_ADDR,
> + BYTE_52_CQE_CNT,
> + BYTE_56_CQE_PERIOD_MAXCNT,
> + CQE_REPORT_TIMER,
> + BYTE_64_SE_CQE_IDX,
> + CQC_CONTEXT_MAX
> +};
> +
> +static const char * const cq_names[CQC_CONTEXT_MAX] = {
> + [BYTE_4_PG_CEQN] = "byte_4_pg_ceqn:",
> + [BYTE_8_CQN] = "byte_8_cqn:",
> + [CQE_CUR_BLK_ADDR] = "cqe_cur_blk_addr:",
> + [BYTE_16_HOP_ADDR] = "byte_16_hop_addr:",
> + [CQE_NXT_BLK_ADDR] = "cqe_nxt_blk_addr:",
> + [BYTE_24_PGSZ_ADDR] = "byte_24_pgsz_addr:",
> + [BYTE_28_CQ_PI] = "byte_28_cq_pi:",
> + [BYTE_32_CQ_CI] = "byte_32_cq_ci:",
> + [CQE_BA] = "cqe_ba:",
> + [BYTE_40_CQE_BA] = "byte_40_cqe_ba:",
> + [BYTE_44_DB_RECORD] = "byte_44_db_record:",
> + [DB_RECORD_ADDR] = "db_record_addr:",
> + [BYTE_52_CQE_CNT] = "byte_52_cqe_cnt:",
> + [BYTE_56_CQE_PERIOD_MAXCNT] = "56_cqe_period_maxcnt:",
> + [CQE_REPORT_TIMER] = "cqe_report_timer:",
> + [BYTE_64_SE_CQE_IDX] = "byte_64_se_cqe_idx:",
> +};
Please remove ":" from the names and all "*_addr" parameters should need
CAP_NET_ADMIN to see them. Also, I don't think that all this data is
really needed. It is unclear to me why don't you test your driver at the
house so users won't need to see all this crap.
> +
> +static int hns_roce_fill_cq(struct sk_buff *msg, int *cqc)
> +{
> + int i;
> +
> + for (i = 0; i < CQC_CONTEXT_MAX; i++) {
> + if (rdma_nl_put_driver_u32_hex(msg, cq_names[i], *(cqc + i)))
You have addresses, which are 64b, but you are setting u32.
> + return -EMSGSIZE;
> + }
> +
> + return 0;
> +}
> +
> +static int hns_roce_fill_res_cq_entry(struct sk_buff *msg,
> + struct rdma_restrack_entry *res)
> +{
> + struct ib_cq *ib_cq = container_of(res, struct ib_cq, res);
> + struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq->device);
> + struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq);
> + int buffer[hr_dev->caps.cqc_entry_sz];
> + struct nlattr *table_attr;
> + int ret;
> +
> + if (!hr_dev->dfx->query_cqc_info)
> + return -1;
"-1" ????
> +
> + ret = hr_dev->dfx->query_cqc_info(hr_dev, hr_cq->cqn, buffer);
> + if (ret)
> + return ret;
> +
> + table_attr = nla_nest_start(msg, RDMA_NLDEV_ATTR_DRIVER);
> + if (!table_attr)
> + goto err;
> +
> + if (hns_roce_fill_cq(msg, buffer))
> + goto err_cancel_table;
> +
> + nla_nest_end(msg, table_attr);
> +
> + return 0;
> +
> +err_cancel_table:
> + nla_nest_cancel(msg, table_attr);
> +err:
> + return -EMSGSIZE;
> +}
> +
> +hns_roce_restrack *hns_roce_restrack_funcs[RDMA_RESTRACK_MAX] = {
> + [RDMA_RESTRACK_CQ] = hns_roce_fill_res_cq_entry,
> +};
> +
> --
> 2.7.4
>
Attachment:
signature.asc
Description: PGP signature
