[PATCH v3 for-next 1/3] RDMA/hns: Add SCC context allocation support for hip08

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



From: Yangyang Li <liyangyang20@xxxxxxxxxx>

This patch adds SCC context allocation and initialization support
for DCQCN in kernel space driver.

Signed-off-by: Yangyang Li <liyangyang20@xxxxxxxxxx>
---
 drivers/infiniband/hw/hns/hns_roce_cmd.h    |  4 ++++
 drivers/infiniband/hw/hns/hns_roce_device.h |  6 +++++
 drivers/infiniband/hw/hns/hns_roce_hem.c    | 26 ++++++++++++++++++---
 drivers/infiniband/hw/hns/hns_roce_hem.h    |  1 +
 drivers/infiniband/hw/hns/hns_roce_hw_v2.c  | 35 ++++++++++++++++++++++++++++-
 drivers/infiniband/hw/hns/hns_roce_hw_v2.h  | 23 +++++++++++++++++--
 drivers/infiniband/hw/hns/hns_roce_main.c   | 18 +++++++++++++++
 drivers/infiniband/hw/hns/hns_roce_qp.c     | 20 ++++++++++++++++-
 8 files changed, 126 insertions(+), 7 deletions(-)

diff --git a/drivers/infiniband/hw/hns/hns_roce_cmd.h b/drivers/infiniband/hw/hns/hns_roce_cmd.h
index 927701d..5348282 100644
--- a/drivers/infiniband/hw/hns/hns_roce_cmd.h
+++ b/drivers/infiniband/hw/hns/hns_roce_cmd.h
@@ -89,6 +89,10 @@ enum {
 	HNS_ROCE_CMD_DESTROY_SRQC_BT1	= 0x39,
 	HNS_ROCE_CMD_DESTROY_SRQC_BT2	= 0x3a,
 
+	/* SCC CTX BT commands */
+	HNS_ROCE_CMD_READ_SCC_CTX_BT0	= 0xa4,
+	HNS_ROCE_CMD_WRITE_SCC_CTX_BT0	= 0xa5,
+
 	/* EQC commands */
 	HNS_ROCE_CMD_CREATE_AEQC	= 0x80,
 	HNS_ROCE_CMD_MODIFY_AEQC	= 0x81,
diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h
index 779dd4c4..718b415 100644
--- a/drivers/infiniband/hw/hns/hns_roce_device.h
+++ b/drivers/infiniband/hw/hns/hns_roce_device.h
@@ -482,6 +482,7 @@ struct hns_roce_qp_table {
 	struct hns_roce_hem_table	qp_table;
 	struct hns_roce_hem_table	irrl_table;
 	struct hns_roce_hem_table	trrl_table;
+	struct hns_roce_hem_table	scc_ctx_table;
 };
 
 struct hns_roce_cq_table {
@@ -768,6 +769,7 @@ struct hns_roce_caps {
 	int		irrl_entry_sz;
 	int		trrl_entry_sz;
 	int		cqc_entry_sz;
+	int		scc_ctx_entry_sz;
 	int		srqc_entry_sz;
 	int		idx_entry_sz;
 	u32		pbl_ba_pg_sz;
@@ -780,6 +782,7 @@ struct hns_roce_caps {
 	u32		srqc_bt_num;
 	u32		cqc_bt_num;
 	u32		mpt_bt_num;
+	u32		scc_ctx_bt_num;
 	u32		qpc_ba_pg_sz;
 	u32		qpc_buf_pg_sz;
 	u32		qpc_hop_num;
@@ -795,6 +798,9 @@ struct hns_roce_caps {
 	u32		mtt_ba_pg_sz;
 	u32		mtt_buf_pg_sz;
 	u32		mtt_hop_num;
+	u32		scc_ctx_ba_pg_sz;
+	u32		scc_ctx_buf_pg_sz;
+	u32		scc_ctx_hop_num;
 	u32		cqe_ba_pg_sz;
 	u32		cqe_buf_pg_sz;
 	u32		cqe_hop_num;
diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.c b/drivers/infiniband/hw/hns/hns_roce_hem.c
index 4cdbcaf..9e4a2ba 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hem.c
+++ b/drivers/infiniband/hw/hns/hns_roce_hem.c
@@ -45,6 +45,7 @@ bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type)
 	    (hr_dev->caps.mpt_hop_num && type == HEM_TYPE_MTPT) ||
 	    (hr_dev->caps.cqc_hop_num && type == HEM_TYPE_CQC) ||
 	    (hr_dev->caps.srqc_hop_num && type == HEM_TYPE_SRQC) ||
+	    (hr_dev->caps.scc_ctx_hop_num && type == HEM_TYPE_SCC_CTX) ||
 	    (hr_dev->caps.cqe_hop_num && type == HEM_TYPE_CQE) ||
 	    (hr_dev->caps.mtt_hop_num && type == HEM_TYPE_MTT) ||
 	    (hr_dev->caps.srqwqe_hop_num && type == HEM_TYPE_SRQWQE) ||
@@ -125,6 +126,14 @@ int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev,
 		mhop->ba_l0_num = hr_dev->caps.cqc_bt_num;
 		mhop->hop_num = hr_dev->caps.cqc_hop_num;
 		break;
+	case HEM_TYPE_SCC_CTX:
+		mhop->buf_chunk_size = 1 << (hr_dev->caps.scc_ctx_buf_pg_sz
+					     + PAGE_SHIFT);
+		mhop->bt_chunk_size = 1 << (hr_dev->caps.scc_ctx_ba_pg_sz
+					    + PAGE_SHIFT);
+		mhop->ba_l0_num = hr_dev->caps.scc_ctx_bt_num;
+		mhop->hop_num = hr_dev->caps.scc_ctx_hop_num;
+		break;
 	case HEM_TYPE_SRQC:
 		mhop->buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz
 					     + PAGE_SHIFT);
@@ -175,7 +184,7 @@ int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev,
 		return 0;
 
 	/*
-	 * QPC/MTPT/CQC/SRQC alloc hem for buffer pages.
+	 * QPC/MTPT/CQC/SRQC/SCC_CTX alloc hem for buffer pages.
 	 * MTT/CQE alloc hem for bt pages.
 	 */
 	bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num);
@@ -486,7 +495,7 @@ static int hns_roce_table_mhop_get(struct hns_roce_dev *hr_dev,
 	}
 
 	/*
-	 * alloc buffer space chunk for QPC/MTPT/CQC/SRQC.
+	 * alloc buffer space chunk for QPC/MTPT/CQC/SRQC/SCC_CTX.
 	 * alloc bt space chunk for MTT/CQE.
 	 */
 	size = table->type < HEM_TYPE_MTT ? buf_chunk_size : bt_chunk_size;
@@ -658,7 +667,7 @@ static void hns_roce_table_mhop_put(struct hns_roce_dev *hr_dev,
 	}
 
 	/*
-	 * free buffer space chunk for QPC/MTPT/CQC/SRQC.
+	 * free buffer space chunk for QPC/MTPT/CQC/SRQC/SCC_CTX.
 	 * free bt space chunk for MTT/CQE.
 	 */
 	hns_roce_free_hem(hr_dev, table->hem[hem_idx]);
@@ -904,6 +913,14 @@ int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev,
 			num_bt_l0 = hr_dev->caps.cqc_bt_num;
 			hop_num = hr_dev->caps.cqc_hop_num;
 			break;
+		case HEM_TYPE_SCC_CTX:
+			buf_chunk_size = 1 << (hr_dev->caps.scc_ctx_buf_pg_sz
+					+ PAGE_SHIFT);
+			bt_chunk_size = 1 << (hr_dev->caps.scc_ctx_ba_pg_sz
+					+ PAGE_SHIFT);
+			num_bt_l0 = hr_dev->caps.scc_ctx_bt_num;
+			hop_num = hr_dev->caps.scc_ctx_hop_num;
+			break;
 		case HEM_TYPE_SRQC:
 			buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz
 					+ PAGE_SHIFT);
@@ -1081,6 +1098,9 @@ void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev)
 		hns_roce_cleanup_hem_table(hr_dev,
 					   &hr_dev->srq_table.table);
 	hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table);
+	if (hr_dev->caps.scc_ctx_entry_sz)
+		hns_roce_cleanup_hem_table(hr_dev,
+					   &hr_dev->qp_table.scc_ctx_table);
 	if (hr_dev->caps.trrl_entry_sz)
 		hns_roce_cleanup_hem_table(hr_dev,
 					   &hr_dev->qp_table.trrl_table);
diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.h b/drivers/infiniband/hw/hns/hns_roce_hem.h
index a650278..6e4fe97 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hem.h
+++ b/drivers/infiniband/hw/hns/hns_roce_hem.h
@@ -44,6 +44,7 @@ enum {
 	HEM_TYPE_MTPT,
 	HEM_TYPE_CQC,
 	HEM_TYPE_SRQC,
+	HEM_TYPE_SCC_CTX,
 
 	 /* UNMAP HEM */
 	HEM_TYPE_MTT,
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
index 835b783..77cfd9b 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c
@@ -1078,6 +1078,9 @@ static int hns_roce_query_pf_resource(struct hns_roce_dev *hr_dev)
 	hr_dev->caps.sl_num = roce_get_field(req_b->qid_idx_sl_num,
 					     PF_RES_DATA_3_PF_SL_NUM_M,
 					     PF_RES_DATA_3_PF_SL_NUM_S);
+	hr_dev->caps.scc_ctx_bt_num = roce_get_field(req_b->scc_ctx_bt_idx_num,
+					     PF_RES_DATA_4_PF_SCC_CTX_BT_NUM_M,
+					     PF_RES_DATA_4_PF_SCC_CTX_BT_NUM_S);
 
 	return 0;
 }
@@ -1178,6 +1181,7 @@ static int hns_roce_v2_set_bt(struct hns_roce_dev *hr_dev)
 	u8 qpc_hop_num = hr_dev->caps.qpc_hop_num;
 	u8 cqc_hop_num = hr_dev->caps.cqc_hop_num;
 	u8 mpt_hop_num = hr_dev->caps.mpt_hop_num;
+	u8 scc_ctx_hop_num = hr_dev->caps.scc_ctx_hop_num;
 	struct hns_roce_cfg_bt_attr *req;
 	struct hns_roce_cmq_desc desc;
 
@@ -1225,6 +1229,20 @@ static int hns_roce_v2_set_bt(struct hns_roce_dev *hr_dev)
 		       CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_S,
 		       mpt_hop_num == HNS_ROCE_HOP_NUM_0 ? 0 : mpt_hop_num);
 
+	roce_set_field(req->vf_scc_ctx_cfg,
+		       CFG_BT_ATTR_DATA_4_VF_SCC_CTX_BA_PGSZ_M,
+		       CFG_BT_ATTR_DATA_4_VF_SCC_CTX_BA_PGSZ_S,
+		       hr_dev->caps.scc_ctx_ba_pg_sz + PG_SHIFT_OFFSET);
+	roce_set_field(req->vf_scc_ctx_cfg,
+		       CFG_BT_ATTR_DATA_4_VF_SCC_CTX_BUF_PGSZ_M,
+		       CFG_BT_ATTR_DATA_4_VF_SCC_CTX_BUF_PGSZ_S,
+		       hr_dev->caps.scc_ctx_buf_pg_sz + PG_SHIFT_OFFSET);
+	roce_set_field(req->vf_scc_ctx_cfg,
+		       CFG_BT_ATTR_DATA_4_VF_SCC_CTX_HOPNUM_M,
+		       CFG_BT_ATTR_DATA_4_VF_SCC_CTX_HOPNUM_S,
+		       scc_ctx_hop_num ==
+			      HNS_ROCE_HOP_NUM_0 ? 0 : scc_ctx_hop_num);
+
 	return hns_roce_cmq_send(hr_dev, &desc, 1);
 }
 
@@ -1372,9 +1390,14 @@ static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev)
 	caps->max_srq_wrs	= HNS_ROCE_V2_MAX_SRQ_WR;
 	caps->max_srq_sges	= HNS_ROCE_V2_MAX_SRQ_SGE;
 
-	if (hr_dev->pci_dev->revision == 0x21)
+	if (hr_dev->pci_dev->revision == 0x21) {
 		caps->flags |= HNS_ROCE_CAP_FLAG_ATOMIC |
 			       HNS_ROCE_CAP_FLAG_SRQ;
+		caps->scc_ctx_entry_sz	= HNS_ROCE_V2_SCC_CTX_ENTRY_SZ;
+		caps->scc_ctx_ba_pg_sz	= 0;
+		caps->scc_ctx_buf_pg_sz = 0;
+		caps->scc_ctx_hop_num	= HNS_ROCE_SCC_CTX_HOP_NUM;
+	}
 
 	ret = hns_roce_v2_set_bt(hr_dev);
 	if (ret)
@@ -2618,6 +2641,13 @@ static int hns_roce_v2_set_hem(struct hns_roce_dev *hr_dev,
 	case HEM_TYPE_SRQC:
 		op = HNS_ROCE_CMD_WRITE_SRQC_BT0;
 		break;
+	case HEM_TYPE_SCC_CTX:
+		if (step_idx) {
+			/* No need to notify Hardware when step_idx is 1 or 2*/
+			return 0;
+		}
+		op = HNS_ROCE_CMD_WRITE_SCC_CTX_BT0;
+		break;
 	default:
 		dev_warn(dev, "Table %d not to be written by mailbox!\n",
 			 table->type);
@@ -2677,6 +2707,9 @@ static int hns_roce_v2_clear_hem(struct hns_roce_dev *hr_dev,
 	case HEM_TYPE_CQC:
 		op = HNS_ROCE_CMD_DESTROY_CQC_BT0;
 		break;
+	case HEM_TYPE_SCC_CTX:
+		/* there is no need to destroy scc ctx */
+		return 0;
 	case HEM_TYPE_SRQC:
 		op = HNS_ROCE_CMD_DESTROY_SRQC_BT0;
 		break;
diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
index c48301c..b92eb30 100644
--- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
+++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h
@@ -83,6 +83,7 @@
 #define HNS_ROCE_V2_MTPT_ENTRY_SZ		64
 #define HNS_ROCE_V2_MTT_ENTRY_SZ		64
 #define HNS_ROCE_V2_CQE_ENTRY_SIZE		32
+#define HNS_ROCE_V2_SCC_CTX_ENTRY_SZ		32
 #define HNS_ROCE_V2_PAGE_SIZE_SUPPORTED		0xFFFFF000
 #define HNS_ROCE_V2_MAX_INNER_MTPT_NUM		2
 #define HNS_ROCE_INVALID_LKEY			0x100
@@ -91,6 +92,7 @@
 #define HNS_ROCE_V2_RSV_QPS			8
 
 #define HNS_ROCE_CONTEXT_HOP_NUM		1
+#define HNS_ROCE_SCC_CTX_HOP_NUM		1
 #define HNS_ROCE_MTT_HOP_NUM			1
 #define HNS_ROCE_CQE_HOP_NUM			1
 #define HNS_ROCE_SRQWQE_HOP_NUM			1
@@ -1295,7 +1297,8 @@ struct hns_roce_pf_res_b {
 	__le32	smac_idx_num;
 	__le32	sgid_idx_num;
 	__le32	qid_idx_sl_num;
-	__le32	rsv[2];
+	__le32	scc_ctx_bt_idx_num;
+	__le32	rsv;
 };
 
 #define PF_RES_DATA_1_PF_SMAC_IDX_S 0
@@ -1316,6 +1319,12 @@ struct hns_roce_pf_res_b {
 #define PF_RES_DATA_3_PF_SL_NUM_S 16
 #define PF_RES_DATA_3_PF_SL_NUM_M GENMASK(26, 16)
 
+#define PF_RES_DATA_4_PF_SCC_CTX_BT_IDX_S 0
+#define PF_RES_DATA_4_PF_SCC_CTX_BT_IDX_M GENMASK(8, 0)
+
+#define PF_RES_DATA_4_PF_SCC_CTX_BT_NUM_S 9
+#define PF_RES_DATA_4_PF_SCC_CTX_BT_NUM_M GENMASK(17, 9)
+
 struct hns_roce_vf_res_a {
 	__le32 vf_id;
 	__le32 vf_qpc_bt_idx_num;
@@ -1389,7 +1398,8 @@ struct hns_roce_cfg_bt_attr {
 	__le32 vf_srqc_cfg;
 	__le32 vf_cqc_cfg;
 	__le32 vf_mpt_cfg;
-	__le32 rsv[2];
+	__le32 vf_scc_ctx_cfg;
+	__le32 rsv;
 };
 
 #define CFG_BT_ATTR_DATA_0_VF_QPC_BA_PGSZ_S 0
@@ -1428,6 +1438,15 @@ struct hns_roce_cfg_bt_attr {
 #define CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_S 8
 #define CFG_BT_ATTR_DATA_3_VF_MPT_HOPNUM_M GENMASK(9, 8)
 
+#define CFG_BT_ATTR_DATA_4_VF_SCC_CTX_BA_PGSZ_S 0
+#define CFG_BT_ATTR_DATA_4_VF_SCC_CTX_BA_PGSZ_M GENMASK(3, 0)
+
+#define CFG_BT_ATTR_DATA_4_VF_SCC_CTX_BUF_PGSZ_S 4
+#define CFG_BT_ATTR_DATA_4_VF_SCC_CTX_BUF_PGSZ_M GENMASK(7, 4)
+
+#define CFG_BT_ATTR_DATA_4_VF_SCC_CTX_HOPNUM_S 8
+#define CFG_BT_ATTR_DATA_4_VF_SCC_CTX_HOPNUM_M GENMASK(9, 8)
+
 struct hns_roce_cfg_sgid_tb {
 	__le32	table_idx_rsv;
 	__le32	vf_sgid_l;
diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c
index 65ba43c..199ab0d 100644
--- a/drivers/infiniband/hw/hns/hns_roce_main.c
+++ b/drivers/infiniband/hw/hns/hns_roce_main.c
@@ -704,8 +704,26 @@ static int hns_roce_init_hem(struct hns_roce_dev *hr_dev)
 		}
 	}
 
+	if (hr_dev->caps.scc_ctx_entry_sz) {
+		ret = hns_roce_init_hem_table(hr_dev,
+					      &hr_dev->qp_table.scc_ctx_table,
+					      HEM_TYPE_SCC_CTX,
+					      hr_dev->caps.scc_ctx_entry_sz,
+					      hr_dev->caps.num_qps, 1);
+		if (ret) {
+			dev_err(dev,
+			      "Failed to init SCC context memory, aborting.\n");
+			goto err_unmap_idx;
+		}
+	}
+
 	return 0;
 
+err_unmap_idx:
+	if (hr_dev->caps.num_idx_segs)
+		hns_roce_cleanup_hem_table(hr_dev,
+					   &hr_dev->mr_table.mtt_idx_table);
+
 err_unmap_srqwqe:
 	if (hr_dev->caps.num_srqwqe_segs)
 		hns_roce_cleanup_hem_table(hr_dev,
diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c
index 52d2b29..895274a 100644
--- a/drivers/infiniband/hw/hns/hns_roce_qp.c
+++ b/drivers/infiniband/hw/hns/hns_roce_qp.c
@@ -209,13 +209,23 @@ static int hns_roce_qp_alloc(struct hns_roce_dev *hr_dev, unsigned long qpn,
 		}
 	}
 
+	if (hr_dev->caps.scc_ctx_entry_sz) {
+		/* Alloc memory for SCC CTX */
+		ret = hns_roce_table_get(hr_dev, &qp_table->scc_ctx_table,
+					 hr_qp->qpn);
+		if (ret) {
+			dev_err(dev, "SCC CTX table get failed\n");
+			goto err_put_trrl;
+		}
+	}
+
 	spin_lock_irq(&qp_table->lock);
 	ret = radix_tree_insert(&hr_dev->qp_table_tree,
 				hr_qp->qpn & (hr_dev->caps.num_qps - 1), hr_qp);
 	spin_unlock_irq(&qp_table->lock);
 	if (ret) {
 		dev_err(dev, "QPC radix_tree_insert failed\n");
-		goto err_put_trrl;
+		goto err_put_scc_ctx;
 	}
 
 	atomic_set(&hr_qp->refcount, 1);
@@ -223,6 +233,11 @@ static int hns_roce_qp_alloc(struct hns_roce_dev *hr_dev, unsigned long qpn,
 
 	return 0;
 
+err_put_scc_ctx:
+	if (hr_dev->caps.scc_ctx_entry_sz)
+		hns_roce_table_put(hr_dev, &qp_table->scc_ctx_table,
+				   hr_qp->qpn);
+
 err_put_trrl:
 	if (hr_dev->caps.trrl_entry_sz)
 		hns_roce_table_put(hr_dev, &qp_table->trrl_table, hr_qp->qpn);
@@ -258,6 +273,9 @@ void hns_roce_qp_free(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
 	wait_for_completion(&hr_qp->free);
 
 	if ((hr_qp->ibqp.qp_type) != IB_QPT_GSI) {
+		if (hr_dev->caps.scc_ctx_entry_sz)
+			hns_roce_table_put(hr_dev, &qp_table->scc_ctx_table,
+					   hr_qp->qpn);
 		if (hr_dev->caps.trrl_entry_sz)
 			hns_roce_table_put(hr_dev, &qp_table->trrl_table,
 					   hr_qp->qpn);
-- 
1.9.1




[Index of Archives]     [Linux USB Devel]     [Video for Linux]     [Linux Audio Users]     [Photo]     [Yosemite News]     [Yosemite Photos]     [Linux Kernel]     [Linux SCSI]     [XFree86]

  Powered by Linux