mlx5_ib_map_mr_sg_pi() will map the PI and data dma mapped SG lists to the
mlx5 memory region prior to the registration operation. In the new
API, the mlx5 driver will allocate an internal memory region for the
UMR operation to register both PI and data SG lists. The internal MR
will use KLM mode in order to map 2 (possibly non-contiguous/non-align)
SG lists using 1 memory key. In the new API, each ULP will use 1 memory
region for the signature operation (instead of 3 in the old API). This
memory region will have a key that will be exposed to remote server to
perform RDMA operation. The internal memory key that will map the SG lists
will stay private.
Signed-off-by: Max Gurtovoy <maxg@xxxxxxxxxxxx>
Signed-off-by: Israel Rukshin <israelr@xxxxxxxxxxxx>
---
drivers/infiniband/hw/mlx5/main.c | 2 +
drivers/infiniband/hw/mlx5/mlx5_ib.h | 9 ++
drivers/infiniband/hw/mlx5/mr.c | 183 ++++++++++++++++++++++++++++++++---
3 files changed, 179 insertions(+), 15 deletions(-)
diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c
index 94fe253d4956..c4b2d9db5d07 100644
--- a/drivers/infiniband/hw/mlx5/main.c
+++ b/drivers/infiniband/hw/mlx5/main.c
@@ -5837,6 +5837,7 @@ static void mlx5_ib_stage_flow_db_cleanup(struct mlx5_ib_dev *dev)
static const struct ib_device_ops mlx5_ib_dev_ops = {
.add_gid = mlx5_ib_add_gid,
.alloc_mr = mlx5_ib_alloc_mr,
+ .alloc_mr_integrity = mlx5_ib_alloc_mr_integrity,
.alloc_pd = mlx5_ib_alloc_pd,
.alloc_ucontext = mlx5_ib_alloc_ucontext,
.attach_mcast = mlx5_ib_mcg_attach,
@@ -5866,6 +5867,7 @@ static const struct ib_device_ops mlx5_ib_dev_ops = {
.get_dma_mr = mlx5_ib_get_dma_mr,
.get_link_layer = mlx5_ib_port_link_layer,
.map_mr_sg = mlx5_ib_map_mr_sg,
+ .map_mr_sg_pi = mlx5_ib_map_mr_sg_pi,
.mmap = mlx5_ib_mmap,
.modify_cq = mlx5_ib_modify_cq,
.modify_device = mlx5_ib_modify_device,
diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h
index 33b0d042ef05..7ef1c0a3c886 100644
--- a/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@ -567,6 +567,9 @@ struct mlx5_ib_mr {
void *descs;
dma_addr_t desc_map;
int ndescs;
+ int data_length;
+ int meta_ndescs;
+ int meta_length;
int max_descs;
int desc_size;
int access_mode;
@@ -585,6 +588,7 @@ struct mlx5_ib_mr {
int access_flags; /* Needed for rereg MR */
struct mlx5_ib_mr *parent;
+ struct mlx5_ib_mr *pi_mr; /* Needed for IB_MR_TYPE_PI type */
atomic_t num_leaf_free;
wait_queue_head_t q_leaf_free;
};
@@ -1107,7 +1111,12 @@ int mlx5_ib_dereg_mr(struct ib_mr *ibmr);
struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd,
enum ib_mr_type mr_type,
u32 max_num_sg);
+struct ib_mr *mlx5_ib_alloc_mr_integrity(struct ib_pd *pd,
+ u32 max_num_sg,
+ u32 max_num_meta_sg);
int mlx5_ib_map_mr_sg(struct ib_mr *ibmr, struct ib_scatterlist *ib_sg);
+int mlx5_ib_map_mr_sg_pi(struct ib_mr *ibmr, struct ib_scatterlist *data_ib_sg,
+ struct ib_scatterlist *meta_ib_sg);
int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num,
const struct ib_wc *in_wc, const struct ib_grh *in_grh,
const struct ib_mad_hdr *in, size_t in_mad_size,
diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c
index 659d39734523..b17e7078bdc4 100644
--- a/drivers/infiniband/hw/mlx5/mr.c
+++ b/drivers/infiniband/hw/mlx5/mr.c
@@ -1684,17 +1684,22 @@ static void dereg_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr)
int mlx5_ib_dereg_mr(struct ib_mr *ibmr)
{
- dereg_mr(to_mdev(ibmr->device), to_mmr(ibmr));
+ struct mlx5_ib_mr *mmr = to_mmr(ibmr);
+
+ if (ibmr->type == IB_MR_TYPE_PI)
+ dereg_mr(to_mdev(mmr->pi_mr->ibmr.device), mmr->pi_mr);
+
+ dereg_mr(to_mdev(ibmr->device), mmr);
+
return 0;
}
-struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd,
- enum ib_mr_type mr_type,
- u32 max_num_sg)
+static struct mlx5_ib_mr *mlx5_ib_alloc_pi_mr(struct ib_pd *pd,
+ u32 max_num_sg, u32 max_num_meta_sg)
{
struct mlx5_ib_dev *dev = to_mdev(pd->device);
int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
- int ndescs = ALIGN(max_num_sg, 4);
+ int ndescs = ALIGN(max_num_sg + max_num_meta_sg, 4);
struct mlx5_ib_mr *mr;
void *mkc;
u32 *in;
@@ -1716,8 +1721,72 @@ struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd,
MLX5_SET(mkc, mkc, qpn, 0xffffff);
MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn);
+ mr->access_mode = MLX5_MKC_ACCESS_MODE_KLMS;
+
+ err = mlx5_alloc_priv_descs(pd->device, mr,
+ ndescs, sizeof(struct mlx5_klm));
+ if (err)
+ goto err_free_in;
+ mr->desc_size = sizeof(struct mlx5_klm);
+ mr->max_descs = ndescs;
+
+ MLX5_SET(mkc, mkc, access_mode_1_0, mr->access_mode & 0x3);
+ MLX5_SET(mkc, mkc, access_mode_4_2, (mr->access_mode >> 2) & 0x7);
+ MLX5_SET(mkc, mkc, umr_en, 1);
+
+ mr->ibmr.pd = pd;
+ mr->ibmr.device = pd->device;
+ err = mlx5_core_create_mkey(dev->mdev, &mr->mmkey, in, inlen);
+ if (err)
+ goto err_priv_descs;
+
+ mr->mmkey.type = MLX5_MKEY_MR;
+ mr->ibmr.lkey = mr->mmkey.key;
+ mr->ibmr.rkey = mr->mmkey.key;
+ mr->umem = NULL;
+ kfree(in);
+
+ return mr;
+
+err_priv_descs:
+ mlx5_free_priv_descs(mr);
+err_free_in:
+ kfree(in);
+err_free:
+ kfree(mr);
+ return ERR_PTR(err);
+}
+
+static struct ib_mr *__mlx5_ib_alloc_mr(struct ib_pd *pd,
+ enum ib_mr_type mr_type,
+ u32 max_num_sg, u32 max_num_meta_sg)
+{
+ struct mlx5_ib_dev *dev = to_mdev(pd->device);
+ int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
+ int ndescs = ALIGN(max_num_sg, 4);
+ struct mlx5_ib_mr *mr;
+ void *mkc;
+ u32 *in;
+ int err;
+
+ mr = kzalloc(sizeof(*mr), GFP_KERNEL);
+ if (!mr)
+ return ERR_PTR(-ENOMEM);
+
+ in = kzalloc(inlen, GFP_KERNEL);
+ if (!in) {
+ err = -ENOMEM;
+ goto err_free;
+ }
+
+ mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
+ MLX5_SET(mkc, mkc, free, 1);
+ MLX5_SET(mkc, mkc, qpn, 0xffffff);
+ MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn);
+
if (mr_type == IB_MR_TYPE_MEM_REG) {
mr->access_mode = MLX5_MKC_ACCESS_MODE_MTT;
+ MLX5_SET(mkc, mkc, translations_octword_size, ndescs);
MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT);
err = mlx5_alloc_priv_descs(pd->device, mr,
ndescs, sizeof(struct mlx5_mtt));
@@ -1728,6 +1797,7 @@ struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd,
mr->max_descs = ndescs;
} else if (mr_type == IB_MR_TYPE_SG_GAPS) {
mr->access_mode = MLX5_MKC_ACCESS_MODE_KLMS;
+ MLX5_SET(mkc, mkc, translations_octword_size, ndescs);
err = mlx5_alloc_priv_descs(pd->device, mr,
ndescs, sizeof(struct mlx5_klm));
@@ -1735,11 +1805,13 @@ struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd,
goto err_free_in;
mr->desc_size = sizeof(struct mlx5_klm);
mr->max_descs = ndescs;
- } else if (mr_type == IB_MR_TYPE_SIGNATURE) {
+ } else if (mr_type == IB_MR_TYPE_SIGNATURE ||
+ mr_type == IB_MR_TYPE_PI) {
u32 psv_index[2];
MLX5_SET(mkc, mkc, bsf_en, 1);
MLX5_SET(mkc, mkc, bsf_octword_size, MLX5_MKEY_BSF_OCTO_SIZE);
+ MLX5_SET(mkc, mkc, translations_octword_size, 4);
mr->sig = kzalloc(sizeof(*mr->sig), GFP_KERNEL);
if (!mr->sig) {
err = -ENOMEM;
@@ -1760,6 +1832,14 @@ struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd,
mr->sig->sig_err_exists = false;
/* Next UMR, Arm SIGERR */
++mr->sig->sigerr_count;
+ if (mr_type == IB_MR_TYPE_PI) {
+ mr->pi_mr = mlx5_ib_alloc_pi_mr(pd, max_num_sg,
+ max_num_meta_sg);
+ if (IS_ERR(mr->pi_mr)) {
+ err = PTR_ERR(mr->pi_mr);
+ goto err_destroy_psv;
+ }
+ }
} else {
mlx5_ib_warn(dev, "Invalid mr type %d\n", mr_type);
err = -EINVAL;
@@ -1773,7 +1853,7 @@ struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd,
mr->ibmr.device = pd->device;
err = mlx5_core_create_mkey(dev->mdev, &mr->mmkey, in, inlen);
if (err)
- goto err_destroy_psv;
+ goto err_free_pi_mr;
mr->mmkey.type = MLX5_MKEY_MR;
mr->ibmr.lkey = mr->mmkey.key;
@@ -1783,6 +1863,11 @@ struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd,
return &mr->ibmr;
+err_free_pi_mr:
+ if (mr->pi_mr) {
+ dereg_mr(to_mdev(mr->pi_mr->ibmr.device), mr->pi_mr);
+ mr->pi_mr = NULL;
+ }
err_destroy_psv:
if (mr->sig) {
if (mlx5_core_destroy_psv(dev->mdev,
@@ -1804,6 +1889,20 @@ struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd,
return ERR_PTR(err);
}
+struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd,
+ enum ib_mr_type mr_type,
+ u32 max_num_sg)
+{
+ return __mlx5_ib_alloc_mr(pd, mr_type, max_num_sg, 0);
+}
+
+struct ib_mr *mlx5_ib_alloc_mr_integrity(struct ib_pd *pd,
+ u32 max_num_sg, u32 max_num_meta_sg)
+{
+ return __mlx5_ib_alloc_mr(pd, IB_MR_TYPE_PI, max_num_sg,
+ max_num_meta_sg);
+}
+
struct ib_mw *mlx5_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,
struct ib_udata *udata)
{
@@ -1934,18 +2033,19 @@ int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask,
static int
mlx5_ib_sg_to_klms(struct mlx5_ib_mr *mr,
- struct ib_scatterlist *ib_sgl)
+ struct ib_scatterlist *data_ib_sgl,
+ struct ib_scatterlist *meta_ib_sgl)
{
- struct scatterlist *sg = ib_sgl->sg;
+ struct scatterlist *sg = data_ib_sgl->sg;
struct mlx5_klm *klms = mr->descs;
- unsigned int sg_offset = ib_sgl->offset;
+ unsigned int sg_offset = data_ib_sgl->offset;
u32 lkey = mr->ibmr.pd->local_dma_lkey;
- int i;
+ int i, j = 0;
mr->ibmr.iova = sg_dma_address(sg) + sg_offset;
mr->ibmr.length = 0;
- for_each_sg(ib_sgl->sg, sg, ib_sgl->dma_nents, i) {
+ for_each_sg(data_ib_sgl->sg, sg, data_ib_sgl->dma_nents, i) {
if (unlikely(i >= mr->max_descs))
break;
klms[i].va = cpu_to_be64(sg_dma_address(sg) + sg_offset);
@@ -1955,11 +2055,34 @@ mlx5_ib_sg_to_klms(struct mlx5_ib_mr *mr,
sg_offset = 0;
}
+
+ data_ib_sgl->offset = sg_offset;
+
mr->ndescs = i;
+ mr->data_length = mr->ibmr.length;
+
+ if (meta_ib_sgl && meta_ib_sgl->dma_nents) {
+ sg = meta_ib_sgl->sg;
+ sg_offset = meta_ib_sgl->offset;
+ for_each_sg(meta_ib_sgl->sg, sg, meta_ib_sgl->dma_nents, j) {
+ if (unlikely(i + j >= mr->max_descs))
+ break;
+ klms[i + j].va = cpu_to_be64(sg_dma_address(sg) +
+ sg_offset);
+ klms[i + j].bcount = cpu_to_be32(sg_dma_len(sg) -
+ sg_offset);
+ klms[i + j].key = cpu_to_be32(lkey);
+ mr->ibmr.length += sg_dma_len(sg) - sg_offset;
+
+ sg_offset = 0;
+ }
+ meta_ib_sgl->offset = sg_offset;
- ib_sgl->offset = sg_offset;
+ mr->meta_ndescs = j;
+ mr->meta_length = mr->ibmr.length - mr->data_length;
+ }
- return i;
+ return i + j;
}
static int mlx5_set_page(struct ib_mr *ibmr, u64 addr)
@@ -1976,6 +2099,36 @@ static int mlx5_set_page(struct ib_mr *ibmr, u64 addr)
return 0;
}
+int mlx5_ib_map_mr_sg_pi(struct ib_mr *ibmr, struct ib_scatterlist *data_ib_sg,
+ struct ib_scatterlist *meta_ib_sg)
+{
+ struct mlx5_ib_mr *mr = to_mmr(ibmr);
+ struct mlx5_ib_mr *pi_mr = mr->pi_mr;
+ int n;
+
+ WARN_ON(ibmr->type != IB_MR_TYPE_PI);
+
+ pi_mr->ndescs = 0;
+ pi_mr->meta_ndescs = 0;
+
+ ib_dma_sync_single_for_cpu(ibmr->device, pi_mr->desc_map,
+ pi_mr->desc_size * pi_mr->max_descs,
+ DMA_TO_DEVICE);
+
+ n = mlx5_ib_sg_to_klms(pi_mr, data_ib_sg, meta_ib_sg);
+
+ /* This is zero-based memory region */
+ pi_mr->ibmr.iova = 0;
+ ibmr->length = pi_mr->ibmr.length;
+ ibmr->iova = pi_mr->ibmr.iova;
