Expose DV APIs to create and destroy indirect mkey, the internal
implementation is done over the DEVX API.
Reviewed-by: Artemy Kovalyov <artemyko@xxxxxxxxxxxx>
Signed-off-by: Yishai Hadas <yishaih@xxxxxxxxxxxx>
---
debian/ibverbs-providers.symbols | 2 +
providers/mlx5/libmlx5.map | 2 +
providers/mlx5/man/CMakeLists.txt | 2 +
providers/mlx5/man/mlx5dv_create_mkey.3.md | 75 +++++++++++++++++++++++++
providers/mlx5/mlx5.h | 6 ++
providers/mlx5/mlx5_ifc.h | 89 ++++++++++++++++++++++++++++++
providers/mlx5/mlx5dv.h | 18 ++++++
providers/mlx5/verbs.c | 61 ++++++++++++++++++++
8 files changed, 255 insertions(+)
create mode 100644 providers/mlx5/man/mlx5dv_create_mkey.3.md
diff --git a/debian/ibverbs-providers.symbols b/debian/ibverbs-providers.symbols
index 309bbef..f2f64ae 100644
--- a/debian/ibverbs-providers.symbols
+++ b/debian/ibverbs-providers.symbols
@@ -58,4 +58,6 @@ libmlx5.so.1 ibverbs-providers #MINVER#
mlx5dv_devx_destroy_cmd_comp@MLX5_1.9 23
mlx5dv_devx_get_async_cmd_comp@MLX5_1.9 23
mlx5dv_devx_obj_query_async@MLX5_1.9 23
+ mlx5dv_create_mkey@MLX5_1.10 24
+ mlx5dv_destroy_mkey@MLX5_1.10 24
mlx5dv_qp_ex_from_ibv_qp_ex@MLX5_1.10 24
diff --git a/providers/mlx5/libmlx5.map b/providers/mlx5/libmlx5.map
index 862cb38..c97874a 100644
--- a/providers/mlx5/libmlx5.map
+++ b/providers/mlx5/libmlx5.map
@@ -83,5 +83,7 @@ MLX5_1.9 {
MLX5_1.10 {
global:
+ mlx5dv_create_mkey;
+ mlx5dv_destroy_mkey;
mlx5dv_qp_ex_from_ibv_qp_ex;
} MLX5_1.9;
diff --git a/providers/mlx5/man/CMakeLists.txt b/providers/mlx5/man/CMakeLists.txt
index 24bd5d8..dedfd98 100644
--- a/providers/mlx5/man/CMakeLists.txt
+++ b/providers/mlx5/man/CMakeLists.txt
@@ -4,6 +4,7 @@ rdma_man_pages(
mlx5dv_create_flow_action_modify_header.3.md
mlx5dv_create_flow_action_packet_reformat.3.md
mlx5dv_create_flow_matcher.3.md
+ mlx5dv_create_mkey.3.md
mlx5dv_create_qp.3.md
mlx5dv_devx_alloc_uar.3.md
mlx5dv_devx_create_cmd_comp.3.md
@@ -22,6 +23,7 @@ rdma_man_pages(
mlx5dv.7
)
rdma_alias_man_pages(
+ mlx5dv_create_mkey.3 mlx5dv_destroy_mkey.3
mlx5dv_devx_alloc_uar.3 mlx5dv_devx_free_uar.3
mlx5dv_devx_create_cmd_comp.3 mlx5dv_devx_destroy_cmd_comp.3
mlx5dv_devx_create_cmd_comp.3 mlx5dv_devx_get_async_cmd_comp.3
diff --git a/providers/mlx5/man/mlx5dv_create_mkey.3.md b/providers/mlx5/man/mlx5dv_create_mkey.3.md
new file mode 100644
index 0000000..0a03d26
--- /dev/null
+++ b/providers/mlx5/man/mlx5dv_create_mkey.3.md
@@ -0,0 +1,75 @@
+---
+layout: page
+title: mlx5dv_create_mkey / mlx5dv_destroy_mkey
+section: 3
+tagline: Verbs
+---
+
+# NAME
+
+mlx5dv_create_mkey - Creates an indirect mkey
+
+mlx5dv_create_mkey - Destroys an indirect mkey
+
+# SYNOPSIS
+
+```c
+#include <infiniband/mlx5dv.h>
+
+struct mlx5dv_mkey_init_attr {
+ struct ibv_pd *pd;
+ uint32_t create_flags;
+ uint16_t max_entries;
+};
+
+struct mlx5dv_mkey {
+ uint32_t lkey;
+ uint32_t rkey;
+};
+
+struct mlx5dv_mkey *
+mlx5dv_create_mkey(struct mlx5dv_mkey_init_attr *mkey_init_attr);
+
+int mlx5dv_destroy_mkey(struct mlx5dv_mkey *mkey);
+
+```
+
+# DESCRIPTION
+
+Create / destroy an indirect mkey.
+
+Create an indirect mkey to enable application uses its specific device functionality.
+
+# ARGUMENTS
+
+##mkey_init_attr##
+
+*pd*
+: ibv protection domain.
+
+*create_flags*
+: MLX5DV_MKEY_INIT_ATTR_FLAGS_INDIRECT:
+ Indirect mkey is being created.
+
+*max_entries*
+: Requested max number of pointed entries by this indirect mkey.
+ The function will update the *mkey_init_attr->max_entries* with the actual mkey value that was created; it will be greater than or equal to the value requested.
+
+# RETURN VALUE
+
+Upon success *mlx5dv_create_mkey* will return a new *struct
+mlx5dv_mkey* on error NULL will be returned and errno will be set.
+
+Upon success destroy 0 is returned or the value of errno on a failure.
+
+# Notes
+
+To let this functionality works a DEVX context should be opened by using *mlx5dv_open_device*.
+
+# SEE ALSO
+
+**mlx5dv_open_device**
+
+#AUTHOR
+
+Yishai Hadas <yishaih@xxxxxxxxxxxx>
diff --git a/providers/mlx5/mlx5.h b/providers/mlx5/mlx5.h
index c7c54fd..14a610c 100644
--- a/providers/mlx5/mlx5.h
+++ b/providers/mlx5/mlx5.h
@@ -601,6 +601,12 @@ struct mlx5_devx_umem {
uint32_t handle;
};
+struct mlx5_mkey {
+ struct mlx5dv_mkey dv_mkey;
+ struct mlx5dv_devx_obj *devx_obj;
+ uint16_t num_desc;
+};
+
static inline int mlx5_ilog2(int n)
{
int t;
diff --git a/providers/mlx5/mlx5_ifc.h b/providers/mlx5/mlx5_ifc.h
index 5cf89d6..72f735a 100644
--- a/providers/mlx5/mlx5_ifc.h
+++ b/providers/mlx5/mlx5_ifc.h
@@ -38,6 +38,7 @@ enum mlx5_cap_mode {
enum {
MLX5_CMD_OP_QUERY_HCA_CAP = 0x100,
+ MLX5_CMD_OP_CREATE_MKEY = 0x200,
};
struct mlx5_ifc_atomic_caps_bits {
@@ -98,3 +99,91 @@ struct mlx5_ifc_query_hca_cap_in_bits {
enum mlx5_cap_type {
MLX5_CAP_ATOMIC = 3,
};
+
+enum {
+ MLX5_MKC_ACCESS_MODE_KLMS = 0x2,
+};
+
+struct mlx5_ifc_mkc_bits {
+ u8 reserved_at_0[0x1];
+ u8 free[0x1];
+ u8 reserved_at_2[0x1];
+ u8 access_mode_4_2[0x3];
+ u8 reserved_at_6[0x7];
+ u8 relaxed_ordering_write[0x1];
+ u8 reserved_at_e[0x1];
+ u8 small_fence_on_rdma_read_response[0x1];
+ u8 umr_en[0x1];
+ u8 a[0x1];
+ u8 rw[0x1];
+ u8 rr[0x1];
+ u8 lw[0x1];
+ u8 lr[0x1];
+ u8 access_mode_1_0[0x2];
+ u8 reserved_at_18[0x8];
+
+ u8 qpn[0x18];
+ u8 mkey_7_0[0x8];
+
+ u8 reserved_at_40[0x20];
+
+ u8 length64[0x1];
+ u8 bsf_en[0x1];
+ u8 sync_umr[0x1];
+ u8 reserved_at_63[0x2];
+ u8 expected_sigerr_count[0x1];
+ u8 reserved_at_66[0x1];
+ u8 en_rinval[0x1];
+ u8 pd[0x18];
+
+ u8 start_addr[0x40];
+
+ u8 len[0x40];
+
+ u8 bsf_octword_size[0x20];
+
+ u8 reserved_at_120[0x80];
+
+ u8 translations_octword_size[0x20];
+
+ u8 reserved_at_1c0[0x1b];
+ u8 log_page_size[0x5];
+
+ u8 reserved_at_1e0[0x20];
+};
+
+struct mlx5_ifc_create_mkey_out_bits {
+ u8 status[0x8];
+ u8 reserved_at_8[0x18];
+
+ u8 syndrome[0x20];
+
+ u8 reserved_at_40[0x8];
+ u8 mkey_index[0x18];
+
+ u8 reserved_at_60[0x20];
+};
+
+struct mlx5_ifc_create_mkey_in_bits {
+ u8 opcode[0x10];
+ u8 reserved_at_10[0x10];
+
+ u8 reserved_at_20[0x10];
+ u8 op_mod[0x10];
+
+ u8 reserved_at_40[0x20];
+
+ u8 pg_access[0x1];
+ u8 mkey_umem_valid[0x1];
+ u8 reserved_at_62[0x1e];
+
+ struct mlx5_ifc_mkc_bits memory_key_mkey_entry;
+
+ u8 reserved_at_280[0x80];
+
+ u8 translations_octword_actual_size[0x20];
+
+ u8 reserved_at_320[0x560];
+
+ u8 klm_pas_mtt[0][0x20];
+};
diff --git a/providers/mlx5/mlx5dv.h b/providers/mlx5/mlx5dv.h
index de4018c..ce033dc 100644
--- a/providers/mlx5/mlx5dv.h
+++ b/providers/mlx5/mlx5dv.h
@@ -168,6 +168,24 @@ enum mlx5dv_qp_create_flags {
MLX5DV_QP_CREATE_PACKET_BASED_CREDIT_MODE = 1 << 5,
};
+enum mlx5dv_mkey_init_attr_flags {
+ MLX5DV_MKEY_INIT_ATTR_FLAGS_INDIRECT = 1 << 0,
+};
+
+struct mlx5dv_mkey_init_attr {
+ struct ibv_pd *pd;
+ uint32_t create_flags; /* Use enum mlx5dv_mkey_init_attr_flags */
+ uint16_t max_entries; /* Requested max number of pointed entries by this indirect mkey */
+};
+
+struct mlx5dv_mkey {
+ uint32_t lkey;
+ uint32_t rkey;
+};
+
+struct mlx5dv_mkey *mlx5dv_create_mkey(struct mlx5dv_mkey_init_attr *mkey_init_attr);
+int mlx5dv_destroy_mkey(struct mlx5dv_mkey *mkey);
+
enum mlx5dv_qp_init_attr_mask {
MLX5DV_QP_INIT_ATTR_MASK_QP_CREATE_FLAGS = 1 << 0,
MLX5DV_QP_INIT_ATTR_MASK_DC = 1 << 1,
diff --git a/providers/mlx5/verbs.c b/providers/mlx5/verbs.c
index abbbf5a..839f43c 100644
--- a/providers/mlx5/verbs.c
+++ b/providers/mlx5/verbs.c
@@ -4500,3 +4500,64 @@ int mlx5dv_devx_get_async_cmd_comp(struct mlx5dv_devx_cmd_comp *cmd_comp,
return 0;
}
+struct mlx5dv_mkey *mlx5dv_create_mkey(struct mlx5dv_mkey_init_attr *mkey_init_attr)
+{
+ uint32_t out[DEVX_ST_SZ_DW(create_mkey_out)] = {};
+ uint32_t in[DEVX_ST_SZ_DW(create_mkey_in)] = {};
+ struct mlx5_mkey *mkey;
+ void *mkc;
+
+ if (!mkey_init_attr->create_flags ||
+ !check_comp_mask(mkey_init_attr->create_flags,
+ MLX5DV_MKEY_INIT_ATTR_FLAGS_INDIRECT)) {
+ errno = EOPNOTSUPP;
+ return NULL;
+ }
+
+ mkey = calloc(1, sizeof(*mkey));
+ if (!mkey) {
+ errno = ENOMEM;
+ return NULL;
+ }
+
+ mkey->num_desc = align(mkey_init_attr->max_entries, 4);
+ DEVX_SET(create_mkey_in, in, opcode, MLX5_CMD_OP_CREATE_MKEY);
+ mkc = DEVX_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
+ DEVX_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_KLMS);
+ DEVX_SET(mkc, mkc, free, 1);
+ DEVX_SET(mkc, mkc, umr_en, 1);
+ DEVX_SET(mkc, mkc, pd, to_mpd(mkey_init_attr->pd)->pdn);
+ DEVX_SET(mkc, mkc, translations_octword_size, mkey->num_desc);
+ DEVX_SET(mkc, mkc, lr, 1);
+ DEVX_SET(mkc, mkc, qpn, 0xffffff);
+ DEVX_SET(mkc, mkc, mkey_7_0, 0);
+
+ mkey->devx_obj = mlx5dv_devx_obj_create(mkey_init_attr->pd->context,
+ in, sizeof(in), out, sizeof(out));
+ if (!mkey->devx_obj)
+ goto end;
+
+ mkey_init_attr->max_entries = mkey->num_desc;
+ mkey->dv_mkey.lkey = (DEVX_GET(create_mkey_out, out, mkey_index) << 8) | 0;
+ mkey->dv_mkey.rkey = mkey->dv_mkey.lkey;
+
+ return &mkey->dv_mkey;
+end:
+ free(mkey);
+ return NULL;
+}
+
+int mlx5dv_destroy_mkey(struct mlx5dv_mkey *dv_mkey)
+{
+ struct mlx5_mkey *mkey = container_of(dv_mkey, struct mlx5_mkey,
+ dv_mkey);
+ int ret;
+
+ ret = mlx5dv_devx_obj_destroy(mkey->devx_obj);
+ if (ret)
+ return ret;
+
+ free(mkey);
+ return 0;
+}
+
--
1.8.3.1
