From: Michael Guralnik <michaelgur@xxxxxxxxxxxx>
Query the PCI atomic capabilities using the DEVX infrastructure and
report it through device query.
Signed-off-by: Michael Guralnik <michaelgur@xxxxxxxxxxxx>
Signed-off-by: Yishai Hadas <yishaih@xxxxxxxxxxxx>
---
providers/mlx5/mlx5_ifc.h | 100 ++++++++++++++++++++++++++++++++++++++++++++++
providers/mlx5/verbs.c | 31 ++++++++++++++
2 files changed, 131 insertions(+)
create mode 100644 providers/mlx5/mlx5_ifc.h
diff --git a/providers/mlx5/mlx5_ifc.h b/providers/mlx5/mlx5_ifc.h
new file mode 100644
index 0000000..5cf89d6
--- /dev/null
+++ b/providers/mlx5/mlx5_ifc.h
@@ -0,0 +1,100 @@
+/*
+ * Copyright (c) 2019 Mellanox Technologies, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses. You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ * Redistribution and use in source and binary forms, with or
+ * without modification, are permitted provided that the following
+ * conditions are met:
+ *
+ * - Redistributions of source code must retain the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer.
+ *
+ * - Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following
+ * disclaimer in the documentation and/or other materials
+ * provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#define u8 uint8_t
+
+enum mlx5_cap_mode {
+ HCA_CAP_OPMOD_GET_CUR = 1,
+};
+
+enum {
+ MLX5_CMD_OP_QUERY_HCA_CAP = 0x100,
+};
+
+struct mlx5_ifc_atomic_caps_bits {
+ u8 reserved_at_0[0x40];
+
+ u8 atomic_req_8B_endianness_mode[0x2];
+ u8 reserved_at_42[0x4];
+ u8 supported_atomic_req_8B_endianness_mode_1[0x1];
+
+ u8 reserved_at_47[0x19];
+
+ u8 reserved_at_60[0x20];
+
+ u8 reserved_at_80[0x10];
+ u8 atomic_operations[0x10];
+
+ u8 reserved_at_a0[0x10];
+ u8 atomic_size_qp[0x10];
+
+ u8 reserved_at_c0[0x10];
+ u8 atomic_size_dc[0x10];
+
+ u8 reserved_at_e0[0x1a0];
+
+ u8 fetch_add_pci_atomic[0x10];
+ u8 swap_pci_atomic[0x10];
+ u8 compare_swap_pci_atomic[0x10];
+
+ u8 reserved_at_2b0[0x550];
+};
+
+union mlx5_ifc_hca_cap_union_bits {
+ struct mlx5_ifc_atomic_caps_bits atomic_caps;
+ u8 reserved_at_0[0x8000];
+};
+
+struct mlx5_ifc_query_hca_cap_out_bits {
+ u8 status[0x8];
+ u8 reserved_at_8[0x18];
+
+ u8 syndrome[0x20];
+
+ u8 reserved_at_40[0x40];
+
+ union mlx5_ifc_hca_cap_union_bits capability;
+};
+
+struct mlx5_ifc_query_hca_cap_in_bits {
+ u8 opcode[0x10];
+ u8 reserved_at_10[0x10];
+
+ u8 reserved_at_20[0x10];
+ u8 op_mod[0x10];
+
+ u8 reserved_at_40[0x40];
+};
+
+enum mlx5_cap_type {
+ MLX5_CAP_ATOMIC = 3,
+};
diff --git a/providers/mlx5/verbs.c b/providers/mlx5/verbs.c
index bab675f..4bc3eac 100644
--- a/providers/mlx5/verbs.c
+++ b/providers/mlx5/verbs.c
@@ -55,6 +55,7 @@
#include "mlx5.h"
#include "mlx5-abi.h"
#include "wqe.h"
+#include "mlx5_ifc.h"
int mlx5_single_threaded = 0;
@@ -2816,6 +2817,32 @@ err:
return NULL;
}
+static void get_pci_atomic_caps(struct ibv_context *context,
+ struct ibv_device_attr_ex *attr)
+{
+ uint32_t in[DEVX_ST_SZ_DW(query_hca_cap_in)] = {};
+ uint32_t out[DEVX_ST_SZ_DW(query_hca_cap_out)] = {};
+ uint16_t opmod = (MLX5_CAP_ATOMIC << 1) | HCA_CAP_OPMOD_GET_CUR;
+ int ret;
+
+ DEVX_SET(query_hca_cap_in, in, opcode, MLX5_CMD_OP_QUERY_HCA_CAP);
+ DEVX_SET(query_hca_cap_in, in, op_mod, opmod);
+
+ ret = mlx5dv_devx_general_cmd(context, in, sizeof(in), out,
+ sizeof(out));
+ if (!ret) {
+ attr->pci_atomic_caps.fetch_add =
+ DEVX_GET(query_hca_cap_out, out,
+ capability.atomic_caps.fetch_add_pci_atomic);
+ attr->pci_atomic_caps.swap =
+ DEVX_GET(query_hca_cap_out, out,
+ capability.atomic_caps.swap_pci_atomic);
+ attr->pci_atomic_caps.compare_swap =
+ DEVX_GET(query_hca_cap_out, out,
+ capability.atomic_caps.compare_swap_pci_atomic);
+ }
+}
+
int mlx5_query_device_ex(struct ibv_context *context,
const struct ibv_query_device_ex_input *input,
struct ibv_device_attr_ex *attr,
@@ -2894,6 +2921,10 @@ int mlx5_query_device_ex(struct ibv_context *context,
snprintf(a->fw_ver, sizeof(a->fw_ver), "%d.%d.%04d",
major, minor, sub_minor);
+ if (attr_size >= offsetof(struct ibv_device_attr_ex, pci_atomic_caps) +
+ sizeof(attr->pci_atomic_caps))
+ get_pci_atomic_caps(context, attr);
+
return 0;
}
--
1.8.3.1
