From: Raed Salem <raeds@xxxxxxxxxxxx>
The user supplies counters instance and a reference to an output
array of uint64_t.
The driver reads the hardware counters values and writes them to
the output index location in the user supplied array.
All counters values are represented as uint64_t types.
The read counter values in the array are defined according to the
configuration defined by the user in the ibv_attach_counters_point_xxx
functions.
Signed-off-by: Raed Salem <raeds@xxxxxxxxxxxx>
Signed-off-by: Yishai Hadas <yishaih@xxxxxxxxxxxx>
---
libibverbs/cmd_counters.c | 19 ++++
libibverbs/driver.h | 9 ++
libibverbs/dummy_ops.c | 10 ++
libibverbs/libibverbs.map.in | 1 +
libibverbs/man/CMakeLists.txt | 1 +
libibverbs/man/ibv_read_counters.3.md | 186 ++++++++++++++++++++++++++++++++++
libibverbs/verbs.h | 22 ++++
7 files changed, 248 insertions(+)
create mode 100644 libibverbs/man/ibv_read_counters.3.md
diff --git a/libibverbs/cmd_counters.c b/libibverbs/cmd_counters.c
index 03e8a8e..e1cd0f0 100644
--- a/libibverbs/cmd_counters.c
+++ b/libibverbs/cmd_counters.c
@@ -69,3 +69,22 @@ int ibv_cmd_destroy_counters(struct verbs_counters *vcounters)
fill_attr_in_obj(cmd, UVERBS_ATTR_DESTROY_COUNTERS_HANDLE, vcounters->handle);
return execute_ioctl(vcounters->counters.context, cmd);
}
+
+int ibv_cmd_read_counters(struct verbs_counters *vcounters,
+ uint64_t *counters_value,
+ uint32_t ncounters,
+ uint32_t flags,
+ struct ibv_command_buffer *link)
+{
+ DECLARE_COMMAND_BUFFER_LINK(cmd, UVERBS_OBJECT_COUNTERS,
+ UVERBS_METHOD_COUNTERS_READ,
+ 4,
+ link);
+
+ fill_attr_in_obj(cmd, UVERBS_ATTR_READ_COUNTERS_HANDLE, vcounters->handle);
+ fill_attr_out(cmd, UVERBS_ATTR_READ_COUNTERS_BUFF, counters_value,
+ ncounters * sizeof(uint64_t));
+ fill_attr_in_uint32(cmd, UVERBS_ATTR_READ_COUNTERS_FLAGS, flags);
+
+ return execute_ioctl(vcounters->counters.context, cmd);
+}
diff --git a/libibverbs/driver.h b/libibverbs/driver.h
index 1030b0c..f296c6b 100644
--- a/libibverbs/driver.h
+++ b/libibverbs/driver.h
@@ -303,6 +303,10 @@ struct verbs_context_ops {
int (*query_rt_values)(struct ibv_context *context,
struct ibv_values_ex *values);
int (*query_srq)(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr);
+ int (*read_counters)(struct ibv_counters *counters,
+ uint64_t *counters_value,
+ uint32_t ncounters,
+ uint32_t flags);
struct ibv_mr *(*reg_dm_mr)(struct ibv_pd *pd, struct ibv_dm *dm,
uint64_t dm_offset, size_t length,
unsigned int access);
@@ -542,6 +546,11 @@ int ibv_cmd_create_counters(struct ibv_context *context,
struct verbs_counters *vcounters,
struct ibv_command_buffer *link);
int ibv_cmd_destroy_counters(struct verbs_counters *vcounters);
+int ibv_cmd_read_counters(struct verbs_counters *vcounters,
+ uint64_t *counters_value,
+ uint32_t ncounters,
+ uint32_t flags,
+ struct ibv_command_buffer *link);
int ibv_dontfork_range(void *base, size_t size);
int ibv_dofork_range(void *base, size_t size);
int ibv_cmd_alloc_dm(struct ibv_context *ctx,
diff --git a/libibverbs/dummy_ops.c b/libibverbs/dummy_ops.c
index 1ccb5b3..6d931f2 100644
--- a/libibverbs/dummy_ops.c
+++ b/libibverbs/dummy_ops.c
@@ -378,6 +378,14 @@ static int query_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr)
return ENOSYS;
}
+static int read_counters(struct ibv_counters *counters,
+ uint64_t *counters_value,
+ uint32_t ncounters,
+ uint32_t flags)
+{
+ return ENOSYS;
+}
+
static struct ibv_mr *reg_dm_mr(struct ibv_pd *pd, struct ibv_dm *dm,
uint64_t dm_offset, size_t length,
unsigned int access)
@@ -480,6 +488,7 @@ const struct verbs_context_ops verbs_dummy_ops = {
query_qp,
query_rt_values,
query_srq,
+ read_counters,
reg_dm_mr,
reg_mr,
req_notify_cq,
@@ -567,6 +576,7 @@ void verbs_set_ops(struct verbs_context *vctx,
SET_OP(ctx, query_qp);
SET_OP(vctx, query_rt_values);
SET_OP(ctx, query_srq);
+ SET_OP(vctx, read_counters);
SET_OP(vctx, reg_dm_mr);
SET_OP(ctx, reg_mr);
SET_OP(ctx, req_notify_cq);
diff --git a/libibverbs/libibverbs.map.in b/libibverbs/libibverbs.map.in
index d66a8d3..8cd8751 100644
--- a/libibverbs/libibverbs.map.in
+++ b/libibverbs/libibverbs.map.in
@@ -164,6 +164,7 @@ IBVERBS_PRIVATE_@IBVERBS_PABI_VERSION@ {
ibv_cmd_query_port;
ibv_cmd_query_qp;
ibv_cmd_query_srq;
+ ibv_cmd_read_counters;
ibv_cmd_reg_dm_mr;
ibv_cmd_reg_mr;
ibv_cmd_req_notify_cq;
diff --git a/libibverbs/man/CMakeLists.txt b/libibverbs/man/CMakeLists.txt
index c4fec80..5cd30d2 100644
--- a/libibverbs/man/CMakeLists.txt
+++ b/libibverbs/man/CMakeLists.txt
@@ -57,6 +57,7 @@ rdma_man_pages(
ibv_rate_to_mbps.3.md
ibv_rate_to_mult.3.md
ibv_rc_pingpong.1
+ ibv_read_counters.3.md
ibv_reg_mr.3
ibv_req_notify_cq.3.md
ibv_rereg_mr.3.md
diff --git a/libibverbs/man/ibv_read_counters.3.md b/libibverbs/man/ibv_read_counters.3.md
new file mode 100644
index 0000000..e9f74dd
--- /dev/null
+++ b/libibverbs/man/ibv_read_counters.3.md
@@ -0,0 +1,186 @@
+---
+date: 2018-04-02
+footer: libibverbs
+header: "Libibverbs Programmer's Manual"
+layout: page
+license: 'Licensed under the OpenIB.org BSD license (FreeBSD Variant) - See COPYING.md'
+section: 3
+title: IBV_READ_COUNTERS
+---
+
+# NAME
+
+**ibv_read_counters**(3) -- Read counter values
+
+# SYNOPSIS
+
+```c
+#include <infiniband/verbs.h>
+
+int ibv_read_counters(struct ibv_counters *counters,
+ uint64_t *counters_value,
+ uint32_t ncounters,
+ uint32_t flags);
+```
+
+# DESCRIPTION
+
+**ibv_read_counters**() returns the values of the chosen counters into
+*counters_value* array of which can accumulate *ncounters*.
+The values are filled according to the configuration defined by the
+user in the **ibv_attach_counters_point_xxx** functions.
+
+# ARGUMENTS
+
+*counters*
+: Counters object to read.
+
+*counters_value*
+: Input buffer to hold read result.
+
+*ncounters*
+: Number of counters to fill.
+
+*flags*
+: Use enum ibv_read_counters_flags.
+
+## *flags* Argument
+
+```c
+enum ibv_read_counters_flags {
+ IBV_READ_COUNTERS_ATTR_PREFER_CACHED = 1 << 0,
+};
+```
+
+IBV_READ_COUNTERS_ATTR_PREFER_CACHED
+: Will prefer reading the values from driver cache, else it will do volatile hardware access which is the default.
+
+# RETURN VALUE
+
+**ibv_read_counters**() returns 0 on success, or the value of errno on failure
+(which indicates the failure reason)
+
+# EXAMPLE
+
+Example: Statically attach counters to a new flow
+
+This example demonstrates the use of counters which are attached statically with
+the creation of a new flow.
+The counters are read from hardware periodically, and finally all resources are released.
+```c
+/* create counters object and define its counters points */
+/* create simple L2 flow with hardcoded MAC, and a count action */
+/* read counters periodically, every 1sec, until loop ends */
+/* assumes user prepared a RAW_PACKET QP as input */
+/* only limited error checking in run time for code simplicity */
+
+#include <inttypes.h>
+#include <infiniband/verbs.h>
+
+/* the below MAC should be replaced by user */
+#define FLOW_SPEC_ETH_MAC_VAL {
+ .dst_mac = { 0x00, 0x01, 0x02, 0x03, 0x04,0x05},
+ .src_mac = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00},
+ .ether_type = 0, .vlan_tag = 0, }
+#define FLOW_SPEC_ETH_MAC_MASK {
+ .dst_mac = { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
+ .src_mac = { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF},
+ .ether_type = 0, .vlan_tag = 0, }
+
+void example_create_flow_with_counters_on_raw_qp(struct ibv_qp *qp) {
+ int idx = 0;
+ int loop = 10;
+ struct ibv_flow *flow = NULL;
+ struct ibv_counters *counters = NULL;
+ struct ibv_counters_init_attr init_attr = {0};
+ struct ibv_counter_attach_attr attach_attr = {0};
+
+ /* create single coutners handle */
+ counters = ibv_create_counters(qp->context, &init_attr);
+
+ /* define counters points */
+ attach_attr.counter_desc = IBV_COUNTER_PACKETS;
+ attach_attr.index = idx++;
+ ret = ibv_attach_counters_point_flow(counters, &attach_attr, NULL);
+ if (ret == ENOTSUP) {
+ fprintf(stderr, "Attaching IBV_COUNTER_PACKETS to flow is not \
+supported");
+ exit(1);
+ }
+ attach_attr.counter_desc = IBV_COUNTER_BYTES;
+ attach_attr.index = idx++;
+ ibv_attach_counters_point_flow(counters, &attach_attr, NULL);
+ if (ret == ENOTSUP) {
+ fprintf(stderr, "Attaching IBV_COUNTER_BYTES to flow is not \
+supported");
+ exit(1);
+ }
+
+ /* define a new flow attr that includes the counters handle */
+ struct raw_eth_flow_attr {
+ struct ibv_flow_attr attr;
+ struct ibv_flow_spec_eth spec_eth;
+ struct ibv_flow_spec_counter_action spec_count;
+ } flow_attr = {
+ .attr = {
+ .comp_mask = 0,
+ .type = IBV_FLOW_ATTR_NORMAL,
+ .size = sizeof(flow_attr),
+ .priority = 0,
+ .num_of_specs = 2, /* ETH + COUNT */
+ .port = 1,
+ .flags = 0,
+ },
+ .spec_eth = {
+ .type = IBV_EXP_FLOW_SPEC_ETH,
+ .size = sizeof(struct ibv_flow_spec_eth),
+ .val = FLOW_SPEC_ETH_MAC_VAL,
+ .mask = FLOW_SPEC_ETH_MAC_MASK,
+ },
+ .spec_count = {
+ .type = IBV_FLOW_SPEC_ACTION_COUNT,
+ .size = sizeof(struct ibv_flow_spec_counter_action),
+ .counters = counters, /* attached this counters handle
+to the newly created ibv_flow */ } };
+
+ /* create the flow */
+ flow = ibv_create_flow(qp, &flow_attr.attr);
+
+ /* allocate array for counters value reading */
+ uint64_t *counters_value = malloc(sizeof(uint64_t) * idx);
+
+ /* periodical read and print of flow counters */
+ while (--loop) {
+ sleep(1);
+
+ /* read hardware counters values */
+ ibv_read_counters(counters, counters_value, idx,
+ IBV_READ_COUNTERS_ATTR_PREFER_CACHED);
+
+ printf("PACKETS = %"PRIu64", BYTES = %"PRIu64 \n",
+ counters_value[0], counters_value[1] );
+ }
+
+ /* all done, release all */
+ free(counters_value);
+
+ /* destroy flow and detach counters */
+ ibv_destroy_flow(flow);
+
+ /* destroy counters handle */
+ ibv_destroy_counters(counters);
+
+ return;
+}
+```
+
+# SEE ALSO
+
+**ibv_create_counters**(3), **ibv_destroy_counters**(3),
+**ibv_attach_counters_point_flow**(3), **ibv_create_flow**(3)
+
+# AUTHORS
+
+Raed Salem <raeds@xxxxxxxxxxxx>
+
+Alex Rosenbaum <alexr@xxxxxxxxxxxx>
diff --git a/libibverbs/verbs.h b/libibverbs/verbs.h
index 066396f..bcb1af0 100644
--- a/libibverbs/verbs.h
+++ b/libibverbs/verbs.h
@@ -1737,6 +1737,10 @@ struct ibv_counter_attach_attr {
uint32_t comp_mask;
};
+enum ibv_read_counters_flags {
+ IBV_READ_COUNTERS_ATTR_PREFER_CACHED = 1 << 0,
+};
+
enum ibv_values_mask {
IBV_VALUES_MASK_RAW_CLOCK = 1 << 0,
IBV_VALUES_MASK_RESERVED = 1 << 1
@@ -1749,6 +1753,10 @@ struct ibv_values_ex {
struct verbs_context {
/* "grows up" - new fields go here */
+ int (*read_counters)(struct ibv_counters *counters,
+ uint64_t *counters_value,
+ uint32_t ncounters,
+ uint32_t flags);
int (*attach_counters_point_flow)(struct ibv_counters *counters,
struct ibv_counter_attach_attr *attr,
struct ibv_flow *flow);
@@ -2860,6 +2868,20 @@ static inline int ibv_attach_counters_point_flow(struct ibv_counters *counters,
return vctx->attach_counters_point_flow(counters, attr, flow);
}
+static inline int ibv_read_counters(struct ibv_counters *counters,
+ uint64_t *counters_value,
+ uint32_t ncounters,
+ uint32_t flags)
+{
+ struct verbs_context *vctx;
+
+ vctx = verbs_get_ctx_op(counters->context, read_counters);
+ if (!vctx)
+ return ENOSYS;
+
+ return vctx->read_counters(counters, counters_value, ncounters, flags);
+}
+
#ifdef __cplusplus
}
#endif
--
1.8.3.1
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
