From: Bodong Wang <bodong@xxxxxxxxxxxx>
Allow user space verbs application finer grain control of QP send rate
limit by setting a max_burst_sz [bytes] and typical_pkt_sz [bytes].
Expose new ibv_modify_qp_rate_limit() user space verb which gets
a structure with below user input:
rate_limit:
Rate limit in kbps for packet pacing.
max_burst_sz:
Max total burst size in bytes, 0 for device default.
Used by the device to calculate inter-burst gap delay.
typical_pkt_sz:
Bytes value of typical packet size sent in burst, 0 to use device
default, which is port's MTU. Used by the device to calculate number of
packets (WR) in a burst.
Signed-off-by: Bodong Wang <bodong@xxxxxxxxxxxx>
Signed-off-by: Alex Rosenbaum <alexr@xxxxxxxxxxxx>
Reviewed-by: Yishai Hadas <yishaih@xxxxxxxxxxxx>
---
libibverbs/driver.h | 2 +
libibverbs/dummy_ops.c | 8 ++++
libibverbs/man/CMakeLists.txt | 1 +
libibverbs/man/ibv_modify_qp_rate_limit.3 | 67 +++++++++++++++++++++++++++++++
libibverbs/verbs.h | 27 +++++++++++++
5 files changed, 105 insertions(+)
create mode 100644 libibverbs/man/ibv_modify_qp_rate_limit.3
diff --git a/libibverbs/driver.h b/libibverbs/driver.h
index 39b6bca..4d36c66 100644
--- a/libibverbs/driver.h
+++ b/libibverbs/driver.h
@@ -233,6 +233,8 @@ struct verbs_context_ops {
int (*modify_cq)(struct ibv_cq *cq, struct ibv_modify_cq_attr *attr);
int (*modify_qp)(struct ibv_qp *qp, struct ibv_qp_attr *attr,
int attr_mask);
+ int (*modify_qp_rate_limit)(struct ibv_qp *qp,
+ struct ibv_qp_rate_limit_attr *attr);
int (*modify_srq)(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr,
int srq_attr_mask);
int (*modify_wq)(struct ibv_wq *wq, struct ibv_wq_attr *wq_attr);
diff --git a/libibverbs/dummy_ops.c b/libibverbs/dummy_ops.c
index 6e1d474..c67ad83 100644
--- a/libibverbs/dummy_ops.c
+++ b/libibverbs/dummy_ops.c
@@ -233,6 +233,12 @@ static int modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr, int attr_mask)
return ENOSYS;
}
+static int modify_qp_rate_limit(struct ibv_qp *qp,
+ struct ibv_qp_rate_limit_attr *attr)
+{
+ return ENOSYS;
+}
+
static int modify_srq(struct ibv_srq *srq, struct ibv_srq_attr *srq_attr,
int srq_attr_mask)
{
@@ -393,6 +399,7 @@ const struct verbs_context_ops verbs_dummy_ops = {
get_srq_num,
modify_cq,
modify_qp,
+ modify_qp_rate_limit,
modify_srq,
modify_wq,
open_qp,
@@ -470,6 +477,7 @@ void verbs_set_ops(struct verbs_context *vctx,
SET_OP(vctx, get_srq_num);
SET_OP(vctx, modify_cq);
SET_OP(ctx, modify_qp);
+ SET_OP(vctx, modify_qp_rate_limit);
SET_OP(ctx, modify_srq);
SET_OP(vctx, modify_wq);
SET_OP(vctx, open_qp);
diff --git a/libibverbs/man/CMakeLists.txt b/libibverbs/man/CMakeLists.txt
index 53dbb06..0967ee8 100644
--- a/libibverbs/man/CMakeLists.txt
+++ b/libibverbs/man/CMakeLists.txt
@@ -31,6 +31,7 @@ rdma_man_pages(
ibv_get_srq_num.3
ibv_inc_rkey.3
ibv_modify_qp.3
+ ibv_modify_qp_rate_limit.3
ibv_modify_srq.3
ibv_modify_wq.3
ibv_open_device.3
diff --git a/libibverbs/man/ibv_modify_qp_rate_limit.3 b/libibverbs/man/ibv_modify_qp_rate_limit.3
new file mode 100644
index 0000000..f6f8efa
--- /dev/null
+++ b/libibverbs/man/ibv_modify_qp_rate_limit.3
@@ -0,0 +1,67 @@
+.\" -*- nroff -*-
+.\" Licensed under the OpenIB.org BSD license (FreeBSD Variant) - See COPYING.md
+.\"
+.TH IBV_MODIFY_QP_RATE_LIMIT 3 2018-01-09 libibverbs "Libibverbs Programmer's Manual"
+.SH "NAME"
+ibv_modify_qp_rate_limit \- modify the send rate limits attributes of a queue pair (QP)
+.SH "SYNOPSIS"
+.nf
+.B #include <infiniband/verbs.h>
+.sp
+.BI "int ibv_modify_qp_rate_limit(struct ibv_qp " "*qp" ", struct ibv_qp_rate_limit_attr " "*attr");
+.fi
+.SH "DESCRIPTION"
+.B ibv_modify_qp_rate_limit()
+modifies the send rate limiting packet pacing attributes of QP
+.I qp
+with the attributes in
+.I attr\fR.
+The argument \fIattr\fR is an ibv_qp_rate_limit_attr struct, as defined in <infiniband/verbs.h>.
+.PP
+The
+.I rate_limit
+defines the MAX send rate this QP will send as long as the link in not blocked and there are work requestes in send queue.
+.PP
+Finer control for shaping the rate limit of a QP is achieved by defining the
+.I max_burst_sz\fR,
+single burst max bytes size and the
+.I typical_pkt_sz\fR,
+typical packet bytes size. These allow the device to adjust the inter-burst gap delay required to correctly shape the scheduling of sends to the wire in order to reach for requested application requirements.
+.PP
+Setting a value of 0 for
+.I max_burst_sz
+or
+.I typical_pkt_sz
+will use the devices defaults.
+.I typical_pkt_sz
+will default to the port's MTU value.
+.PP
+.nf
+struct ibv_qp_rate_limit_attr {
+.in +8
+uint32_t rate_limit; /* kbps */
+uint32_t max_burst_sz; /* bytes */
+uint16_t typical_pkt_sz; /* bytes */
+.in -8
+};
+.fi
+.PP
+.SH "RETURN VALUE"
+.B ibv_modify_qp_rate_limit()
+returns 0 on success, or the value of errno on failure (which indicates the failure reason).
+.SH "ERRORS"
+.SS EINVAL
+Invalid arguments.
+.SS ENOSYS
+Function is not implemented for this device.
+.PP
+.SH "SEE ALSO"
+.BR ibv_create_qp (3),
+.BR ibv_destroy_qp (3),
+.BR ibv_modify_qp (3),
+.BR ibv_query_qp (3)
+.SH "AUTHORS"
+.TP
+Alex Rosenbaum <alexr@xxxxxxxxxxxx>
+.TP
+Bodong Wang <bodong@xxxxxxxxxxxx>
diff --git a/libibverbs/verbs.h b/libibverbs/verbs.h
index cb96f33..af82a8d 100644
--- a/libibverbs/verbs.h
+++ b/libibverbs/verbs.h
@@ -969,6 +969,13 @@ struct ibv_qp_attr {
uint32_t rate_limit;
};
+struct ibv_qp_rate_limit_attr {
+ uint32_t rate_limit; /* in kbps */
+ uint32_t max_burst_sz; /* total burst size in bytes */
+ uint16_t typical_pkt_sz; /* typical send packet size in bytes */
+ uint32_t comp_mask;
+};
+
enum ibv_wr_opcode {
IBV_WR_RDMA_WRITE,
IBV_WR_RDMA_WRITE_WITH_IMM,
@@ -1657,6 +1664,8 @@ struct ibv_values_ex {
struct verbs_context {
/* "grows up" - new fields go here */
+ int (*modify_qp_rate_limit)(struct ibv_qp *qp,
+ struct ibv_qp_rate_limit_attr *attr);
struct ibv_pd *(*alloc_parent_domain)(struct ibv_context *context,
struct ibv_parent_domain_init_attr *attr);
int (*dealloc_td)(struct ibv_td *td);
@@ -2321,6 +2330,24 @@ int ibv_modify_qp(struct ibv_qp *qp, struct ibv_qp_attr *attr,
int attr_mask);
/**
+ * ibv_modify_qp_rate_limit - Modify a queue pair rate limit values
+ * @qp - QP object to modify
+ * @attr - Attributes to configure the rate limiting values of the QP
+ */
+static inline int
+ibv_modify_qp_rate_limit(struct ibv_qp *qp,
+ struct ibv_qp_rate_limit_attr *attr)
+{
+ struct verbs_context *vctx;
+
+ vctx = verbs_get_ctx_op(qp->context, modify_qp_rate_limit);
+ if (!vctx)
+ return ENOSYS;
+
+ return vctx->modify_qp_rate_limit(qp, attr);
+}
+
+/**
* ibv_query_qp - Returns the attribute list and current values for the
* specified QP.
* @qp: The QP to query.
--
1.8.3.1
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
