From: Alex Rosenbaum <alexr@xxxxxxxxxxxx>
Introduce a new set of QP send operations which allow extensibility for
new send opcodes. This concept lets to add vendor specific send opcodes
or even vendor specific QP types easily. Users can require a set of
supported QP send ops specified during QP creation. The new 'struct
ibv_qp_ex' will expose support to the required post send ops.
The post send ops API's are broken down into:
1. Create a new send WR entry on the QP's send queue buffer
2. Set WR attributes for specific fields depending on QP type and create
opcode
3. Commit the open send buffer queued entries to HW and ring the doorbell.
The application can 'create' multiple new WR entries on the QP's send
queue buffer before ringing the doorbell
Signed-off-by: Alex Rosenbaum <alexr@xxxxxxxxxxxx>
Signed-off-by: Jason Gunthorpe <jgg@xxxxxxxxxxxx>
Signed-off-by: Guy Levi <guyle@xxxxxxxxxxxx>
Signed-off-by: Yishai Hadas <yishaih@xxxxxxxxxxxx>
---
debian/libibverbs1.symbols | 2 +
libibverbs/CMakeLists.txt | 2 +-
libibverbs/cmd.c | 14 +-
libibverbs/driver.h | 8 +-
libibverbs/libibverbs.map.in | 5 +
libibverbs/man/CMakeLists.txt | 21 +++
libibverbs/man/ibv_create_qp_ex.3 | 26 ++-
libibverbs/man/ibv_wr_post.3.md | 333 ++++++++++++++++++++++++++++++++++++++
libibverbs/verbs.c | 9 ++
libibverbs/verbs.h | 191 +++++++++++++++++++++-
10 files changed, 604 insertions(+), 7 deletions(-)
create mode 100644 libibverbs/man/ibv_wr_post.3.md
diff --git a/debian/libibverbs1.symbols b/debian/libibverbs1.symbols
index 774862e..a077f9a 100644
--- a/debian/libibverbs1.symbols
+++ b/debian/libibverbs1.symbols
@@ -3,6 +3,7 @@ libibverbs.so.1 libibverbs1 #MINVER#
IBVERBS_1.0@IBVERBS_1.0 1.1.6
IBVERBS_1.1@IBVERBS_1.1 1.1.6
IBVERBS_1.5@IBVERBS_1.5 20
+ IBVERBS_1.6@IBVERBS_1.6 24
(symver)IBVERBS_PRIVATE_22 22
ibv_ack_async_event@IBVERBS_1.0 1.1.6
ibv_ack_async_event@IBVERBS_1.1 1.1.6
@@ -70,6 +71,7 @@ libibverbs.so.1 libibverbs1 #MINVER#
ibv_open_device@IBVERBS_1.0 1.1.6
ibv_open_device@IBVERBS_1.1 1.1.6
ibv_port_state_str@IBVERBS_1.1 1.1.6
+ ibv_qp_to_qp_ex@IBVERBS_1.6 24
ibv_query_device@IBVERBS_1.0 1.1.6
ibv_query_device@IBVERBS_1.1 1.1.6
ibv_query_gid@IBVERBS_1.0 1.1.6
diff --git a/libibverbs/CMakeLists.txt b/libibverbs/CMakeLists.txt
index 0671977..76aa522 100644
--- a/libibverbs/CMakeLists.txt
+++ b/libibverbs/CMakeLists.txt
@@ -27,7 +27,7 @@ configure_file("libibverbs.map.in"
rdma_library(ibverbs "${CMAKE_CURRENT_BINARY_DIR}/libibverbs.map"
# See Documentation/versioning.md
- 1 1.5.${PACKAGE_VERSION}
+ 1 1.6.${PACKAGE_VERSION}
all_providers.c
cmd.c
cmd_ah.c
diff --git a/libibverbs/cmd.c b/libibverbs/cmd.c
index d976842..5291abf 100644
--- a/libibverbs/cmd.c
+++ b/libibverbs/cmd.c
@@ -806,7 +806,14 @@ int ibv_cmd_create_qp_ex2(struct ibv_context *context,
struct verbs_xrcd *vxrcd = NULL;
int err;
- if (qp_attr->comp_mask >= IBV_QP_INIT_ATTR_RESERVED)
+ if (!check_comp_mask(qp_attr->comp_mask,
+ IBV_QP_INIT_ATTR_PD |
+ IBV_QP_INIT_ATTR_XRCD |
+ IBV_QP_INIT_ATTR_CREATE_FLAGS |
+ IBV_QP_INIT_ATTR_MAX_TSO_HEADER |
+ IBV_QP_INIT_ATTR_IND_TABLE |
+ IBV_QP_INIT_ATTR_RX_HASH |
+ IBV_QP_INIT_ATTR_SEND_OPS_FLAGS))
return EINVAL;
memset(&cmd->core_payload, 0, sizeof(cmd->core_payload));
@@ -850,7 +857,10 @@ int ibv_cmd_create_qp_ex(struct ibv_context *context,
struct verbs_xrcd *vxrcd = NULL;
int err;
- if (attr_ex->comp_mask > (IBV_QP_INIT_ATTR_XRCD | IBV_QP_INIT_ATTR_PD))
+ if (!check_comp_mask(attr_ex->comp_mask,
+ IBV_QP_INIT_ATTR_PD |
+ IBV_QP_INIT_ATTR_XRCD |
+ IBV_QP_INIT_ATTR_SEND_OPS_FLAGS))
return ENOSYS;
err = create_qp_ex_common(qp, attr_ex, vxrcd,
diff --git a/libibverbs/driver.h b/libibverbs/driver.h
index b9f648c..e4d624b 100644
--- a/libibverbs/driver.h
+++ b/libibverbs/driver.h
@@ -77,7 +77,7 @@ struct verbs_srq {
enum verbs_qp_mask {
VERBS_QP_XRCD = 1 << 0,
- VERBS_QP_RESERVED = 1 << 1
+ VERBS_QP_EX = 1 << 1,
};
enum ibv_gid_type {
@@ -101,10 +101,14 @@ static inline struct verbs_mr *verbs_get_mr(struct ibv_mr *mr)
}
struct verbs_qp {
- struct ibv_qp qp;
+ union {
+ struct ibv_qp qp;
+ struct ibv_qp_ex qp_ex;
+ };
uint32_t comp_mask;
struct verbs_xrcd *xrcd;
};
+static_assert(offsetof(struct ibv_qp_ex, qp_base) == 0, "Invalid qp layout");
enum ibv_flow_action_type {
IBV_FLOW_ACTION_UNSPECIFIED,
diff --git a/libibverbs/libibverbs.map.in b/libibverbs/libibverbs.map.in
index 4bffb1b..87a1b9f 100644
--- a/libibverbs/libibverbs.map.in
+++ b/libibverbs/libibverbs.map.in
@@ -111,6 +111,11 @@ IBVERBS_1.5 {
ibv_get_pkey_index;
} IBVERBS_1.1;
+IBVERBS_1.6 {
+ global:
+ ibv_qp_to_qp_ex;
+} IBVERBS_1.5;
+
/* If any symbols in this stanza change ABI then the entire staza gets a new symbol
version. See the top level CMakeLists.txt for this setting. */
diff --git a/libibverbs/man/CMakeLists.txt b/libibverbs/man/CMakeLists.txt
index 4d5abef..e1d5edf 100644
--- a/libibverbs/man/CMakeLists.txt
+++ b/libibverbs/man/CMakeLists.txt
@@ -68,6 +68,7 @@ rdma_man_pages(
ibv_srq_pingpong.1
ibv_uc_pingpong.1
ibv_ud_pingpong.1
+ ibv_wr_post.3.md
ibv_xsrq_pingpong.1
)
rdma_alias_man_pages(
@@ -101,4 +102,24 @@ rdma_alias_man_pages(
ibv_rate_to_mbps.3 mbps_to_ibv_rate.3
ibv_rate_to_mult.3 mult_to_ibv_rate.3
ibv_reg_mr.3 ibv_dereg_mr.3
+ ibv_wr_post.3 ibv_wr_abort.3
+ ibv_wr_post.3 ibv_wr_complete.3
+ ibv_wr_post.3 ibv_wr_start.3
+ ibv_wr_post.3 ibv_wr_atomic_cmp_swp.3
+ ibv_wr_post.3 ibv_wr_atomic_fetch_add.3
+ ibv_wr_post.3 ibv_wr_bind_mw.3
+ ibv_wr_post.3 ibv_wr_local_inv.3
+ ibv_wr_post.3 ibv_wr_rdma_read.3
+ ibv_wr_post.3 ibv_wr_rdma_write.3
+ ibv_wr_post.3 ibv_wr_rdma_write_imm.3
+ ibv_wr_post.3 ibv_wr_send.3
+ ibv_wr_post.3 ibv_wr_send_imm.3
+ ibv_wr_post.3 ibv_wr_send_inv.3
+ ibv_wr_post.3 ibv_wr_send_tso.3
+ ibv_wr_post.3 ibv_wr_set_inline_data.3
+ ibv_wr_post.3 ibv_wr_set_inline_data_list.3
+ ibv_wr_post.3 ibv_wr_set_sge.3
+ ibv_wr_post.3 ibv_wr_set_sge_list.3
+ ibv_wr_post.3 ibv_wr_set_ud_addr.3
+ ibv_wr_post.3 ibv_wr_set_xrc_srqn.3
)
diff --git a/libibverbs/man/ibv_create_qp_ex.3 b/libibverbs/man/ibv_create_qp_ex.3
index 47e0d9e..277e9fa 100644
--- a/libibverbs/man/ibv_create_qp_ex.3
+++ b/libibverbs/man/ibv_create_qp_ex.3
@@ -39,6 +39,7 @@ uint16_t max_tso_header; /* Maximum TSO header size */
struct ibv_rwq_ind_table *rwq_ind_tbl; /* Indirection table to be associated with the QP */
struct ibv_rx_hash_conf rx_hash_conf; /* RX hash configuration to be used */
uint32_t source_qpn; /* Source QP number, creation flag IBV_QP_CREATE_SOURCE_QPN should be set, few NOTEs below */
+uint64_t send_ops_flags; /* Select which QP send ops will be defined in struct ibv_qp_ex. Use enum ibv_qp_create_send_ops_flags */
.in -8
};
.sp
@@ -62,6 +63,7 @@ IBV_QP_CREATE_SOURCE_QPN = 1 << 10, /* The created QP will use th
IBV_QP_CREATE_PCI_WRITE_END_PADDING = 1 << 11, /* Incoming packets will be padded to cacheline size */
.in -8
};
+.fi
.nf
struct ibv_rx_hash_conf {
.in +8
@@ -72,7 +74,6 @@ uint64_t rx_hash_fields_mask; /* RX fields that should particip
.in -8
};
.fi
-
.nf
enum ibv_rx_hash_fields {
.in +8
@@ -92,6 +93,23 @@ IBV_RX_HASH_INNER = (1UL << 31),
.in -8
};
.fi
+.nf
+struct ibv_qp_create_send_ops_flags {
+.in +8
+IBV_QP_EX_WITH_RDMA_WRITE = 1 << 0,
+IBV_QP_EX_WITH_RDMA_WRITE_WITH_IMM = 1 << 1,
+IBV_QP_EX_WITH_SEND = 1 << 2,
+IBV_QP_EX_WITH_SEND_WITH_IMM = 1 << 3,
+IBV_QP_EX_WITH_RDMA_READ = 1 << 4,
+IBV_QP_EX_WITH_ATOMIC_CMP_AND_SWP = 1 << 5,
+IBV_QP_EX_WITH_ATOMIC_FETCH_AND_ADD = 1 << 6,
+IBV_QP_EX_WITH_LOCAL_INV = 1 << 7,
+IBV_QP_EX_WITH_BIND_MW = 1 << 8,
+IBV_QP_EX_WITH_SEND_WITH_INV = 1 << 9,
+IBV_QP_EX_WITH_TSO = 1 << 10,
+.in -8
+};
+.fi
.PP
The function
@@ -119,6 +137,12 @@ if the QP is to be associated with an SRQ.
.PP
The attribute source_qpn is supported only on UD QP, without flow steering RX should not be possible.
.PP
+Use
+.B ibv_qp_to_qp_ex()
+to get the
+.I ibv_qp_ex
+for accessing the send ops iterator interface, when QP create attr IBV_QP_INIT_ATTR_SEND_OPS_FLAGS is used.
+.PP
.B ibv_destroy_qp()
fails if the QP is attached to a multicast group.
.PP
diff --git a/libibverbs/man/ibv_wr_post.3.md b/libibverbs/man/ibv_wr_post.3.md
new file mode 100644
index 0000000..1fc1acb
--- /dev/null
+++ b/libibverbs/man/ibv_wr_post.3.md
@@ -0,0 +1,333 @@
+---
+date: 2018-11-27
+footer: libibverbs
+header: "Libibverbs Programmer's Manual"
+layout: page
+license: 'Licensed under the OpenIB.org BSD license (FreeBSD Variant) - See COPYING.md'
+section: 3
+title: IBV_WR API
+---
+
+# NAME
+
+ibv_wr_abort, ibv_wr_complete, ibv_wr_start - Manage regions allowed to post work
+
+ibv_wr_atomic_cmp_swp, ibv_wr_atomic_fetch_add - Post remote atomic operation work requests
+
+ibv_wr_bind_mw, ibv_wr_local_inv - Post work requests for memory windows
+
+ibv_wr_rdma_read, ibv_wr_rdma_write, ibv_wr_rdma_write_imm - Post RDMA work requests
+
+ibv_wr_send, ibv_wr_send_imm, ibv_wr_send_inv - Post send work requests
+
+ibv_wr_send_tso - Post segmentation offload work requests
+
+ibv_wr_set_inline_data, ibv_wr_set_inline_data_list - Attach inline data to the last work request
+
+ibv_wr_set_sge, ibv_wr_set_sge_list - Attach data to the last work request
+
+ibv_wr_set_ud_addr - Attach UD addressing info to the last work request
+
+ibv_wr_set_xrc_srqn - Attach an XRC SRQN to the last work request
+
+# SYNOPSIS
+
+```c
+#include <infiniband/verbs.h>
+
+void ibv_wr_abort(struct ibv_qp_ex *qp);
+int ibv_wr_complete(struct ibv_qp_ex *qp);
+void ibv_wr_start(struct ibv_qp_ex *qp);
+
+void ibv_wr_atomic_cmp_swp(struct ibv_qp_ex *qp, uint32_t rkey,
+ uint64_t remote_addr, uint64_t compare,
+ uint64_t swap);
+void ibv_wr_atomic_fetch_add(struct ibv_qp_ex *qp, uint32_t rkey,
+ uint64_t remote_addr, uint64_t add);
+
+void ibv_wr_bind_mw(struct ibv_qp_ex *qp, struct ibv_mw *mw, uint32_t rkey,
+ const struct ibv_mw_bind_info *bind_info);
+void ibv_wr_local_inv(struct ibv_qp_ex *qp, uint32_t invalidate_rkey);
+
+void ibv_wr_rdma_read(struct ibv_qp_ex *qp, uint32_t rkey,
+ uint64_t remote_addr);
+void ibv_wr_rdma_write(struct ibv_qp_ex *qp, uint32_t rkey,
+ uint64_t remote_addr);
+void ibv_wr_rdma_write_imm(struct ibv_qp_ex *qp, uint32_t rkey,
+ uint64_t remote_addr, __be32 imm_data);
+
+void ibv_wr_send(struct ibv_qp_ex *qp);
+void ibv_wr_send_imm(struct ibv_qp_ex *qp, __be32 imm_data);
+void ibv_wr_send_inv(struct ibv_qp_ex *qp, uint32_t invalidate_rkey);
+void ibv_wr_send_tso(struct ibv_qp_ex *qp, void *hdr, uint16_t hdr_sz,
+ uint16_t mss);
+
+void ibv_wr_set_inline_data(struct ibv_qp_ex *qp, void *addr, size_t length);
+void ibv_wr_set_inline_data_list(struct ibv_qp_ex *qp, size_t num_buf,
+ const struct ibv_data_buf *buf_list);
+void ibv_wr_set_sge(struct ibv_qp_ex *qp, uint32_t lkey, uint64_t addr,
+ uint32_t length);
+void ibv_wr_set_sge_list(struct ibv_qp_ex *qp, size_t num_sge,
+ const struct ibv_sge *sg_list);
+
+void ibv_wr_set_ud_addr(struct ibv_qp_ex *qp, struct ibv_ah *ah,
+ uint32_t remote_qpn, uint32_t remote_qkey);
+void ibv_wr_set_xrc_srqn(struct ibv_qp_ex *qp, uint32_t remote_srqn);
+```
+
+# DESCRIPTION
+
+The verbs work request API (ibv_wr_\*) allows efficient posting of work to a send
+queue using function calls instead of the struct based *ibv_post_send()*
+scheme. This approach is designed to minimize CPU branching and locking during
+the posting process.
+
+This API is intended to be used to access additional functionality beyond
+what is provided by *ibv_post_send()*.
+
+WRs batches of *ibv_post_send()* and this API WRs batches can interleave
+together just if they are not posted within the critical region of each other.
+(A critical region in this API formed by *ibv_wr_start()* and
+*ibv_wr_complete()*/*ibv_wr_abort()*)
+
+# USAGE
+
+To use these APIs the QP must be created using ibv_create_qp_ex() which allows
+setting the **IBV_QP_INIT_ATTR_SEND_OPS_FLAGS** in *comp_mask*. The
+*send_ops_flags* should be set to the OR of the work request types that will
+be posted to the QP.
+
+If the QP does not support all the requested work request types then QP
+creation will fail.
+
+Posting work requests to the QP is done within the critical region formed by
+*ibv_wr_start()* and *ibv_wr_complete()*/*ibv_wr_abort()* (see CONCURRENCY below).
+
+Each work request is created by calling a WR builder function (see the table
+column WR builder below) to start creating the work request, followed by
+allowed/required setter functions described below.
+
+The WR builder and setter combination can be called multiple times to
+efficiently post multiple work requests within a single critical region.
+
+Each WR builder will use the *wr_id* member of *struct ibv_qp_ex* to set the
+value to be returned in the completion. Some operations will also use the
+*wr_flags* member to influence operation (see Flags below). These values
+should be set before invoking the WR builder function.
+
+For example a simple send could be formed as follows:
+
+```C
+qpx->wr_id = 1;
+ibv_wr_send(qpx);
+ibv_wr_set_sge(qpx, lkey, &data, sizeof(data));
+```
+
+The section WORK REQUESTS describes the various WR builders and setters in
+details.
+
+Posting work is completed by calling *ibv_wr_complete()* or *ibv_wr_abort()*.
+No work is executed to the queue until *ibv_wr_complete()* returns
+success. *ibv_wr_abort()* will discard all work prepared since *ibv_wr_start()*.
+
+# WORK REQUESTS
+
+Many of the operations match the opcodes available for *ibv_post_send()*. Each
+operation has a WR builder function, a list of allowed setters, and a flag bit
+to request the operation with *send_ops_flags* in *struct
+ibv_qp_init_attr_ex* (see the EXAMPLE below).
+
+| Operation | WR builder | QP Type Supported | setters |
+|----------------------|---------------------------|----------------------------------|----------|
+| ATOMIC_CMP_AND_SWP | ibv_wr_atomic_cmp_swp() | RC, XRC_SEND | DATA, QP |
+| ATOMIC_FETCH_AND_ADD | ibv_wr_atomic_fetch_add() | RC, XRC_SEND | DATA, QP |
+| BIND_MW | ibv_wr_bind_mw() | UC, RC, XRC_SEND | NONE |
+| LOCAL_INV | ibv_wr_local_inv() | UC, RC, XRC_SEND | NONE |
+| RDMA_READ | ibv_wr_rdma_read() | RC, XRC_SEND | DATA, QP |
+| RDMA_WRITE | ibv_wr_rdma_write() | UC, RC, XRC_SEND | DATA, QP |
+| RDMA_WRITE_WITH_IMM | ibv_wr_rdma_write_imm() | UC, RC, XRC_SEND | DATA, QP |
+| SEND | ibv_wr_send() | UD, UC, RC, XRC_SEND, RAW_PACKET | DATA, QP |
+| SEND_WITH_IMM | ibv_wr_send_imm() | UD, UC, RC, SRC SEND | DATA, QP |
+| SEND_WITH_INV | ibv_wr_send_inv() | UC, RC, XRC_SEND | DATA, QP |
+| TSO | ibv_wr_send_tso() | UD, RAW_PACKET | DATA, QP |
+
+
+## Atomic operations
+
+Atomic operations are only atomic so long as all writes to memory go only
+through the same RDMA hardware. It is not atomic with writes performed by the
+CPU, or by other RDMA hardware in the system.
+
+*ibv_wr_atomic_cmp_swp()*
+: If the remote 64 bit memory location specified by *rkey* and *remote_addr*
+ equals *compare* then set it to *swap*.
+
+*ibv_wr_atomic_fetch_add()*
+: Add *add* to the 64 bit memory location specified *rkey* and *remote_addr*.
+
+## Memory Windows
+
+Memory window type 2 operations (See man page for ibv_alloc_mw).
+
+*ibv_wr_bind_mw()*
+: Bind a MW type 2 specified by **mw**, set a new **rkey** and set its
+ properties by **bind_info**.
+
+*ibv_wr_local_inv()*
+: Invalidate a MW type 2 which is associated with **rkey**.
+
+## RDMA
+
+*ibv_wr_rdma_read()*
+: Read from the remote memory location specified *rkey* and
+ *remote_addr*. The number of bytes to read, and the local location to
+ store the data, is determined by the DATA buffers set after this call.
+
+*ibv_wr_rdma_write()*, *ibv_wr_rdma_write_imm()*
+: Write to the remote memory location specified *rkey* and
+ *remote_addr*. The number of bytes to read, and the local location to get
+ the data, is determined by the DATA buffers set after this call.
+
+ The _imm version causes the remote side to get a IBV_WC_RECV_RDMA_WITH_IMM
+ containing the 32 bits of immediate data.
+
+## Message Send
+
+*ibv_wr_send()*, *ibv_wr_send_imm()*
+: Send a message. The number of bytes to send, and the local location to get
+ the data, is determined by the DATA buffers set after this call.
+
+ The _imm version causes the remote side to get a IBV_WC_RECV_RDMA_WITH_IMM
+ containing the 32 bits of immediate data.
+
+*ibv_wr_send_inv()*
+: The data transfer is the same as for *ibv_wr_send()*, however the remote
+ side will invalidate the MR specified by *invalidate_rkey* before
+ delivering a completion.
+
+*ibv_wr_send_tso()*
+: Produce multiple SEND messages using TCP Segmentation Offload. The SGE
+ points to a TCP Stream buffer which will be segmented into
+ MSS size SENDs. The hdr includes the entire network headers up to and
+ including the TCP header and is prefixed before each segment.
+
+## QP Specific setters
+
+Certain QP types require each post to be accompanied by additional setters,
+these setters are mandatory for any operation listing a QP setter in the above
+table.
+
+*UD* QPs
+: *ibv_wr_set_ud_addr()* must be called to set the destination address of
+ the work.
+
+*XRC_SEND* QPs
+: *ibv_wr_set_xrc_srqn()* must be called to set the destination SRQN field.
+
+## DATA transfer setters
+
+For work that requires to transfer data one of the following setters should
+be called once after the WR builder:
+
+*ibv_wr_set_sge()*
+: Transfer data to/from a single buffer given by the lkey, addr and
+ length. This is equivalent to *ibv_wr_set_sge_list()* with a single
+ element.
+
+*ibv_wr_set_sge_list()*
+: Transfer data to/from a list of buffers, logically concatenated
+ together. Each buffer is specified by an element in an array of *struct
+ ibv_sge*.
+
+Inline setters will copy the send data during the setter and allows the caller
+to immediately re-use the buffer. This behavior is identical to the
+IBV_SEND_INLINE flag. Generally this copy is done in a way that optimizes
+SEND latency and is suitable for small messages. The provider will limit the
+amount of data it can support in a single operation. This limit is requested
+in the *max_inline_data* member of *struct ibv_qp_init_attr*. Valid only
+for SEND and RDMA_WRITE.
+
+*ibv_wr_set_inline_data()*
+: Copy send data from a single buffer given by the addr and length.
+ This is equivalent to *ibv_wr_set_inline_data_list()* with a single
+ element.
+
+*ibv_wr_set_inline_data_list()*
+: Copy send data from a list of buffers, logically concatenated
+ together. Each buffer is specified by an element in an array of *struct
+ ibv_inl_data*.
+
+## Flags
+
+A bit mask of flags may be specified in *wr_flags* to control the behavior of
+the work request.
+
+**IBV_SEND_FENCE**
+: Do not start this work request until prior work has completed.
+
+**IBV_SEND_IP_CSUM**
+: Offload the IPv4 and TCP/UDP checksum calculation
+
+**IBV_SEND_SIGNALED**
+: A completion will be generated in the completion queue for the operation.
+
+**IBV_SEND_SOLICTED**
+: Set the solicted bit in the RDMA packet. This informs the other side to
+ generate a completion event upon receiving the RDMA operation.
+
+# CONCURRENCY
+
+The provider will provide locking to ensure that *ibv_wr_start()* and
+*ibv_wr_complete()/abort()* form a per-QP critical section where no other
+threads can enter.
+
+If an *ibv_td* is provided during QP creation then no locking will be perfomed
+and it is up to the caller to ensure that only one thread can be within the
+critical region at a time.
+
+# RETURN VALUE
+
+Applications should use this API in a way that does not create failures. The
+individual APIs do not return a failure indication to avoid branching.
+
+If a failure is detected during operation, for instance due to an invalid
+argument, then *ibv_wr_complete()* will return failure and the entire posting
+will be aborted.
+
+# EXAMPLE
+
+```c
+/* create RC QP type and specify the required send opcodes */
+qp_init_attr_ex.qp_type = IBV_QPT_RC;
+qp_init_attr_ex.comp_mask |= IBV_QP_INIT_ATTR_SEND_OPS_FLAGS;
+qp_init_attr_ex.send_ops_flags |= IBV_QP_EX_WITH_RDMA_WRITE;
+qp_init_attr_ex.send_ops_flags |= IBV_QP_EX_WITH_RDMA_WRITE_WITH_IMM;
+
+ibv_qp *qp = ibv_create_qp_ex(ctx, qp_init_attr_ex);
+ibv_qp_ex *qpx = ibv_qp_to_qp_ex(qp);
+
+ibv_wr_start(qpx);
+
+/* create 1st WRITE WR entry */
+qpx->wr_id = my_wr_id_1;
+ibv_wr_rdma_write(qpx, rkey, remote_addr_1);
+ibv_wr_set_sge(qpx, lkey, local_addr_1, length_1);
+
+/* create 2nd WRITE_WITH_IMM WR entry */
+qpx->wr_id = my_wr_id_2;
+qpx->send_flags = IBV_SEND_SIGNALED;
+ibv_wr_rdma_write_imm(qpx, rkey, remote_addr_2, htonl(0x1234));
+ibv_set_wr_sge(qpx, lkey, local_addr_2, length_2);
+
+/* Begin processing WRs */
+ret = ibv_wr_complete(qpx);
+```
+
+# SEE ALSO
+
+**ibv_post_send**(3), **ibv_create_qp_ex(3)**.
+
+# AUTHOR
+
+Jason Gunthorpe <jgg@xxxxxxxxxxxx>
+Guy Levi <guyle@xxxxxxxxxxxx>
diff --git a/libibverbs/verbs.c b/libibverbs/verbs.c
index 188bccb..1766b9f 100644
--- a/libibverbs/verbs.c
+++ b/libibverbs/verbs.c
@@ -589,6 +589,15 @@ LATEST_SYMVER_FUNC(ibv_create_qp, 1_1, "IBVERBS_1.1",
return qp;
}
+struct ibv_qp_ex *ibv_qp_to_qp_ex(struct ibv_qp *qp)
+{
+ struct verbs_qp *vqp = (struct verbs_qp *)qp;
+
+ if (vqp->comp_mask & VERBS_QP_EX)
+ return &vqp->qp_ex;
+ return NULL;
+}
+
LATEST_SYMVER_FUNC(ibv_query_qp, 1_1, "IBVERBS_1.1",
int,
struct ibv_qp *qp, struct ibv_qp_attr *attr,
diff --git a/libibverbs/verbs.h b/libibverbs/verbs.h
index 243cd8c..a2bae25 100644
--- a/libibverbs/verbs.h
+++ b/libibverbs/verbs.h
@@ -889,7 +889,7 @@ enum ibv_qp_init_attr_mask {
IBV_QP_INIT_ATTR_MAX_TSO_HEADER = 1 << 3,
IBV_QP_INIT_ATTR_IND_TABLE = 1 << 4,
IBV_QP_INIT_ATTR_RX_HASH = 1 << 5,
- IBV_QP_INIT_ATTR_RESERVED = 1 << 6
+ IBV_QP_INIT_ATTR_SEND_OPS_FLAGS = 1 << 6,
};
enum ibv_qp_create_flags {
@@ -900,6 +900,20 @@ enum ibv_qp_create_flags {
IBV_QP_CREATE_PCI_WRITE_END_PADDING = 1 << 11,
};
+enum ibv_qp_create_send_ops_flags {
+ IBV_QP_EX_WITH_RDMA_WRITE = 1 << 0,
+ IBV_QP_EX_WITH_RDMA_WRITE_WITH_IMM = 1 << 1,
+ IBV_QP_EX_WITH_SEND = 1 << 2,
+ IBV_QP_EX_WITH_SEND_WITH_IMM = 1 << 3,
+ IBV_QP_EX_WITH_RDMA_READ = 1 << 4,
+ IBV_QP_EX_WITH_ATOMIC_CMP_AND_SWP = 1 << 5,
+ IBV_QP_EX_WITH_ATOMIC_FETCH_AND_ADD = 1 << 6,
+ IBV_QP_EX_WITH_LOCAL_INV = 1 << 7,
+ IBV_QP_EX_WITH_BIND_MW = 1 << 8,
+ IBV_QP_EX_WITH_SEND_WITH_INV = 1 << 9,
+ IBV_QP_EX_WITH_TSO = 1 << 10,
+};
+
struct ibv_rx_hash_conf {
/* enum ibv_rx_hash_function_flags */
uint8_t rx_hash_function;
@@ -926,6 +940,8 @@ struct ibv_qp_init_attr_ex {
struct ibv_rwq_ind_table *rwq_ind_tbl;
struct ibv_rx_hash_conf rx_hash_conf;
uint32_t source_qpn;
+ /* See enum ibv_qp_create_send_ops_flags */
+ uint64_t send_ops_flags;
};
enum ibv_qp_open_attr_mask {
@@ -1051,6 +1067,11 @@ enum ibv_send_flags {
IBV_SEND_IP_CSUM = 1 << 4
};
+struct ibv_data_buf {
+ void *addr;
+ size_t length;
+};
+
struct ibv_sge {
uint64_t addr;
uint32_t length;
@@ -1206,6 +1227,174 @@ struct ibv_qp {
uint32_t events_completed;
};
+struct ibv_qp_ex {
+ struct ibv_qp qp_base;
+ uint64_t comp_mask;
+
+ uint64_t wr_id;
+ /* bitmask from enum ibv_send_flags */
+ unsigned int wr_flags;
+
+ void (*wr_atomic_cmp_swp)(struct ibv_qp_ex *qp, uint32_t rkey,
+ uint64_t remote_addr, uint64_t compare,
+ uint64_t swap);
+ void (*wr_atomic_fetch_add)(struct ibv_qp_ex *qp, uint32_t rkey,
+ uint64_t remote_addr, uint64_t add);
+ void (*wr_bind_mw)(struct ibv_qp_ex *qp, struct ibv_mw *mw,
+ uint32_t rkey,
+ const struct ibv_mw_bind_info *bind_info);
+ void (*wr_local_inv)(struct ibv_qp_ex *qp, uint32_t invalidate_rkey);
+ void (*wr_rdma_read)(struct ibv_qp_ex *qp, uint32_t rkey,
+ uint64_t remote_addr);
+ void (*wr_rdma_write)(struct ibv_qp_ex *qp, uint32_t rkey,
+ uint64_t remote_addr);
+ void (*wr_rdma_write_imm)(struct ibv_qp_ex *qp, uint32_t rkey,
+ uint64_t remote_addr, __be32 imm_data);
+
+ void (*wr_send)(struct ibv_qp_ex *qp);
+ void (*wr_send_imm)(struct ibv_qp_ex *qp, __be32 imm_data);
+ void (*wr_send_inv)(struct ibv_qp_ex *qp, uint32_t invalidate_rkey);
+ void (*wr_send_tso)(struct ibv_qp_ex *qp, void *hdr, uint16_t hdr_sz,
+ uint16_t mss);
+
+ void (*wr_set_ud_addr)(struct ibv_qp_ex *qp, struct ibv_ah *ah,
+ uint32_t remote_qpn, uint32_t remote_qkey);
+ void (*wr_set_xrc_srqn)(struct ibv_qp_ex *qp, uint32_t remote_srqn);
+
+ void (*wr_set_inline_data)(struct ibv_qp_ex *qp, void *addr,
+ size_t length);
+ void (*wr_set_inline_data_list)(struct ibv_qp_ex *qp, size_t num_buf,
+ const struct ibv_data_buf *buf_list);
+ void (*wr_set_sge)(struct ibv_qp_ex *qp, uint32_t lkey, uint64_t addr,
+ uint32_t length);
+ void (*wr_set_sge_list)(struct ibv_qp_ex *qp, size_t num_sge,
+ const struct ibv_sge *sg_list);
+
+ void (*wr_start)(struct ibv_qp_ex *qp);
+ int (*wr_complete)(struct ibv_qp_ex *qp);
+ void (*wr_abort)(struct ibv_qp_ex *qp);
+};
+
+struct ibv_qp_ex *ibv_qp_to_qp_ex(struct ibv_qp *qp);
+
+static inline void ibv_wr_atomic_cmp_swp(struct ibv_qp_ex *qp, uint32_t rkey,
+ uint64_t remote_addr, uint64_t compare,
+ uint64_t swap)
+{
+ qp->wr_atomic_cmp_swp(qp, rkey, remote_addr, compare, swap);
+}
+
+static inline void ibv_wr_atomic_fetch_add(struct ibv_qp_ex *qp, uint32_t rkey,
+ uint64_t remote_addr, uint64_t add)
+{
+ qp->wr_atomic_fetch_add(qp, rkey, remote_addr, add);
+}
+
+static inline void ibv_wr_bind_mw(struct ibv_qp_ex *qp, struct ibv_mw *mw,
+ uint32_t rkey,
+ const struct ibv_mw_bind_info *bind_info)
+{
+ qp->wr_bind_mw(qp, mw, rkey, bind_info);
+}
+
+static inline void ibv_wr_local_inv(struct ibv_qp_ex *qp,
+ uint32_t invalidate_rkey)
+{
+ qp->wr_local_inv(qp, invalidate_rkey);
+}
+
+static inline void ibv_wr_rdma_read(struct ibv_qp_ex *qp, uint32_t rkey,
+ uint64_t remote_addr)
+{
+ qp->wr_rdma_read(qp, rkey, remote_addr);
+}
+
+static inline void ibv_wr_rdma_write(struct ibv_qp_ex *qp, uint32_t rkey,
+ uint64_t remote_addr)
+{
+ qp->wr_rdma_write(qp, rkey, remote_addr);
+}
+
+static inline void ibv_wr_rdma_write_imm(struct ibv_qp_ex *qp, uint32_t rkey,
+ uint64_t remote_addr, __be32 imm_data)
+{
+ qp->wr_rdma_write_imm(qp, rkey, remote_addr, imm_data);
+}
+
+static inline void ibv_wr_send(struct ibv_qp_ex *qp)
+{
+ qp->wr_send(qp);
+}
+
+static inline void ibv_wr_send_imm(struct ibv_qp_ex *qp, __be32 imm_data)
+{
+ qp->wr_send_imm(qp, imm_data);
+}
+
+static inline void ibv_wr_send_inv(struct ibv_qp_ex *qp,
+ uint32_t invalidate_rkey)
+{
+ qp->wr_send_inv(qp, invalidate_rkey);
+}
+
+static inline void ibv_wr_send_tso(struct ibv_qp_ex *qp, void *hdr,
+ uint16_t hdr_sz, uint16_t mss)
+{
+ qp->wr_send_tso(qp, hdr, hdr_sz, mss);
+}
+
+static inline void ibv_wr_set_ud_addr(struct ibv_qp_ex *qp, struct ibv_ah *ah,
+ uint32_t remote_qpn, uint32_t remote_qkey)
+{
+ qp->wr_set_ud_addr(qp, ah, remote_qpn, remote_qkey);
+}
+
+static inline void ibv_wr_set_xrc_srqn(struct ibv_qp_ex *qp,
+ uint32_t remote_srqn)
+{
+ qp->wr_set_xrc_srqn(qp, remote_srqn);
+}
+
+static inline void ibv_wr_set_inline_data(struct ibv_qp_ex *qp, void *addr,
+ size_t length)
+{
+ qp->wr_set_inline_data(qp, addr, length);
+}
+
+static inline void ibv_wr_set_inline_data_list(struct ibv_qp_ex *qp,
+ size_t num_buf,
+ const struct ibv_data_buf *buf_list)
+{
+ qp->wr_set_inline_data_list(qp, num_buf, buf_list);
+}
+
+static inline void ibv_wr_set_sge(struct ibv_qp_ex *qp, uint32_t lkey,
+ uint64_t addr, uint32_t length)
+{
+ qp->wr_set_sge(qp, lkey, addr, length);
+}
+
+static inline void ibv_wr_set_sge_list(struct ibv_qp_ex *qp, size_t num_sge,
+ const struct ibv_sge *sg_list)
+{
+ qp->wr_set_sge_list(qp, num_sge, sg_list);
+}
+
+static inline void ibv_wr_start(struct ibv_qp_ex *qp)
+{
+ qp->wr_start(qp);
+}
+
+static inline int ibv_wr_complete(struct ibv_qp_ex *qp)
+{
+ return qp->wr_complete(qp);
+}
+
+static inline void ibv_wr_abort(struct ibv_qp_ex *qp)
+{
+ qp->wr_abort(qp);
+}
+
struct ibv_comp_channel {
struct ibv_context *context;
int fd;
--
1.8.3.1
