On Tue, Oct 27, 2015 at 6:52 PM, Matan Barak <matanb@xxxxxxxxxxxx> wrote:
> Add an implementation for verb_poll_cq extension verb.
> This patch implements the new API via the standard
> function mlx4_poll_one.
>
> Signed-off-by: Matan Barak <matanb@xxxxxxxxxxxx>
> ---
> src/cq.c | 307 ++++++++++++++++++++++++++++++++++++++++++++++++++++++------
> src/mlx4.c | 1 +
> src/mlx4.h | 4 +
> src/verbs.c | 1 +
> 4 files changed, 284 insertions(+), 29 deletions(-)
>
> diff --git a/src/cq.c b/src/cq.c
> index 32c9070..c86e824 100644
> --- a/src/cq.c
> +++ b/src/cq.c
> @@ -52,6 +52,7 @@ enum {
> };
>
> enum {
> + CQ_CONTINUE = 1,
> CQ_OK = 0,
> CQ_EMPTY = -1,
> CQ_POLL_ERR = -2
> @@ -121,7 +122,9 @@ static void update_cons_index(struct mlx4_cq *cq)
> *cq->set_ci_db = htonl(cq->cons_index & 0xffffff);
> }
>
> -static void mlx4_handle_error_cqe(struct mlx4_err_cqe *cqe, struct ibv_wc *wc)
> +static void mlx4_handle_error_cqe(struct mlx4_err_cqe *cqe,
> + enum ibv_wc_status *status,
> + enum ibv_wc_opcode *vendor_err)
> {
> if (cqe->syndrome == MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR)
> printf(PFX "local QP operation err "
> @@ -133,64 +136,68 @@ static void mlx4_handle_error_cqe(struct mlx4_err_cqe *cqe, struct ibv_wc *wc)
>
> switch (cqe->syndrome) {
> case MLX4_CQE_SYNDROME_LOCAL_LENGTH_ERR:
> - wc->status = IBV_WC_LOC_LEN_ERR;
> + *status = IBV_WC_LOC_LEN_ERR;
> break;
> case MLX4_CQE_SYNDROME_LOCAL_QP_OP_ERR:
> - wc->status = IBV_WC_LOC_QP_OP_ERR;
> + *status = IBV_WC_LOC_QP_OP_ERR;
> break;
> case MLX4_CQE_SYNDROME_LOCAL_PROT_ERR:
> - wc->status = IBV_WC_LOC_PROT_ERR;
> + *status = IBV_WC_LOC_PROT_ERR;
> break;
> case MLX4_CQE_SYNDROME_WR_FLUSH_ERR:
> - wc->status = IBV_WC_WR_FLUSH_ERR;
> + *status = IBV_WC_WR_FLUSH_ERR;
> break;
> case MLX4_CQE_SYNDROME_MW_BIND_ERR:
> - wc->status = IBV_WC_MW_BIND_ERR;
> + *status = IBV_WC_MW_BIND_ERR;
> break;
> case MLX4_CQE_SYNDROME_BAD_RESP_ERR:
> - wc->status = IBV_WC_BAD_RESP_ERR;
> + *status = IBV_WC_BAD_RESP_ERR;
> break;
> case MLX4_CQE_SYNDROME_LOCAL_ACCESS_ERR:
> - wc->status = IBV_WC_LOC_ACCESS_ERR;
> + *status = IBV_WC_LOC_ACCESS_ERR;
> break;
> case MLX4_CQE_SYNDROME_REMOTE_INVAL_REQ_ERR:
> - wc->status = IBV_WC_REM_INV_REQ_ERR;
> + *status = IBV_WC_REM_INV_REQ_ERR;
> break;
> case MLX4_CQE_SYNDROME_REMOTE_ACCESS_ERR:
> - wc->status = IBV_WC_REM_ACCESS_ERR;
> + *status = IBV_WC_REM_ACCESS_ERR;
> break;
> case MLX4_CQE_SYNDROME_REMOTE_OP_ERR:
> - wc->status = IBV_WC_REM_OP_ERR;
> + *status = IBV_WC_REM_OP_ERR;
> break;
> case MLX4_CQE_SYNDROME_TRANSPORT_RETRY_EXC_ERR:
> - wc->status = IBV_WC_RETRY_EXC_ERR;
> + *status = IBV_WC_RETRY_EXC_ERR;
> break;
> case MLX4_CQE_SYNDROME_RNR_RETRY_EXC_ERR:
> - wc->status = IBV_WC_RNR_RETRY_EXC_ERR;
> + *status = IBV_WC_RNR_RETRY_EXC_ERR;
> break;
> case MLX4_CQE_SYNDROME_REMOTE_ABORTED_ERR:
> - wc->status = IBV_WC_REM_ABORT_ERR;
> + *status = IBV_WC_REM_ABORT_ERR;
> break;
> default:
> - wc->status = IBV_WC_GENERAL_ERR;
> + *status = IBV_WC_GENERAL_ERR;
> break;
> }
>
> - wc->vendor_err = cqe->vendor_err;
> + *vendor_err = cqe->vendor_err;
> }
>
> -static int mlx4_poll_one(struct mlx4_cq *cq,
> - struct mlx4_qp **cur_qp,
> - struct ibv_wc *wc)
> +static inline int mlx4_handle_cq(struct mlx4_cq *cq,
> + struct mlx4_qp **cur_qp,
> + uint64_t *wc_wr_id,
> + enum ibv_wc_status *wc_status,
> + uint32_t *wc_vendor_err,
> + struct mlx4_cqe **pcqe,
> + uint32_t *pqpn,
> + int *pis_send)
> {
> struct mlx4_wq *wq;
> struct mlx4_cqe *cqe;
> struct mlx4_srq *srq;
> uint32_t qpn;
> - uint32_t g_mlpath_rqpn;
> - uint16_t wqe_index;
> int is_error;
> int is_send;
> + uint16_t wqe_index;
>
> cqe = next_cqe_sw(cq);
> if (!cqe)
> @@ -201,7 +208,7 @@ static int mlx4_poll_one(struct mlx4_cq *cq,
>
> ++cq->cons_index;
>
> - VALGRIND_MAKE_MEM_DEFINED(cqe, sizeof *cqe);
> + VALGRIND_MAKE_MEM_DEFINED(cqe, sizeof(*cqe));
>
> /*
> * Make sure we read CQ entry contents after we've checked the
> @@ -210,7 +217,6 @@ static int mlx4_poll_one(struct mlx4_cq *cq,
> rmb();
>
> qpn = ntohl(cqe->vlan_my_qpn) & MLX4_CQE_QPN_MASK;
> - wc->qp_num = qpn;
>
> is_send = cqe->owner_sr_opcode & MLX4_CQE_IS_SEND_MASK;
> is_error = (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) ==
> @@ -243,26 +249,50 @@ static int mlx4_poll_one(struct mlx4_cq *cq,
> if (is_send) {
> wq = &(*cur_qp)->sq;
> wqe_index = ntohs(cqe->wqe_index);
> - wq->tail += (uint16_t) (wqe_index - (uint16_t) wq->tail);
> - wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
> + wq->tail += (uint16_t)(wqe_index - (uint16_t)wq->tail);
> + *wc_wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
> ++wq->tail;
> } else if (srq) {
> wqe_index = htons(cqe->wqe_index);
> - wc->wr_id = srq->wrid[wqe_index];
> + *wc_wr_id = srq->wrid[wqe_index];
> mlx4_free_srq_wqe(srq, wqe_index);
> } else {
> wq = &(*cur_qp)->rq;
> - wc->wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
> + *wc_wr_id = wq->wrid[wq->tail & (wq->wqe_cnt - 1)];
> ++wq->tail;
> }
>
> if (is_error) {
> - mlx4_handle_error_cqe((struct mlx4_err_cqe *) cqe, wc);
> + mlx4_handle_error_cqe((struct mlx4_err_cqe *)cqe,
> + wc_status, wc_vendor_err);
> return CQ_OK;
> }
>
> - wc->status = IBV_WC_SUCCESS;
> + *wc_status = IBV_WC_SUCCESS;
>
> + *pcqe = cqe;
> + *pqpn = qpn;
> + *pis_send = is_send;
> +
> + return CQ_CONTINUE;
> +}
> +
> +static int mlx4_poll_one(struct mlx4_cq *cq,
> + struct mlx4_qp **cur_qp,
> + struct ibv_wc *wc)
> +{
> + struct mlx4_cqe *cqe;
> + uint32_t qpn;
> + uint32_t g_mlpath_rqpn;
> + int is_send;
> + int err;
> +
> + err = mlx4_handle_cq(cq, cur_qp, &wc->wr_id, &wc->status,
> + &wc->vendor_err, &cqe, &qpn, &is_send);
> + if (err != CQ_CONTINUE)
> + return err;
> +
> + wc->qp_num = qpn;
> if (is_send) {
> wc->wc_flags = 0;
> switch (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) {
> @@ -340,6 +370,195 @@ static int mlx4_poll_one(struct mlx4_cq *cq,
> return CQ_OK;
> }
>
> +union wc_buffer {
> + uint8_t *b8;
> + uint16_t *b16;
> + uint32_t *b32;
> + uint64_t *b64;
> +};
> +
> +static inline int _mlx4_poll_one_ex(struct mlx4_cq *cq,
> + struct mlx4_qp **cur_qp,
> + struct ibv_wc_ex **pwc_ex,
> + uint64_t wc_flags)
> +{
> + struct mlx4_cqe *cqe;
> + uint32_t qpn;
> + uint32_t g_mlpath_rqpn;
> + int is_send;
> + struct ibv_wc_ex *wc_ex = *pwc_ex;
> + union wc_buffer wc_buffer;
> + int err;
> + uint64_t wc_flags_out = 0;
> +
> + wc_buffer.b64 = (uint64_t *)&wc_ex->buffer;
> + wc_ex->wc_flags = 0;
> + wc_ex->reserved = 0;
> + err = mlx4_handle_cq(cq, cur_qp, &wc_ex->wr_id, &wc_ex->status,
> + &wc_ex->vendor_err, &cqe, &qpn, &is_send);
> + if (err != CQ_CONTINUE)
> + return err;
> +
> + if (is_send) {
> + switch (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) {
> + case MLX4_OPCODE_RDMA_WRITE_IMM:
> + wc_flags_out |= IBV_WC_EX_IMM;
> + case MLX4_OPCODE_RDMA_WRITE:
> + wc_ex->opcode = IBV_WC_RDMA_WRITE;
> + if (wc_flags & IBV_WC_EX_WITH_BYTE_LEN)
> + wc_buffer.b32++;
> + if (wc_flags & IBV_WC_EX_WITH_IMM)
> + wc_buffer.b32++;
> + break;
> + case MLX4_OPCODE_SEND_IMM:
> + wc_flags_out |= IBV_WC_EX_IMM;
> + case MLX4_OPCODE_SEND:
> + wc_ex->opcode = IBV_WC_SEND;
> + if (wc_flags & IBV_WC_EX_WITH_BYTE_LEN)
> + wc_buffer.b32++;
> + if (wc_flags & IBV_WC_EX_WITH_IMM)
> + wc_buffer.b32++;
> + break;
> + case MLX4_OPCODE_RDMA_READ:
> + wc_ex->opcode = IBV_WC_RDMA_READ;
> + if (wc_flags & IBV_WC_EX_WITH_BYTE_LEN) {
> + *wc_buffer.b32++ = ntohl(cqe->byte_cnt);
> + wc_flags_out |= IBV_WC_EX_WITH_BYTE_LEN;
> + }
> + if (wc_flags & IBV_WC_EX_WITH_IMM)
> + wc_buffer.b32++;
> + break;
> + case MLX4_OPCODE_ATOMIC_CS:
> + wc_ex->opcode = IBV_WC_COMP_SWAP;
> + if (wc_flags & IBV_WC_EX_WITH_BYTE_LEN) {
> + *wc_buffer.b32++ = 8;
> + wc_flags_out |= IBV_WC_EX_WITH_BYTE_LEN;
> + }
> + if (wc_flags & IBV_WC_EX_WITH_IMM)
> + wc_buffer.b32++;
> + break;
> + case MLX4_OPCODE_ATOMIC_FA:
> + wc_ex->opcode = IBV_WC_FETCH_ADD;
> + if (wc_flags & IBV_WC_EX_WITH_BYTE_LEN) {
> + *wc_buffer.b32++ = 8;
> + wc_flags_out |= IBV_WC_EX_WITH_BYTE_LEN;
> + }
> + if (wc_flags & IBV_WC_EX_WITH_IMM)
> + wc_buffer.b32++;
> + break;
> + case MLX4_OPCODE_BIND_MW:
> + wc_ex->opcode = IBV_WC_BIND_MW;
> + if (wc_flags & IBV_WC_EX_WITH_BYTE_LEN)
> + wc_buffer.b32++;
> + if (wc_flags & IBV_WC_EX_WITH_IMM)
> + wc_buffer.b32++;
> + break;
> + default:
> + /* assume it's a send completion */
> + wc_ex->opcode = IBV_WC_SEND;
> + if (wc_flags & IBV_WC_EX_WITH_BYTE_LEN)
> + wc_buffer.b32++;
> + if (wc_flags & IBV_WC_EX_WITH_IMM)
> + wc_buffer.b32++;
> + break;
> + }
> +
> + if (wc_flags & IBV_WC_EX_WITH_QP_NUM) {
> + *wc_buffer.b32++ = qpn;
> + wc_flags_out |= IBV_WC_EX_WITH_QP_NUM;
> + }
> + if (wc_flags & IBV_WC_EX_WITH_SRC_QP)
> + wc_buffer.b32++;
> + if (wc_flags & IBV_WC_EX_WITH_PKEY_INDEX)
> + wc_buffer.b16++;
> + if (wc_flags & IBV_WC_EX_WITH_SLID)
> + wc_buffer.b16++;
> + if (wc_flags & IBV_WC_EX_WITH_SL)
> + wc_buffer.b8++;
> + if (wc_flags & IBV_WC_EX_WITH_DLID_PATH_BITS)
> + wc_buffer.b8++;
> + } else {
> + if (wc_flags & IBV_WC_EX_WITH_BYTE_LEN) {
> + *wc_buffer.b32++ = ntohl(cqe->byte_cnt);
> + wc_flags_out |= IBV_WC_EX_WITH_BYTE_LEN;
> + }
> +
> + switch (cqe->owner_sr_opcode & MLX4_CQE_OPCODE_MASK) {
> + case MLX4_RECV_OPCODE_RDMA_WRITE_IMM:
> + wc_ex->opcode = IBV_WC_RECV_RDMA_WITH_IMM;
> + wc_flags_out |= IBV_WC_EX_IMM;
> + if (wc_flags & IBV_WC_EX_WITH_IMM) {
> + *wc_buffer.b32++ = cqe->immed_rss_invalid;
> + wc_flags_out |= IBV_WC_EX_WITH_IMM;
> + }
> + break;
> + case MLX4_RECV_OPCODE_SEND:
> + wc_ex->opcode = IBV_WC_RECV;
> + if (wc_flags & IBV_WC_EX_WITH_IMM)
> + wc_buffer.b32++;
> + break;
> + case MLX4_RECV_OPCODE_SEND_IMM:
> + wc_ex->opcode = IBV_WC_RECV;
> + wc_flags_out |= IBV_WC_EX_IMM;
> + if (wc_flags & IBV_WC_EX_WITH_IMM) {
> + *wc_buffer.b32++ = cqe->immed_rss_invalid;
> + wc_flags_out |= IBV_WC_EX_WITH_IMM;
> + }
> + break;
> + }
> +
> + if (wc_flags & IBV_WC_EX_WITH_QP_NUM) {
> + *wc_buffer.b32++ = qpn;
> + wc_flags_out |= IBV_WC_EX_WITH_QP_NUM;
> + }
> + g_mlpath_rqpn = ntohl(cqe->g_mlpath_rqpn);
> + if (wc_flags & IBV_WC_EX_WITH_SRC_QP) {
> + *wc_buffer.b32++ = g_mlpath_rqpn & 0xffffff;
> + wc_flags_out |= IBV_WC_EX_WITH_SRC_QP;
> + }
> + if (wc_flags & IBV_WC_EX_WITH_PKEY_INDEX) {
> + *wc_buffer.b16++ = ntohl(cqe->immed_rss_invalid) & 0x7f;
> + wc_flags_out |= IBV_WC_EX_WITH_PKEY_INDEX;
> + }
> + if (wc_flags & IBV_WC_EX_WITH_SLID) {
> + *wc_buffer.b16++ = ntohs(cqe->rlid);
> + wc_flags_out |= IBV_WC_EX_WITH_SLID;
> + }
> + if (wc_flags & IBV_WC_EX_WITH_SL) {
> + wc_flags_out |= IBV_WC_EX_WITH_SL;
> + if ((*cur_qp) && (*cur_qp)->link_layer == IBV_LINK_LAYER_ETHERNET)
> + *wc_buffer.b8++ = ntohs(cqe->sl_vid) >> 13;
> + else
> + *wc_buffer.b8++ = ntohs(cqe->sl_vid) >> 12;
> + }
> + if (wc_flags & IBV_WC_EX_WITH_DLID_PATH_BITS) {
> + *wc_buffer.b8++ = (g_mlpath_rqpn >> 24) & 0x7f;
> + wc_flags_out |= IBV_WC_EX_WITH_DLID_PATH_BITS;
> + }
> + wc_flags_out |= g_mlpath_rqpn & 0x80000000 ? IBV_WC_EX_GRH : 0;
> + /* When working with xrc srqs, don't have qp to check link layer.
> + * Using IB SL, should consider Roce. (TBD)
> + */
> + }
> +
> + wc_ex->wc_flags = wc_flags_out;
> + /* Align the WC ex to the next 64bit. This is mandatory as ibv_wc_ex is
> + * 64bit aligned. pwc_ex is used to write to the next wc and thus we
> + * need to align it.
> + */
> + *pwc_ex = (struct ibv_wc_ex *)((uintptr_t)(wc_buffer.b8 + sizeof(uint64_t) - 1) &
> + ~(sizeof(uint64_t) - 1));
> +
> + return CQ_OK;
> +}
> +
> +int mlx4_poll_one_ex(struct mlx4_cq *cq,
> + struct mlx4_qp **cur_qp,
> + struct ibv_wc_ex **pwc_ex)
> +{
> + return _mlx4_poll_one_ex(cq, cur_qp, pwc_ex, cq->wc_flags);
> +}
> +
> int mlx4_poll_cq(struct ibv_cq *ibcq, int ne, struct ibv_wc *wc)
> {
> struct mlx4_cq *cq = to_mcq(ibcq);
> @@ -363,6 +582,36 @@ int mlx4_poll_cq(struct ibv_cq *ibcq, int ne, struct ibv_wc *wc)
> return err == CQ_POLL_ERR ? err : npolled;
> }
>
> +int mlx4_poll_cq_ex(struct ibv_cq *ibcq,
> + struct ibv_wc_ex *wc,
> + struct ibv_poll_cq_ex_attr *attr)
> +{
> + struct mlx4_cq *cq = to_mcq(ibcq);
> + struct mlx4_qp *qp = NULL;
> + int npolled;
> + int err = CQ_OK;
> + unsigned int ne = attr->max_entries;
> + uint64_t wc_flags = cq->wc_flags;
> +
> + if (attr->comp_mask)
> + return -EINVAL;
> +
> + pthread_spin_lock(&cq->lock);
> +
> + for (npolled = 0; npolled < ne; ++npolled) {
> + err = _mlx4_poll_one_ex(cq, &qp, &wc, wc_flags);
> + if (err != CQ_OK)
> + break;
> + }
> +
> + if (npolled || err == CQ_POLL_ERR)
> + update_cons_index(cq);
> +
> + pthread_spin_unlock(&cq->lock);
> +
> + return err == CQ_POLL_ERR ? err : npolled;
> +}
> +
> int mlx4_arm_cq(struct ibv_cq *ibvcq, int solicited)
> {
> struct mlx4_cq *cq = to_mcq(ibvcq);
> diff --git a/src/mlx4.c b/src/mlx4.c
> index 9cfd013..cc1211f 100644
> --- a/src/mlx4.c
> +++ b/src/mlx4.c
> @@ -209,6 +209,7 @@ static int mlx4_init_context(struct verbs_device *v_device,
> verbs_set_ctx_op(verbs_ctx, ibv_destroy_flow, ibv_cmd_destroy_flow);
> verbs_set_ctx_op(verbs_ctx, query_device_ex, mlx4_query_device_ex);
> verbs_set_ctx_op(verbs_ctx, create_cq_ex, mlx4_create_cq_ex);
> + verbs_set_ctx_op(verbs_ctx, poll_cq_ex, mlx4_poll_cq_ex);
>
> return 0;
>
> diff --git a/src/mlx4.h b/src/mlx4.h
> index 91eb79c..e22f879 100644
> --- a/src/mlx4.h
> +++ b/src/mlx4.h
> @@ -213,6 +213,7 @@ struct mlx4_pd {
>
> struct mlx4_cq {
> struct ibv_cq ibv_cq;
> + uint64_t wc_flags;
> struct mlx4_buf buf;
> struct mlx4_buf resize_buf;
> pthread_spinlock_t lock;
> @@ -410,6 +411,9 @@ int mlx4_alloc_cq_buf(struct mlx4_device *dev, struct mlx4_buf *buf, int nent,
> int mlx4_resize_cq(struct ibv_cq *cq, int cqe);
> int mlx4_destroy_cq(struct ibv_cq *cq);
> int mlx4_poll_cq(struct ibv_cq *cq, int ne, struct ibv_wc *wc);
> +int mlx4_poll_cq_ex(struct ibv_cq *ibcq,
> + struct ibv_wc_ex *wc,
> + struct ibv_poll_cq_ex_attr *attr);
> int mlx4_arm_cq(struct ibv_cq *cq, int solicited);
> void mlx4_cq_event(struct ibv_cq *cq);
> void __mlx4_cq_clean(struct mlx4_cq *cq, uint32_t qpn, struct mlx4_srq *srq);
> diff --git a/src/verbs.c b/src/verbs.c
> index 3290b86..0dcdc87 100644
> --- a/src/verbs.c
> +++ b/src/verbs.c
> @@ -387,6 +387,7 @@ static struct ibv_cq *create_cq(struct ibv_context *context,
> goto err_db;
>
> cq->creation_flags = cmd_e.ibv_cmd.flags;
> + cq->wc_flags = cq_attr->wc_flags;
> cq->cqn = resp.cqn;
>
> return &cq->ibv_cq;
> --
> 2.1.0
>
> --
> To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
> the body of a message to majordomo@xxxxxxxxxxxxxxx
> More majordomo info at http://vger.kernel.org/majordomo-info.html
This should have libmlx4 prefix.
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
