In rdma-core mlx5 there's a lock over the srq_table. See below the mlx5_store_srq():
pthread_mutex_lock(&ctx->srq_table_mutex);
ret = ibv_cmd_create_srq(pd, ibsrq, attr, &cmd.ibv_cmd, sizeof(cmd),
&resp.ibv_resp, sizeof(resp));
if (ret)
goto err_db;
ret = mlx5_store_srq(ctx, resp.srqn, srq);
if (ret)
goto err_destroy;
pthread_mutex_unlock(&ctx->srq_table_mutex);
For some reason, mlx5_clear_srq() isn't invoked under a mutex lock:
int mlx5_destroy_srq(struct ibv_srq *srq)
{
int ret;
struct mlx5_srq *msrq = to_msrq(srq);
struct mlx5_context *ctx = to_mctx(srq->context);
if (msrq->cmd_qp) {
ret = mlx5_destroy_qp(msrq->cmd_qp);
if (ret)
return ret;
msrq->cmd_qp = NULL;
}
ret = ibv_cmd_destroy_srq(srq);
if (ret)
return ret;
if (ctx->cqe_version && msrq->rsc.type == MLX5_RSC_TYPE_XSRQ)
mlx5_clear_uidx(ctx, msrq->rsc.rsn);
else
mlx5_clear_srq(ctx, msrq->srqn);
mlx5_free_db(ctx, msrq->db);
mlx5_free_buf(&msrq->buf);
free(msrq->tm_list);
free(msrq->wrid);
free(msrq->op);
free(msrq);
return 0;
}
The mlx5_find_srq() isn't protected as well. This is fast path.
You cannot promise that poll_cq for a relevant CQE won't be invoked during or after the XRC SRQ removal.
This calls for something like CQEs that are being removed/marked as invalid following the QP is destruction.
Cheers,
Ram
--
To unsubscribe from this list: send the line "unsubscribe linux-rdma" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
