On Wed, May 13, 2020 at 02:52:40PM +0300, Yamin Friedman wrote:
> A pre-step for adding shared CQs. Add the infrastructure to prevent
> shared CQ users from altering the CQ configurations. For now all cqs are
> marked as private (non-shared). The core driver should use the new force
> functions to perform resize/destroy/moderation changes that are not
> allowed for users of shared CQs.
>
> Signed-off-by: Yamin Friedman <yaminf@xxxxxxxxxxxx>
> Reviewed-by: Or Gerlitz <ogerlitz@xxxxxxxxxxxx>
> Reviewed-by: Max Gurtovoy <maxg@xxxxxxxxxxxx>
> ---
> drivers/infiniband/core/cq.c | 18 ++++++++++++------
> drivers/infiniband/core/verbs.c | 9 +++++++++
> include/rdma/ib_verbs.h | 8 ++++++++
> 3 files changed, 29 insertions(+), 6 deletions(-)
>
> diff --git a/drivers/infiniband/core/cq.c b/drivers/infiniband/core/cq.c
> index 4f25b24..04046eb 100644
> --- a/drivers/infiniband/core/cq.c
> +++ b/drivers/infiniband/core/cq.c
> @@ -300,12 +300,7 @@ struct ib_cq *__ib_alloc_cq_any(struct ib_device *dev, void *private,
> }
> EXPORT_SYMBOL(__ib_alloc_cq_any);
>
> -/**
> - * ib_free_cq_user - free a completion queue
> - * @cq: completion queue to free.
> - * @udata: User data or NULL for kernel object
> - */
> -void ib_free_cq_user(struct ib_cq *cq, struct ib_udata *udata)
> +static void _ib_free_cq_user(struct ib_cq *cq, struct ib_udata *udata)
> {
> if (WARN_ON_ONCE(atomic_read(&cq->usecnt)))
> return;
> @@ -333,4 +328,15 @@ void ib_free_cq_user(struct ib_cq *cq, struct ib_udata *udata)
> kfree(cq->wc);
> kfree(cq);
> }
> +
> +/**
> + * ib_free_cq_user - free a completion queue
> + * @cq: completion queue to free.
> + * @udata: User data or NULL for kernel object
> + */
> +void ib_free_cq_user(struct ib_cq *cq, struct ib_udata *udata)
> +{
> + if (!cq->shared)
> + _ib_free_cq_user(cq, udata);
> +}
> EXPORT_SYMBOL(ib_free_cq_user);
All CQs from the pool will have "shared" boolean set and you don't need
to create an extra wrapper, simply put this "if (cq->shared) return;"
line in original ib_free_cq_user(). Just don't forget to set "shared"
to be false in ib_cq_pool_destroy().
> diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c
> index bf0249f..d1bacd7 100644
> --- a/drivers/infiniband/core/verbs.c
> +++ b/drivers/infiniband/core/verbs.c
> @@ -1990,6 +1990,9 @@ struct ib_cq *__ib_create_cq(struct ib_device *device,
>
> int rdma_set_cq_moderation(struct ib_cq *cq, u16 cq_count, u16 cq_period)
> {
> + if (cq->shared)
> + return -EOPNOTSUPP;
> +
> return cq->device->ops.modify_cq ?
> cq->device->ops.modify_cq(cq, cq_count,
> cq_period) : -EOPNOTSUPP;
> @@ -1998,6 +2001,9 @@ int rdma_set_cq_moderation(struct ib_cq *cq, u16 cq_count, u16 cq_period)
>
> int ib_destroy_cq_user(struct ib_cq *cq, struct ib_udata *udata)
> {
> + if (cq->shared)
> + return -EOPNOTSUPP;
> +
> if (atomic_read(&cq->usecnt))
> return -EBUSY;
>
> @@ -2010,6 +2016,9 @@ int ib_destroy_cq_user(struct ib_cq *cq, struct ib_udata *udata)
>
> int ib_resize_cq(struct ib_cq *cq, int cqe)
> {
> + if (cq->shared)
> + return -EOPNOTSUPP;
> +
> return cq->device->ops.resize_cq ?
> cq->device->ops.resize_cq(cq, cqe, NULL) : -EOPNOTSUPP;
> }
> diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
> index 4c488ca..b79737b 100644
> --- a/include/rdma/ib_verbs.h
> +++ b/include/rdma/ib_verbs.h
> @@ -1582,6 +1582,7 @@ struct ib_cq {
> * Implementation details of the RDMA core, don't use in drivers:
> */
> struct rdma_restrack_entry res;
> + bool shared;
Lets do "u8 shared:1;" from the beginning.
> };
>
> struct ib_srq {
> @@ -3824,6 +3825,8 @@ static inline struct ib_cq *ib_alloc_cq_any(struct ib_device *dev,
> * ib_free_cq_user - Free kernel/user CQ
> * @cq: The CQ to free
> * @udata: Valid user data or NULL for kernel objects
> + *
> + * NOTE: this will fail for shared cqs
> */
> void ib_free_cq_user(struct ib_cq *cq, struct ib_udata *udata);
>
> @@ -3832,6 +3835,7 @@ static inline struct ib_cq *ib_alloc_cq_any(struct ib_device *dev,
> * @cq: The CQ to free
> *
> * NOTE: for user cq use ib_free_cq_user with valid udata!
> + * NOTE: this will fail for shared cqs
> */
> static inline void ib_free_cq(struct ib_cq *cq)
> {
> @@ -3868,6 +3872,7 @@ struct ib_cq *__ib_create_cq(struct ib_device *device,
> * @cqe: The minimum size of the CQ.
> *
> * Users can examine the cq structure to determine the actual CQ size.
> + * NOTE: Will fail for shared CQs.
> */
> int ib_resize_cq(struct ib_cq *cq, int cqe);
>
> @@ -3877,6 +3882,7 @@ struct ib_cq *__ib_create_cq(struct ib_device *device,
> * @cq_count: number of CQEs that will trigger an event
> * @cq_period: max period of time in usec before triggering an event
> *
> + * NOTE: Will fail for shared CQs.
> */
> int rdma_set_cq_moderation(struct ib_cq *cq, u16 cq_count, u16 cq_period);
>
> @@ -3884,6 +3890,8 @@ struct ib_cq *__ib_create_cq(struct ib_device *device,
> * ib_destroy_cq_user - Destroys the specified CQ.
> * @cq: The CQ to destroy.
> * @udata: Valid user data or NULL for kernel objects
> + *
> + * NOTE: Will fail for shared CQs.
Let's add WARN_ON_ONCE() to catch mistakes. No one should call to
ib_destroy_cq_user() for the shared CQs.
> */
> int ib_destroy_cq_user(struct ib_cq *cq, struct ib_udata *udata);
>
> --
> 1.8.3.1
>
