On 5/18/2020 10:54 AM, Leon Romanovsky wrote:
On Wed, May 13, 2020 at 02:52:40PM +0300, Yamin Friedman wrote:A pre-step for adding shared CQs. Add the infrastructure to prevent shared CQ users from altering the CQ configurations. For now all cqs are marked as private (non-shared). The core driver should use the new force functions to perform resize/destroy/moderation changes that are not allowed for users of shared CQs. Signed-off-by: Yamin Friedman <yaminf@xxxxxxxxxxxx> Reviewed-by: Or Gerlitz <ogerlitz@xxxxxxxxxxxx> Reviewed-by: Max Gurtovoy <maxg@xxxxxxxxxxxx> --- drivers/infiniband/core/cq.c | 18 ++++++++++++------ drivers/infiniband/core/verbs.c | 9 +++++++++ include/rdma/ib_verbs.h | 8 ++++++++ 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/drivers/infiniband/core/cq.c b/drivers/infiniband/core/cq.c index 4f25b24..04046eb 100644 --- a/drivers/infiniband/core/cq.c +++ b/drivers/infiniband/core/cq.c @@ -300,12 +300,7 @@ struct ib_cq *__ib_alloc_cq_any(struct ib_device *dev, void *private, } EXPORT_SYMBOL(__ib_alloc_cq_any); -/** - * ib_free_cq_user - free a completion queue - * @cq: completion queue to free. - * @udata: User data or NULL for kernel object - */ -void ib_free_cq_user(struct ib_cq *cq, struct ib_udata *udata) +static void _ib_free_cq_user(struct ib_cq *cq, struct ib_udata *udata) { if (WARN_ON_ONCE(atomic_read(&cq->usecnt))) return; @@ -333,4 +328,15 @@ void ib_free_cq_user(struct ib_cq *cq, struct ib_udata *udata) kfree(cq->wc); kfree(cq); } + +/** + * ib_free_cq_user - free a completion queue + * @cq: completion queue to free. + * @udata: User data or NULL for kernel object + */ +void ib_free_cq_user(struct ib_cq *cq, struct ib_udata *udata) +{ + if (!cq->shared) + _ib_free_cq_user(cq, udata); +} EXPORT_SYMBOL(ib_free_cq_user);All CQs from the pool will have "shared" boolean set and you don't need to create an extra wrapper, simply put this "if (cq->shared) return;" line in original ib_free_cq_user(). Just don't forget to set "shared" to be false in ib_cq_pool_destroy().
Makes sense.
diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c index bf0249f..d1bacd7 100644 --- a/drivers/infiniband/core/verbs.c +++ b/drivers/infiniband/core/verbs.c @@ -1990,6 +1990,9 @@ struct ib_cq *__ib_create_cq(struct ib_device *device, int rdma_set_cq_moderation(struct ib_cq *cq, u16 cq_count, u16 cq_period) { + if (cq->shared) + return -EOPNOTSUPP; + return cq->device->ops.modify_cq ? cq->device->ops.modify_cq(cq, cq_count, cq_period) : -EOPNOTSUPP; @@ -1998,6 +2001,9 @@ int rdma_set_cq_moderation(struct ib_cq *cq, u16 cq_count, u16 cq_period) int ib_destroy_cq_user(struct ib_cq *cq, struct ib_udata *udata) { + if (cq->shared) + return -EOPNOTSUPP; + if (atomic_read(&cq->usecnt)) return -EBUSY; @@ -2010,6 +2016,9 @@ int ib_destroy_cq_user(struct ib_cq *cq, struct ib_udata *udata) int ib_resize_cq(struct ib_cq *cq, int cqe) { + if (cq->shared) + return -EOPNOTSUPP; + return cq->device->ops.resize_cq ? cq->device->ops.resize_cq(cq, cqe, NULL) : -EOPNOTSUPP; } diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h index 4c488ca..b79737b 100644 --- a/include/rdma/ib_verbs.h +++ b/include/rdma/ib_verbs.h @@ -1582,6 +1582,7 @@ struct ib_cq { * Implementation details of the RDMA core, don't use in drivers: */ struct rdma_restrack_entry res; + bool shared;Lets do "u8 shared:1;" from the beginning.
Not sure what you mean "from the beginning".
}; struct ib_srq { @@ -3824,6 +3825,8 @@ static inline struct ib_cq *ib_alloc_cq_any(struct ib_device *dev, * ib_free_cq_user - Free kernel/user CQ * @cq: The CQ to free * @udata: Valid user data or NULL for kernel objects + * + * NOTE: this will fail for shared cqs */ void ib_free_cq_user(struct ib_cq *cq, struct ib_udata *udata); @@ -3832,6 +3835,7 @@ static inline struct ib_cq *ib_alloc_cq_any(struct ib_device *dev, * @cq: The CQ to free * * NOTE: for user cq use ib_free_cq_user with valid udata! + * NOTE: this will fail for shared cqs */ static inline void ib_free_cq(struct ib_cq *cq) { @@ -3868,6 +3872,7 @@ struct ib_cq *__ib_create_cq(struct ib_device *device, * @cqe: The minimum size of the CQ. * * Users can examine the cq structure to determine the actual CQ size. + * NOTE: Will fail for shared CQs. */ int ib_resize_cq(struct ib_cq *cq, int cqe); @@ -3877,6 +3882,7 @@ struct ib_cq *__ib_create_cq(struct ib_device *device, * @cq_count: number of CQEs that will trigger an event * @cq_period: max period of time in usec before triggering an event * + * NOTE: Will fail for shared CQs. */ int rdma_set_cq_moderation(struct ib_cq *cq, u16 cq_count, u16 cq_period); @@ -3884,6 +3890,8 @@ struct ib_cq *__ib_create_cq(struct ib_device *device, * ib_destroy_cq_user - Destroys the specified CQ. * @cq: The CQ to destroy. * @udata: Valid user data or NULL for kernel objects + * + * NOTE: Will fail for shared CQs.Let's add WARN_ON_ONCE() to catch mistakes. No one should call to ib_destroy_cq_user() for the shared CQs.
Alright.
*/ int ib_destroy_cq_user(struct ib_cq *cq, struct ib_udata *udata); -- 1.8.3.1
