On Thu, Aug 29, 2024 at 4:33 PM Leon Romanovsky <leon@xxxxxxxxxx> wrote:
>
> On Wed, Aug 28, 2024 at 01:47:12AM -0700, Selvin Xavier wrote:
> > From: Chandramohan Akula <chandramohan.akula@xxxxxxxxxxxx>
> >
> > Gen P7 adapters needs to share a toggle bits information received
> > in kernel driver with the user space. User space needs this
> > info to arm the SRQ.
> >
> > User space application can get this page using the
> > UAPI routines. Library will mmap this page and get the
> > toggle bits to be used in the next ARM Doorbell.
> >
> > Uses a hash list to map the SRQ structure from the SRQ ID.
> > SRQ structure is retrieved from the hash list while the
> > library calls the UAPI routine to get the toggle page
> > mapping. Currently the full page is mapped per SRQ. This
> > can be optimized to enable multiple SRQs from the same
> > application share the same page and different offsets
> > in the page
> >
> > Signed-off-by: Chandramohan Akula <chandramohan.akula@xxxxxxxxxxxx>
> > Signed-off-by: Selvin Xavier <selvin.xavier@xxxxxxxxxxxx>
> > ---
> > drivers/infiniband/hw/bnxt_re/bnxt_re.h | 2 ++
> > drivers/infiniband/hw/bnxt_re/ib_verbs.c | 34 +++++++++++++++++++++++++++++++-
> > drivers/infiniband/hw/bnxt_re/ib_verbs.h | 1 +
> > drivers/infiniband/hw/bnxt_re/main.c | 6 +++++-
> > include/uapi/rdma/bnxt_re-abi.h | 5 +++++
> > 5 files changed, 46 insertions(+), 2 deletions(-)
> >
> > diff --git a/drivers/infiniband/hw/bnxt_re/bnxt_re.h b/drivers/infiniband/hw/bnxt_re/bnxt_re.h
> > index 0912d2f..2be9a62 100644
> > --- a/drivers/infiniband/hw/bnxt_re/bnxt_re.h
> > +++ b/drivers/infiniband/hw/bnxt_re/bnxt_re.h
> > @@ -141,6 +141,7 @@ struct bnxt_re_pacing {
> > #define BNXT_RE_GRC_FIFO_REG_BASE 0x2000
> >
> > #define MAX_CQ_HASH_BITS (16)
> > +#define MAX_SRQ_HASH_BITS (16)
> > struct bnxt_re_dev {
> > struct ib_device ibdev;
> > struct list_head list;
> > @@ -196,6 +197,7 @@ struct bnxt_re_dev {
> > struct work_struct dbq_fifo_check_work;
> > struct delayed_work dbq_pacing_work;
> > DECLARE_HASHTABLE(cq_hash, MAX_CQ_HASH_BITS);
> > + DECLARE_HASHTABLE(srq_hash, MAX_SRQ_HASH_BITS);
> > };
> >
> > #define to_bnxt_re_dev(ptr, member) \
> > diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
> > index 1e76093..0219c8a 100644
> > --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c
> > +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c
> > @@ -1685,6 +1685,10 @@ int bnxt_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata)
> >
> > if (qplib_srq->cq)
> > nq = qplib_srq->cq->nq;
> > + if (rdev->chip_ctx->modes.toggle_bits & BNXT_QPLIB_SRQ_TOGGLE_BIT) {
> > + free_page((unsigned long)srq->uctx_srq_page);
> > + hash_del(&srq->hash_entry);
> > + }
> > bnxt_qplib_destroy_srq(&rdev->qplib_res, qplib_srq);
> > ib_umem_release(srq->umem);
> > atomic_dec(&rdev->stats.res.srq_count);
> > @@ -1789,9 +1793,18 @@ int bnxt_re_create_srq(struct ib_srq *ib_srq,
> > }
> >
> > if (udata) {
> > - struct bnxt_re_srq_resp resp;
> > + struct bnxt_re_srq_resp resp = {};
> >
> > resp.srqid = srq->qplib_srq.id;
> > + if (rdev->chip_ctx->modes.toggle_bits & BNXT_QPLIB_SRQ_TOGGLE_BIT) {
> > + hash_add(rdev->srq_hash, &srq->hash_entry, srq->qplib_srq.id);
> > + srq->uctx_srq_page = (void *)get_zeroed_page(GFP_KERNEL);
> > + if (!srq->uctx_srq_page) {
> > + rc = -ENOMEM;
> > + goto fail;
> > + }
> > + resp.comp_mask |= BNXT_RE_SRQ_TOGGLE_PAGE_SUPPORT;
> > + }
> > rc = ib_copy_to_udata(udata, &resp, sizeof(resp));
> > if (rc) {
> > ibdev_err(&rdev->ibdev, "SRQ copy to udata failed!");
> > @@ -4266,6 +4279,19 @@ static struct bnxt_re_cq *bnxt_re_search_for_cq(struct bnxt_re_dev *rdev, u32 cq
> > return cq;
> > }
> >
> > +static struct bnxt_re_srq *bnxt_re_search_for_srq(struct bnxt_re_dev *rdev, u32 srq_id)
> > +{
> > + struct bnxt_re_srq *srq = NULL, *tmp_srq;
> > +
> > + hash_for_each_possible(rdev->srq_hash, tmp_srq, hash_entry, srq_id) {
> > + if (tmp_srq->qplib_srq.id == srq_id) {
> > + srq = tmp_srq;
> > + break;
> > + }
> > + }
> > + return srq;
> > +}
> > +
> > /* Helper function to mmap the virtual memory from user app */
> > int bnxt_re_mmap(struct ib_ucontext *ib_uctx, struct vm_area_struct *vma)
> > {
> > @@ -4494,6 +4520,7 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_GET_TOGGLE_MEM)(struct uverbs_attr_bund
> > struct bnxt_re_ucontext *uctx;
> > struct ib_ucontext *ib_uctx;
> > struct bnxt_re_dev *rdev;
> > + struct bnxt_re_srq *srq;
> > u32 length = PAGE_SIZE;
> > struct bnxt_re_cq *cq;
> > u64 mem_offset;
> > @@ -4525,6 +4552,11 @@ static int UVERBS_HANDLER(BNXT_RE_METHOD_GET_TOGGLE_MEM)(struct uverbs_attr_bund
> > addr = (u64)cq->uctx_cq_page;
> > break;
> > case BNXT_RE_SRQ_TOGGLE_MEM:
> > + srq = bnxt_re_search_for_srq(rdev, res_id);
> > + if (!srq)
> > + return -EINVAL;
> > +
> > + addr = (u64)srq->uctx_srq_page;
> > break;
> >
> > default:
> > diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.h b/drivers/infiniband/hw/bnxt_re/ib_verbs.h
> > index 4e113b9..9c74dfe 100644
> > --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.h
> > +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.h
> > @@ -78,6 +78,7 @@ struct bnxt_re_srq {
> > struct ib_umem *umem;
> > spinlock_t lock; /* protect srq */
> > void *uctx_srq_page;
> > + struct hlist_node hash_entry;
> > };
> >
> > struct bnxt_re_qp {
> > diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c
> > index 9714b9a..1211fe5 100644
> > --- a/drivers/infiniband/hw/bnxt_re/main.c
> > +++ b/drivers/infiniband/hw/bnxt_re/main.c
> > @@ -139,8 +139,10 @@ static void bnxt_re_set_drv_mode(struct bnxt_re_dev *rdev, u8 mode)
> > if (bnxt_re_hwrm_qcaps(rdev))
> > dev_err(rdev_to_dev(rdev),
> > "Failed to query hwrm qcaps\n");
> > - if (bnxt_qplib_is_chip_gen_p7(rdev->chip_ctx))
> > + if (bnxt_qplib_is_chip_gen_p7(rdev->chip_ctx)) {
> > cctx->modes.toggle_bits |= BNXT_QPLIB_CQ_TOGGLE_BIT;
> > + cctx->modes.toggle_bits |= BNXT_QPLIB_SRQ_TOGGLE_BIT;
> > + }
> > }
> >
> > static void bnxt_re_destroy_chip_ctx(struct bnxt_re_dev *rdev)
> > @@ -1771,6 +1773,8 @@ static int bnxt_re_dev_init(struct bnxt_re_dev *rdev, u8 wqe_mode)
> > bnxt_re_vf_res_config(rdev);
> > }
> > hash_init(rdev->cq_hash);
> > + if (rdev->chip_ctx->modes.toggle_bits & BNXT_QPLIB_SRQ_TOGGLE_BIT)
> > + hash_init(rdev->srq_hash);
> >
> > return 0;
> > free_sctx:
> > diff --git a/include/uapi/rdma/bnxt_re-abi.h b/include/uapi/rdma/bnxt_re-abi.h
> > index e61104f..84917a9 100644
> > --- a/include/uapi/rdma/bnxt_re-abi.h
> > +++ b/include/uapi/rdma/bnxt_re-abi.h
> > @@ -134,8 +134,13 @@ struct bnxt_re_srq_req {
> > __aligned_u64 srq_handle;
> > };
> >
> > +enum bnxt_re_srq_mask {
> > + BNXT_RE_SRQ_TOGGLE_PAGE_SUPPORT = 0x1,
> > +};
> > +
> > struct bnxt_re_srq_resp {
> > __u32 srqid;
>
> I think that you should add __u32 reserved field here to align the
> struct.
Sure. will post a v2 soon.
Thanks,
>
> > + __aligned_u64 comp_mask;
> > };
> >
> > enum bnxt_re_shpg_offt {
> > --
> > 2.5.5
> >
Attachment:
smime.p7s
Description: S/MIME Cryptographic Signature
