This helper iterates over a DMA-mapped SGL and returns contiguous
memory blocks aligned to a HW supported page size.
The implementation is intended to work for HW that support
single page sizes or mixed page sizes.
Suggested-by: Jason Gunthorpe <jgg@xxxxxxxx>
Cc: Gal Pressman <galpress@xxxxxxxxxx>
Signed-off-by: Shiraz Saleem <shiraz.saleem@xxxxxxxxx>
---
drivers/infiniband/core/verbs.c | 68 +++++++++++++++++++++++++++++++++++++++++
include/rdma/ib_verbs.h | 57 ++++++++++++++++++++++++++++++++++
2 files changed, 125 insertions(+)
diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c
index 7313edc..fa9725d 100644
--- a/drivers/infiniband/core/verbs.c
+++ b/drivers/infiniband/core/verbs.c
@@ -2711,3 +2711,71 @@ int rdma_init_netdev(struct ib_device *device, u8 port_num,
netdev, params.param);
}
EXPORT_SYMBOL(rdma_init_netdev);
+
+static unsigned int rdma_find_mixed_pg_bit(struct ib_block_iter *biter)
+{
+ if (biter->__sg == biter->__sgl_head) {
+ return rdma_find_pg_bit(sg_dma_address(biter->__sg) +
+ sg_dma_len(biter->__sg),
+ biter->pgsz_bitmap);
+ } else if (sg_is_last(biter->__sg)) {
+ return rdma_find_pg_bit(sg_dma_address(biter->__sg),
+ biter->pgsz_bitmap);
+ } else {
+ unsigned int remaining =
+ sg_dma_address(biter->__sg) + sg_dma_len(biter->__sg) -
+ biter->__dma_addr;
+ unsigned int pg_bit = rdma_find_pg_bit(biter->__dma_addr,
+ biter->pgsz_bitmap);
+ if (remaining < BIT_ULL(biter->__pg_bit))
+ pg_bit = rdma_find_pg_bit(remaining,
+ biter->pgsz_bitmap);
+
+ return pg_bit;
+ }
+}
+
+void __rdma_block_iter_start(struct ib_block_iter *biter,
+ struct scatterlist *sglist, unsigned int nents,
+ unsigned long pgsz_bitmap)
+{
+ memset(biter, 0, sizeof(struct ib_block_iter));
+ biter->__sg = sglist;
+ biter->pgsz_bitmap = pgsz_bitmap;
+ biter->__sg_nents = nents;
+
+ /* Driver provides best block size to use */
+ if (hweight_long(pgsz_bitmap) == 1) {
+ biter->__pg_bit = __fls(pgsz_bitmap);
+ } else {
+ /* mixed block size support. compute best block size to use */
+ WARN_ON(!(pgsz_bitmap & GENMASK(PAGE_SHIFT, 0)));
+ biter->__sgl_head = &sglist[0];
+ biter->__mixed = true;
+ }
+}
+EXPORT_SYMBOL(__rdma_block_iter_start);
+
+bool __rdma_block_iter_next(struct ib_block_iter *biter)
+{
+ unsigned int block_offset;
+
+ if (!biter->__sg_nents || !biter->__sg)
+ return false;
+
+ biter->__dma_addr = sg_dma_address(biter->__sg) + biter->__sg_advance;
+ if (biter->__mixed)
+ biter->__pg_bit = rdma_find_mixed_pg_bit(biter);
+
+ block_offset = biter->__dma_addr & (BIT_ULL(biter->__pg_bit) - 1);
+ biter->__sg_advance += BIT_ULL(biter->__pg_bit) - block_offset;
+
+ if (biter->__sg_advance >= sg_dma_len(biter->__sg)) {
+ biter->__sg_advance = 0;
+ biter->__sg = sg_next(biter->__sg);
+ biter->__sg_nents--;
+ }
+
+ return true;
+}
+EXPORT_SYMBOL(__rdma_block_iter_next);
diff --git a/include/rdma/ib_verbs.h b/include/rdma/ib_verbs.h
index 720ce23..3398348 100644
--- a/include/rdma/ib_verbs.h
+++ b/include/rdma/ib_verbs.h
@@ -2681,6 +2681,31 @@ struct ib_client {
u8 no_kverbs_req:1;
};
+/*
+ * IB block DMA iterator
+ *
+ * Iterates the DMA-mapped SGL in contiguous memory blocks aligned
+ * to a HW supported page size.
+ */
+struct ib_block_iter {
+ unsigned long pgsz_bitmap; /* bitmap of supported HW page sizes.
+ * HW that can handle only blocks of a
+ * single page size must just provide
+ * the best page size to use in pgsz_bitmap
+ */
+
+ /* internal states */
+ struct scatterlist *__sg; /* sg holding the current aligned block */
+ struct scatterlist *__sgl_head; /* scatterlist head */
+ dma_addr_t __dma_addr; /* unaligned DMA address of this block */
+ unsigned int __sg_nents; /* number of SG entries */
+ unsigned int __sg_advance; /* number of bytes to advance in sg in next step */
+ unsigned int __pg_bit; /* alignment of current block */
+ u8 __mixed; /* HW supports single block size or mixed
+ * block sizes
+ */
+};
+
struct ib_device *_ib_alloc_device(size_t size);
#define ib_alloc_device(drv_struct, member) \
container_of(_ib_alloc_device(sizeof(struct drv_struct) + \
@@ -2701,6 +2726,38 @@ struct ib_client {
int ib_register_client (struct ib_client *client);
void ib_unregister_client(struct ib_client *client);
+void __rdma_block_iter_start(struct ib_block_iter *biter,
+ struct scatterlist *sglist,
+ unsigned int nents,
+ unsigned long pgsz_bitmap);
+bool __rdma_block_iter_next(struct ib_block_iter *biter);
+
+/**
+ * rdma_block_iter_dma_address - get the aligned dma address of the current
+ * block held by the block iterator.
+ * @biter: block iterator holding the memory block
+ */
+static inline dma_addr_t
+rdma_block_iter_dma_address(struct ib_block_iter *biter)
+{
+ return biter->__dma_addr & ~(BIT_ULL(biter->__pg_bit) - 1);
+}
+
+/**
+ * rdma_for_each_block - iterate over contiguous memory blocks of the sg list
+ * @sglist: sglist to iterate over
+ * @biter: block iterator holding the memory block
+ * @nents: maximum number of sg entries to iterate over
+ * @pgsz_bitmap: bitmap of HW supported page sizes
+ *
+ * Callers may use rdma_block_iter_dma_address() to get each
+ * blocks aligned DMA address.
+ */
+#define rdma_for_each_block(sglist, biter, nents, pgsz_bitmap) \
+ for (__rdma_block_iter_start(biter, sglist, nents, \
+ pgsz_bitmap); \
+ __rdma_block_iter_next(biter);)
+
/**
* ib_get_client_data - Get IB client context
* @device:Device to get context for
--
1.8.3.1
