Re: [PATCH v2 1/2] RDMA/rxe: Remove struct rxe_phys_buf

[Yang, Xiao/杨 晓 <yangx.jy@xxxxxxxxxxx>]

Hi Jason, Ira and others

Kindly ping. I hope you can review them.
Thank you in advance.

Best Regards,
Xiao Yang

On 2022/11/11 15:51, Xiao Yang wrote:
> 1) Remove rxe_phys_buf[n].size by using ibmr.page_size.
> 2) Replace rxe_phys_buf[n].buf with addrs[n].
>
> Signed-off-by: Xiao Yang <yangx.jy@xxxxxxxxxxx>
> ---
>   drivers/infiniband/sw/rxe/rxe_mr.c    | 45 +++++++++------------------
>   drivers/infiniband/sw/rxe/rxe_verbs.c |  6 +---
>   drivers/infiniband/sw/rxe/rxe_verbs.h |  9 ++----
>   3 files changed, 18 insertions(+), 42 deletions(-)
>
> diff --git a/drivers/infiniband/sw/rxe/rxe_mr.c b/drivers/infiniband/sw/rxe/rxe_mr.c
> index bc081002bddc..4438eb8a3727 100644
> --- a/drivers/infiniband/sw/rxe/rxe_mr.c
> +++ b/drivers/infiniband/sw/rxe/rxe_mr.c
> @@ -115,7 +115,6 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,
>   		     int access, struct rxe_mr *mr)
>   {
>   	struct rxe_map		**map;
> -	struct rxe_phys_buf	*buf = NULL;
>   	struct ib_umem		*umem;
>   	struct sg_page_iter	sg_iter;
>   	int			num_buf;
> @@ -144,16 +143,14 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,
>   
>   	mr->page_shift = PAGE_SHIFT;
>   	mr->page_mask = PAGE_SIZE - 1;
> +	mr->ibmr.page_size = PAGE_SIZE;
>   
> -	num_buf			= 0;
> +	num_buf	= 0;
>   	map = mr->map;
>   	if (length > 0) {
> -		buf = map[0]->buf;
> -
>   		for_each_sgtable_page (&umem->sgt_append.sgt, &sg_iter, 0) {
>   			if (num_buf >= RXE_BUF_PER_MAP) {
>   				map++;
> -				buf = map[0]->buf;
>   				num_buf = 0;
>   			}
>   
> @@ -165,10 +162,8 @@ int rxe_mr_init_user(struct rxe_dev *rxe, u64 start, u64 length, u64 iova,
>   				goto err_cleanup_map;
>   			}
>   
> -			buf->addr = (uintptr_t)vaddr;
> -			buf->size = PAGE_SIZE;
> +			map[0]->addrs[num_buf] = (uintptr_t)vaddr;
>   			num_buf++;
> -			buf++;
>   
>   		}
>   	}
> @@ -216,9 +211,9 @@ static void lookup_iova(struct rxe_mr *mr, u64 iova, int *m_out, int *n_out,
>   			size_t *offset_out)
>   {
>   	size_t offset = iova - mr->ibmr.iova + mr->offset;
> +	u64 length = mr->ibmr.page_size;
>   	int			map_index;
> -	int			buf_index;
> -	u64			length;
> +	int			addr_index;
>   
>   	if (likely(mr->page_shift)) {
>   		*offset_out = offset & mr->page_mask;
> @@ -227,23 +222,20 @@ static void lookup_iova(struct rxe_mr *mr, u64 iova, int *m_out, int *n_out,
>   		*m_out = offset >> mr->map_shift;
>   	} else {
>   		map_index = 0;
> -		buf_index = 0;
> -
> -		length = mr->map[map_index]->buf[buf_index].size;
> +		addr_index = 0;
>   
>   		while (offset >= length) {
>   			offset -= length;
> -			buf_index++;
> +			addr_index++;
>   
> -			if (buf_index == RXE_BUF_PER_MAP) {
> +			if (addr_index == RXE_BUF_PER_MAP) {
>   				map_index++;
> -				buf_index = 0;
> +				addr_index = 0;
>   			}
> -			length = mr->map[map_index]->buf[buf_index].size;
>   		}
>   
>   		*m_out = map_index;
> -		*n_out = buf_index;
> +		*n_out = addr_index;
>   		*offset_out = offset;
>   	}
>   }
> @@ -273,13 +265,13 @@ void *iova_to_vaddr(struct rxe_mr *mr, u64 iova, int length)
>   
>   	lookup_iova(mr, iova, &m, &n, &offset);
>   
> -	if (offset + length > mr->map[m]->buf[n].size) {
> +	if (offset + length > mr->ibmr.page_size) {
>   		pr_warn("crosses page boundary\n");
>   		addr = NULL;
>   		goto out;
>   	}
>   
> -	addr = (void *)(uintptr_t)mr->map[m]->buf[n].addr + offset;
> +	addr = (void *)(uintptr_t)mr->map[m]->addrs[n] + offset;
>   
>   out:
>   	return addr;
> @@ -294,8 +286,6 @@ int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length,
>   	int			err;
>   	int			bytes;
>   	u8			*va;
> -	struct rxe_map		**map;
> -	struct rxe_phys_buf	*buf;
>   	int			m;
>   	int			i;
>   	size_t			offset;
> @@ -325,17 +315,14 @@ int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length,
>   
>   	lookup_iova(mr, iova, &m, &i, &offset);
>   
> -	map = mr->map + m;
> -	buf	= map[0]->buf + i;
> -
>   	while (length > 0) {
>   		u8 *src, *dest;
>   
> -		va	= (u8 *)(uintptr_t)buf->addr + offset;
> +		va	= (u8 *)(uintptr_t)mr->map[m]->addrs[i] + offset;
>   		src = (dir == RXE_TO_MR_OBJ) ? addr : va;
>   		dest = (dir == RXE_TO_MR_OBJ) ? va : addr;
>   
> -		bytes	= buf->size - offset;
> +		bytes	= mr->ibmr.page_size - offset;
>   
>   		if (bytes > length)
>   			bytes = length;
> @@ -346,13 +333,11 @@ int rxe_mr_copy(struct rxe_mr *mr, u64 iova, void *addr, int length,
>   		addr	+= bytes;
>   
>   		offset	= 0;
> -		buf++;
>   		i++;
>   
>   		if (i == RXE_BUF_PER_MAP) {
>   			i = 0;
> -			map++;
> -			buf = map[0]->buf;
> +			m++;
>   		}
>   	}
>   
> diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.c b/drivers/infiniband/sw/rxe/rxe_verbs.c
> index bcdfdadaebbc..13e4d660cb02 100644
> --- a/drivers/infiniband/sw/rxe/rxe_verbs.c
> +++ b/drivers/infiniband/sw/rxe/rxe_verbs.c
> @@ -948,16 +948,12 @@ static int rxe_set_page(struct ib_mr *ibmr, u64 addr)
>   {
>   	struct rxe_mr *mr = to_rmr(ibmr);
>   	struct rxe_map *map;
> -	struct rxe_phys_buf *buf;
>   
>   	if (unlikely(mr->nbuf == mr->num_buf))
>   		return -ENOMEM;
>   
>   	map = mr->map[mr->nbuf / RXE_BUF_PER_MAP];
> -	buf = &map->buf[mr->nbuf % RXE_BUF_PER_MAP];
> -
> -	buf->addr = addr;
> -	buf->size = ibmr->page_size;
> +	map->addrs[mr->nbuf % RXE_BUF_PER_MAP] = addr;
>   	mr->nbuf++;
>   
>   	return 0;
> diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h
> index 22a299b0a9f0..d136f02d5b56 100644
> --- a/drivers/infiniband/sw/rxe/rxe_verbs.h
> +++ b/drivers/infiniband/sw/rxe/rxe_verbs.h
> @@ -277,15 +277,10 @@ enum rxe_mr_lookup_type {
>   	RXE_LOOKUP_REMOTE,
>   };
>   
> -#define RXE_BUF_PER_MAP		(PAGE_SIZE / sizeof(struct rxe_phys_buf))
> -
> -struct rxe_phys_buf {
> -	u64      addr;
> -	u64      size;
> -};
> +#define RXE_BUF_PER_MAP		(PAGE_SIZE / sizeof(u64))
>   
>   struct rxe_map {
> -	struct rxe_phys_buf	buf[RXE_BUF_PER_MAP];
> +	u64 addrs[RXE_BUF_PER_MAP];
>   };
>   
>   static inline int rkey_is_mw(u32 rkey)