Re: [PATCH v3 08/17] rdma_rxe: Added mw object

[Zhu Yanjun <zyjzyj2000@xxxxxxxxx>]

On 8/21/2020 6:46 AM, Bob Pearson wrote:
> Set max_mw device attribute.
> Changed rxe_param.c so we can create mw objects (RXE_MAX_MW != 0).
> Changed mw_pool.c to use struct rxe_mw size for the mw pool.
> Added new mw struct to rxe_verbs.h. Can share state enum with mr.
>
> Signed-off-by: Bob Pearson <rpearson@xxxxxxx>
> ---
>   drivers/infiniband/sw/rxe/rxe.c       |  1 +
>   drivers/infiniband/sw/rxe/rxe_param.h | 10 ++++++----
>   drivers/infiniband/sw/rxe/rxe_pool.c  |  2 +-
>   drivers/infiniband/sw/rxe/rxe_verbs.h | 13 +++++++++++++
>   4 files changed, 21 insertions(+), 5 deletions(-)
>
> diff --git a/drivers/infiniband/sw/rxe/rxe.c b/drivers/infiniband/sw/rxe/rxe.c
> index 6c2100c71874..10f441ac7203 100644
> --- a/drivers/infiniband/sw/rxe/rxe.c
> +++ b/drivers/infiniband/sw/rxe/rxe.c
> @@ -54,6 +54,7 @@ static void rxe_init_device_param(struct rxe_dev *rxe)
>   	rxe->attr.max_cq			= RXE_MAX_CQ;
>   	rxe->attr.max_cqe			= (1 << RXE_MAX_LOG_CQE) - 1;
>   	rxe->attr.max_mr			= RXE_MAX_MR;
> +	rxe->attr.max_mw			= RXE_MAX_MW;
>   	rxe->attr.max_pd			= RXE_MAX_PD;
>   	rxe->attr.max_qp_rd_atom		= RXE_MAX_QP_RD_ATOM;
>   	rxe->attr.max_res_rd_atom		= RXE_MAX_RES_RD_ATOM;
> diff --git a/drivers/infiniband/sw/rxe/rxe_param.h b/drivers/infiniband/sw/rxe/rxe_param.h
> index 96516e251bc1..e1d485bdd6af 100644
> --- a/drivers/infiniband/sw/rxe/rxe_param.h
> +++ b/drivers/infiniband/sw/rxe/rxe_param.h
> @@ -60,7 +60,8 @@ enum rxe_device_param {
>   	RXE_MAX_SGE_RD			= 32,
>   	RXE_MAX_CQ			= 16384,
>   	RXE_MAX_LOG_CQE			= 15,
> -	RXE_MAX_MR			= 256 * 1024,
> +	RXE_MAX_MR			= 0x40000,
> +	RXE_MAX_MW			= 0x40000,

How to get this 0x40000?

Thanks


>   	RXE_MAX_PD			= 0x7ffc,
>   	RXE_MAX_QP_RD_ATOM		= 128,
>   	RXE_MAX_RES_RD_ATOM		= 0x3f000,
> @@ -89,9 +90,10 @@ enum rxe_device_param {
>   	RXE_MAX_SRQ_INDEX		= 0x00040000,
>   
>   	RXE_MIN_MR_INDEX		= 0x00000001,
> -	RXE_MAX_MR_INDEX		= 0x00040000,
> -	RXE_MIN_MW_INDEX		= 0x00040001,
> -	RXE_MAX_MW_INDEX		= 0x00060000,
> +	RXE_MAX_MR_INDEX		= RXE_MIN_MR_INDEX + RXE_MAX_MR - 1,
> +	RXE_MIN_MW_INDEX		= RXE_MIN_MR_INDEX + RXE_MAX_MR,
> +	RXE_MAX_MW_INDEX		= RXE_MIN_MW_INDEX + RXE_MAX_MW - 1,
> +
>   	RXE_MAX_PKT_PER_ACK		= 64,
>   
>   	RXE_MAX_UNACKED_PSNS		= 128,
> diff --git a/drivers/infiniband/sw/rxe/rxe_pool.c b/drivers/infiniband/sw/rxe/rxe_pool.c
> index fe652ce488f3..ad6f50e8b57a 100644
> --- a/drivers/infiniband/sw/rxe/rxe_pool.c
> +++ b/drivers/infiniband/sw/rxe/rxe_pool.c
> @@ -60,7 +60,7 @@ struct rxe_type_info rxe_type_info[RXE_NUM_TYPES] = {
>   	},
>   	[RXE_TYPE_MW] = {
>   		.name		= "rxe-mw",
> -		.size		= sizeof(struct rxe_mr),
> +		.size		= sizeof(struct rxe_mw),
>   		.flags		= RXE_POOL_INDEX,
>   		.max_index	= RXE_MAX_MW_INDEX,
>   		.min_index	= RXE_MIN_MW_INDEX,
> diff --git a/drivers/infiniband/sw/rxe/rxe_verbs.h b/drivers/infiniband/sw/rxe/rxe_verbs.h
> index c866fc5a80d6..44da74a41ccb 100644
> --- a/drivers/infiniband/sw/rxe/rxe_verbs.h
> +++ b/drivers/infiniband/sw/rxe/rxe_verbs.h
> @@ -264,6 +264,7 @@ struct rxe_qp {
>   	struct execute_work	cleanup_work;
>   };
>   
> +/* common state values for mr and mw */
>   enum rxe_mem_state {
>   	RXE_MEM_STATE_ZOMBIE,
>   	RXE_MEM_STATE_INVALID,
> @@ -321,6 +322,18 @@ struct rxe_mr {
>   	struct rxe_map		**map;
>   };
>   
> +struct rxe_mw {
> +	struct rxe_pool_entry	pelem;
> +	struct ib_mw		ibmw;
> +	struct rxe_qp		*qp;	/* type 2B only */
> +	struct rxe_mr		*mr;
> +	spinlock_t		lock;
> +	enum rxe_mem_state	state;
> +	u32			access;
> +	u64			addr;
> +	u64			length;
> +};
> +
>   struct rxe_mc_grp {
>   	struct rxe_pool_entry	pelem;
>   	spinlock_t		mcg_lock; /* guard group */