On Tue, Sep 16, 2008 at 06:34:30AM -0500, Tom Tucker wrote:
> Add data types to track Fast Reg Memory Regions. The core data type is
> svc_rdma_fastreg_mr that associates a device MR with a host kva and page
> list. A field is added to the WR context to keep track of the FRMR
> used to map the local memory for an RPC.
>
> An FRMR list and spin lock are added to the transport instance to keep
> track of all FRMR allocated for the transport. Also added are device
> capability flags to indicate what the memory registration
> capabilities are for the underlying device and whether or not fast
> memory registration is supported.
>
> Signed-off-by: Tom Tucker <tom@xxxxxxxxxxxxxxxxxxxxx>
>
> ---
> include/linux/sunrpc/svc_rdma.h | 27 ++++++++++++++++++++++++++-
> 1 files changed, 26 insertions(+), 1 deletions(-)
>
> diff --git a/include/linux/sunrpc/svc_rdma.h b/include/linux/sunrpc/svc_rdma.h
> index dc05b54..295ebbc 100644
> --- a/include/linux/sunrpc/svc_rdma.h
> +++ b/include/linux/sunrpc/svc_rdma.h
> @@ -72,6 +72,7 @@ extern atomic_t rdma_stat_sq_prod;
> */
> struct svc_rdma_op_ctxt {
> struct svc_rdma_op_ctxt *read_hdr;
> + struct svc_rdma_fastreg_mr *frmr;
> int hdr_count;
> struct xdr_buf arg;
> struct list_head dto_q;
> @@ -103,16 +104,35 @@ struct svc_rdma_chunk_sge {
> int start; /* sge no for this chunk */
> int count; /* sge count for this chunk */
> };
> +struct svc_rdma_fastreg_mr {
> + struct ib_mr *mr;
> + void *kva;
> + struct ib_fast_reg_page_list *page_list;
> + int page_list_len;
> + unsigned long access_flags;
> + unsigned long map_len;
> + enum dma_data_direction direction;
> + struct list_head frmr_list;
> +};
> struct svc_rdma_req_map {
> + struct svc_rdma_fastreg_mr *frmr;
> unsigned long count;
> union {
> struct kvec sge[RPCSVC_MAXPAGES];
> struct svc_rdma_chunk_sge ch[RPCSVC_MAXPAGES];
> };
> };
> -
> +#define RDMACTXT_F_FAST_UNREG 1
> #define RDMACTXT_F_LAST_CTXT 2
>
> +enum svcrdma_dev_cap { /* device supports: */
> + SVCRDMA_DEVCAP_FAST_REG = 1, /* - fast mr registration */
> + SVCRDMA_DEVCAP_READ_W_INV = 2, /* - read w/ invalidate */
> + SVCRDMA_DEVCAP_LCL_DMA_MR = 4, /* - lcl dma lkey */
> + SVCRDMA_DEVCAP_LCL_WR_REQ = 8, /* - data sink requires remote */
> + /* write */
> +};
Nit: I suppose that works, but I thought the idea of an enum was that it
enumerated every possible value of the given type? If you're expecting
to "or" these together I would have expected just some kind of unsigned
int with some #define's.
--b.
> +
> struct svcxprt_rdma {
> struct svc_xprt sc_xprt; /* SVC transport structure */
> struct rdma_cm_id *sc_cm_id; /* RDMA connection id */
> @@ -136,6 +156,11 @@ struct svcxprt_rdma {
> struct ib_cq *sc_rq_cq;
> struct ib_cq *sc_sq_cq;
> struct ib_mr *sc_phys_mr; /* MR for server memory */
> + enum svcrdma_dev_cap sc_dev_caps; /* distilled device caps */
> + u32 sc_dma_lkey; /* local dma key */
> + unsigned int sc_frmr_pg_list_len;
> + struct list_head sc_frmr_q;
> + spinlock_t sc_frmr_q_lock;
>
> spinlock_t sc_lock; /* transport lock */
>
--
To unsubscribe from this list: send the line "unsubscribe linux-nfs" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
