On Wed, 7 Sep 2016 15:42:22 +0300 Saeed Mahameed <saeedm@xxxxxxxxxxxx> wrote:
> From: Tariq Toukan <tariqt@xxxxxxxxxxxx>
>
> To improve the memory consumption scheme, we omit the flow that
> demands and splits high-order pages in Striding RQ, and stay
> with a single Striding RQ flow that uses order-0 pages.
Thanks you for doing this! MM-list people thanks you!
For others to understand what this means: This driver was doing
split_page() on high-order pages (for Striding RQ). This was really bad
because it will cause fragmenting the page-allocator, and depleting the
high-order pages available quickly.
(I've left rest of patch intact below, if some MM people should be
interested in looking at the changes).
There is even a funny comment in split_page() relevant to this:
/* [...]
* Note: this is probably too low level an operation for use in drivers.
* Please consult with lkml before using this in your driver.
*/
> Moving to fragmented memory allows the use of larger MPWQEs,
> which reduces the number of UMR posts and filler CQEs.
>
> Moving to a single flow allows several optimizations that improve
> performance, especially in production servers where we would
> anyway fallback to order-0 allocations:
> - inline functions that were called via function pointers.
> - improve the UMR post process.
>
> This patch alone is expected to give a slight performance reduction.
> However, the new memory scheme gives the possibility to use a page-cache
> of a fair size, that doesn't inflate the memory footprint, which will
> dramatically fix the reduction and even give a huge gain.
>
> We ran pktgen single-stream benchmarks, with iptables-raw-drop:
>
> Single stride, 64 bytes:
> * 4,739,057 - baseline
> * 4,749,550 - this patch
> no reduction
>
> Larger packets, no page cross, 1024 bytes:
> * 3,982,361 - baseline
> * 3,845,682 - this patch
> 3.5% reduction
>
> Larger packets, every 3rd packet crosses a page, 1500 bytes:
> * 3,731,189 - baseline
> * 3,579,414 - this patch
> 4% reduction
>
Well, the reduction does not really matter than much, because your
baseline benchmarks are from a freshly booted system, where you have
not fragmented and depleted the high-order pages yet... ;-)
> Fixes: 461017cb006a ("net/mlx5e: Support RX multi-packet WQE (Striding RQ)")
> Fixes: bc77b240b3c5 ("net/mlx5e: Add fragmented memory support for RX multi packet WQE")
> Signed-off-by: Tariq Toukan <tariqt@xxxxxxxxxxxx>
> Signed-off-by: Saeed Mahameed <saeedm@xxxxxxxxxxxx>
> ---
> drivers/net/ethernet/mellanox/mlx5/core/en.h | 54 ++--
> drivers/net/ethernet/mellanox/mlx5/core/en_main.c | 136 ++++++++--
> drivers/net/ethernet/mellanox/mlx5/core/en_rx.c | 292 ++++-----------------
> drivers/net/ethernet/mellanox/mlx5/core/en_stats.h | 4 -
> drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c | 2 +-
> 5 files changed, 184 insertions(+), 304 deletions(-)
>
> diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h
> index bf722aa..075cdfc 100644
> --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h
> +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h
> @@ -62,12 +62,12 @@
> #define MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE 0xd
>
> #define MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW 0x1
> -#define MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE_MPW 0x4
> +#define MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE_MPW 0x3
> #define MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE_MPW 0x6
>
> #define MLX5_MPWRQ_LOG_STRIDE_SIZE 6 /* >= 6, HW restriction */
> #define MLX5_MPWRQ_LOG_STRIDE_SIZE_CQE_COMPRESS 8 /* >= 6, HW restriction */
> -#define MLX5_MPWRQ_LOG_WQE_SZ 17
> +#define MLX5_MPWRQ_LOG_WQE_SZ 18
> #define MLX5_MPWRQ_WQE_PAGE_ORDER (MLX5_MPWRQ_LOG_WQE_SZ - PAGE_SHIFT > 0 ? \
> MLX5_MPWRQ_LOG_WQE_SZ - PAGE_SHIFT : 0)
> #define MLX5_MPWRQ_PAGES_PER_WQE BIT(MLX5_MPWRQ_WQE_PAGE_ORDER)
> @@ -293,8 +293,8 @@ struct mlx5e_rq {
> u32 wqe_sz;
> struct sk_buff **skb;
> struct mlx5e_mpw_info *wqe_info;
> + void *mtt_no_align;
> __be32 mkey_be;
> - __be32 umr_mkey_be;
>
> struct device *pdev;
> struct net_device *netdev;
> @@ -323,32 +323,15 @@ struct mlx5e_rq {
>
> struct mlx5e_umr_dma_info {
> __be64 *mtt;
> - __be64 *mtt_no_align;
> dma_addr_t mtt_addr;
> - struct mlx5e_dma_info *dma_info;
> + struct mlx5e_dma_info dma_info[MLX5_MPWRQ_PAGES_PER_WQE];
> + struct mlx5e_umr_wqe wqe;
> };
>
> struct mlx5e_mpw_info {
> - union {
> - struct mlx5e_dma_info dma_info;
> - struct mlx5e_umr_dma_info umr;
> - };
> + struct mlx5e_umr_dma_info umr;
> u16 consumed_strides;
> u16 skbs_frags[MLX5_MPWRQ_PAGES_PER_WQE];
> -
> - void (*dma_pre_sync)(struct device *pdev,
> - struct mlx5e_mpw_info *wi,
> - u32 wqe_offset, u32 len);
> - void (*add_skb_frag)(struct mlx5e_rq *rq,
> - struct sk_buff *skb,
> - struct mlx5e_mpw_info *wi,
> - u32 page_idx, u32 frag_offset, u32 len);
> - void (*copy_skb_header)(struct device *pdev,
> - struct sk_buff *skb,
> - struct mlx5e_mpw_info *wi,
> - u32 page_idx, u32 offset,
> - u32 headlen);
> - void (*free_wqe)(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi);
> };
>
> struct mlx5e_tx_wqe_info {
> @@ -706,24 +689,11 @@ void mlx5e_handle_rx_cqe(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe);
> void mlx5e_handle_rx_cqe_mpwrq(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe);
> bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq);
> int mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq, struct mlx5e_rx_wqe *wqe, u16 ix);
> -int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_rx_wqe *wqe, u16 ix);
> +int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_rx_wqe *wqe, u16 ix);
> void mlx5e_dealloc_rx_wqe(struct mlx5e_rq *rq, u16 ix);
> void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix);
> -void mlx5e_post_rx_fragmented_mpwqe(struct mlx5e_rq *rq);
> -void mlx5e_complete_rx_linear_mpwqe(struct mlx5e_rq *rq,
> - struct mlx5_cqe64 *cqe,
> - u16 byte_cnt,
> - struct mlx5e_mpw_info *wi,
> - struct sk_buff *skb);
> -void mlx5e_complete_rx_fragmented_mpwqe(struct mlx5e_rq *rq,
> - struct mlx5_cqe64 *cqe,
> - u16 byte_cnt,
> - struct mlx5e_mpw_info *wi,
> - struct sk_buff *skb);
> -void mlx5e_free_rx_linear_mpwqe(struct mlx5e_rq *rq,
> - struct mlx5e_mpw_info *wi);
> -void mlx5e_free_rx_fragmented_mpwqe(struct mlx5e_rq *rq,
> - struct mlx5e_mpw_info *wi);
> +void mlx5e_post_rx_mpwqe(struct mlx5e_rq *rq);
> +void mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi);
> struct mlx5_cqe64 *mlx5e_get_cqe(struct mlx5e_cq *cq);
>
> void mlx5e_rx_am(struct mlx5e_rq *rq);
> @@ -810,6 +780,12 @@ static inline void mlx5e_cq_arm(struct mlx5e_cq *cq)
> mlx5_cq_arm(mcq, MLX5_CQ_DB_REQ_NOT, mcq->uar->map, NULL, cq->wq.cc);
> }
>
> +static inline u32 mlx5e_get_wqe_mtt_offset(struct mlx5e_rq *rq, u16 wqe_ix)
> +{
> + return rq->mpwqe_mtt_offset +
> + wqe_ix * ALIGN(MLX5_MPWRQ_PAGES_PER_WQE, 8);
> +}
> +
> static inline int mlx5e_get_max_num_channels(struct mlx5_core_dev *mdev)
> {
> return min_t(int, mdev->priv.eq_table.num_comp_vectors,
> diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
> index 2459c7f..0db4d3b 100644
> --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
> +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
> @@ -138,7 +138,6 @@ static void mlx5e_update_sw_counters(struct mlx5e_priv *priv)
> s->rx_csum_unnecessary_inner += rq_stats->csum_unnecessary_inner;
> s->rx_wqe_err += rq_stats->wqe_err;
> s->rx_mpwqe_filler += rq_stats->mpwqe_filler;
> - s->rx_mpwqe_frag += rq_stats->mpwqe_frag;
> s->rx_buff_alloc_err += rq_stats->buff_alloc_err;
> s->rx_cqe_compress_blks += rq_stats->cqe_compress_blks;
> s->rx_cqe_compress_pkts += rq_stats->cqe_compress_pkts;
> @@ -298,6 +297,107 @@ static void mlx5e_disable_async_events(struct mlx5e_priv *priv)
> #define MLX5E_HW2SW_MTU(hwmtu) (hwmtu - (ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN))
> #define MLX5E_SW2HW_MTU(swmtu) (swmtu + (ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN))
>
> +static inline int mlx5e_get_wqe_mtt_sz(void)
> +{
> + /* UMR copies MTTs in units of MLX5_UMR_MTT_ALIGNMENT bytes.
> + * To avoid copying garbage after the mtt array, we allocate
> + * a little more.
> + */
> + return ALIGN(MLX5_MPWRQ_PAGES_PER_WQE * sizeof(__be64),
> + MLX5_UMR_MTT_ALIGNMENT);
> +}
> +
> +static inline void mlx5e_build_umr_wqe(struct mlx5e_rq *rq, struct mlx5e_sq *sq,
> + struct mlx5e_umr_wqe *wqe, u16 ix)
> +{
> + struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
> + struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl;
> + struct mlx5_wqe_data_seg *dseg = &wqe->data;
> + struct mlx5e_mpw_info *wi = &rq->wqe_info[ix];
> + u8 ds_cnt = DIV_ROUND_UP(sizeof(*wqe), MLX5_SEND_WQE_DS);
> + u32 umr_wqe_mtt_offset = mlx5e_get_wqe_mtt_offset(rq, ix);
> +
> + cseg->qpn_ds = cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) |
> + ds_cnt);
> + cseg->fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE;
> + cseg->imm = rq->mkey_be;
> +
> + ucseg->flags = MLX5_UMR_TRANSLATION_OFFSET_EN;
> + ucseg->klm_octowords =
> + cpu_to_be16(MLX5_MTT_OCTW(MLX5_MPWRQ_PAGES_PER_WQE));
> + ucseg->bsf_octowords =
> + cpu_to_be16(MLX5_MTT_OCTW(umr_wqe_mtt_offset));
> + ucseg->mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE);
> +
> + dseg->lkey = sq->mkey_be;
> + dseg->addr = cpu_to_be64(wi->umr.mtt_addr);
> +}
> +
> +static int mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq *rq,
> + struct mlx5e_channel *c)
> +{
> + int wq_sz = mlx5_wq_ll_get_size(&rq->wq);
> + int mtt_sz = mlx5e_get_wqe_mtt_sz();
> + int mtt_alloc = mtt_sz + MLX5_UMR_ALIGN - 1;
> + int i;
> +
> + rq->wqe_info = kzalloc_node(wq_sz * sizeof(*rq->wqe_info),
> + GFP_KERNEL, cpu_to_node(c->cpu));
> + if (!rq->wqe_info)
> + goto err_out;
> +
> + /* We allocate more than mtt_sz as we will align the pointer */
> + rq->mtt_no_align = kzalloc_node(mtt_alloc * wq_sz, GFP_KERNEL,
> + cpu_to_node(c->cpu));
> + if (unlikely(!rq->mtt_no_align))
> + goto err_free_wqe_info;
> +
> + for (i = 0; i < wq_sz; i++) {
> + struct mlx5e_mpw_info *wi = &rq->wqe_info[i];
> +
> + wi->umr.mtt = PTR_ALIGN(rq->mtt_no_align + i * mtt_alloc,
> + MLX5_UMR_ALIGN);
> + wi->umr.mtt_addr = dma_map_single(c->pdev, wi->umr.mtt, mtt_sz,
> + PCI_DMA_TODEVICE);
> + if (unlikely(dma_mapping_error(c->pdev, wi->umr.mtt_addr)))
> + goto err_unmap_mtts;
> +
> + mlx5e_build_umr_wqe(rq, &c->icosq, &wi->umr.wqe, i);
> + }
> +
> + return 0;
> +
> +err_unmap_mtts:
> + while (--i >= 0) {
> + struct mlx5e_mpw_info *wi = &rq->wqe_info[i];
> +
> + dma_unmap_single(c->pdev, wi->umr.mtt_addr, mtt_sz,
> + PCI_DMA_TODEVICE);
> + }
> + kfree(rq->mtt_no_align);
> +err_free_wqe_info:
> + kfree(rq->wqe_info);
> +
> +err_out:
> + return -ENOMEM;
> +}
> +
> +static void mlx5e_rq_free_mpwqe_info(struct mlx5e_rq *rq)
> +{
> + int wq_sz = mlx5_wq_ll_get_size(&rq->wq);
> + int mtt_sz = mlx5e_get_wqe_mtt_sz();
> + int i;
> +
> + for (i = 0; i < wq_sz; i++) {
> + struct mlx5e_mpw_info *wi = &rq->wqe_info[i];
> +
> + dma_unmap_single(rq->pdev, wi->umr.mtt_addr, mtt_sz,
> + PCI_DMA_TODEVICE);
> + }
> + kfree(rq->mtt_no_align);
> + kfree(rq->wqe_info);
> +}
> +
> static int mlx5e_create_rq(struct mlx5e_channel *c,
> struct mlx5e_rq_param *param,
> struct mlx5e_rq *rq)
> @@ -322,14 +422,16 @@ static int mlx5e_create_rq(struct mlx5e_channel *c,
>
> wq_sz = mlx5_wq_ll_get_size(&rq->wq);
>
> + rq->wq_type = priv->params.rq_wq_type;
> + rq->pdev = c->pdev;
> + rq->netdev = c->netdev;
> + rq->tstamp = &priv->tstamp;
> + rq->channel = c;
> + rq->ix = c->ix;
> + rq->priv = c->priv;
> +
> switch (priv->params.rq_wq_type) {
> case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
> - rq->wqe_info = kzalloc_node(wq_sz * sizeof(*rq->wqe_info),
> - GFP_KERNEL, cpu_to_node(c->cpu));
> - if (!rq->wqe_info) {
> - err = -ENOMEM;
> - goto err_rq_wq_destroy;
> - }
> rq->handle_rx_cqe = mlx5e_handle_rx_cqe_mpwrq;
> rq->alloc_wqe = mlx5e_alloc_rx_mpwqe;
> rq->dealloc_wqe = mlx5e_dealloc_rx_mpwqe;
> @@ -341,6 +443,10 @@ static int mlx5e_create_rq(struct mlx5e_channel *c,
> rq->mpwqe_num_strides = BIT(priv->params.mpwqe_log_num_strides);
> rq->wqe_sz = rq->mpwqe_stride_sz * rq->mpwqe_num_strides;
> byte_count = rq->wqe_sz;
> + rq->mkey_be = cpu_to_be32(c->priv->umr_mkey.key);
> + err = mlx5e_rq_alloc_mpwqe_info(rq, c);
> + if (err)
> + goto err_rq_wq_destroy;
> break;
> default: /* MLX5_WQ_TYPE_LINKED_LIST */
> rq->skb = kzalloc_node(wq_sz * sizeof(*rq->skb), GFP_KERNEL,
> @@ -359,27 +465,19 @@ static int mlx5e_create_rq(struct mlx5e_channel *c,
> rq->wqe_sz = SKB_DATA_ALIGN(rq->wqe_sz);
> byte_count = rq->wqe_sz;
> byte_count |= MLX5_HW_START_PADDING;
> + rq->mkey_be = c->mkey_be;
> }
>
> for (i = 0; i < wq_sz; i++) {
> struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, i);
>
> wqe->data.byte_count = cpu_to_be32(byte_count);
> + wqe->data.lkey = rq->mkey_be;
> }
>
> INIT_WORK(&rq->am.work, mlx5e_rx_am_work);
> rq->am.mode = priv->params.rx_cq_period_mode;
>
> - rq->wq_type = priv->params.rq_wq_type;
> - rq->pdev = c->pdev;
> - rq->netdev = c->netdev;
> - rq->tstamp = &priv->tstamp;
> - rq->channel = c;
> - rq->ix = c->ix;
> - rq->priv = c->priv;
> - rq->mkey_be = c->mkey_be;
> - rq->umr_mkey_be = cpu_to_be32(c->priv->umr_mkey.key);
> -
> return 0;
>
> err_rq_wq_destroy:
> @@ -392,7 +490,7 @@ static void mlx5e_destroy_rq(struct mlx5e_rq *rq)
> {
> switch (rq->wq_type) {
> case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
> - kfree(rq->wqe_info);
> + mlx5e_rq_free_mpwqe_info(rq);
> break;
> default: /* MLX5_WQ_TYPE_LINKED_LIST */
> kfree(rq->skb);
> @@ -530,7 +628,7 @@ static void mlx5e_free_rx_descs(struct mlx5e_rq *rq)
>
> /* UMR WQE (if in progress) is always at wq->head */
> if (test_bit(MLX5E_RQ_STATE_UMR_WQE_IN_PROGRESS, &rq->state))
> - mlx5e_free_rx_fragmented_mpwqe(rq, &rq->wqe_info[wq->head]);
> + mlx5e_free_rx_mpwqe(rq, &rq->wqe_info[wq->head]);
>
> while (!mlx5_wq_ll_is_empty(wq)) {
> wqe_ix_be = *wq->tail_next;
> diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
> index b6f8ebb..8ad4d32 100644
> --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
> +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
> @@ -200,7 +200,6 @@ int mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq, struct mlx5e_rx_wqe *wqe, u16 ix)
>
> *((dma_addr_t *)skb->cb) = dma_addr;
> wqe->data.addr = cpu_to_be64(dma_addr);
> - wqe->data.lkey = rq->mkey_be;
>
> rq->skb[ix] = skb;
>
> @@ -231,44 +230,11 @@ static inline int mlx5e_mpwqe_strides_per_page(struct mlx5e_rq *rq)
> return rq->mpwqe_num_strides >> MLX5_MPWRQ_WQE_PAGE_ORDER;
> }
>
> -static inline void
> -mlx5e_dma_pre_sync_linear_mpwqe(struct device *pdev,
> - struct mlx5e_mpw_info *wi,
> - u32 wqe_offset, u32 len)
> -{
> - dma_sync_single_for_cpu(pdev, wi->dma_info.addr + wqe_offset,
> - len, DMA_FROM_DEVICE);
> -}
> -
> -static inline void
> -mlx5e_dma_pre_sync_fragmented_mpwqe(struct device *pdev,
> - struct mlx5e_mpw_info *wi,
> - u32 wqe_offset, u32 len)
> -{
> - /* No dma pre sync for fragmented MPWQE */
> -}
> -
> -static inline void
> -mlx5e_add_skb_frag_linear_mpwqe(struct mlx5e_rq *rq,
> - struct sk_buff *skb,
> - struct mlx5e_mpw_info *wi,
> - u32 page_idx, u32 frag_offset,
> - u32 len)
> -{
> - unsigned int truesize = ALIGN(len, rq->mpwqe_stride_sz);
> -
> - wi->skbs_frags[page_idx]++;
> - skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
> - &wi->dma_info.page[page_idx], frag_offset,
> - len, truesize);
> -}
> -
> -static inline void
> -mlx5e_add_skb_frag_fragmented_mpwqe(struct mlx5e_rq *rq,
> - struct sk_buff *skb,
> - struct mlx5e_mpw_info *wi,
> - u32 page_idx, u32 frag_offset,
> - u32 len)
> +static inline void mlx5e_add_skb_frag_mpwqe(struct mlx5e_rq *rq,
> + struct sk_buff *skb,
> + struct mlx5e_mpw_info *wi,
> + u32 page_idx, u32 frag_offset,
> + u32 len)
> {
> unsigned int truesize = ALIGN(len, rq->mpwqe_stride_sz);
>
> @@ -282,24 +248,11 @@ mlx5e_add_skb_frag_fragmented_mpwqe(struct mlx5e_rq *rq,
> }
>
> static inline void
> -mlx5e_copy_skb_header_linear_mpwqe(struct device *pdev,
> - struct sk_buff *skb,
> - struct mlx5e_mpw_info *wi,
> - u32 page_idx, u32 offset,
> - u32 headlen)
> -{
> - struct page *page = &wi->dma_info.page[page_idx];
> -
> - skb_copy_to_linear_data(skb, page_address(page) + offset,
> - ALIGN(headlen, sizeof(long)));
> -}
> -
> -static inline void
> -mlx5e_copy_skb_header_fragmented_mpwqe(struct device *pdev,
> - struct sk_buff *skb,
> - struct mlx5e_mpw_info *wi,
> - u32 page_idx, u32 offset,
> - u32 headlen)
> +mlx5e_copy_skb_header_mpwqe(struct device *pdev,
> + struct sk_buff *skb,
> + struct mlx5e_mpw_info *wi,
> + u32 page_idx, u32 offset,
> + u32 headlen)
> {
> u16 headlen_pg = min_t(u32, headlen, PAGE_SIZE - offset);
> struct mlx5e_dma_info *dma_info = &wi->umr.dma_info[page_idx];
> @@ -324,46 +277,9 @@ mlx5e_copy_skb_header_fragmented_mpwqe(struct device *pdev,
> }
> }
>
> -static u32 mlx5e_get_wqe_mtt_offset(struct mlx5e_rq *rq, u16 wqe_ix)
> -{
> - return rq->mpwqe_mtt_offset +
> - wqe_ix * ALIGN(MLX5_MPWRQ_PAGES_PER_WQE, 8);
> -}
> -
> -static void mlx5e_build_umr_wqe(struct mlx5e_rq *rq,
> - struct mlx5e_sq *sq,
> - struct mlx5e_umr_wqe *wqe,
> - u16 ix)
> +static inline void mlx5e_post_umr_wqe(struct mlx5e_rq *rq, u16 ix)
> {
> - struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
> - struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl;
> - struct mlx5_wqe_data_seg *dseg = &wqe->data;
> struct mlx5e_mpw_info *wi = &rq->wqe_info[ix];
> - u8 ds_cnt = DIV_ROUND_UP(sizeof(*wqe), MLX5_SEND_WQE_DS);
> - u32 umr_wqe_mtt_offset = mlx5e_get_wqe_mtt_offset(rq, ix);
> -
> - memset(wqe, 0, sizeof(*wqe));
> - cseg->opmod_idx_opcode =
> - cpu_to_be32((sq->pc << MLX5_WQE_CTRL_WQE_INDEX_SHIFT) |
> - MLX5_OPCODE_UMR);
> - cseg->qpn_ds = cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) |
> - ds_cnt);
> - cseg->fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE;
> - cseg->imm = rq->umr_mkey_be;
> -
> - ucseg->flags = MLX5_UMR_TRANSLATION_OFFSET_EN;
> - ucseg->klm_octowords =
> - cpu_to_be16(MLX5_MTT_OCTW(MLX5_MPWRQ_PAGES_PER_WQE));
> - ucseg->bsf_octowords =
> - cpu_to_be16(MLX5_MTT_OCTW(umr_wqe_mtt_offset));
> - ucseg->mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE);
> -
> - dseg->lkey = sq->mkey_be;
> - dseg->addr = cpu_to_be64(wi->umr.mtt_addr);
> -}
> -
> -static void mlx5e_post_umr_wqe(struct mlx5e_rq *rq, u16 ix)
> -{
> struct mlx5e_sq *sq = &rq->channel->icosq;
> struct mlx5_wq_cyc *wq = &sq->wq;
> struct mlx5e_umr_wqe *wqe;
> @@ -378,30 +294,22 @@ static void mlx5e_post_umr_wqe(struct mlx5e_rq *rq, u16 ix)
> }
>
> wqe = mlx5_wq_cyc_get_wqe(wq, pi);
> - mlx5e_build_umr_wqe(rq, sq, wqe, ix);
> + memcpy(wqe, &wi->umr.wqe, sizeof(*wqe));
> + wqe->ctrl.opmod_idx_opcode =
> + cpu_to_be32((sq->pc << MLX5_WQE_CTRL_WQE_INDEX_SHIFT) |
> + MLX5_OPCODE_UMR);
> +
> sq->ico_wqe_info[pi].opcode = MLX5_OPCODE_UMR;
> sq->ico_wqe_info[pi].num_wqebbs = num_wqebbs;
> sq->pc += num_wqebbs;
> mlx5e_tx_notify_hw(sq, &wqe->ctrl, 0);
> }
>
> -static inline int mlx5e_get_wqe_mtt_sz(void)
> -{
> - /* UMR copies MTTs in units of MLX5_UMR_MTT_ALIGNMENT bytes.
> - * To avoid copying garbage after the mtt array, we allocate
> - * a little more.
> - */
> - return ALIGN(MLX5_MPWRQ_PAGES_PER_WQE * sizeof(__be64),
> - MLX5_UMR_MTT_ALIGNMENT);
> -}
> -
> -static int mlx5e_alloc_and_map_page(struct mlx5e_rq *rq,
> - struct mlx5e_mpw_info *wi,
> - int i)
> +static inline int mlx5e_alloc_and_map_page(struct mlx5e_rq *rq,
> + struct mlx5e_mpw_info *wi,
> + int i)
> {
> - struct page *page;
> -
> - page = dev_alloc_page();
> + struct page *page = dev_alloc_page();
> if (unlikely(!page))
> return -ENOMEM;
>
> @@ -417,47 +325,25 @@ static int mlx5e_alloc_and_map_page(struct mlx5e_rq *rq,
> return 0;
> }
>
> -static int mlx5e_alloc_rx_fragmented_mpwqe(struct mlx5e_rq *rq,
> - struct mlx5e_rx_wqe *wqe,
> - u16 ix)
> +static int mlx5e_alloc_rx_umr_mpwqe(struct mlx5e_rq *rq,
> + struct mlx5e_rx_wqe *wqe,
> + u16 ix)
> {
> struct mlx5e_mpw_info *wi = &rq->wqe_info[ix];
> - int mtt_sz = mlx5e_get_wqe_mtt_sz();
> u64 dma_offset = (u64)mlx5e_get_wqe_mtt_offset(rq, ix) << PAGE_SHIFT;
> + int pg_strides = mlx5e_mpwqe_strides_per_page(rq);
> + int err;
> int i;
>
> - wi->umr.dma_info = kmalloc(sizeof(*wi->umr.dma_info) *
> - MLX5_MPWRQ_PAGES_PER_WQE,
> - GFP_ATOMIC);
> - if (unlikely(!wi->umr.dma_info))
> - goto err_out;
> -
> - /* We allocate more than mtt_sz as we will align the pointer */
> - wi->umr.mtt_no_align = kzalloc(mtt_sz + MLX5_UMR_ALIGN - 1,
> - GFP_ATOMIC);
> - if (unlikely(!wi->umr.mtt_no_align))
> - goto err_free_umr;
> -
> - wi->umr.mtt = PTR_ALIGN(wi->umr.mtt_no_align, MLX5_UMR_ALIGN);
> - wi->umr.mtt_addr = dma_map_single(rq->pdev, wi->umr.mtt, mtt_sz,
> - PCI_DMA_TODEVICE);
> - if (unlikely(dma_mapping_error(rq->pdev, wi->umr.mtt_addr)))
> - goto err_free_mtt;
> -
> for (i = 0; i < MLX5_MPWRQ_PAGES_PER_WQE; i++) {
> - if (unlikely(mlx5e_alloc_and_map_page(rq, wi, i)))
> + err = mlx5e_alloc_and_map_page(rq, wi, i);
> + if (unlikely(err))
> goto err_unmap;
> - page_ref_add(wi->umr.dma_info[i].page,
> - mlx5e_mpwqe_strides_per_page(rq));
> + page_ref_add(wi->umr.dma_info[i].page, pg_strides);
> wi->skbs_frags[i] = 0;
> }
>
> wi->consumed_strides = 0;
> - wi->dma_pre_sync = mlx5e_dma_pre_sync_fragmented_mpwqe;
> - wi->add_skb_frag = mlx5e_add_skb_frag_fragmented_mpwqe;
> - wi->copy_skb_header = mlx5e_copy_skb_header_fragmented_mpwqe;
> - wi->free_wqe = mlx5e_free_rx_fragmented_mpwqe;
> - wqe->data.lkey = rq->umr_mkey_be;
> wqe->data.addr = cpu_to_be64(dma_offset);
>
> return 0;
> @@ -466,41 +352,28 @@ err_unmap:
> while (--i >= 0) {
> dma_unmap_page(rq->pdev, wi->umr.dma_info[i].addr, PAGE_SIZE,
> PCI_DMA_FROMDEVICE);
> - page_ref_sub(wi->umr.dma_info[i].page,
> - mlx5e_mpwqe_strides_per_page(rq));
> + page_ref_sub(wi->umr.dma_info[i].page, pg_strides);
> put_page(wi->umr.dma_info[i].page);
> }
> - dma_unmap_single(rq->pdev, wi->umr.mtt_addr, mtt_sz, PCI_DMA_TODEVICE);
> -
> -err_free_mtt:
> - kfree(wi->umr.mtt_no_align);
> -
> -err_free_umr:
> - kfree(wi->umr.dma_info);
>
> -err_out:
> - return -ENOMEM;
> + return err;
> }
>
> -void mlx5e_free_rx_fragmented_mpwqe(struct mlx5e_rq *rq,
> - struct mlx5e_mpw_info *wi)
> +void mlx5e_free_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_mpw_info *wi)
> {
> - int mtt_sz = mlx5e_get_wqe_mtt_sz();
> + int pg_strides = mlx5e_mpwqe_strides_per_page(rq);
> int i;
>
> for (i = 0; i < MLX5_MPWRQ_PAGES_PER_WQE; i++) {
> dma_unmap_page(rq->pdev, wi->umr.dma_info[i].addr, PAGE_SIZE,
> PCI_DMA_FROMDEVICE);
> page_ref_sub(wi->umr.dma_info[i].page,
> - mlx5e_mpwqe_strides_per_page(rq) - wi->skbs_frags[i]);
> + pg_strides - wi->skbs_frags[i]);
> put_page(wi->umr.dma_info[i].page);
> }
> - dma_unmap_single(rq->pdev, wi->umr.mtt_addr, mtt_sz, PCI_DMA_TODEVICE);
> - kfree(wi->umr.mtt_no_align);
> - kfree(wi->umr.dma_info);
> }
>
> -void mlx5e_post_rx_fragmented_mpwqe(struct mlx5e_rq *rq)
> +void mlx5e_post_rx_mpwqe(struct mlx5e_rq *rq)
> {
> struct mlx5_wq_ll *wq = &rq->wq;
> struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(wq, wq->head);
> @@ -508,12 +381,11 @@ void mlx5e_post_rx_fragmented_mpwqe(struct mlx5e_rq *rq)
> clear_bit(MLX5E_RQ_STATE_UMR_WQE_IN_PROGRESS, &rq->state);
>
> if (unlikely(test_bit(MLX5E_RQ_STATE_FLUSH, &rq->state))) {
> - mlx5e_free_rx_fragmented_mpwqe(rq, &rq->wqe_info[wq->head]);
> + mlx5e_free_rx_mpwqe(rq, &rq->wqe_info[wq->head]);
> return;
> }
>
> mlx5_wq_ll_push(wq, be16_to_cpu(wqe->next.next_wqe_index));
> - rq->stats.mpwqe_frag++;
>
> /* ensure wqes are visible to device before updating doorbell record */
> dma_wmb();
> @@ -521,84 +393,23 @@ void mlx5e_post_rx_fragmented_mpwqe(struct mlx5e_rq *rq)
> mlx5_wq_ll_update_db_record(wq);
> }
>
> -static int mlx5e_alloc_rx_linear_mpwqe(struct mlx5e_rq *rq,
> - struct mlx5e_rx_wqe *wqe,
> - u16 ix)
> -{
> - struct mlx5e_mpw_info *wi = &rq->wqe_info[ix];
> - gfp_t gfp_mask;
> - int i;
> -
> - gfp_mask = GFP_ATOMIC | __GFP_COLD | __GFP_MEMALLOC;
> - wi->dma_info.page = alloc_pages_node(NUMA_NO_NODE, gfp_mask,
> - MLX5_MPWRQ_WQE_PAGE_ORDER);
> - if (unlikely(!wi->dma_info.page))
> - return -ENOMEM;
> -
> - wi->dma_info.addr = dma_map_page(rq->pdev, wi->dma_info.page, 0,
> - rq->wqe_sz, PCI_DMA_FROMDEVICE);
> - if (unlikely(dma_mapping_error(rq->pdev, wi->dma_info.addr))) {
> - put_page(wi->dma_info.page);
> - return -ENOMEM;
> - }
> -
> - /* We split the high-order page into order-0 ones and manage their
> - * reference counter to minimize the memory held by small skb fragments
> - */
> - split_page(wi->dma_info.page, MLX5_MPWRQ_WQE_PAGE_ORDER);
> - for (i = 0; i < MLX5_MPWRQ_PAGES_PER_WQE; i++) {
> - page_ref_add(&wi->dma_info.page[i],
> - mlx5e_mpwqe_strides_per_page(rq));
> - wi->skbs_frags[i] = 0;
> - }
> -
> - wi->consumed_strides = 0;
> - wi->dma_pre_sync = mlx5e_dma_pre_sync_linear_mpwqe;
> - wi->add_skb_frag = mlx5e_add_skb_frag_linear_mpwqe;
> - wi->copy_skb_header = mlx5e_copy_skb_header_linear_mpwqe;
> - wi->free_wqe = mlx5e_free_rx_linear_mpwqe;
> - wqe->data.lkey = rq->mkey_be;
> - wqe->data.addr = cpu_to_be64(wi->dma_info.addr);
> -
> - return 0;
> -}
> -
> -void mlx5e_free_rx_linear_mpwqe(struct mlx5e_rq *rq,
> - struct mlx5e_mpw_info *wi)
> -{
> - int i;
> -
> - dma_unmap_page(rq->pdev, wi->dma_info.addr, rq->wqe_sz,
> - PCI_DMA_FROMDEVICE);
> - for (i = 0; i < MLX5_MPWRQ_PAGES_PER_WQE; i++) {
> - page_ref_sub(&wi->dma_info.page[i],
> - mlx5e_mpwqe_strides_per_page(rq) - wi->skbs_frags[i]);
> - put_page(&wi->dma_info.page[i]);
> - }
> -}
> -
> -int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_rx_wqe *wqe, u16 ix)
> +int mlx5e_alloc_rx_mpwqe(struct mlx5e_rq *rq, struct mlx5e_rx_wqe *wqe, u16 ix)
> {
> int err;
>
> - err = mlx5e_alloc_rx_linear_mpwqe(rq, wqe, ix);
> - if (unlikely(err)) {
> - err = mlx5e_alloc_rx_fragmented_mpwqe(rq, wqe, ix);
> - if (unlikely(err))
> - return err;
> - set_bit(MLX5E_RQ_STATE_UMR_WQE_IN_PROGRESS, &rq->state);
> - mlx5e_post_umr_wqe(rq, ix);
> - return -EBUSY;
> - }
> -
> - return 0;
> + err = mlx5e_alloc_rx_umr_mpwqe(rq, wqe, ix);
> + if (unlikely(err))
> + return err;
> + set_bit(MLX5E_RQ_STATE_UMR_WQE_IN_PROGRESS, &rq->state);
> + mlx5e_post_umr_wqe(rq, ix);
> + return -EBUSY;
> }
>
> void mlx5e_dealloc_rx_mpwqe(struct mlx5e_rq *rq, u16 ix)
> {
> struct mlx5e_mpw_info *wi = &rq->wqe_info[ix];
>
> - wi->free_wqe(rq, wi);
> + mlx5e_free_rx_mpwqe(rq, wi);
> }
>
> #define RQ_CANNOT_POST(rq) \
> @@ -617,9 +428,10 @@ bool mlx5e_post_rx_wqes(struct mlx5e_rq *rq)
> int err;
>
> err = rq->alloc_wqe(rq, wqe, wq->head);
> + if (err == -EBUSY)
> + return true;
> if (unlikely(err)) {
> - if (err != -EBUSY)
> - rq->stats.buff_alloc_err++;
> + rq->stats.buff_alloc_err++;
> break;
> }
>
> @@ -823,7 +635,6 @@ static inline void mlx5e_mpwqe_fill_rx_skb(struct mlx5e_rq *rq,
> u32 cqe_bcnt,
> struct sk_buff *skb)
> {
> - u32 consumed_bytes = ALIGN(cqe_bcnt, rq->mpwqe_stride_sz);
> u16 stride_ix = mpwrq_get_cqe_stride_index(cqe);
> u32 wqe_offset = stride_ix * rq->mpwqe_stride_sz;
> u32 head_offset = wqe_offset & (PAGE_SIZE - 1);
> @@ -837,21 +648,20 @@ static inline void mlx5e_mpwqe_fill_rx_skb(struct mlx5e_rq *rq,
> page_idx++;
> frag_offset -= PAGE_SIZE;
> }
> - wi->dma_pre_sync(rq->pdev, wi, wqe_offset, consumed_bytes);
>
> while (byte_cnt) {
> u32 pg_consumed_bytes =
> min_t(u32, PAGE_SIZE - frag_offset, byte_cnt);
>
> - wi->add_skb_frag(rq, skb, wi, page_idx, frag_offset,
> - pg_consumed_bytes);
> + mlx5e_add_skb_frag_mpwqe(rq, skb, wi, page_idx, frag_offset,
> + pg_consumed_bytes);
> byte_cnt -= pg_consumed_bytes;
> frag_offset = 0;
> page_idx++;
> }
> /* copy header */
> - wi->copy_skb_header(rq->pdev, skb, wi, head_page_idx, head_offset,
> - headlen);
> + mlx5e_copy_skb_header_mpwqe(rq->pdev, skb, wi, head_page_idx,
> + head_offset, headlen);
> /* skb linear part was allocated with headlen and aligned to long */
> skb->tail += headlen;
> skb->len += headlen;
> @@ -896,7 +706,7 @@ mpwrq_cqe_out:
> if (likely(wi->consumed_strides < rq->mpwqe_num_strides))
> return;
>
> - wi->free_wqe(rq, wi);
> + mlx5e_free_rx_mpwqe(rq, wi);
> mlx5_wq_ll_pop(&rq->wq, cqe->wqe_id, &wqe->next.next_wqe_index);
> }
>
> diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h
> index 499487c..1f56543 100644
> --- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h
> +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h
> @@ -73,7 +73,6 @@ struct mlx5e_sw_stats {
> u64 tx_xmit_more;
> u64 rx_wqe_err;
> u64 rx_mpwqe_filler;
> - u64 rx_mpwqe_frag;
> u64 rx_buff_alloc_err;
> u64 rx_cqe_compress_blks;
> u64 rx_cqe_compress_pkts;
> @@ -105,7 +104,6 @@ static const struct counter_desc sw_stats_desc[] = {
> { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_xmit_more) },
> { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_wqe_err) },
> { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_mpwqe_filler) },
> - { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_mpwqe_frag) },
> { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_buff_alloc_err) },
> { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cqe_compress_blks) },
> { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_cqe_compress_pkts) },
> @@ -274,7 +272,6 @@ struct mlx5e_rq_stats {
> u64 lro_bytes;
> u64 wqe_err;
> u64 mpwqe_filler;
> - u64 mpwqe_frag;
> u64 buff_alloc_err;
> u64 cqe_compress_blks;
> u64 cqe_compress_pkts;
> @@ -290,7 +287,6 @@ static const struct counter_desc rq_stats_desc[] = {
> { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, lro_bytes) },
> { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, wqe_err) },
> { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, mpwqe_filler) },
> - { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, mpwqe_frag) },
> { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, buff_alloc_err) },
> { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cqe_compress_blks) },
> { MLX5E_DECLARE_RX_STAT(struct mlx5e_rq_stats, cqe_compress_pkts) },
> diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
> index 9bf33bb..08d8b0c 100644
> --- a/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
> +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_txrx.c
> @@ -87,7 +87,7 @@ static void mlx5e_poll_ico_cq(struct mlx5e_cq *cq)
> case MLX5_OPCODE_NOP:
> break;
> case MLX5_OPCODE_UMR:
> - mlx5e_post_rx_fragmented_mpwqe(&sq->channel->rq);
> + mlx5e_post_rx_mpwqe(&sq->channel->rq);
> break;
> default:
> WARN_ONCE(true,
--
Best regards,
Jesper Dangaard Brouer
MSc.CS, Principal Kernel Engineer at Red Hat
Author of http://www.iptv-analyzer.org
LinkedIn: http://www.linkedin.com/in/brouer
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@xxxxxxxxx. For more info on Linux MM,
see: http://www.linux-mm.org/ .
Don't email: <a href=mailto:"dont@xxxxxxxxx";> email@xxxxxxxxx </a>