On Fri, 2022-09-23 at 18:45 +0300, Ioana Ciornei wrote:
> From: Robert-Ionut Alexa <robert-ionut.alexa@xxxxxxx>
>
> This patch adds the support for receiving packets via the AF_XDP
> zero-copy mechanism in the dpaa2-eth driver. The support is available
> only on the LX2160A SoC and variants because we are relying on the HW
> capability to associate a buffer pool to a specific queue (QDBIN), only
> available on newer WRIOP versions.
>
> On the control path, the dpaa2_xsk_enable_pool() function is responsible
> to allocate a buffer pool (BP), setup this new BP to be used only on the
> requested queue and change the consume function to point to the XSK ZC
> one.
> We are forced to call dev_close() in order to change the queue to buffer
> pool association (dpaa2_xsk_set_bp_per_qdbin) . This also works in our
> favor since at dev_close() the buffer pools will be drained and at the
> later dev_open() call they will be again seeded, this time with buffers
> allocated from the XSK pool if needed.
>
> On the data path, a new software annotation type is defined to be used
> only for the XSK scenarios. This will enable us to pass keep necessary
> information about a packet buffer between the moment in which it was
> seeded and when it's received by the driver. In the XSK case, we are
> keeping the associated xdp_buff.
> Depending on the action returned by the BPF program, we will do the
> following:
> - XDP_PASS: copy the contents of the packet into a brand new skb,
> recycle the initial buffer.
> - XDP_TX: just enqueue the same frame descriptor back into the Tx path,
> the buffer will get automatically released into the initial BP.
> - XDP_REDIRECT: call xdp_do_redirect() and exit.
>
> Signed-off-by: Robert-Ionut Alexa <robert-ionut.alexa@xxxxxxx>
> Signed-off-by: Ioana Ciornei <ioana.ciornei@xxxxxxx>
> ---
> Changes in v2:
> - none
>
> MAINTAINERS | 1 +
> drivers/net/ethernet/freescale/dpaa2/Makefile | 2 +-
> .../net/ethernet/freescale/dpaa2/dpaa2-eth.c | 131 ++++---
> .../net/ethernet/freescale/dpaa2/dpaa2-eth.h | 36 +-
> .../net/ethernet/freescale/dpaa2/dpaa2-xsk.c | 327 ++++++++++++++++++
> 5 files changed, 452 insertions(+), 45 deletions(-)
> create mode 100644 drivers/net/ethernet/freescale/dpaa2/dpaa2-xsk.c
>
> diff --git a/MAINTAINERS b/MAINTAINERS
> index 1415a1498d68..004411566f48 100644
> --- a/MAINTAINERS
> +++ b/MAINTAINERS
> @@ -6290,6 +6290,7 @@ F: drivers/net/ethernet/freescale/dpaa2/Kconfig
> F: drivers/net/ethernet/freescale/dpaa2/Makefile
> F: drivers/net/ethernet/freescale/dpaa2/dpaa2-eth*
> F: drivers/net/ethernet/freescale/dpaa2/dpaa2-mac*
> +F: drivers/net/ethernet/freescale/dpaa2/dpaa2-xsk*
> F: drivers/net/ethernet/freescale/dpaa2/dpkg.h
> F: drivers/net/ethernet/freescale/dpaa2/dpmac*
> F: drivers/net/ethernet/freescale/dpaa2/dpni*
> diff --git a/drivers/net/ethernet/freescale/dpaa2/Makefile b/drivers/net/ethernet/freescale/dpaa2/Makefile
> index 3d9842af7f10..1b05ba8d1cbf 100644
> --- a/drivers/net/ethernet/freescale/dpaa2/Makefile
> +++ b/drivers/net/ethernet/freescale/dpaa2/Makefile
> @@ -7,7 +7,7 @@ obj-$(CONFIG_FSL_DPAA2_ETH) += fsl-dpaa2-eth.o
> obj-$(CONFIG_FSL_DPAA2_PTP_CLOCK) += fsl-dpaa2-ptp.o
> obj-$(CONFIG_FSL_DPAA2_SWITCH) += fsl-dpaa2-switch.o
>
> -fsl-dpaa2-eth-objs := dpaa2-eth.o dpaa2-ethtool.o dpni.o dpaa2-mac.o dpmac.o dpaa2-eth-devlink.o
> +fsl-dpaa2-eth-objs := dpaa2-eth.o dpaa2-ethtool.o dpni.o dpaa2-mac.o dpmac.o dpaa2-eth-devlink.o dpaa2-xsk.o
> fsl-dpaa2-eth-${CONFIG_FSL_DPAA2_ETH_DCB} += dpaa2-eth-dcb.o
> fsl-dpaa2-eth-${CONFIG_DEBUG_FS} += dpaa2-eth-debugfs.o
> fsl-dpaa2-ptp-objs := dpaa2-ptp.o dprtc.o
> diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c
> index d786740d1bdd..1e94506bf9e6 100644
> --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c
> +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.c
> @@ -19,6 +19,7 @@
> #include <net/pkt_cls.h>
> #include <net/sock.h>
> #include <net/tso.h>
> +#include <net/xdp_sock_drv.h>
>
> #include "dpaa2-eth.h"
>
> @@ -104,8 +105,8 @@ static void dpaa2_ptp_onestep_reg_update_method(struct dpaa2_eth_priv *priv)
> priv->dpaa2_set_onestep_params_cb = dpaa2_update_ptp_onestep_direct;
> }
>
> -static void *dpaa2_iova_to_virt(struct iommu_domain *domain,
> - dma_addr_t iova_addr)
> +void *dpaa2_iova_to_virt(struct iommu_domain *domain,
> + dma_addr_t iova_addr)
> {
> phys_addr_t phys_addr;
>
> @@ -279,23 +280,33 @@ static struct sk_buff *dpaa2_eth_build_frag_skb(struct dpaa2_eth_priv *priv,
> * be released in the pool
> */
> static void dpaa2_eth_free_bufs(struct dpaa2_eth_priv *priv, u64 *buf_array,
> - int count)
> + int count, bool xsk_zc)
> {
> struct device *dev = priv->net_dev->dev.parent;
> + struct dpaa2_eth_swa *swa;
> + struct xdp_buff *xdp_buff;
> void *vaddr;
> int i;
>
> for (i = 0; i < count; i++) {
> vaddr = dpaa2_iova_to_virt(priv->iommu_domain, buf_array[i]);
> - dma_unmap_page(dev, buf_array[i], priv->rx_buf_size,
> - DMA_BIDIRECTIONAL);
> - free_pages((unsigned long)vaddr, 0);
> +
> + if (!xsk_zc) {
> + dma_unmap_page(dev, buf_array[i], priv->rx_buf_size,
> + DMA_BIDIRECTIONAL);
> + free_pages((unsigned long)vaddr, 0);
> + } else {
> + swa = (struct dpaa2_eth_swa *)
> + (vaddr + DPAA2_ETH_RX_HWA_SIZE);
> + xdp_buff = swa->xsk.xdp_buff;
> + xsk_buff_free(xdp_buff);
> + }
> }
> }
>
> -static void dpaa2_eth_recycle_buf(struct dpaa2_eth_priv *priv,
> - struct dpaa2_eth_channel *ch,
> - dma_addr_t addr)
> +void dpaa2_eth_recycle_buf(struct dpaa2_eth_priv *priv,
> + struct dpaa2_eth_channel *ch,
> + dma_addr_t addr)
> {
> int retries = 0;
> int err;
> @@ -313,7 +324,8 @@ static void dpaa2_eth_recycle_buf(struct dpaa2_eth_priv *priv,
> }
>
> if (err) {
> - dpaa2_eth_free_bufs(priv, ch->recycled_bufs, ch->recycled_bufs_cnt);
> + dpaa2_eth_free_bufs(priv, ch->recycled_bufs,
> + ch->recycled_bufs_cnt, ch->xsk_zc);
> ch->buf_count -= ch->recycled_bufs_cnt;
> }
>
> @@ -377,10 +389,10 @@ static void dpaa2_eth_xdp_tx_flush(struct dpaa2_eth_priv *priv,
> fq->xdp_tx_fds.num = 0;
> }
>
> -static void dpaa2_eth_xdp_enqueue(struct dpaa2_eth_priv *priv,
> - struct dpaa2_eth_channel *ch,
> - struct dpaa2_fd *fd,
> - void *buf_start, u16 queue_id)
> +void dpaa2_eth_xdp_enqueue(struct dpaa2_eth_priv *priv,
> + struct dpaa2_eth_channel *ch,
> + struct dpaa2_fd *fd,
> + void *buf_start, u16 queue_id)
> {
> struct dpaa2_faead *faead;
> struct dpaa2_fd *dest_fd;
> @@ -1652,37 +1664,63 @@ static int dpaa2_eth_set_tx_csum(struct dpaa2_eth_priv *priv, bool enable)
> static int dpaa2_eth_add_bufs(struct dpaa2_eth_priv *priv,
> struct dpaa2_eth_channel *ch)
> {
> + struct xdp_buff *xdp_buffs[DPAA2_ETH_BUFS_PER_CMD];
> struct device *dev = priv->net_dev->dev.parent;
> u64 buf_array[DPAA2_ETH_BUFS_PER_CMD];
> + struct dpaa2_eth_swa *swa;
> struct page *page;
> dma_addr_t addr;
> int retries = 0;
> - int i, err;
> -
> - for (i = 0; i < DPAA2_ETH_BUFS_PER_CMD; i++) {
> - /* Allocate buffer visible to WRIOP + skb shared info +
> - * alignment padding
> - */
> - /* allocate one page for each Rx buffer. WRIOP sees
> - * the entire page except for a tailroom reserved for
> - * skb shared info
> + int i = 0, err;
> + u32 batch;
> +
> + /* Allocate buffers visible to WRIOP */
> + if (!ch->xsk_zc) {
> + for (i = 0; i < DPAA2_ETH_BUFS_PER_CMD; i++) {
> + /* Also allocate skb shared info and alignment padding.
> + * There is one page for each Rx buffer. WRIOP sees
> + * the entire page except for a tailroom reserved for
> + * skb shared info
> + */
> + page = dev_alloc_pages(0);
> + if (!page)
> + goto err_alloc;
> +
> + addr = dma_map_page(dev, page, 0, priv->rx_buf_size,
> + DMA_BIDIRECTIONAL);
> + if (unlikely(dma_mapping_error(dev, addr)))
> + goto err_map;
> +
> + buf_array[i] = addr;
> +
> + /* tracing point */
> + trace_dpaa2_eth_buf_seed(priv->net_dev,
> + page_address(page),
> + DPAA2_ETH_RX_BUF_RAW_SIZE,
> + addr, priv->rx_buf_size,
> + ch->bp->bpid);
> + }
> + } else if (xsk_buff_can_alloc(ch->xsk_pool, DPAA2_ETH_BUFS_PER_CMD)) {
> + /* Allocate XSK buffers for AF_XDP fast path in batches
> + * of DPAA2_ETH_BUFS_PER_CMD. Bail out if the UMEM cannot
> + * provide enough buffers at the moment
> */
> - page = dev_alloc_pages(0);
> - if (!page)
> + batch = xsk_buff_alloc_batch(ch->xsk_pool, xdp_buffs,
> + DPAA2_ETH_BUFS_PER_CMD);
> + if (!batch)
> goto err_alloc;
>
> - addr = dma_map_page(dev, page, 0, priv->rx_buf_size,
> - DMA_BIDIRECTIONAL);
> - if (unlikely(dma_mapping_error(dev, addr)))
> - goto err_map;
> + for (i = 0; i < batch; i++) {
> + swa = (struct dpaa2_eth_swa *)(xdp_buffs[i]->data_hard_start +
> + DPAA2_ETH_RX_HWA_SIZE);
> + swa->xsk.xdp_buff = xdp_buffs[i];
>
> - buf_array[i] = addr;
> + addr = xsk_buff_xdp_get_frame_dma(xdp_buffs[i]);
> + if (unlikely(dma_mapping_error(dev, addr)))
> + goto err_map;
>
> - /* tracing point */
> - trace_dpaa2_eth_buf_seed(priv->net_dev, page_address(page),
> - DPAA2_ETH_RX_BUF_RAW_SIZE,
> - addr, priv->rx_buf_size,
> - ch->bp->bpid);
> + buf_array[i] = addr;
> + }
> }
>
> release_bufs:
> @@ -1698,14 +1736,19 @@ static int dpaa2_eth_add_bufs(struct dpaa2_eth_priv *priv,
> * not much else we can do about it
> */
> if (err) {
> - dpaa2_eth_free_bufs(priv, buf_array, i);
> + dpaa2_eth_free_bufs(priv, buf_array, i, ch->xsk_zc);
> return 0;
> }
>
> return i;
>
> err_map:
> - __free_pages(page, 0);
> + if (!ch->xsk_zc) {
> + __free_pages(page, 0);
> + } else {
> + for (; i < batch; i++)
> + xsk_buff_free(xdp_buffs[i]);
> + }
> err_alloc:
> /* If we managed to allocate at least some buffers,
> * release them to hardware
> @@ -1764,8 +1807,13 @@ static void dpaa2_eth_drain_bufs(struct dpaa2_eth_priv *priv, int bpid,
> int count)
> {
> u64 buf_array[DPAA2_ETH_BUFS_PER_CMD];
> + bool xsk_zc = false;
> int retries = 0;
> - int ret;
> + int i, ret;
> +
> + for (i = 0; i < priv->num_channels; i++)
> + if (priv->channel[i]->bp->bpid == bpid)
> + xsk_zc = priv->channel[i]->xsk_zc;
>
> do {
> ret = dpaa2_io_service_acquire(NULL, bpid, buf_array, count);
> @@ -1776,7 +1824,7 @@ static void dpaa2_eth_drain_bufs(struct dpaa2_eth_priv *priv, int bpid,
> netdev_err(priv->net_dev, "dpaa2_io_service_acquire() failed\n");
> return;
> }
> - dpaa2_eth_free_bufs(priv, buf_array, ret);
> + dpaa2_eth_free_bufs(priv, buf_array, ret, xsk_zc);
> retries = 0;
> } while (ret);
> }
> @@ -2694,6 +2742,8 @@ static int dpaa2_eth_xdp(struct net_device *dev, struct netdev_bpf *xdp)
> switch (xdp->command) {
> case XDP_SETUP_PROG:
> return dpaa2_eth_setup_xdp(dev, xdp->prog);
> + case XDP_SETUP_XSK_POOL:
> + return dpaa2_xsk_setup_pool(dev, xdp->xsk.pool, xdp->xsk.queue_id);
> default:
> return -EINVAL;
> }
> @@ -2924,6 +2974,7 @@ static const struct net_device_ops dpaa2_eth_ops = {
> .ndo_change_mtu = dpaa2_eth_change_mtu,
> .ndo_bpf = dpaa2_eth_xdp,
> .ndo_xdp_xmit = dpaa2_eth_xdp_xmit,
> + .ndo_xsk_wakeup = dpaa2_xsk_wakeup,
> .ndo_setup_tc = dpaa2_eth_setup_tc,
> .ndo_vlan_rx_add_vid = dpaa2_eth_rx_add_vid,
> .ndo_vlan_rx_kill_vid = dpaa2_eth_rx_kill_vid
> @@ -4246,8 +4297,8 @@ static int dpaa2_eth_bind_dpni(struct dpaa2_eth_priv *priv)
> {
> struct dpaa2_eth_bp *bp = priv->bp[DPAA2_ETH_DEFAULT_BP_IDX];
> struct net_device *net_dev = priv->net_dev;
> + struct dpni_pools_cfg pools_params = { 0 };
> struct device *dev = net_dev->dev.parent;
> - struct dpni_pools_cfg pools_params;
> struct dpni_error_cfg err_cfg;
> int err = 0;
> int i;
> diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.h b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.h
> index 3c4fc46b1324..38f67b98865f 100644
> --- a/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.h
> +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-eth.h
> @@ -130,6 +130,7 @@ enum dpaa2_eth_swa_type {
> DPAA2_ETH_SWA_SINGLE,
> DPAA2_ETH_SWA_SG,
> DPAA2_ETH_SWA_XDP,
> + DPAA2_ETH_SWA_XSK,
> DPAA2_ETH_SWA_SW_TSO,
> };
>
> @@ -151,6 +152,9 @@ struct dpaa2_eth_swa {
> int dma_size;
> struct xdp_frame *xdpf;
> } xdp;
> + struct {
> + struct xdp_buff *xdp_buff;
> + } xsk;
> struct {
> struct sk_buff *skb;
> int num_sg;
> @@ -429,12 +433,19 @@ enum dpaa2_eth_fq_type {
> };
>
> struct dpaa2_eth_priv;
> +struct dpaa2_eth_channel;
> +struct dpaa2_eth_fq;
>
> struct dpaa2_eth_xdp_fds {
> struct dpaa2_fd fds[DEV_MAP_BULK_SIZE];
> ssize_t num;
> };
>
> +typedef void dpaa2_eth_consume_cb_t(struct dpaa2_eth_priv *priv,
> + struct dpaa2_eth_channel *ch,
> + const struct dpaa2_fd *fd,
> + struct dpaa2_eth_fq *fq);
> +
> struct dpaa2_eth_fq {
> u32 fqid;
> u32 tx_qdbin;
> @@ -447,10 +458,7 @@ struct dpaa2_eth_fq {
> struct dpaa2_eth_channel *channel;
> enum dpaa2_eth_fq_type type;
>
> - void (*consume)(struct dpaa2_eth_priv *priv,
> - struct dpaa2_eth_channel *ch,
> - const struct dpaa2_fd *fd,
> - struct dpaa2_eth_fq *fq);
> + dpaa2_eth_consume_cb_t *consume;
> struct dpaa2_eth_fq_stats stats;
>
> struct dpaa2_eth_xdp_fds xdp_redirect_fds;
> @@ -486,6 +494,8 @@ struct dpaa2_eth_channel {
> u64 recycled_bufs[DPAA2_ETH_BUFS_PER_CMD];
> int recycled_bufs_cnt;
>
> + bool xsk_zc;
> + struct xsk_buff_pool *xsk_pool;
> struct dpaa2_eth_bp *bp;
> };
>
> @@ -808,4 +818,22 @@ void dpaa2_eth_rx(struct dpaa2_eth_priv *priv,
> struct dpaa2_eth_channel *ch,
> const struct dpaa2_fd *fd,
> struct dpaa2_eth_fq *fq);
> +
> +struct dpaa2_eth_bp *dpaa2_eth_allocate_dpbp(struct dpaa2_eth_priv *priv);
> +void dpaa2_eth_free_dpbp(struct dpaa2_eth_priv *priv,
> + struct dpaa2_eth_bp *bp);
> +
> +void *dpaa2_iova_to_virt(struct iommu_domain *domain, dma_addr_t iova_addr);
> +void dpaa2_eth_recycle_buf(struct dpaa2_eth_priv *priv,
> + struct dpaa2_eth_channel *ch,
> + dma_addr_t addr);
> +
> +void dpaa2_eth_xdp_enqueue(struct dpaa2_eth_priv *priv,
> + struct dpaa2_eth_channel *ch,
> + struct dpaa2_fd *fd,
> + void *buf_start, u16 queue_id);
> +
> +int dpaa2_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags);
> +int dpaa2_xsk_setup_pool(struct net_device *dev, struct xsk_buff_pool *pool, u16 qid);
> +
> #endif /* __DPAA2_H */
> diff --git a/drivers/net/ethernet/freescale/dpaa2/dpaa2-xsk.c b/drivers/net/ethernet/freescale/dpaa2/dpaa2-xsk.c
> new file mode 100644
> index 000000000000..2df7bffec5a7
> --- /dev/null
> +++ b/drivers/net/ethernet/freescale/dpaa2/dpaa2-xsk.c
> @@ -0,0 +1,327 @@
> +// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
> +/* Copyright 2022 NXP
> + */
> +#include <linux/filter.h>
> +#include <linux/compiler.h>
> +#include <linux/bpf_trace.h>
> +#include <net/xdp.h>
> +#include <net/xdp_sock_drv.h>
> +
> +#include "dpaa2-eth.h"
> +
> +static void dpaa2_eth_setup_consume_func(struct dpaa2_eth_priv *priv,
> + struct dpaa2_eth_channel *ch,
> + enum dpaa2_eth_fq_type type,
> + dpaa2_eth_consume_cb_t *consume)
> +{
> + struct dpaa2_eth_fq *fq;
> + int i;
> +
> + for (i = 0; i < priv->num_fqs; i++) {
> + fq = &priv->fq[i];
> +
> + if (fq->type != type)
> + continue;
> + if (fq->channel != ch)
> + continue;
> +
> + fq->consume = consume;
> + }
> +}
> +
> +static u32 dpaa2_xsk_run_xdp(struct dpaa2_eth_priv *priv,
> + struct dpaa2_eth_channel *ch,
> + struct dpaa2_eth_fq *rx_fq,
> + struct dpaa2_fd *fd, void *vaddr)
> +{
> + dma_addr_t addr = dpaa2_fd_get_addr(fd);
> + struct bpf_prog *xdp_prog;
> + struct xdp_buff *xdp_buff;
> + struct dpaa2_eth_swa *swa;
> + u32 xdp_act = XDP_PASS;
> + int err;
> +
> + xdp_prog = READ_ONCE(ch->xdp.prog);
> + if (!xdp_prog)
> + goto out;
> +
> + swa = (struct dpaa2_eth_swa *)(vaddr + DPAA2_ETH_RX_HWA_SIZE +
> + ch->xsk_pool->umem->headroom);
> + xdp_buff = swa->xsk.xdp_buff;
> +
> + xdp_buff->data_hard_start = vaddr;
> + xdp_buff->data = vaddr + dpaa2_fd_get_offset(fd);
> + xdp_buff->data_end = xdp_buff->data + dpaa2_fd_get_len(fd);
> + xdp_set_data_meta_invalid(xdp_buff);
> + xdp_buff->rxq = &ch->xdp_rxq;
> +
> + xsk_buff_dma_sync_for_cpu(xdp_buff, ch->xsk_pool);
> + xdp_act = bpf_prog_run_xdp(xdp_prog, xdp_buff);
> +
> + /* xdp.data pointer may have changed */
> + dpaa2_fd_set_offset(fd, xdp_buff->data - vaddr);
> + dpaa2_fd_set_len(fd, xdp_buff->data_end - xdp_buff->data);
> +
> + if (likely(xdp_act == XDP_REDIRECT)) {
> + err = xdp_do_redirect(priv->net_dev, xdp_buff, xdp_prog);
> + if (unlikely(err)) {
> + ch->stats.xdp_drop++;
> + dpaa2_eth_recycle_buf(priv, ch, addr);
> + } else {
> + ch->buf_count--;
> + ch->stats.xdp_redirect++;
> + }
> +
> + goto xdp_redir;
> + }
> +
> + switch (xdp_act) {
> + case XDP_PASS:
> + break;
> + case XDP_TX:
> + dpaa2_eth_xdp_enqueue(priv, ch, fd, vaddr, rx_fq->flowid);
> + break;
> + default:
> + bpf_warn_invalid_xdp_action(priv->net_dev, xdp_prog, xdp_act);
> + fallthrough;
> + case XDP_ABORTED:
> + trace_xdp_exception(priv->net_dev, xdp_prog, xdp_act);
> + fallthrough;
> + case XDP_DROP:
> + dpaa2_eth_recycle_buf(priv, ch, addr);
> + ch->stats.xdp_drop++;
> + break;
> + }
> +
> +xdp_redir:
> + ch->xdp.res |= xdp_act;
> +out:
> + return xdp_act;
> +}
> +
> +/* Rx frame processing routine for the AF_XDP fast path */
> +static void dpaa2_xsk_rx(struct dpaa2_eth_priv *priv,
> + struct dpaa2_eth_channel *ch,
> + const struct dpaa2_fd *fd,
> + struct dpaa2_eth_fq *fq)
> +{
> + dma_addr_t addr = dpaa2_fd_get_addr(fd);
> + u8 fd_format = dpaa2_fd_get_format(fd);
> + struct rtnl_link_stats64 *percpu_stats;
> + u32 fd_length = dpaa2_fd_get_len(fd);
> + struct sk_buff *skb;
> + void *vaddr;
> + u32 xdp_act;
> +
> + vaddr = dpaa2_iova_to_virt(priv->iommu_domain, addr);
> + percpu_stats = this_cpu_ptr(priv->percpu_stats);
> +
> + if (fd_format != dpaa2_fd_single) {
> + WARN_ON(priv->xdp_prog);
> + /* AF_XDP doesn't support any other formats */
> + goto err_frame_format;
> + }
> +
> + xdp_act = dpaa2_xsk_run_xdp(priv, ch, fq, (struct dpaa2_fd *)fd, vaddr);
> + if (xdp_act != XDP_PASS) {
> + percpu_stats->rx_packets++;
> + percpu_stats->rx_bytes += dpaa2_fd_get_len(fd);
> + return;
> + }
> +
> + /* Build skb */
> + skb = dpaa2_eth_alloc_skb(priv, ch, fd, fd_length, vaddr);
> + if (!skb)
> + /* Nothing else we can do, recycle the buffer and
> + * drop the frame.
> + */
> + goto err_alloc_skb;
> +
> + /* Send the skb to the Linux networking stack */
> + dpaa2_eth_receive_skb(priv, ch, fd, vaddr, fq, percpu_stats, skb);
> +
> + return;
> +
> +err_alloc_skb:
> + dpaa2_eth_recycle_buf(priv, ch, addr);
> +err_frame_format:
> + percpu_stats->rx_dropped++;
> +}
> +
> +static void dpaa2_xsk_set_bp_per_qdbin(struct dpaa2_eth_priv *priv,
> + struct dpni_pools_cfg *pools_params)
> +{
> + int curr_bp = 0, i, j;
> +
> + pools_params->pool_options = DPNI_POOL_ASSOC_QDBIN;
> + for (i = 0; i < priv->num_bps; i++) {
> + for (j = 0; j < priv->num_channels; j++)
> + if (priv->bp[i] == priv->channel[j]->bp)
> + pools_params->pools[curr_bp].priority_mask |= (1 << j);
> + if (!pools_params->pools[curr_bp].priority_mask)
> + continue;
> +
> + pools_params->pools[curr_bp].dpbp_id = priv->bp[i]->bpid;
> + pools_params->pools[curr_bp].buffer_size = priv->rx_buf_size;
> + pools_params->pools[curr_bp++].backup_pool = 0;
> + }
> + pools_params->num_dpbp = curr_bp;
> +}
> +
> +static int dpaa2_xsk_disable_pool(struct net_device *dev, u16 qid)
> +{
> + struct xsk_buff_pool *pool = xsk_get_pool_from_qid(dev, qid);
> + struct dpaa2_eth_priv *priv = netdev_priv(dev);
> + struct dpni_pools_cfg pools_params = { 0 };
> + struct dpaa2_eth_channel *ch;
> + int err;
> + bool up;
> +
> + ch = priv->channel[qid];
> + if (!ch->xsk_pool)
> + return -EINVAL;
> +
> + up = netif_running(dev);
> + if (up)
> + dev_close(dev);
> +
> + xsk_pool_dma_unmap(pool, 0);
> + err = xdp_rxq_info_reg_mem_model(&ch->xdp_rxq,
> + MEM_TYPE_PAGE_ORDER0, NULL);
> + if (err)
> + netdev_err(dev, "xsk_rxq_info_reg_mem_model() failed (err = %d)\n",
> + err);
> +
> + dpaa2_eth_free_dpbp(priv, ch->bp);
> +
> + ch->xsk_zc = false;
> + ch->xsk_pool = NULL;
> + ch->bp = priv->bp[DPAA2_ETH_DEFAULT_BP_IDX];
> +
> + dpaa2_eth_setup_consume_func(priv, ch, DPAA2_RX_FQ, dpaa2_eth_rx);
> +
> + dpaa2_xsk_set_bp_per_qdbin(priv, &pools_params);
> + err = dpni_set_pools(priv->mc_io, 0, priv->mc_token, &pools_params);
> + if (err)
> + netdev_err(dev, "dpni_set_pools() failed\n");
> +
> + if (up) {
> + err = dev_open(dev, NULL);
> + if (err)
> + return err;
Same problem of patch 7/12, it's better to avoid the device changing
status for this kind of operations - even on unlikely error paths.
Thanks,
Paolo
