5.15-stable review patch. If anyone has any objections, please let me know.
------------------
From: Robert Hancock <robert.hancock@xxxxxxxxxx>
[ Upstream commit cc37610caaf8d13a6ecb8afd1fe2ebc2424ff622 ]
Implement NAPI and GRO receive. In addition to better performance, this
also avoids handling RX packets in hard IRQ context, which reduces the
IRQ latency impact to other devices.
Signed-off-by: Robert Hancock <robert.hancock@xxxxxxxxxx>
Signed-off-by: David S. Miller <davem@xxxxxxxxxxxxx>
Stable-dep-of: 5a6caa2cfabb ("net: xilinx: axienet: Fix packet counting")
Signed-off-by: Sasha Levin <sashal@xxxxxxxxxx>
---
drivers/net/ethernet/xilinx/xilinx_axienet.h | 6 ++
.../net/ethernet/xilinx/xilinx_axienet_main.c | 81 ++++++++++++-------
2 files changed, 59 insertions(+), 28 deletions(-)
diff --git a/drivers/net/ethernet/xilinx/xilinx_axienet.h b/drivers/net/ethernet/xilinx/xilinx_axienet.h
index bdda836115095..e8a210201f744 100644
--- a/drivers/net/ethernet/xilinx/xilinx_axienet.h
+++ b/drivers/net/ethernet/xilinx/xilinx_axienet.h
@@ -386,6 +386,7 @@ struct axidma_bd {
* @phy_node: Pointer to device node structure
* @phylink: Pointer to phylink instance
* @phylink_config: phylink configuration settings
+ * @napi: NAPI control structure
* @pcs_phy: Reference to PCS/PMA PHY if used
* @switch_x_sgmii: Whether switchable 1000BaseX/SGMII mode is enabled in the core
* @axi_clk: AXI4-Lite bus clock
@@ -395,6 +396,7 @@ struct axidma_bd {
* @regs_start: Resource start for axienet device addresses
* @regs: Base address for the axienet_local device address space
* @dma_regs: Base address for the axidma device address space
+ * @rx_dma_cr: Nominal content of RX DMA control register
* @dma_err_task: Work structure to process Axi DMA errors
* @tx_irq: Axidma TX IRQ number
* @rx_irq: Axidma RX IRQ number
@@ -434,6 +436,8 @@ struct axienet_local {
struct phylink *phylink;
struct phylink_config phylink_config;
+ struct napi_struct napi;
+
struct mdio_device *pcs_phy;
bool switch_x_sgmii;
@@ -448,6 +452,8 @@ struct axienet_local {
void __iomem *regs;
void __iomem *dma_regs;
+ u32 rx_dma_cr;
+
struct work_struct dma_err_task;
int tx_irq;
diff --git a/drivers/net/ethernet/xilinx/xilinx_axienet_main.c b/drivers/net/ethernet/xilinx/xilinx_axienet_main.c
index 7bb8d04c997e7..a33e860861d55 100644
--- a/drivers/net/ethernet/xilinx/xilinx_axienet_main.c
+++ b/drivers/net/ethernet/xilinx/xilinx_axienet_main.c
@@ -7,7 +7,7 @@
* Copyright (c) 2008-2009 Secret Lab Technologies Ltd.
* Copyright (c) 2010 - 2011 Michal Simek <monstr@xxxxxxxxx>
* Copyright (c) 2010 - 2011 PetaLogix
- * Copyright (c) 2019 SED Systems, a division of Calian Ltd.
+ * Copyright (c) 2019 - 2022 Calian Advanced Technologies
* Copyright (c) 2010 - 2012 Xilinx, Inc. All rights reserved.
*
* This is a driver for the Xilinx Axi Ethernet which is used in the Virtex6
@@ -232,18 +232,18 @@ static void axienet_dma_bd_release(struct net_device *ndev)
*/
static void axienet_dma_start(struct axienet_local *lp)
{
- u32 rx_cr, tx_cr;
+ u32 tx_cr;
/* Start updating the Rx channel control register */
- rx_cr = (lp->coalesce_count_rx << XAXIDMA_COALESCE_SHIFT) |
- XAXIDMA_IRQ_IOC_MASK | XAXIDMA_IRQ_ERROR_MASK;
+ lp->rx_dma_cr = (lp->coalesce_count_rx << XAXIDMA_COALESCE_SHIFT) |
+ XAXIDMA_IRQ_IOC_MASK | XAXIDMA_IRQ_ERROR_MASK;
/* Only set interrupt delay timer if not generating an interrupt on
* the first RX packet. Otherwise leave at 0 to disable delay interrupt.
*/
if (lp->coalesce_count_rx > 1)
- rx_cr |= (XAXIDMA_DFT_RX_WAITBOUND << XAXIDMA_DELAY_SHIFT) |
- XAXIDMA_IRQ_DELAY_MASK;
- axienet_dma_out32(lp, XAXIDMA_RX_CR_OFFSET, rx_cr);
+ lp->rx_dma_cr |= (XAXIDMA_DFT_RX_WAITBOUND << XAXIDMA_DELAY_SHIFT) |
+ XAXIDMA_IRQ_DELAY_MASK;
+ axienet_dma_out32(lp, XAXIDMA_RX_CR_OFFSET, lp->rx_dma_cr);
/* Start updating the Tx channel control register */
tx_cr = (lp->coalesce_count_tx << XAXIDMA_COALESCE_SHIFT) |
@@ -260,8 +260,8 @@ static void axienet_dma_start(struct axienet_local *lp)
* halted state. This will make the Rx side ready for reception.
*/
axienet_dma_out_addr(lp, XAXIDMA_RX_CDESC_OFFSET, lp->rx_bd_p);
- rx_cr |= XAXIDMA_CR_RUNSTOP_MASK;
- axienet_dma_out32(lp, XAXIDMA_RX_CR_OFFSET, rx_cr);
+ lp->rx_dma_cr |= XAXIDMA_CR_RUNSTOP_MASK;
+ axienet_dma_out32(lp, XAXIDMA_RX_CR_OFFSET, lp->rx_dma_cr);
axienet_dma_out_addr(lp, XAXIDMA_RX_TDESC_OFFSET, lp->rx_bd_p +
(sizeof(*lp->rx_bd_v) * (lp->rx_bd_num - 1)));
@@ -876,28 +876,26 @@ axienet_start_xmit(struct sk_buff *skb, struct net_device *ndev)
}
/**
- * axienet_recv - Is called from Axi DMA Rx Isr to complete the received
- * BD processing.
- * @ndev: Pointer to net_device structure.
+ * axienet_poll - Triggered by RX ISR to complete the received BD processing.
+ * @napi: Pointer to NAPI structure.
+ * @budget: Max number of packets to process.
*
- * This function is invoked from the Axi DMA Rx isr to process the Rx BDs. It
- * does minimal processing and invokes "netif_rx" to complete further
- * processing.
+ * Return: Number of RX packets processed.
*/
-static void axienet_recv(struct net_device *ndev)
+static int axienet_poll(struct napi_struct *napi, int budget)
{
u32 length;
u32 csumstatus;
u32 size = 0;
- u32 packets = 0;
+ int packets = 0;
dma_addr_t tail_p = 0;
- struct axienet_local *lp = netdev_priv(ndev);
- struct sk_buff *skb, *new_skb;
struct axidma_bd *cur_p;
+ struct sk_buff *skb, *new_skb;
+ struct axienet_local *lp = container_of(napi, struct axienet_local, napi);
cur_p = &lp->rx_bd_v[lp->rx_bd_ci];
- while ((cur_p->status & XAXIDMA_BD_STS_COMPLETE_MASK)) {
+ while (packets < budget && (cur_p->status & XAXIDMA_BD_STS_COMPLETE_MASK)) {
dma_addr_t phys;
/* Ensure we see complete descriptor update */
@@ -919,7 +917,7 @@ static void axienet_recv(struct net_device *ndev)
DMA_FROM_DEVICE);
skb_put(skb, length);
- skb->protocol = eth_type_trans(skb, ndev);
+ skb->protocol = eth_type_trans(skb, lp->ndev);
/*skb_checksum_none_assert(skb);*/
skb->ip_summed = CHECKSUM_NONE;
@@ -938,13 +936,13 @@ static void axienet_recv(struct net_device *ndev)
skb->ip_summed = CHECKSUM_COMPLETE;
}
- netif_rx(skb);
+ napi_gro_receive(napi, skb);
size += length;
packets++;
}
- new_skb = netdev_alloc_skb_ip_align(ndev, lp->max_frm_size);
+ new_skb = netdev_alloc_skb_ip_align(lp->ndev, lp->max_frm_size);
if (!new_skb)
break;
@@ -953,7 +951,7 @@ static void axienet_recv(struct net_device *ndev)
DMA_FROM_DEVICE);
if (unlikely(dma_mapping_error(lp->dev, phys))) {
if (net_ratelimit())
- netdev_err(ndev, "RX DMA mapping error\n");
+ netdev_err(lp->ndev, "RX DMA mapping error\n");
dev_kfree_skb(new_skb);
break;
}
@@ -973,11 +971,20 @@ static void axienet_recv(struct net_device *ndev)
cur_p = &lp->rx_bd_v[lp->rx_bd_ci];
}
- ndev->stats.rx_packets += packets;
- ndev->stats.rx_bytes += size;
+ lp->ndev->stats.rx_packets += packets;
+ lp->ndev->stats.rx_bytes += size;
if (tail_p)
axienet_dma_out_addr(lp, XAXIDMA_RX_TDESC_OFFSET, tail_p);
+
+ if (packets < budget && napi_complete_done(napi, packets)) {
+ /* Re-enable RX completion interrupts. This should
+ * cause an immediate interrupt if any RX packets are
+ * already pending.
+ */
+ axienet_dma_out32(lp, XAXIDMA_RX_CR_OFFSET, lp->rx_dma_cr);
+ }
+ return packets;
}
/**
@@ -1023,7 +1030,7 @@ static irqreturn_t axienet_tx_irq(int irq, void *_ndev)
*
* Return: IRQ_HANDLED if device generated a RX interrupt, IRQ_NONE otherwise.
*
- * This is the Axi DMA Rx Isr. It invokes "axienet_recv" to complete the BD
+ * This is the Axi DMA Rx Isr. It invokes NAPI polling to complete the RX BD
* processing.
*/
static irqreturn_t axienet_rx_irq(int irq, void *_ndev)
@@ -1046,7 +1053,15 @@ static irqreturn_t axienet_rx_irq(int irq, void *_ndev)
(lp->rx_bd_v[lp->rx_bd_ci]).phys);
schedule_work(&lp->dma_err_task);
} else {
- axienet_recv(lp->ndev);
+ /* Disable further RX completion interrupts and schedule
+ * NAPI receive.
+ */
+ u32 cr = lp->rx_dma_cr;
+
+ cr &= ~(XAXIDMA_IRQ_IOC_MASK | XAXIDMA_IRQ_DELAY_MASK);
+ axienet_dma_out32(lp, XAXIDMA_RX_CR_OFFSET, cr);
+
+ napi_schedule(&lp->napi);
}
return IRQ_HANDLED;
@@ -1122,6 +1137,8 @@ static int axienet_open(struct net_device *ndev)
/* Enable worker thread for Axi DMA error handling */
INIT_WORK(&lp->dma_err_task, axienet_dma_err_handler);
+ napi_enable(&lp->napi);
+
/* Enable interrupts for Axi DMA Tx */
ret = request_irq(lp->tx_irq, axienet_tx_irq, IRQF_SHARED,
ndev->name, ndev);
@@ -1147,6 +1164,7 @@ static int axienet_open(struct net_device *ndev)
err_rx_irq:
free_irq(lp->tx_irq, ndev);
err_tx_irq:
+ napi_disable(&lp->napi);
phylink_stop(lp->phylink);
phylink_disconnect_phy(lp->phylink);
cancel_work_sync(&lp->dma_err_task);
@@ -1170,6 +1188,8 @@ static int axienet_stop(struct net_device *ndev)
dev_dbg(&ndev->dev, "axienet_close()\n");
+ napi_disable(&lp->napi);
+
phylink_stop(lp->phylink);
phylink_disconnect_phy(lp->phylink);
@@ -1743,6 +1763,8 @@ static void axienet_dma_err_handler(struct work_struct *work)
dma_err_task);
struct net_device *ndev = lp->ndev;
+ napi_disable(&lp->napi);
+
axienet_setoptions(ndev, lp->options &
~(XAE_OPTION_TXEN | XAE_OPTION_RXEN));
@@ -1807,6 +1829,7 @@ static void axienet_dma_err_handler(struct work_struct *work)
axienet_set_mac_address(ndev, NULL);
axienet_set_multicast_list(ndev);
axienet_setoptions(ndev, lp->options);
+ napi_enable(&lp->napi);
}
/**
@@ -1855,6 +1878,8 @@ static int axienet_probe(struct platform_device *pdev)
lp->rx_bd_num = RX_BD_NUM_DEFAULT;
lp->tx_bd_num = TX_BD_NUM_DEFAULT;
+ netif_napi_add(ndev, &lp->napi, axienet_poll, NAPI_POLL_WEIGHT);
+
lp->axi_clk = devm_clk_get_optional(&pdev->dev, "s_axi_lite_clk");
if (!lp->axi_clk) {
/* For backward compatibility, if named AXI clock is not present,
--
2.43.0
