Handle the XDP_REDIRECT_XSK action on the SKB path by retrieving the
handle to the socket from the netdev_rx_queue struct and immediately
calling the xsk_generic_rcv function. Also, prepare for supporting
this action in the drivers by exposing the xsk_rcv and xsk_flush functions
so they can be used directly in the drivers.
Signed-off-by: Ciara Loftus <ciara.loftus@xxxxxxxxx>
---
include/net/xdp_sock_drv.h | 21 +++++++++++++++++++++
net/core/dev.c | 14 ++++++++++++++
net/core/filter.c | 4 ++++
net/xdp/xsk.c | 6 ++++--
4 files changed, 43 insertions(+), 2 deletions(-)
diff --git a/include/net/xdp_sock_drv.h b/include/net/xdp_sock_drv.h
index 443d45951564..e923f5d1adb6 100644
--- a/include/net/xdp_sock_drv.h
+++ b/include/net/xdp_sock_drv.h
@@ -22,6 +22,8 @@ void xsk_set_tx_need_wakeup(struct xsk_buff_pool *pool);
void xsk_clear_rx_need_wakeup(struct xsk_buff_pool *pool);
void xsk_clear_tx_need_wakeup(struct xsk_buff_pool *pool);
bool xsk_uses_need_wakeup(struct xsk_buff_pool *pool);
+int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp);
+void xsk_flush(struct xdp_sock *xs);
static inline u32 xsk_pool_get_headroom(struct xsk_buff_pool *pool)
{
@@ -130,6 +132,11 @@ static inline void xsk_buff_raw_dma_sync_for_device(struct xsk_buff_pool *pool,
xp_dma_sync_for_device(pool, dma, size);
}
+static inline struct xdp_sock *xsk_get_redirect_xsk(struct netdev_rx_queue *q)
+{
+ return READ_ONCE(q->xsk);
+}
+
#else
static inline void xsk_tx_completed(struct xsk_buff_pool *pool, u32 nb_entries)
@@ -179,6 +186,15 @@ static inline bool xsk_uses_need_wakeup(struct xsk_buff_pool *pool)
return false;
}
+static inline int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
+{
+ return 0;
+}
+
+static inline void xsk_flush(struct xdp_sock *xs)
+{
+}
+
static inline u32 xsk_pool_get_headroom(struct xsk_buff_pool *pool)
{
return 0;
@@ -264,6 +280,11 @@ static inline void xsk_buff_raw_dma_sync_for_device(struct xsk_buff_pool *pool,
{
}
+static inline struct xdp_sock *xsk_get_redirect_xsk(struct netdev_rx_queue *q)
+{
+ return NULL;
+}
+
#endif /* CONFIG_XDP_SOCKETS */
#endif /* _LINUX_XDP_SOCK_DRV_H */
diff --git a/net/core/dev.c b/net/core/dev.c
index edeb811c454e..9b38b50f1f97 100644
--- a/net/core/dev.c
+++ b/net/core/dev.c
@@ -106,6 +106,7 @@
#include <net/pkt_sched.h>
#include <net/pkt_cls.h>
#include <net/checksum.h>
+#include <net/xdp_sock.h>
#include <net/xfrm.h>
#include <linux/highmem.h>
#include <linux/init.h>
@@ -4771,6 +4772,7 @@ u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp,
* kfree_skb in response to actions it cannot handle/XDP_DROP).
*/
switch (act) {
+ case XDP_REDIRECT_XSK:
case XDP_REDIRECT:
case XDP_TX:
__skb_push(skb, mac_len);
@@ -4819,6 +4821,7 @@ static u32 netif_receive_generic_xdp(struct sk_buff *skb,
act = bpf_prog_run_generic_xdp(skb, xdp, xdp_prog);
switch (act) {
+ case XDP_REDIRECT_XSK:
case XDP_REDIRECT:
case XDP_TX:
case XDP_PASS:
@@ -4875,6 +4878,17 @@ int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff *skb)
act = netif_receive_generic_xdp(skb, &xdp, xdp_prog);
if (act != XDP_PASS) {
switch (act) {
+#ifdef CONFIG_XDP_SOCKETS
+ case XDP_REDIRECT_XSK:
+ struct xdp_sock *xs =
+ READ_ONCE(skb->dev->_rx[xdp.rxq->queue_index].xsk);
+
+ err = xsk_generic_rcv(xs, &xdp);
+ if (err)
+ goto out_redir;
+ consume_skb(skb);
+ break;
+#endif
case XDP_REDIRECT:
err = xdp_do_generic_redirect(skb->dev, skb,
&xdp, xdp_prog);
diff --git a/net/core/filter.c b/net/core/filter.c
index 4497ad046790..c65262722c64 100644
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -8203,7 +8203,11 @@ static bool xdp_is_valid_access(int off, int size,
void bpf_warn_invalid_xdp_action(u32 act)
{
+#ifdef CONFIG_XDP_SOCKETS
+ const u32 act_max = XDP_REDIRECT_XSK;
+#else
const u32 act_max = XDP_REDIRECT;
+#endif
WARN_ONCE(1, "%s XDP return value %u, expect packet loss!\n",
act > act_max ? "Illegal" : "Driver unsupported",
diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c
index 94ee524b9ca8..ce004f5fae64 100644
--- a/net/xdp/xsk.c
+++ b/net/xdp/xsk.c
@@ -226,12 +226,13 @@ static int xsk_rcv_check(struct xdp_sock *xs, struct xdp_buff *xdp)
return 0;
}
-static void xsk_flush(struct xdp_sock *xs)
+void xsk_flush(struct xdp_sock *xs)
{
xskq_prod_submit(xs->rx);
__xskq_cons_release(xs->pool->fq);
sock_def_readable(&xs->sk);
}
+EXPORT_SYMBOL(xsk_flush);
int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
{
@@ -247,7 +248,7 @@ int xsk_generic_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
return err;
}
-static int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
+int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
{
int err;
u32 len;
@@ -266,6 +267,7 @@ static int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
xdp_return_buff(xdp);
return err;
}
+EXPORT_SYMBOL(xsk_rcv);
int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp)
{
--
2.17.1
