receive_merageable() use virtnet_xdp_handler()
Meanwhile, support Multi Buffer XDP.
Signed-off-by: Xuan Zhuo <xuanzhuo@xxxxxxxxxxxxxxxxx>
---
drivers/net/virtio/main.c | 88 +++++++++++++++------------------------
1 file changed, 33 insertions(+), 55 deletions(-)
diff --git a/drivers/net/virtio/main.c b/drivers/net/virtio/main.c
index d7a856bd8862..fb82035a0b7f 100644
--- a/drivers/net/virtio/main.c
+++ b/drivers/net/virtio/main.c
@@ -483,8 +483,10 @@ int virtnet_xdp_handler(struct bpf_prog *xdp_prog, struct xdp_buff *xdp,
unsigned int *xdp_xmit,
struct virtnet_rq_stats *stats)
{
+ struct skb_shared_info *shinfo;
struct xdp_frame *xdpf;
- int err;
+ struct page *xdp_page;
+ int err, i;
u32 act;
act = bpf_prog_run_xdp(xdp_prog, xdp);
@@ -527,6 +529,13 @@ int virtnet_xdp_handler(struct bpf_prog *xdp_prog, struct xdp_buff *xdp,
trace_xdp_exception(dev, xdp_prog, act);
fallthrough;
case XDP_DROP:
+ if (xdp_buff_has_frags(xdp)) {
+ shinfo = xdp_get_shared_info_from_buff(xdp);
+ for (i = 0; i < shinfo->nr_frags; i++) {
+ xdp_page = skb_frag_page(&shinfo->frags[i]);
+ put_page(xdp_page);
+ }
+ }
return VIRTNET_XDP_RES_DROP;
}
}
@@ -809,7 +818,7 @@ static int virtnet_build_xdp_buff_mrg(struct net_device *dev,
unsigned int xdp_frags_truesz = 0;
struct page *page;
skb_frag_t *frag;
- int offset;
+ int offset, i;
void *ctx;
xdp_init_buff(xdp, frame_sz, &rq->xdp_rxq);
@@ -842,7 +851,7 @@ static int virtnet_build_xdp_buff_mrg(struct net_device *dev,
dev->name, *num_buf,
virtio16_to_cpu(vi->vdev, hdr->num_buffers));
dev->stats.rx_length_errors++;
- return -EINVAL;
+ goto err;
}
stats->bytes += len;
@@ -861,7 +870,7 @@ static int virtnet_build_xdp_buff_mrg(struct net_device *dev,
pr_debug("%s: rx error: len %u exceeds truesize %lu\n",
dev->name, len, (unsigned long)(truesize - room));
dev->stats.rx_length_errors++;
- return -EINVAL;
+ goto err;
}
frag = &shinfo->frags[shinfo->nr_frags++];
@@ -876,6 +885,14 @@ static int virtnet_build_xdp_buff_mrg(struct net_device *dev,
*xdp_frags_truesize = xdp_frags_truesz;
return 0;
+
+err:
+ for (i = 0; i < shinfo->nr_frags; i++) {
+ page = skb_frag_page(&shinfo->frags[i]);
+ put_page(page);
+ }
+
+ return -EINVAL;
}
static struct sk_buff *receive_mergeable(struct net_device *dev,
@@ -919,13 +936,10 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
xdp_prog = rcu_dereference(rq->xdp_prog);
if (xdp_prog) {
unsigned int xdp_frags_truesz = 0;
- struct skb_shared_info *shinfo;
- struct xdp_frame *xdpf;
struct page *xdp_page;
struct xdp_buff xdp;
void *data;
u32 act;
- int i;
/* Transient failure which in theory could occur if
* in-flight packets from before XDP was enabled reach
@@ -983,69 +997,33 @@ static struct sk_buff *receive_mergeable(struct net_device *dev,
err = virtnet_build_xdp_buff_mrg(dev, vi, rq, &xdp, data, len, frame_sz,
&num_buf, &xdp_frags_truesz, stats);
if (unlikely(err))
- goto err_xdp_frags;
+ goto err_xdp;
- act = bpf_prog_run_xdp(xdp_prog, &xdp);
- stats->xdp_packets++;
+ act = virtnet_xdp_handler(xdp_prog, &xdp, dev, xdp_xmit, stats);
switch (act) {
- case XDP_PASS:
+ case VIRTNET_XDP_RES_PASS:
if (unlikely(xdp_page != page))
put_page(page);
+
head_skb = build_skb_from_xdp_buff(dev, vi, &xdp, xdp_frags_truesz);
rcu_read_unlock();
return head_skb;
- case XDP_TX:
- stats->xdp_tx++;
- xdpf = xdp_convert_buff_to_frame(&xdp);
- if (unlikely(!xdpf)) {
- netdev_dbg(dev, "convert buff to frame failed for xdp\n");
- goto err_xdp_frags;
- }
- err = virtnet_xdp_xmit(dev, 1, &xdpf, 0);
- if (unlikely(!err)) {
- xdp_return_frame_rx_napi(xdpf);
- } else if (unlikely(err < 0)) {
- trace_xdp_exception(vi->dev, xdp_prog, act);
- goto err_xdp_frags;
- }
- *xdp_xmit |= VIRTIO_XDP_TX;
- if (unlikely(xdp_page != page))
- put_page(page);
- rcu_read_unlock();
- goto xdp_xmit;
- case XDP_REDIRECT:
- stats->xdp_redirects++;
- err = xdp_do_redirect(dev, &xdp, xdp_prog);
- if (err)
- goto err_xdp_frags;
- *xdp_xmit |= VIRTIO_XDP_REDIR;
+
+ case VIRTNET_XDP_RES_CONSUMED:
if (unlikely(xdp_page != page))
put_page(page);
+
rcu_read_unlock();
goto xdp_xmit;
- default:
- bpf_warn_invalid_xdp_action(vi->dev, xdp_prog, act);
- fallthrough;
- case XDP_ABORTED:
- trace_xdp_exception(vi->dev, xdp_prog, act);
- fallthrough;
- case XDP_DROP:
- goto err_xdp_frags;
- }
-err_xdp_frags:
- if (unlikely(xdp_page != page))
- __free_pages(xdp_page, 0);
- if (xdp_buff_has_frags(&xdp)) {
- shinfo = xdp_get_shared_info_from_buff(&xdp);
- for (i = 0; i < shinfo->nr_frags; i++) {
- xdp_page = skb_frag_page(&shinfo->frags[i]);
+ case VIRTNET_XDP_RES_DROP:
+ if (unlikely(xdp_page != page))
put_page(xdp_page);
- }
- }
- goto err_xdp;
+ rcu_read_unlock();
+ goto err_xdp;
+ }
}
rcu_read_unlock();
--
2.32.0.3.g01195cf9f
