The per-CPU flush lists are accessed from within the NAPI callback
(xdp_do_flush() for instance). They are subject to the same problem as struct
bpf_redirect_info.
Add the per-CPU lists cpu_map_flush_list, dev_map_flush_list and
xskmap_map_flush_list to struct bpf_xdp_storage. Add wrappers for the
access. Use it only on PREEMPT_RT, keep the per-CPU lists on
non-PREEMPT_RT builds.
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@xxxxxxxxxxxxx>
---
include/linux/filter.h | 34 ++++++++++++++++++++++++++++++++++
kernel/bpf/cpumap.c | 33 ++++++++++++++++++++++++++-------
kernel/bpf/devmap.c | 33 ++++++++++++++++++++++++++-------
net/xdp/xsk.c | 33 +++++++++++++++++++++++++++------
4 files changed, 113 insertions(+), 20 deletions(-)
diff --git a/include/linux/filter.h b/include/linux/filter.h
index 97c9be9cabfd6..231ecdc431a00 100644
--- a/include/linux/filter.h
+++ b/include/linux/filter.h
@@ -725,6 +725,9 @@ static inline struct bpf_redirect_info *xdp_storage_get_ri(void)
struct bpf_xdp_storage {
struct bpf_redirect_info ri;
+ struct list_head cpu_map_flush_list;
+ struct list_head dev_map_flush_list;
+ struct list_head xskmap_map_flush_list;
};
static inline struct bpf_xdp_storage *xdp_storage_set(struct bpf_xdp_storage *xdp_store)
@@ -734,6 +737,9 @@ static inline struct bpf_xdp_storage *xdp_storage_set(struct bpf_xdp_storage *xd
tsk = current;
if (tsk->bpf_xdp_storage != NULL)
return NULL;
+ INIT_LIST_HEAD(&xdp_store->cpu_map_flush_list);
+ INIT_LIST_HEAD(&xdp_store->dev_map_flush_list);
+ INIT_LIST_HEAD(&xdp_store->xskmap_map_flush_list);
tsk->bpf_xdp_storage = xdp_store;
return xdp_store;
}
@@ -764,6 +770,34 @@ static inline struct bpf_redirect_info *xdp_storage_get_ri(void)
return NULL;
return &xdp_store->ri;
}
+
+static inline struct list_head *xdp_storage_get_cpu_map_flush_list(void)
+{
+ struct bpf_xdp_storage *xdp_store = xdp_storage_get();
+
+ if (!xdp_store)
+ return NULL;
+ return &xdp_store->cpu_map_flush_list;
+}
+
+static inline struct list_head *xdp_storage_get_dev_flush_list(void)
+{
+ struct bpf_xdp_storage *xdp_store = xdp_storage_get();
+
+ if (!xdp_store)
+ return NULL;
+ return &xdp_store->dev_map_flush_list;
+}
+
+static inline struct list_head *xdp_storage_get_xskmap_flush_list(void)
+{
+ struct bpf_xdp_storage *xdp_store = xdp_storage_get();
+
+ if (!xdp_store)
+ return NULL;
+ return &xdp_store->xskmap_map_flush_list;
+}
+
#endif
DEFINE_FREE(xdp_storage_clear, struct bpf_xdp_storage *, if (_T) xdp_storage_clear(_T));
diff --git a/kernel/bpf/cpumap.c b/kernel/bpf/cpumap.c
index c40ae831ab1a6..ec5be37399c82 100644
--- a/kernel/bpf/cpumap.c
+++ b/kernel/bpf/cpumap.c
@@ -78,8 +78,30 @@ struct bpf_cpu_map {
struct bpf_cpu_map_entry __rcu **cpu_map;
};
+#ifndef CONFIG_PREEMPT_RT
static DEFINE_PER_CPU(struct list_head, cpu_map_flush_list);
+static struct list_head *xdp_storage_get_cpu_map_flush_list(void)
+{
+ return this_cpu_ptr(&cpu_map_flush_list);
+}
+
+static void init_cpu_map_flush_list(void)
+{
+ int cpu;
+
+ for_each_possible_cpu(cpu)
+ INIT_LIST_HEAD(&per_cpu(cpu_map_flush_list, cpu));
+}
+
+#else
+
+static void init_cpu_map_flush_list(void)
+{
+}
+
+#endif
+
static struct bpf_map *cpu_map_alloc(union bpf_attr *attr)
{
u32 value_size = attr->value_size;
@@ -703,7 +725,7 @@ static void bq_flush_to_queue(struct xdp_bulk_queue *bq)
*/
static void bq_enqueue(struct bpf_cpu_map_entry *rcpu, struct xdp_frame *xdpf)
{
- struct list_head *flush_list = this_cpu_ptr(&cpu_map_flush_list);
+ struct list_head *flush_list = xdp_storage_get_cpu_map_flush_list();
struct xdp_bulk_queue *bq = this_cpu_ptr(rcpu->bulkq);
if (unlikely(bq->count == CPU_MAP_BULK_SIZE))
@@ -755,7 +777,7 @@ int cpu_map_generic_redirect(struct bpf_cpu_map_entry *rcpu,
void __cpu_map_flush(void)
{
- struct list_head *flush_list = this_cpu_ptr(&cpu_map_flush_list);
+ struct list_head *flush_list = xdp_storage_get_cpu_map_flush_list();
struct xdp_bulk_queue *bq, *tmp;
list_for_each_entry_safe(bq, tmp, flush_list, flush_node) {
@@ -769,7 +791,7 @@ void __cpu_map_flush(void)
#ifdef CONFIG_DEBUG_NET
bool cpu_map_check_flush(void)
{
- if (list_empty(this_cpu_ptr(&cpu_map_flush_list)))
+ if (list_empty(xdp_storage_get_cpu_map_flush_list()))
return false;
__cpu_map_flush();
return true;
@@ -778,10 +800,7 @@ bool cpu_map_check_flush(void)
static int __init cpu_map_init(void)
{
- int cpu;
-
- for_each_possible_cpu(cpu)
- INIT_LIST_HEAD(&per_cpu(cpu_map_flush_list, cpu));
+ init_cpu_map_flush_list();
return 0;
}
diff --git a/kernel/bpf/devmap.c b/kernel/bpf/devmap.c
index a936c704d4e77..c1af5d9d60381 100644
--- a/kernel/bpf/devmap.c
+++ b/kernel/bpf/devmap.c
@@ -83,7 +83,29 @@ struct bpf_dtab {
u32 n_buckets;
};
+#ifndef CONFIG_PREEMPT_RT
static DEFINE_PER_CPU(struct list_head, dev_flush_list);
+
+static struct list_head *xdp_storage_get_dev_flush_list(void)
+{
+ return this_cpu_ptr(&dev_flush_list);
+}
+
+static void init_dev_flush_list(void)
+{
+ int cpu;
+
+ for_each_possible_cpu(cpu)
+ INIT_LIST_HEAD(&per_cpu(dev_flush_list, cpu));
+}
+
+#else
+
+static void init_dev_flush_list(void)
+{
+}
+#endif
+
static DEFINE_SPINLOCK(dev_map_lock);
static LIST_HEAD(dev_map_list);
@@ -407,7 +429,7 @@ static void bq_xmit_all(struct xdp_dev_bulk_queue *bq, u32 flags)
*/
void __dev_flush(void)
{
- struct list_head *flush_list = this_cpu_ptr(&dev_flush_list);
+ struct list_head *flush_list = xdp_storage_get_dev_flush_list();
struct xdp_dev_bulk_queue *bq, *tmp;
list_for_each_entry_safe(bq, tmp, flush_list, flush_node) {
@@ -421,7 +443,7 @@ void __dev_flush(void)
#ifdef CONFIG_DEBUG_NET
bool dev_check_flush(void)
{
- if (list_empty(this_cpu_ptr(&dev_flush_list)))
+ if (list_empty(xdp_storage_get_dev_flush_list()))
return false;
__dev_flush();
return true;
@@ -452,7 +474,7 @@ static void *__dev_map_lookup_elem(struct bpf_map *map, u32 key)
static void bq_enqueue(struct net_device *dev, struct xdp_frame *xdpf,
struct net_device *dev_rx, struct bpf_prog *xdp_prog)
{
- struct list_head *flush_list = this_cpu_ptr(&dev_flush_list);
+ struct list_head *flush_list = xdp_storage_get_dev_flush_list();
struct xdp_dev_bulk_queue *bq = this_cpu_ptr(dev->xdp_bulkq);
if (unlikely(bq->count == DEV_MAP_BULK_SIZE))
@@ -1155,15 +1177,12 @@ static struct notifier_block dev_map_notifier = {
static int __init dev_map_init(void)
{
- int cpu;
-
/* Assure tracepoint shadow struct _bpf_dtab_netdev is in sync */
BUILD_BUG_ON(offsetof(struct bpf_dtab_netdev, dev) !=
offsetof(struct _bpf_dtab_netdev, dev));
register_netdevice_notifier(&dev_map_notifier);
- for_each_possible_cpu(cpu)
- INIT_LIST_HEAD(&per_cpu(dev_flush_list, cpu));
+ init_dev_flush_list();
return 0;
}
diff --git a/net/xdp/xsk.c b/net/xdp/xsk.c
index b78c0e095e221..3050739cfe1e0 100644
--- a/net/xdp/xsk.c
+++ b/net/xdp/xsk.c
@@ -35,8 +35,30 @@
#define TX_BATCH_SIZE 32
#define MAX_PER_SOCKET_BUDGET (TX_BATCH_SIZE)
+#ifndef CONFIG_PREEMPT_RT
static DEFINE_PER_CPU(struct list_head, xskmap_flush_list);
+static struct list_head *xdp_storage_get_xskmap_flush_list(void)
+{
+ return this_cpu_ptr(&xskmap_flush_list);
+}
+
+static void init_xskmap_flush_list(void)
+{
+ int cpu;
+
+ for_each_possible_cpu(cpu)
+ INIT_LIST_HEAD(&per_cpu(xskmap_flush_list, cpu));
+}
+
+#else
+
+static void init_xskmap_flush_list(void)
+{
+}
+
+#endif
+
void xsk_set_rx_need_wakeup(struct xsk_buff_pool *pool)
{
if (pool->cached_need_wakeup & XDP_WAKEUP_RX)
@@ -372,7 +394,7 @@ static int xsk_rcv(struct xdp_sock *xs, struct xdp_buff *xdp)
int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp)
{
- struct list_head *flush_list = this_cpu_ptr(&xskmap_flush_list);
+ struct list_head *flush_list = xdp_storage_get_xskmap_flush_list();
int err;
err = xsk_rcv(xs, xdp);
@@ -387,7 +409,7 @@ int __xsk_map_redirect(struct xdp_sock *xs, struct xdp_buff *xdp)
void __xsk_map_flush(void)
{
- struct list_head *flush_list = this_cpu_ptr(&xskmap_flush_list);
+ struct list_head *flush_list = xdp_storage_get_xskmap_flush_list();
struct xdp_sock *xs, *tmp;
list_for_each_entry_safe(xs, tmp, flush_list, flush_node) {
@@ -399,7 +421,7 @@ void __xsk_map_flush(void)
#ifdef CONFIG_DEBUG_NET
bool xsk_map_check_flush(void)
{
- if (list_empty(this_cpu_ptr(&xskmap_flush_list)))
+ if (list_empty(xdp_storage_get_xskmap_flush_list()))
return false;
__xsk_map_flush();
return true;
@@ -1770,7 +1792,7 @@ static struct pernet_operations xsk_net_ops = {
static int __init xsk_init(void)
{
- int err, cpu;
+ int err;
err = proto_register(&xsk_proto, 0 /* no slab */);
if (err)
@@ -1788,8 +1810,7 @@ static int __init xsk_init(void)
if (err)
goto out_pernet;
- for_each_possible_cpu(cpu)
- INIT_LIST_HEAD(&per_cpu(xskmap_flush_list, cpu));
+ init_xskmap_flush_list();
return 0;
out_pernet:
--
2.43.0
