Currently, xdp_attachment_setup() uses plain assignments and puts
the previous BPF program before updating the pointer, rendering
itself dangerous for program hot-swaps due to pointer tearing and
potential use-after-free's.
At the same time, &xdp_attachment_info comes handy to use it in
drivers as a main container including hotpath -- the BTF ID and meta
threshold values are now being used there as well, not speaking of
reducing some boilerplate code.
Add an RCU-protected pointer to XDP program to that structure and an
RCU version of xdp_attachment_setup(), which will make sure that all
the values were not corrupted and that old BPF program was freed
only after the pointer was updated. The only thing left is that RCU
read critical sections might happen in between each assignment, but
since the relations between XDP prog, BTF ID and meta threshold are
not vital, it's totally fine to allow this.
A caller must ensure it's being executed under the RTNL lock. Reader
sides must ensure they're being executed under the RCU read lock.
Once all the current users of xdp_attachment_setup() are switched to
the RCU-aware version (with appropriate adjustments), the "regular"
one will be removed.
Partially inspired by commit fe45386a2082 ("net/mlx5e: Use RCU to
protect rq->xdp_prog").
Signed-off-by: Alexander Lobakin <alexandr.lobakin@xxxxxxxxx>
---
include/net/xdp.h | 7 ++++++-
net/bpf/core.c | 28 ++++++++++++++++++++++++++++
2 files changed, 34 insertions(+), 1 deletion(-)
diff --git a/include/net/xdp.h b/include/net/xdp.h
index 5762ce18885f..49e562e4fcca 100644
--- a/include/net/xdp.h
+++ b/include/net/xdp.h
@@ -379,7 +379,10 @@ int xdp_reg_mem_model(struct xdp_mem_info *mem,
void xdp_unreg_mem_model(struct xdp_mem_info *mem);
struct xdp_attachment_info {
- struct bpf_prog *prog;
+ union {
+ struct bpf_prog __rcu *prog_rcu;
+ struct bpf_prog *prog;
+ };
union {
__le64 btf_id_le;
u64 btf_id;
@@ -391,6 +394,8 @@ struct xdp_attachment_info {
struct netdev_bpf;
void xdp_attachment_setup(struct xdp_attachment_info *info,
struct netdev_bpf *bpf);
+void xdp_attachment_setup_rcu(struct xdp_attachment_info *info,
+ struct netdev_bpf *bpf);
#define DEV_MAP_BULK_SIZE XDP_BULK_QUEUE_SIZE
diff --git a/net/bpf/core.c b/net/bpf/core.c
index 65f25019493d..d444d0555057 100644
--- a/net/bpf/core.c
+++ b/net/bpf/core.c
@@ -557,6 +557,34 @@ void xdp_attachment_setup(struct xdp_attachment_info *info,
}
EXPORT_SYMBOL_GPL(xdp_attachment_setup);
+/**
+ * xdp_attachment_setup_rcu - an RCU-powered version of xdp_attachment_setup()
+ * @info: pointer to the target container
+ * @bpf: pointer to the container passed to ::ndo_bpf()
+ *
+ * Protects sensitive values with RCU to allow program how-swaps without
+ * stopping an interface. Write side (this) must be called under the RTNL lock
+ * and reader sides must fetch any data only under the RCU read lock -- old BPF
+ * program will be freed only after a critical section is finished (see
+ * bpf_prog_put()).
+ */
+void xdp_attachment_setup_rcu(struct xdp_attachment_info *info,
+ struct netdev_bpf *bpf)
+{
+ struct bpf_prog *old_prog;
+
+ ASSERT_RTNL();
+
+ old_prog = rcu_replace_pointer(info->prog_rcu, bpf->prog,
+ lockdep_rtnl_is_held());
+ WRITE_ONCE(info->btf_id, bpf->btf_id);
+ WRITE_ONCE(info->meta_thresh, bpf->meta_thresh);
+
+ if (old_prog)
+ bpf_prog_put(old_prog);
+}
+EXPORT_SYMBOL_GPL(xdp_attachment_setup_rcu);
+
struct xdp_frame *xdp_convert_zc_to_xdp_frame(struct xdp_buff *xdp)
{
unsigned int metasize, totsize;
--
2.36.1
