On Thu, Nov 7, 2024 at 9:51 AM Eduard Zingerman <eddyz87@xxxxxxxxx> wrote: > > This commit adds a notion of inlinable kfuncs, compiled both to native > code and BPF. BPF-compiled version is embedded in kernel data section > and is available to verifier. Verifier uses it to replace calls to > such kfuncs with inlined function bodies. > > Inlinable kfuncs are available only if CLANG is used for kernel > compilation. > > In the scope of this commit all inlining happens as last steps of > do_check(), after verification is finished. Follow up commits would > extend this mechanism to allow removal of some conditional branches > inside inlined function bodies. > > The commit consists of the following changes: > - main kernel makefile: > modified to compile a bootstrap version of the bpftool; > - kernel/bpf/Makefile: > - a new file inlinable_kfuncs.c is added; > - makefile is modified to compile this file as BPF elf, > using the following steps: > - use clang with native target to produce LLVM bitcode; > - compile LLVM bitcode to BPF object file; > - resolve relocations inside BPF object file using bpftool as a > linker; > Such arrangement allows including unmodified network related > header files. > - verifier.c: > - generated BPF elf is included as a part of kernel data section; > - at kernel initialization phase: > - the elf is parsed and each function declared within it is > recorded as an instance of 'inlinable_kfunc' structure; > - calls to extern functions within elf file (pointed to by > relocation records) are replaced with kfunc call instructions; > - do_check() is modified to replace calls to kfuncs from inlinable > kfunc table with function bodies: > - replacement happens after main verification pass, so the bodies > of the kfuncs are not analyzed by verifier; > - if kfunc uses callee saved registers r6-r9 the spill/fill pairs > are generated for these register before/after inlined kfunc body > at call site; > - if kfunc uses r10 as a base pointer for load or store > instructions, offsets of these instructions are adjusted; > - if kfunc uses r10 in other instructions, such r10 is considered > as escaping and kfunc is not inlined. > > Signed-off-by: Eduard Zingerman <eddyz87@xxxxxxxxx> > --- > Makefile | 22 +- > kernel/bpf/Makefile | 24 +- > kernel/bpf/inlinable_kfuncs.c | 1 + > kernel/bpf/verifier.c | 652 +++++++++++++++++++++++++++++++++- > 4 files changed, 680 insertions(+), 19 deletions(-) > create mode 100644 kernel/bpf/inlinable_kfuncs.c > > diff --git a/Makefile b/Makefile > index a9a7d9ffaa98..4ded57f4b0c2 100644 > --- a/Makefile > +++ b/Makefile > @@ -496,6 +496,7 @@ CLIPPY_DRIVER = clippy-driver > BINDGEN = bindgen > PAHOLE = pahole > RESOLVE_BTFIDS = $(objtree)/tools/bpf/resolve_btfids/resolve_btfids > +BPFTOOL = $(objtree)/tools/bpf/bpftool/bootstrap/bpftool > LEX = flex > YACC = bison > AWK = awk > @@ -585,7 +586,7 @@ export RUSTC_BOOTSTRAP := 1 > export ARCH SRCARCH CONFIG_SHELL BASH HOSTCC KBUILD_HOSTCFLAGS CROSS_COMPILE LD CC HOSTPKG_CONFIG > export RUSTC RUSTDOC RUSTFMT RUSTC_OR_CLIPPY_QUIET RUSTC_OR_CLIPPY BINDGEN > export HOSTRUSTC KBUILD_HOSTRUSTFLAGS > -export CPP AR NM STRIP OBJCOPY OBJDUMP READELF PAHOLE RESOLVE_BTFIDS LEX YACC AWK INSTALLKERNEL > +export CPP AR NM STRIP OBJCOPY OBJDUMP READELF PAHOLE RESOLVE_BTFIDS BPFTOOL LEX YACC AWK INSTALLKERNEL > export PERL PYTHON3 CHECK CHECKFLAGS MAKE UTS_MACHINE HOSTCXX > export KGZIP KBZIP2 KLZOP LZMA LZ4 XZ ZSTD > export KBUILD_HOSTCXXFLAGS KBUILD_HOSTLDFLAGS KBUILD_HOSTLDLIBS LDFLAGS_MODULE > @@ -1356,6 +1357,25 @@ ifneq ($(wildcard $(resolve_btfids_O)),) > $(Q)$(MAKE) -sC $(srctree)/tools/bpf/resolve_btfids O=$(resolve_btfids_O) clean > endif > > +# TODO: cross compilation? > +# TODO: bootstrap! (to avoid vmlinux.h generation) > +PHONY += bpftool_bootstrap bpftool_clean > +bpftool_O = $(abspath $(objtree))/tools/bpf/bpftool > + > +ifdef CONFIG_BPF > +ifdef CONFIG_CC_IS_CLANG > +prepare: bpftool_bootstrap > +endif > +endif > + > +bpftool_bootstrap: > + $(Q)$(MAKE) -sC $(srctree)/tools/bpf/bpftool O=$(bpftool_O) srctree=$(abspath $(srctree)) bootstrap > + > +bpftool_clean: > +ifneq ($(wildcard $(bpftool_O)),) > + $(Q)$(MAKE) -sC $(srctree)/tools/bpf/bpftool O=$(bpftool_O) srctree=$(abspath $(srctree)) clean > +endif > + > # Clear a bunch of variables before executing the submake > ifeq ($(quiet),silent_) > tools_silent=s > diff --git a/kernel/bpf/Makefile b/kernel/bpf/Makefile > index 105328f0b9c0..3d7ee81c8e2e 100644 > --- a/kernel/bpf/Makefile > +++ b/kernel/bpf/Makefile > @@ -6,7 +6,7 @@ cflags-nogcse-$(CONFIG_X86)$(CONFIG_CC_IS_GCC) := -fno-gcse > endif > CFLAGS_core.o += -Wno-override-init $(cflags-nogcse-yy) > > -obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o helpers.o tnum.o log.o token.o > +obj-$(CONFIG_BPF_SYSCALL) += syscall.o verifier.o inode.o tnum.o log.o token.o > obj-$(CONFIG_BPF_SYSCALL) += bpf_iter.o map_iter.o task_iter.o prog_iter.o link_iter.o > obj-$(CONFIG_BPF_SYSCALL) += hashtab.o arraymap.o percpu_freelist.o bpf_lru_list.o lpm_trie.o map_in_map.o bloom_filter.o > obj-$(CONFIG_BPF_SYSCALL) += local_storage.o queue_stack_maps.o ringbuf.o > @@ -53,3 +53,25 @@ obj-$(CONFIG_BPF_SYSCALL) += relo_core.o > obj-$(CONFIG_BPF_SYSCALL) += btf_iter.o > obj-$(CONFIG_BPF_SYSCALL) += btf_relocate.o > obj-$(CONFIG_BPF_SYSCALL) += kmem_cache_iter.o > +obj-$(CONFIG_BPF_SYSCALL) += helpers.o inlinable_kfuncs.o > + > +ifdef CONFIG_CC_IS_CLANG > + > +LLC ?= $(LLVM_PREFIX)llc$(LLVM_SUFFIX) > + > +# -mfentry -pg is $(CC_FLAGS_FTRACE) > +# -fpatchable-function-entry=16,16 is $(PADDING_CFLAGS) > +CFLAGS_REMOVE_inlinable_kfuncs.bpf.bc.o += $(CC_FLAGS_FTRACE) > +CFLAGS_REMOVE_inlinable_kfuncs.bpf.bc.o += $(PADDING_CFLAGS) > +$(obj)/inlinable_kfuncs.bpf.bc.o: $(src)/inlinable_kfuncs.c > + $(Q)$(CLANG) $(c_flags) -emit-llvm -c $< -o $@ > + > +$(obj)/inlinable_kfuncs.bpf.o: $(obj)/inlinable_kfuncs.bpf.bc.o > + $(Q)$(LLC) -mcpu=v3 --mtriple=bpf --filetype=obj $< -o $@ > + > +$(obj)/inlinable_kfuncs.bpf.linked.o: $(obj)/inlinable_kfuncs.bpf.o > + $(Q)$(BPFTOOL) gen object $@ $< what's the point? Just to strip DWARF? `strip -g` then? but honestly, why even bother embedding entire ELF? Get binary data from .text, separately record ELF symbols (to know function name, size, and where they start). Keep it simple and minimal. > + > +$(obj)/verifier.o: $(obj)/inlinable_kfuncs.bpf.linked.o > + > +endif > diff --git a/kernel/bpf/inlinable_kfuncs.c b/kernel/bpf/inlinable_kfuncs.c > new file mode 100644 > index 000000000000..7b7dc05fa1a4 > --- /dev/null > +++ b/kernel/bpf/inlinable_kfuncs.c > @@ -0,0 +1 @@ > +// SPDX-License-Identifier: GPL-2.0-only > diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c > index 3bae0bbc1da9..fbf51147f319 100644 > --- a/kernel/bpf/verifier.c > +++ b/kernel/bpf/verifier.c > @@ -20509,6 +20509,622 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, > return 0; > } > [...]
