Enable the machine function split optimization for AutoFDO in Clang. Machine function split (MFS) is a pass in the Clang compiler that splits a function into hot and cold parts. The linker groups all cold blocks across functions together. This decreases hot code fragmentation and improves iCache and iTLB utilization. MFS requires a profile so this is enabled only for the AutoFDO builds. Co-developed-by: Han Shen <shenhan@xxxxxxxxxx> Signed-off-by: Han Shen <shenhan@xxxxxxxxxx> Signed-off-by: Rong Xu <xur@xxxxxxxxxx> Suggested-by: Sriraman Tallam <tmsriram@xxxxxxxxxx> Suggested-by: Krzysztof Pszeniczny <kpszeniczny@xxxxxxxxxx> --- include/asm-generic/vmlinux.lds.h | 6 ++++++ scripts/Makefile.autofdo | 2 ++ 2 files changed, 8 insertions(+) diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index 97c8399e5532..7d9dc8a3c046 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h @@ -593,9 +593,14 @@ defined(CONFIG_AUTOFDO_CLANG) __unlikely_text_start = .; \ *(.text.unlikely .text.unlikely.*) \ __unlikely_text_end = .; +#define TEXT_SPLIT \ + __split_text_start = .; \ + *(.text.split .text.split.[0-9a-zA-Z_]*) \ + __split_text_end = .; #else #define TEXT_HOT *(.text.hot .text.hot.*) #define TEXT_UNLIKELY *(.text.unlikely .text.unlikely.*) +#define TEXT_SPLIT #endif /* @@ -611,6 +616,7 @@ defined(CONFIG_AUTOFDO_CLANG) #define TEXT_TEXT \ *(.text.asan.* .text.tsan.*) \ *(.text.unknown .text.unknown.*) \ + TEXT_SPLIT \ TEXT_UNLIKELY \ ALIGN_FUNCTION(); \ TEXT_HOT \ diff --git a/scripts/Makefile.autofdo b/scripts/Makefile.autofdo index f765bd9e81d7..80ad06689947 100644 --- a/scripts/Makefile.autofdo +++ b/scripts/Makefile.autofdo @@ -6,6 +6,7 @@ CFLAGS_AUTOFDO_CLANG := -fdebug-info-for-profiling -mllvm -enable-fs-discriminat ifdef CLANG_AUTOFDO_PROFILE CFLAGS_AUTOFDO_CLANG += -fprofile-sample-use=$(CLANG_AUTOFDO_PROFILE) -ffunction-sections +CFLAGS_AUTOFDO_CLANG += -fsplit-machine-functions endif ifdef CONFIG_LTO_CLANG @@ -14,6 +15,7 @@ ifdef CLANG_AUTOFDO_PROFILE KBUILD_LDFLAGS += --lto-sample-profile=$(CLANG_AUTOFDO_PROFILE) endif KBUILD_LDFLAGS += --mllvm=-enable-fs-discriminator=true --mllvm=-improved-fs-discriminator=true -plugin-opt=thinlto +KBUILD_LDFLAGS += -plugin-opt=-split-machine-functions endif endif -- 2.46.0.rc1.232.g9752f9e123-goog
