Enable the machine function split optimization for AutoFDO in Clang.
Machine function split (MFS) is a pass in the Clang compiler that
splits a function into hot and cold parts. The linker groups all
cold blocks across functions together. This decreases hot code
fragmentation and improves iCache and iTLB utilization.
MFS requires a profile so this is enabled only for the AutoFDO builds.
Co-developed-by: Han Shen <shen...@google.com>
Signed-off-by: Han Shen <shen...@google.com>
Signed-off-by: Rong Xu <x...@google.com>
Suggested-by: Sriraman Tallam <tmsri...@google.com>
Suggested-by: Krzysztof Pszeniczny <kpszenic...@google.com>
Tested-by: Yonghong Song <yonghong.s...@linux.dev>
Tested-by: Yabin Cui <yab...@google.com>
Change-Id: Iece9b36d37162c0cf66b8efba7e81d04b4768254
---
include/asm-generic/vmlinux.lds.h | 7 ++++++-
scripts/Makefile.autofdo | 2 ++
2 files changed, 8 insertions(+), 1 deletion(-)
diff --git a/include/asm-generic/vmlinux.lds.h
b/include/asm-generic/vmlinux.lds.h
index bd64fdedabd2f..8a0bb3946cf05 100644
--- a/include/asm-generic/vmlinux.lds.h
+++ b/include/asm-generic/vmlinux.lds.h
@@ -556,6 +556,11 @@ defined(CONFIG_AUTOFDO_CLANG)
__cpuidle_text_end = .; \
__noinstr_text_end = .;
+#define TEXT_SPLIT \
+ __split_text_start = .; \
+ *(.text.split .text.split.[0-9a-zA-Z_]*) \
+ __split_text_end = .;
+
#define TEXT_UNLIKELY \
__unlikely_text_start = .; \
*(.text.unlikely .text.unlikely.*) \
@@ -582,6 +587,7 @@ defined(CONFIG_AUTOFDO_CLANG)
ALIGN_FUNCTION(); \
*(.text.asan.* .text.tsan.*) \
*(.text.unknown .text.unknown.*) \
+ TEXT_SPLIT \
TEXT_UNLIKELY \
. = ALIGN(PAGE_SIZE); \
TEXT_HOT \
@@ -589,7 +595,6 @@ defined(CONFIG_AUTOFDO_CLANG)
NOINSTR_TEXT \
*(.ref.text)
-
/* sched.text is aling to function alignment to secure we have same
* address even at second ld pass when generating System.map */
#define SCHED_TEXT \
diff --git a/scripts/Makefile.autofdo b/scripts/Makefile.autofdo
index 6155d6fc4ca7f..1caf2457e585c 100644
--- a/scripts/Makefile.autofdo
+++ b/scripts/Makefile.autofdo
@@ -10,6 +10,7 @@ endif
ifdef CLANG_AUTOFDO_PROFILE
CFLAGS_AUTOFDO_CLANG += -fprofile-sample-use=$(CLANG_AUTOFDO_PROFILE)
-ffunction-sections
+ CFLAGS_AUTOFDO_CLANG += -fsplit-machine-functions
endif
ifdef CONFIG_LTO_CLANG_THIN
@@ -17,6 +18,7 @@ ifdef CONFIG_LTO_CLANG_THIN
KBUILD_LDFLAGS += --lto-sample-profile=$(CLANG_AUTOFDO_PROFILE)
endif
KBUILD_LDFLAGS += --mllvm=-enable-fs-discriminator=true
--mllvm=-improved-fs-discriminator=true -plugin-opt=thinlto
+ KBUILD_LDFLAGS += -plugin-opt=-split-machine-functions
endif
export CFLAGS_AUTOFDO_CLANG
--
2.47.0.163.g1226f6d8fa-goog
