https://github.com/rlavaee updated https://github.com/llvm/llvm-project/pull/227880
>From a67b44646c6fd8af9cb31f20cdcdfd03e42b38a7 Mon Sep 17 00:00:00 2001 From: Rahman Lavaee <[email protected]> Date: Sat, 19 Sep 2026 18:56:47 +0000 Subject: [PATCH 1/8] [CodeGen] Replace EnableMachineFunctionSplitter with FunctionSplittingMode Add TargetOptions::FunctionSplitting and a -function-splitting={none,bbsections,all} flag, replacing TargetOptions::EnableMachineFunctionSplitter and the -split-machine-functions / -enable-split-machine-functions flags. MachineFunctionSplitter runs when the mode is 'all'. clang and flang map -fsplit-machine-functions to 'all', including through the LTO plugin-opt. This is the first step towards merging MachineFunctionSplitter into BasicBlockSections. After that, 'none' and 'bbsections' will fully take effect, so splitting can be turned off for some or all functions whether a bbsections or PGO profile is used. --- clang/lib/CodeGen/BackendUtil.cpp | 3 +- clang/lib/Driver/ToolChains/CommonArgs.cpp | 2 +- clang/test/Driver/fsplit-machine-functions.c | 4 +- flang/lib/Frontend/CompilerInstance.cpp | 3 +- llvm/docs/ReleaseNotes.md | 7 +++ llvm/include/llvm/CodeGen/CommandFlags.h | 4 +- .../include/llvm/Target/CGPassBuilderOption.h | 1 - llvm/include/llvm/Target/TargetOptions.h | 21 +++++-- llvm/lib/CodeGen/CodeGenOptions.td | 2 - llvm/lib/CodeGen/CommandFlags.cpp | 24 +++++--- llvm/lib/CodeGen/TargetPassConfig.cpp | 10 +-- .../machine-function-splitter-optnone.ll | 2 +- .../Generic/machine-function-splitter.ll | 38 +++++++----- .../X86/basic-block-address-map-with-mfs.ll | 4 +- .../CodeGen/X86/basic-block-address-map.ll | 2 +- llvm/test/CodeGen/X86/basic-block-sections.ll | 2 +- .../CodeGen/X86/function-splitting-mode.ll | 61 +++++++++++++++++++ .../test/CodeGen/X86/load-sample-profile-2.ll | 2 +- llvm/test/CodeGen/X86/load-sample-profile.ll | 2 +- .../tools/llvm-profgen/X86/func-split.test | 2 +- 20 files changed, 147 insertions(+), 49 deletions(-) create mode 100644 llvm/test/CodeGen/X86/function-splitting-mode.ll diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp index c754c75aa59de5..2cab30fad3d00a 100644 --- a/clang/lib/CodeGen/BackendUtil.cpp +++ b/clang/lib/CodeGen/BackendUtil.cpp @@ -419,7 +419,8 @@ static bool initTargetOptions(const CompilerInstance &CI, Options.BBSectionsFuncListBuf = std::move(*MBOrErr); } - Options.EnableMachineFunctionSplitter = CodeGenOpts.SplitMachineFunctions; + if (CodeGenOpts.SplitMachineFunctions) + Options.FunctionSplitting = llvm::FunctionSplittingMode::All; Options.EnableStaticDataPartitioning = CodeGenOpts.PartitionStaticDataSections; Options.FunctionSections = CodeGenOpts.FunctionSections; diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp index c54a0b1068f437..7c832ef405fa51 100644 --- a/clang/lib/Driver/ToolChains/CommonArgs.cpp +++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp @@ -1303,7 +1303,7 @@ void tools::addLTOOptions(const ToolChain &ToolChain, const ArgList &Args, options::OPT_fno_split_machine_functions)) { if (A->getOption().matches(options::OPT_fsplit_machine_functions)) CmdArgs.push_back(Args.MakeArgString(Twine(PluginOptPrefix) + - "-split-machine-functions")); + "-function-splitting=all")); } if (auto *A = diff --git a/clang/test/Driver/fsplit-machine-functions.c b/clang/test/Driver/fsplit-machine-functions.c index 153aafba8fbd00..80485a32db3b66 100644 --- a/clang/test/Driver/fsplit-machine-functions.c +++ b/clang/test/Driver/fsplit-machine-functions.c @@ -14,8 +14,8 @@ // RUN: %clang -### --target=x86_64-linux -flto -fsplit-machine-functions %s 2>&1 | FileCheck %s --check-prefix=LTO // RUN: %clang -### --target=x86_64-linux -flto -fsplit-machine-functions -fno-split-machine-functions %s 2>&1 | FileCheck %s --check-prefix=LTO-NEG -// LTO: "-plugin-opt=-split-machine-functions" -// LTO-NEG-NOT: "-plugin-opt=-split-machine-functions" +// LTO: "-plugin-opt=-function-splitting=all" +// LTO-NEG-NOT: "-plugin-opt=-function-splitting=all" // RUN: not %clang -### -c --target=arm-unknown-linux -fsplit-machine-functions %s 2>&1 | FileCheck %s --check-prefix=ERR // ERR: error: unsupported option '-fsplit-machine-functions' for target diff --git a/flang/lib/Frontend/CompilerInstance.cpp b/flang/lib/Frontend/CompilerInstance.cpp index 1b09fcfd049afb..f63b7cb1580825 100644 --- a/flang/lib/Frontend/CompilerInstance.cpp +++ b/flang/lib/Frontend/CompilerInstance.cpp @@ -380,7 +380,8 @@ bool CompilerInstance::setUpTargetMachine() { std::optional<llvm::CodeModel::Model> cm = getCodeModel(CGOpts.CodeModel); llvm::TargetOptions tOpts = llvm::TargetOptions(); - tOpts.EnableMachineFunctionSplitter = targetOpts.SplitMachineFunctions; + if (targetOpts.SplitMachineFunctions) + tOpts.FunctionSplitting = llvm::FunctionSplittingMode::All; tOpts.VecLib = convertDriverVectorLibraryToVectorLibrary(CGOpts.getVecLib()); tOpts.MCOptions.DisableIntegratedAS = CGOpts.DisableIntegratedAS; tOpts.FunctionSections = CGOpts.FunctionSections; diff --git a/llvm/docs/ReleaseNotes.md b/llvm/docs/ReleaseNotes.md index 7c09b303442a37..4758b5ef253145 100644 --- a/llvm/docs/ReleaseNotes.md +++ b/llvm/docs/ReleaseNotes.md @@ -181,6 +181,13 @@ Makes programs 10x faster by doing Special New Thing. GNU-vs-EABI distinction is now derived entirely from the target triple's environment (e.g. `arm-none-gnueabi` vs `arm-none-eabi`). +* Replaced `TargetOptions::EnableMachineFunctionSplitter` with + `TargetOptions::FunctionSplitting`, and the `-split-machine-functions` and + `-enable-split-machine-functions` flags with + `-function-splitting={none,bbsections,all}`. Use `-function-splitting=all` + for the previous behavior. Clang's `-fsplit-machine-functions` is + unchanged. + ### Changes to building LLVM * A new `LLVM_ENABLE_LZMA` option (`ON`, `OFF` or `FORCE_ON`; default `ON`) diff --git a/llvm/include/llvm/CodeGen/CommandFlags.h b/llvm/include/llvm/CodeGen/CommandFlags.h index 1a5d44570430fe..bea26529158654 100644 --- a/llvm/include/llvm/CodeGen/CommandFlags.h +++ b/llvm/include/llvm/CodeGen/CommandFlags.h @@ -91,6 +91,8 @@ LLVM_ABI bool getXCOFFTracebackTable(); LLVM_ABI std::string getBBSections(); +LLVM_ABI llvm::FunctionSplittingMode getFunctionSplitting(); + LLVM_ABI unsigned getTLSSize(); LLVM_ABI bool getEmulatedTLS(); @@ -117,8 +119,6 @@ LLVM_ABI bool getEnableCallGraphSection(); LLVM_ABI bool getEmitCallSiteInfo(); -LLVM_ABI bool getEnableMachineFunctionSplitter(); - LLVM_ABI bool getEnableStaticDataPartitioning(); LLVM_ABI bool getEnableDebugEntryValues(); diff --git a/llvm/include/llvm/Target/CGPassBuilderOption.h b/llvm/include/llvm/Target/CGPassBuilderOption.h index b8bffb9e60209e..b143715baacb37 100644 --- a/llvm/include/llvm/Target/CGPassBuilderOption.h +++ b/llvm/include/llvm/Target/CGPassBuilderOption.h @@ -55,7 +55,6 @@ struct CGPassBuilderOption { bool EnableImplicitNullChecks = false; bool EnableBlockPlacementStats = false; bool EnableGlobalMergeFunc = false; - bool EnableMachineFunctionSplitter = false; bool EnableSinkAndFold = false; bool EnableTailMerge = true; /// Enable LoopTermFold immediately after LSR. diff --git a/llvm/include/llvm/Target/TargetOptions.h b/llvm/include/llvm/Target/TargetOptions.h index 6db8eca394a38c..b0db80947fb996 100644 --- a/llvm/include/llvm/Target/TargetOptions.h +++ b/llvm/include/llvm/Target/TargetOptions.h @@ -50,6 +50,19 @@ enum class BasicBlockSection { None // Do not use Basic Block Sections. }; +/// Late function splitting mode. Decides which functions are eligible to have +/// their cold blocks moved into a separate section. +enum class FunctionSplittingMode { + None, // Hard off. Never create a cold section, even for functions + // which have a basic block sections profile. Such functions + // are still laid out using the profile, but are emitted as a + // single contiguous section. + BBSectionsOnly, // Split only functions which have a basic block sections + // profile. This is the default. + All // Split functions using the basic block sections profile + // where it is available, and PGO/AutoFDO elsewhere. +}; + /// Identify a debugger for "tuning" the debug info. /// /// The "debugger tuning" concept allows us to present a more intuitive @@ -113,7 +126,6 @@ class TargetOptions { TrapUnreachable(false), NoTrapAfterNoreturn(false), TLSSize(0), EmulatedTLS(false), EnableTLSDESC(false), EnableIPRA(false), EmitStackSizeSection(false), EnableMachineOutliner(false), - EnableMachineFunctionSplitter(false), EnableStaticDataPartitioning(false), EnableDefaultMachineVerifier(true), EmitAddrsig(false), BBAddrMap(false), EmitCallGraphSection(false), EmitCallSiteInfo(false), EnableDebugEntryValues(false), @@ -208,9 +220,6 @@ class TargetOptions { /// Enables the MachineOutliner pass. unsigned EnableMachineOutliner : 1; - /// Enables the MachineFunctionSplitter pass. - unsigned EnableMachineFunctionSplitter : 1; - /// Enables the StaticDataSplitter pass. unsigned EnableStaticDataPartitioning : 1; @@ -228,6 +237,10 @@ class TargetOptions { /// Emit basic blocks into separate sections. BasicBlockSection BBSections = BasicBlockSection::None; + /// Which functions are eligible for late function splitting. + FunctionSplittingMode FunctionSplitting = + FunctionSplittingMode::BBSectionsOnly; + /// Memory Buffer that contains information on sampled basic blocks and used /// to selectively generate basic block sections. std::shared_ptr<MemoryBuffer> BBSectionsFuncListBuf; diff --git a/llvm/lib/CodeGen/CodeGenOptions.td b/llvm/lib/CodeGen/CodeGenOptions.td index 3a5c4bd5471fa0..132b2ec8a4f7d4 100644 --- a/llvm/lib/CodeGen/CodeGenOptions.td +++ b/llvm/lib/CodeGen/CodeGenOptions.td @@ -151,8 +151,6 @@ defm : BoolField<"enable-implicit-null-checks", "false", "Fold null checks into faulting memory operations">; defm : OptionalBoolField<"enable-ipra", "Enable interprocedural register allocation to reduce load/store at procedure calls">; -defm : BoolField<"enable-split-machine-functions", "false", - "Split out cold blocks from machine functions based on profile information">; // -fast-isel selects FastISel at any optimization level, and -fast-isel=0 // disables it at -O0, where it is used unless GlobalISel is enabled. defm : OptionalBoolField<"fast-isel", diff --git a/llvm/lib/CodeGen/CommandFlags.cpp b/llvm/lib/CodeGen/CommandFlags.cpp index 1bebecda621664..2cd6f8aaf3a868 100644 --- a/llvm/lib/CodeGen/CommandFlags.cpp +++ b/llvm/lib/CodeGen/CommandFlags.cpp @@ -92,6 +92,7 @@ CGOPT(bool, IgnoreXCOFFVisibility) CGOPT(bool, XCOFFTracebackTable) CGOPT(bool, EnableBBAddrMap) CGOPT(std::string, BBSections) +CGOPT(FunctionSplittingMode, FunctionSplitting) CGOPT(unsigned, TLSSize) CGOPT_EXP(bool, EmulatedTLS) CGOPT_EXP(bool, EnableTLSDESC) @@ -104,7 +105,6 @@ CGOPT(bool, EnableStackSizeSection) CGOPT(bool, EnableAddrsig) CGOPT(bool, EnableCallGraphSection) CGOPT(bool, EmitCallSiteInfo) -CGOPT(bool, EnableMachineFunctionSplitter) CGOPT(bool, EnableStaticDataPartitioning) CGOPT(bool, EnableDebugEntryValues) CGOPT(bool, ForceDwarfFrameSection) @@ -432,12 +432,20 @@ codegen::RegisterCodeGenFlags::RegisterCodeGenFlags() { cl::init(false)); CGBINDOPT(EnableDebugEntryValues); - static cl::opt<bool> EnableMachineFunctionSplitter( - "split-machine-functions", - cl::desc("Split out cold basic blocks from machine functions based on " - "profile information"), - cl::init(false)); - CGBINDOPT(EnableMachineFunctionSplitter); + static cl::opt<FunctionSplittingMode> FunctionSplitting( + "function-splitting", + cl::desc("Which functions are eligible for late function splitting"), + cl::init(FunctionSplittingMode::BBSectionsOnly), + cl::values( + clEnumValN(FunctionSplittingMode::None, "none", + "Do not split any function"), + clEnumValN(FunctionSplittingMode::BBSectionsOnly, "bbsections", + "Only split functions which have a basic block sections " + "profile"), + clEnumValN(FunctionSplittingMode::All, "all", + "Split functions using the basic block sections profile " + "where it is available, and PGO/AutoFDO elsewhere"))); + CGBINDOPT(FunctionSplitting); static cl::opt<bool> EnableStaticDataPartitioning( "partition-static-data-sections", @@ -550,7 +558,7 @@ codegen::InitTargetOptionsFromCodeGenFlags(const Triple &TheTriple) { Options.ExceptionModel = getExceptionModel(); Options.VecLib = getVectorLibrary(); Options.EmitStackSizeSection = getEnableStackSizeSection(); - Options.EnableMachineFunctionSplitter = getEnableMachineFunctionSplitter(); + Options.FunctionSplitting = getFunctionSplitting(); Options.EnableStaticDataPartitioning = getEnableStaticDataPartitioning(); Options.EmitAddrsig = getEnableAddrsig(); Options.EmitCallGraphSection = getEnableCallGraphSection(); diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp index 731ffab7e71b41..35e70a53082354 100644 --- a/llvm/lib/CodeGen/TargetPassConfig.cpp +++ b/llvm/lib/CodeGen/TargetPassConfig.cpp @@ -347,7 +347,6 @@ CGPassBuilderOption llvm::getCGPassBuilderOption() { toBoolOrDefault(Opts.debugify_check_and_strip_all_safe); Opt.DisableRAFSProfileLoader = Opts.disable_ra_fsprofile_loader; Opt.DisableCFIFixup = Opts.disable_cfi_fixup; - Opt.EnableMachineFunctionSplitter = Opts.enable_split_machine_functions; return Opt; } @@ -1090,8 +1089,10 @@ void TargetPassConfig::addMachinePasses() { addPass(createMIRAddFSDiscriminatorsPass( sampleprof::FSDiscriminatorPass::PassLast)); - if (TM->Options.EnableMachineFunctionSplitter || - Opts.enable_split_machine_functions || Opts.split_static_data || + const bool SplitFunctions = + TM->Options.FunctionSplitting == FunctionSplittingMode::All; + + if (SplitFunctions || Opts.split_static_data || TM->Options.EnableStaticDataPartitioning) { const std::string ProfileFile = getFSProfileFile(TM); if (!ProfileFile.empty()) { @@ -1114,8 +1115,7 @@ void TargetPassConfig::addMachinePasses() { // feature takes precedence. This means functions eligible for // basic-block-sections optimizations (`=all`, or `=list=` with function // included in the list profile) will get that optimization instead. - if (TM->Options.EnableMachineFunctionSplitter || - Opts.enable_split_machine_functions) + if (SplitFunctions) addPass(createMachineFunctionSplitterPass()); if (Opts.split_static_data || TM->Options.EnableStaticDataPartitioning) { diff --git a/llvm/test/CodeGen/Generic/machine-function-splitter-optnone.ll b/llvm/test/CodeGen/Generic/machine-function-splitter-optnone.ll index 67d2ad72ee2f43..2f53e64adf48df 100644 --- a/llvm/test/CodeGen/Generic/machine-function-splitter-optnone.ll +++ b/llvm/test/CodeGen/Generic/machine-function-splitter-optnone.ll @@ -1,6 +1,6 @@ ; REQUIRES: x86-registered-target -; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -split-machine-functions -O0 -mfs-psi-cutoff=0 -mfs-count-threshold=10000 | FileCheck %s +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -function-splitting=all -O0 -mfs-psi-cutoff=0 -mfs-count-threshold=10000 | FileCheck %s ;; Check that functions with optnone attribute are not split. ; CHECK-LABEL: foo_optnone: diff --git a/llvm/test/CodeGen/Generic/machine-function-splitter.ll b/llvm/test/CodeGen/Generic/machine-function-splitter.ll index d798b2875645b4..cf8abe3a536b29 100644 --- a/llvm/test/CodeGen/Generic/machine-function-splitter.ll +++ b/llvm/test/CodeGen/Generic/machine-function-splitter.ll @@ -2,11 +2,11 @@ ; REQUIRES: x86-registered-target ; COM: Machine function splitting with FDO profiles -; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -split-machine-functions | FileCheck %s -check-prefixes=MFS-DEFAULTS,MFS-DEFAULTS-X86,MFS-NOBBSECTIONS -; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -split-machine-functions -mfs-psi-cutoff=0 -mfs-count-threshold=2000 | FileCheck %s --dump-input=always -check-prefixes=MFS-OPTS1,MFS-OPTS1-X86 -; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -split-machine-functions -mfs-psi-cutoff=950000 | FileCheck %s -check-prefixes=MFS-OPTS2,MFS-OPTS2-X86 -; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -split-machine-functions -mfs-split-ehcode | FileCheck %s -check-prefixes=MFS-EH-SPLIT,MFS-EH-SPLIT-X86 -; RUN: llc < %s -mtriple=x86_64 -split-machine-functions -O0 -mfs-psi-cutoff=0 -mfs-count-threshold=10000 | FileCheck %s -check-prefixes=MFS-O0,MFS-O0-X86 +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -function-splitting=all | FileCheck %s -check-prefixes=MFS-DEFAULTS,MFS-DEFAULTS-X86,MFS-NOBBSECTIONS +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -function-splitting=all -mfs-psi-cutoff=0 -mfs-count-threshold=2000 | FileCheck %s --dump-input=always -check-prefixes=MFS-OPTS1,MFS-OPTS1-X86 +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -function-splitting=all -mfs-psi-cutoff=950000 | FileCheck %s -check-prefixes=MFS-OPTS2,MFS-OPTS2-X86 +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -function-splitting=all -mfs-split-ehcode | FileCheck %s -check-prefixes=MFS-EH-SPLIT,MFS-EH-SPLIT-X86 +; RUN: llc < %s -mtriple=x86_64 -function-splitting=all -O0 -mfs-psi-cutoff=0 -mfs-count-threshold=10000 | FileCheck %s -check-prefixes=MFS-O0,MFS-O0-X86 ; COM: Machine function splitting along with -basic-block-sections profile ; RUN: echo 'v1' > %t @@ -15,19 +15,20 @@ ; RUN: echo 'ffoo22' >> %t ; RUN: echo 'c0 1' >> %t ; RUN: echo 'c2' >> %t -; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -basic-block-sections=%t -split-machine-functions | FileCheck %s --check-prefixes=MFS-BBSECTIONS +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -basic-block-sections=%t -function-splitting=all | FileCheck %s --check-prefixes=MFS-BBSECTIONS,MFS-BBSECTIONS-ALL +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -basic-block-sections=%t -function-splitting=bbsections | FileCheck %s --check-prefixes=MFS-BBSECTIONS,MFS-BBSECTIONS-ONLY -; RUN: llc < %s -mtriple=aarch64-unknown-linux-gnu -aarch64-min-jump-table-entries=4 -enable-split-machine-functions | FileCheck %s -check-prefixes=MFS-DEFAULTS,MFS-DEFAULTS-AARCH64 -; RUN: llc < %s -mtriple=aarch64-unknown-linux-gnu -aarch64-min-jump-table-entries=4 -enable-split-machine-functions -mfs-psi-cutoff=0 -mfs-count-threshold=2000 | FileCheck %s --dump-input=always -check-prefixes=MFS-OPTS1,MFS-OPTS1-AARCH64 -; RUN: llc < %s -mtriple=aarch64-unknown-linux-gnu -aarch64-min-jump-table-entries=4 -enable-split-machine-functions -mfs-psi-cutoff=950000 | FileCheck %s -check-prefixes=MFS-OPTS2,MFS-OPTS2-AARCH64 -; RUN: llc < %s -mtriple=aarch64-unknown-linux-gnu -aarch64-min-jump-table-entries=4 -enable-split-machine-functions -mfs-split-ehcode | FileCheck %s -check-prefixes=MFS-EH-SPLIT,MFS-EH-SPLIT-AARCH64 -; RUN: llc < %s -mtriple=aarch64 -split-machine-functions -O0 -mfs-psi-cutoff=0 -mfs-count-threshold=10000 | FileCheck %s -check-prefixes=MFS-O0,MFS-O0-AARCH64 -; RUN: llc < %s -mtriple=aarch64 -enable-split-machine-functions -aarch64-redzone | FileCheck %s -check-prefixes=MFS-REDZONE-AARCH64 +; RUN: llc < %s -mtriple=aarch64-unknown-linux-gnu -aarch64-min-jump-table-entries=4 -function-splitting=all | FileCheck %s -check-prefixes=MFS-DEFAULTS,MFS-DEFAULTS-AARCH64 +; RUN: llc < %s -mtriple=aarch64-unknown-linux-gnu -aarch64-min-jump-table-entries=4 -function-splitting=all -mfs-psi-cutoff=0 -mfs-count-threshold=2000 | FileCheck %s --dump-input=always -check-prefixes=MFS-OPTS1,MFS-OPTS1-AARCH64 +; RUN: llc < %s -mtriple=aarch64-unknown-linux-gnu -aarch64-min-jump-table-entries=4 -function-splitting=all -mfs-psi-cutoff=950000 | FileCheck %s -check-prefixes=MFS-OPTS2,MFS-OPTS2-AARCH64 +; RUN: llc < %s -mtriple=aarch64-unknown-linux-gnu -aarch64-min-jump-table-entries=4 -function-splitting=all -mfs-split-ehcode | FileCheck %s -check-prefixes=MFS-EH-SPLIT,MFS-EH-SPLIT-AARCH64 +; RUN: llc < %s -mtriple=aarch64 -function-splitting=all -O0 -mfs-psi-cutoff=0 -mfs-count-threshold=10000 | FileCheck %s -check-prefixes=MFS-O0,MFS-O0-AARCH64 +; RUN: llc < %s -mtriple=aarch64 -function-splitting=all -aarch64-redzone | FileCheck %s -check-prefixes=MFS-REDZONE-AARCH64 ; COM: Machine function splitting with AFDO profiles ; RUN: sed 's/InstrProf/SampleProfile/g' %s > %t.ll -; RUN: llc < %t.ll -mtriple=x86_64-unknown-linux-gnu -split-machine-functions | FileCheck %s --check-prefix=FSAFDO-MFS -; RUN: llc < %t.ll -mtriple=x86_64-unknown-linux-gnu -split-machine-functions | FileCheck %s --check-prefix=FSAFDO-MFS2 +; RUN: llc < %t.ll -mtriple=x86_64-unknown-linux-gnu -function-splitting=all | FileCheck %s --check-prefix=FSAFDO-MFS +; RUN: llc < %t.ll -mtriple=x86_64-unknown-linux-gnu -function-splitting=all | FileCheck %s --check-prefix=FSAFDO-MFS2 define void @foo1(i1 zeroext %0) nounwind !prof !14 !section_prefix !15 { ;; Check that cold block is moved to .text.split. @@ -38,6 +39,15 @@ define void @foo1(i1 zeroext %0) nounwind !prof !14 !section_prefix !15 { ; MFS-DEFAULTS-X86-NEXT: callq baz ; MFS-DEFAULTS-AARCH64-NOT: bl bar ; MFS-DEFAULTS-AARCH64-NEXT: bl baz +;; Check that with a basic-block-sections profile, a function outside the +;; profile is still split using PGO with -function-splitting=all. +; MFS-BBSECTIONS-ALL-LABEL: foo1: +; MFS-BBSECTIONS-ALL: .section .text.split.foo1 +; MFS-BBSECTIONS-ALL-NEXT: foo1.cold: +;; Check that with -function-splitting=bbsections, a function outside the +;; basic-block-sections profile is not split, even though it has PGO data. +; MFS-BBSECTIONS-ONLY-LABEL: foo1: +; MFS-BBSECTIONS-ONLY-NOT: foo1.cold: br i1 %0, label %2, label %4, !prof !17 2: ; preds = %1 diff --git a/llvm/test/CodeGen/X86/basic-block-address-map-with-mfs.ll b/llvm/test/CodeGen/X86/basic-block-address-map-with-mfs.ll index f8146bb38f71eb..a03c2ff9ef3d4c 100644 --- a/llvm/test/CodeGen/X86/basic-block-address-map-with-mfs.ll +++ b/llvm/test/CodeGen/X86/basic-block-address-map-with-mfs.ll @@ -1,8 +1,8 @@ ; COM: Emitting basic-block-address-map when machine function splitting is enabled. -; RUN: llc < %s -mtriple=x86_64 -function-sections -split-machine-functions -basic-block-address-map | FileCheck %s --check-prefixes=CHECK,BASIC +; RUN: llc < %s -mtriple=x86_64 -function-sections -function-splitting=all -basic-block-address-map | FileCheck %s --check-prefixes=CHECK,BASIC ; COM: Emitting basic-block-address-map with PGO analysis with machine function splitting enabled. -; RUN: llc < %s -mtriple=x86_64 -function-sections -split-machine-functions -basic-block-address-map -pgo-analysis-map=func-entry-count,bb-freq,br-prob | FileCheck %s --check-prefixes=CHECK,PGO +; RUN: llc < %s -mtriple=x86_64 -function-sections -function-splitting=all -basic-block-address-map -pgo-analysis-map=func-entry-count,bb-freq,br-prob | FileCheck %s --check-prefixes=CHECK,PGO define void @foo(i1 zeroext %0) nounwind !prof !14 { br i1 %0, label %2, label %4, !prof !15 diff --git a/llvm/test/CodeGen/X86/basic-block-address-map.ll b/llvm/test/CodeGen/X86/basic-block-address-map.ll index c716e12a576462..21d9066d2a7db8 100644 --- a/llvm/test/CodeGen/X86/basic-block-address-map.ll +++ b/llvm/test/CodeGen/X86/basic-block-address-map.ll @@ -1,7 +1,7 @@ ; Check the basic block sections labels option ; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=true -basic-block-address-map | FileCheck %s --check-prefixes=CHECK,ELF,UNIQ ; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=false -basic-block-address-map | FileCheck %s --check-prefixes=CHECK,ELF,NOUNIQ -; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=true -basic-block-address-map -split-machine-functions | FileCheck %s --check-prefixes=CHECK,ELF,UNIQ +; RUN: llc < %s -mtriple=x86_64 -function-sections -unique-section-names=true -basic-block-address-map -function-splitting=all | FileCheck %s --check-prefixes=CHECK,ELF,UNIQ ; RUN: llc < %s -mtriple=x86_64-pc-windows-msvc -function-sections -basic-block-address-map | FileCheck %s --check-prefixes=CHECK,COFF define void @_Z3bazb(i1 zeroext, i1 zeroext) personality ptr @__gxx_personality_v0 { diff --git a/llvm/test/CodeGen/X86/basic-block-sections.ll b/llvm/test/CodeGen/X86/basic-block-sections.ll index 8e50416554a0a5..fd1db1e29ef392 100644 --- a/llvm/test/CodeGen/X86/basic-block-sections.ll +++ b/llvm/test/CodeGen/X86/basic-block-sections.ll @@ -1,6 +1,6 @@ ; RUN: llc < %s -mtriple=x86_64-pc-linux -function-sections -basic-block-sections=all -unique-basic-block-section-names | FileCheck %s -check-prefix=LINUX-SECTIONS ; RUN: llc < %s -mtriple=x86_64-pc-linux -basic-block-sections=all -unique-basic-block-section-names | FileCheck %s -check-prefix=LINUX-SECTIONS -; RUN: llc < %s -mtriple=x86_64-pc-linux -function-sections -basic-block-sections=all -unique-basic-block-section-names -split-machine-functions | FileCheck %s -check-prefix=LINUX-SECTIONS +; RUN: llc < %s -mtriple=x86_64-pc-linux -function-sections -basic-block-sections=all -unique-basic-block-section-names -function-splitting=all | FileCheck %s -check-prefix=LINUX-SECTIONS ; RUN: llc < %s -mtriple=i386-unknown-linux-gnu -function-sections -basic-block-sections=all -unique-basic-block-section-names | FileCheck %s -check-prefix=LINUX-SECTIONS ; RUN: llc < %s -mtriple=i386-unknown-linux-gnu -basic-block-sections=all -unique-basic-block-section-names | FileCheck %s -check-prefix=LINUX-SECTIONS diff --git a/llvm/test/CodeGen/X86/function-splitting-mode.ll b/llvm/test/CodeGen/X86/function-splitting-mode.ll new file mode 100644 index 00000000000000..7486f510360de1 --- /dev/null +++ b/llvm/test/CodeGen/X86/function-splitting-mode.ll @@ -0,0 +1,61 @@ +;; Tests for the -function-splitting= late function splitting mode. +; REQUIRES: x86-registered-target + +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -function-splitting=all | FileCheck %s --check-prefix=SPLIT + +;; A function which only has PGO data is not split unless the mode is 'all'. +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu | FileCheck %s --check-prefix=NOSPLIT +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -function-splitting=none | FileCheck %s --check-prefix=NOSPLIT +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -function-splitting=bbsections | FileCheck %s --check-prefix=NOSPLIT + +;; Invalid modes are rejected. +; RUN: not llc < %s -mtriple=x86_64-unknown-linux-gnu -function-splitting=bogus 2>&1 | FileCheck %s --check-prefix=ERR +; ERR: for the --function-splitting option: Cannot find option named 'bogus'! + +define void @foo(i1 zeroext %0) nounwind !prof !14 !section_prefix !15 { +; SPLIT-LABEL: foo +; SPLIT: .section .text.split.foo +; SPLIT-NEXT: foo.cold: +; SPLIT-NOT: callq bar +; SPLIT-NEXT: callq baz +; +; NOSPLIT-LABEL: foo +; NOSPLIT-NOT: .section .text.split.foo +; NOSPLIT-NOT: foo.cold: + br i1 %0, label %2, label %4, !prof !17 + +2: ; preds = %1 + %3 = call i32 @bar() + br label %6 + +4: ; preds = %1 + %5 = call i32 @baz() + br label %6 + +6: ; preds = %4, %2 + %7 = tail call i32 @qux() + ret void +} + +declare i32 @bar() +declare i32 @baz() +declare i32 @qux() + +!llvm.module.flags = !{!0} +!0 = !{i32 1, !"ProfileSummary", !1} +!1 = !{!2, !3, !4, !5, !6, !7, !8, !9} +!2 = !{!"ProfileFormat", !"InstrProf"} +!3 = !{!"TotalCount", i64 10000} +!4 = !{!"MaxCount", i64 10} +!5 = !{!"MaxInternalCount", i64 1} +!6 = !{!"MaxFunctionCount", i64 1000} +!7 = !{!"NumCounts", i64 3} +!8 = !{!"NumFunctions", i64 5} +!9 = !{!"DetailedSummary", !10} +!10 = !{!11, !12, !13} +!11 = !{i32 10000, i64 100, i32 1} +!12 = !{i32 999900, i64 100, i32 1} +!13 = !{i32 999999, i64 1, i32 2} +!14 = !{!"function_entry_count", i64 7000} +!15 = !{!"function_section_prefix", !"hot"} +!17 = !{!"branch_weights", i32 7000, i32 0} diff --git a/llvm/test/CodeGen/X86/load-sample-profile-2.ll b/llvm/test/CodeGen/X86/load-sample-profile-2.ll index f608d22843d568..76cc02e7b35c04 100644 --- a/llvm/test/CodeGen/X86/load-sample-profile-2.ll +++ b/llvm/test/CodeGen/X86/load-sample-profile-2.ll @@ -1,5 +1,5 @@ ;;; MFS with sample profile fails when no -enable-fs-discriminator=true. -; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -fs-profile-file=%S/Inputs/fsloader-mfs.afdo -split-machine-functions -enable-fs-discriminator=false 2>&1 | FileCheck %s --check-prefix=NODISCRIMINATOR +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -fs-profile-file=%S/Inputs/fsloader-mfs.afdo -function-splitting=all -enable-fs-discriminator=false 2>&1 | FileCheck %s --check-prefix=NODISCRIMINATOR ; NODISCRIMINATOR: warning: Using AutoFDO without FSDiscriminator for MFS may regress performance. define void @foo4(i1 zeroext %0, i1 zeroext %1) nounwind { diff --git a/llvm/test/CodeGen/X86/load-sample-profile.ll b/llvm/test/CodeGen/X86/load-sample-profile.ll index 2e3f8809d3882e..3ac1f39b17e388 100644 --- a/llvm/test/CodeGen/X86/load-sample-profile.ll +++ b/llvm/test/CodeGen/X86/load-sample-profile.ll @@ -1,6 +1,6 @@ ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -debug-pass=Structure -enable-fs-discriminator=true -improved-fs-discriminator=true 2>&1 | FileCheck %s --check-prefix=NOPROFILE ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -debug-pass=Structure -fs-profile-file=%S/Inputs/fsloader-mfs.afdo -enable-fs-discriminator=true -improved-fs-discriminator=true 2>&1 | FileCheck %s --check-prefix=PROFILE-NOMFS -; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -debug-pass=Structure -fs-profile-file=%S/Inputs/fsloader-mfs.afdo -split-machine-functions -enable-fs-discriminator=true -improved-fs-discriminator=true 2>&1 | FileCheck %s --check-prefix=PROFILE-MFS +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -debug-pass=Structure -fs-profile-file=%S/Inputs/fsloader-mfs.afdo -function-splitting=all -enable-fs-discriminator=true -improved-fs-discriminator=true 2>&1 | FileCheck %s --check-prefix=PROFILE-MFS ;; No profile is specified, no load passes. ; NOPROFILE: Add FS discriminators in MIR diff --git a/llvm/test/tools/llvm-profgen/X86/func-split.test b/llvm/test/tools/llvm-profgen/X86/func-split.test index 734f39ea898ca1..3393da2e52dcb1 100644 --- a/llvm/test/tools/llvm-profgen/X86/func-split.test +++ b/llvm/test/tools/llvm-profgen/X86/func-split.test @@ -30,7 +30,7 @@ ; clang -g -O3 -fdebug-info-for-profiling func-split.c -mllvm -mfs-count-threshold=0 -; -fprofile-sample-use=profile.txt -fno-inline -mllvm --enable-split-machine-functions=1 +; -fprofile-sample-use=profile.txt -fno-inline -fsplit-machine-functions #include <stdio.h> >From 82898c047456d3defcc6f32e7566fd52c4dff990 Mon Sep 17 00:00:00 2001 From: Rahman Lavaee <[email protected]> Date: Wed, 7 Oct 2026 18:19:37 +0000 Subject: [PATCH 2/8] [CodeGen] Declare -function-splitting in CodeGenOptions.td Move the -function-splitting option from a cl::opt in CommandFlags.cpp to an EnumField in CodeGenOptions.td, replacing the leftover enable-split-machine-functions BoolField. TargetPassConfig applies the option to TargetOptions::FunctionSplitting when it is given, so it also takes effect through -mllvm and libLTO debug options. --- llvm/include/llvm/CodeGen/CommandFlags.h | 2 -- llvm/lib/CodeGen/CodeGenOptions.td | 5 +++++ llvm/lib/CodeGen/CommandFlags.cpp | 16 ---------------- llvm/lib/CodeGen/TargetPassConfig.cpp | 3 +++ llvm/test/CodeGen/X86/function-splitting-mode.ll | 2 +- 5 files changed, 9 insertions(+), 19 deletions(-) diff --git a/llvm/include/llvm/CodeGen/CommandFlags.h b/llvm/include/llvm/CodeGen/CommandFlags.h index bea26529158654..4e09a2c720c170 100644 --- a/llvm/include/llvm/CodeGen/CommandFlags.h +++ b/llvm/include/llvm/CodeGen/CommandFlags.h @@ -91,8 +91,6 @@ LLVM_ABI bool getXCOFFTracebackTable(); LLVM_ABI std::string getBBSections(); -LLVM_ABI llvm::FunctionSplittingMode getFunctionSplitting(); - LLVM_ABI unsigned getTLSSize(); LLVM_ABI bool getEmulatedTLS(); diff --git a/llvm/lib/CodeGen/CodeGenOptions.td b/llvm/lib/CodeGen/CodeGenOptions.td index 132b2ec8a4f7d4..2084cbae77346d 100644 --- a/llvm/lib/CodeGen/CodeGenOptions.td +++ b/llvm/lib/CodeGen/CodeGenOptions.td @@ -151,6 +151,11 @@ defm : BoolField<"enable-implicit-null-checks", "false", "Fold null checks into faulting memory operations">; defm : OptionalBoolField<"enable-ipra", "Enable interprocedural register allocation to reduce load/store at procedure calls">; +defm : EnumField<"function-splitting", "std::optional<FunctionSplittingMode>", "std::nullopt", + "none,bbsections,all", + ["FunctionSplittingMode::None", "FunctionSplittingMode::BBSectionsOnly", + "FunctionSplittingMode::All"], + "Which functions are eligible for late function splitting">; // -fast-isel selects FastISel at any optimization level, and -fast-isel=0 // disables it at -O0, where it is used unless GlobalISel is enabled. defm : OptionalBoolField<"fast-isel", diff --git a/llvm/lib/CodeGen/CommandFlags.cpp b/llvm/lib/CodeGen/CommandFlags.cpp index 2cd6f8aaf3a868..75677b43c46d0b 100644 --- a/llvm/lib/CodeGen/CommandFlags.cpp +++ b/llvm/lib/CodeGen/CommandFlags.cpp @@ -92,7 +92,6 @@ CGOPT(bool, IgnoreXCOFFVisibility) CGOPT(bool, XCOFFTracebackTable) CGOPT(bool, EnableBBAddrMap) CGOPT(std::string, BBSections) -CGOPT(FunctionSplittingMode, FunctionSplitting) CGOPT(unsigned, TLSSize) CGOPT_EXP(bool, EmulatedTLS) CGOPT_EXP(bool, EnableTLSDESC) @@ -432,20 +431,6 @@ codegen::RegisterCodeGenFlags::RegisterCodeGenFlags() { cl::init(false)); CGBINDOPT(EnableDebugEntryValues); - static cl::opt<FunctionSplittingMode> FunctionSplitting( - "function-splitting", - cl::desc("Which functions are eligible for late function splitting"), - cl::init(FunctionSplittingMode::BBSectionsOnly), - cl::values( - clEnumValN(FunctionSplittingMode::None, "none", - "Do not split any function"), - clEnumValN(FunctionSplittingMode::BBSectionsOnly, "bbsections", - "Only split functions which have a basic block sections " - "profile"), - clEnumValN(FunctionSplittingMode::All, "all", - "Split functions using the basic block sections profile " - "where it is available, and PGO/AutoFDO elsewhere"))); - CGBINDOPT(FunctionSplitting); static cl::opt<bool> EnableStaticDataPartitioning( "partition-static-data-sections", @@ -558,7 +543,6 @@ codegen::InitTargetOptionsFromCodeGenFlags(const Triple &TheTriple) { Options.ExceptionModel = getExceptionModel(); Options.VecLib = getVectorLibrary(); Options.EmitStackSizeSection = getEnableStackSizeSection(); - Options.FunctionSplitting = getFunctionSplitting(); Options.EnableStaticDataPartitioning = getEnableStaticDataPartitioning(); Options.EmitAddrsig = getEnableAddrsig(); Options.EmitCallGraphSection = getEnableCallGraphSection(); diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp index 35e70a53082354..e290a315cacde9 100644 --- a/llvm/lib/CodeGen/TargetPassConfig.cpp +++ b/llvm/lib/CodeGen/TargetPassConfig.cpp @@ -449,6 +449,9 @@ TargetPassConfig::TargetPassConfig(TargetMachine &TM, PassManagerBase &PM) if (Opts.global_isel_abort) TM.Options.GlobalISelAbort = *Opts.global_isel_abort; + if (Opts.function_splitting) + TM.Options.FunctionSplitting = *Opts.function_splitting; + setStartStopPasses(); } diff --git a/llvm/test/CodeGen/X86/function-splitting-mode.ll b/llvm/test/CodeGen/X86/function-splitting-mode.ll index 7486f510360de1..6fc1f3ced84335 100644 --- a/llvm/test/CodeGen/X86/function-splitting-mode.ll +++ b/llvm/test/CodeGen/X86/function-splitting-mode.ll @@ -10,7 +10,7 @@ ;; Invalid modes are rejected. ; RUN: not llc < %s -mtriple=x86_64-unknown-linux-gnu -function-splitting=bogus 2>&1 | FileCheck %s --check-prefix=ERR -; ERR: for the --function-splitting option: Cannot find option named 'bogus'! +; ERR: invalid value 'bogus' in '-function-splitting=bogus' define void @foo(i1 zeroext %0) nounwind !prof !14 !section_prefix !15 { ; SPLIT-LABEL: foo >From 0857ad75219fe5ab995c5b7b10e4bd3e20266403 Mon Sep 17 00:00:00 2001 From: Rahman Lavaee <[email protected]> Date: Wed, 7 Oct 2026 18:19:38 +0000 Subject: [PATCH 3/8] [CodeGen] Clarify the FunctionSplittingMode::BBSectionsOnly comment --- llvm/include/llvm/Target/TargetOptions.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/llvm/include/llvm/Target/TargetOptions.h b/llvm/include/llvm/Target/TargetOptions.h index b0db80947fb996..052a8f4db0ca61 100644 --- a/llvm/include/llvm/Target/TargetOptions.h +++ b/llvm/include/llvm/Target/TargetOptions.h @@ -57,8 +57,10 @@ enum class FunctionSplittingMode { // which have a basic block sections profile. Such functions // are still laid out using the profile, but are emitted as a // single contiguous section. - BBSectionsOnly, // Split only functions which have a basic block sections - // profile. This is the default. + BBSectionsOnly, // Split only functions that have a basic block sections + // profile. Functions without one are not split, even if + // they have PGO/SamplePGO data. This is the default, and + // matches the behavior without -fsplit-machine-functions. All // Split functions using the basic block sections profile // where it is available, and PGO/AutoFDO elsewhere. }; >From 069ab74675512bcbcee802f19fd7d3942322ce31 Mon Sep 17 00:00:00 2001 From: Rahman Lavaee <[email protected]> Date: Wed, 7 Oct 2026 18:22:22 +0000 Subject: [PATCH 4/8] [CodeGen] Note that FunctionSplittingMode::None is not implemented yet A basic block sections profile which only lists the entry block of a function still produces a cold section with -function-splitting=none. --- llvm/include/llvm/Target/TargetOptions.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/llvm/include/llvm/Target/TargetOptions.h b/llvm/include/llvm/Target/TargetOptions.h index 052a8f4db0ca61..adf2a03cbc2fc7 100644 --- a/llvm/include/llvm/Target/TargetOptions.h +++ b/llvm/include/llvm/Target/TargetOptions.h @@ -57,6 +57,9 @@ enum class FunctionSplittingMode { // which have a basic block sections profile. Such functions // are still laid out using the profile, but are emitted as a // single contiguous section. + // NOTE: Not implemented yet. A basic block sections + // profile which only lists the entry block of a function + // still produces a cold section in this mode. BBSectionsOnly, // Split only functions that have a basic block sections // profile. Functions without one are not split, even if // they have PGO/SamplePGO data. This is the default, and >From a59469b0fa0f1426d7fbaa1984329683a9fff044 Mon Sep 17 00:00:00 2001 From: Rahman Lavaee <[email protected]> Date: Wed, 7 Oct 2026 18:24:30 +0000 Subject: [PATCH 5/8] [CodeGen] Shorten the FunctionSplittingMode::None note --- llvm/include/llvm/Target/TargetOptions.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/llvm/include/llvm/Target/TargetOptions.h b/llvm/include/llvm/Target/TargetOptions.h index adf2a03cbc2fc7..75e855789dedb3 100644 --- a/llvm/include/llvm/Target/TargetOptions.h +++ b/llvm/include/llvm/Target/TargetOptions.h @@ -57,9 +57,7 @@ enum class FunctionSplittingMode { // which have a basic block sections profile. Such functions // are still laid out using the profile, but are emitted as a // single contiguous section. - // NOTE: Not implemented yet. A basic block sections - // profile which only lists the entry block of a function - // still produces a cold section in this mode. + // NOTE: Not implemented yet. BBSectionsOnly, // Split only functions that have a basic block sections // profile. Functions without one are not split, even if // they have PGO/SamplePGO data. This is the default, and >From f4e1e68997e3b44aa72c9ffec38c1f02dbc3a633 Mon Sep 17 00:00:00 2001 From: Rahman Lavaee <[email protected]> Date: Wed, 7 Oct 2026 18:34:11 +0000 Subject: [PATCH 6/8] [CodeGen] Distinguish frontend and backend tool migration in the release note --- llvm/docs/ReleaseNotes.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/llvm/docs/ReleaseNotes.md b/llvm/docs/ReleaseNotes.md index 4758b5ef253145..f7786e79b5fb2e 100644 --- a/llvm/docs/ReleaseNotes.md +++ b/llvm/docs/ReleaseNotes.md @@ -184,9 +184,13 @@ Makes programs 10x faster by doing Special New Thing. * Replaced `TargetOptions::EnableMachineFunctionSplitter` with `TargetOptions::FunctionSplitting`, and the `-split-machine-functions` and `-enable-split-machine-functions` flags with - `-function-splitting={none,bbsections,all}`. Use `-function-splitting=all` - for the previous behavior. Clang's `-fsplit-machine-functions` is - unchanged. + `-function-splitting={none,bbsections,all}`. + * Backend tools (`llc`, `opt`, LTO plugin options and libLTO debug options): + use `-function-splitting=all` for the previous behavior. + * Clang and Flang: `-fsplit-machine-functions` is unchanged and remains the + recommended spelling. `-mllvm -enable-split-machine-functions` should be + replaced with `-fsplit-machine-functions` or + `-mllvm -function-splitting=all`. ### Changes to building LLVM >From d021dec77b1a062b3bf57242a81cf22e57d9af4d Mon Sep 17 00:00:00 2001 From: Rahman Lavaee <[email protected]> Date: Wed, 7 Oct 2026 18:41:26 +0000 Subject: [PATCH 7/8] [CodeGen] Remove a duplicate blank line in CommandFlags.cpp --- llvm/lib/CodeGen/CommandFlags.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/llvm/lib/CodeGen/CommandFlags.cpp b/llvm/lib/CodeGen/CommandFlags.cpp index 75677b43c46d0b..46a294f3524b94 100644 --- a/llvm/lib/CodeGen/CommandFlags.cpp +++ b/llvm/lib/CodeGen/CommandFlags.cpp @@ -431,7 +431,6 @@ codegen::RegisterCodeGenFlags::RegisterCodeGenFlags() { cl::init(false)); CGBINDOPT(EnableDebugEntryValues); - static cl::opt<bool> EnableStaticDataPartitioning( "partition-static-data-sections", cl::desc("Partition data sections using profile information."), >From b8799e658367ea758315d5bc11fbd1c9ac0d9320 Mon Sep 17 00:00:00 2001 From: Rahman Lavaee <[email protected]> Date: Wed, 30 Sep 2026 21:19:15 +0000 Subject: [PATCH 8/8] [CodeGen] Honor -function-splitting=none in BasicBlockSections With -function-splitting=none, functions with a basic block sections profile are still laid out according to the profile, but all their basic blocks are now emitted in a single section, instead of the profile's clusters and a cold section. --- llvm/include/llvm/Target/TargetOptions.h | 1 - llvm/lib/CodeGen/BasicBlockSections.cpp | 28 +++++++++++++++---- .../Generic/machine-function-splitter.ll | 15 ++++++++++ .../X86/basic-block-sections-clusters.ll | 15 ++++++++++ 4 files changed, 52 insertions(+), 7 deletions(-) diff --git a/llvm/include/llvm/Target/TargetOptions.h b/llvm/include/llvm/Target/TargetOptions.h index 75e855789dedb3..052a8f4db0ca61 100644 --- a/llvm/include/llvm/Target/TargetOptions.h +++ b/llvm/include/llvm/Target/TargetOptions.h @@ -57,7 +57,6 @@ enum class FunctionSplittingMode { // which have a basic block sections profile. Such functions // are still laid out using the profile, but are emitted as a // single contiguous section. - // NOTE: Not implemented yet. BBSectionsOnly, // Split only functions that have a basic block sections // profile. Functions without one are not split, even if // they have PGO/SamplePGO data. This is the default, and diff --git a/llvm/lib/CodeGen/BasicBlockSections.cpp b/llvm/lib/CodeGen/BasicBlockSections.cpp index 9972d431aae8b0..18ecc1bbe2213f 100644 --- a/llvm/lib/CodeGen/BasicBlockSections.cpp +++ b/llvm/lib/CodeGen/BasicBlockSections.cpp @@ -252,13 +252,19 @@ createBBClusterInfoForFunction(MachineFunction &MF, // Additionally, if exception handling landing pads end up in more than one // clusters, they are moved into a single "Exception" section. Eventually, // clusters are ordered in increasing order of their IDs, with the "Exception" -// and "Cold" succeeding all other clusters. +// and "Cold" succeeding all other clusters. With -function-splitting=none, all +// basic blocks stay in the default section instead. // FuncClusterInfo represents the cluster information for basic blocks. It // maps from BBID of basic blocks to their cluster information. static void assignSections(MachineFunction &MF, const DenseMap<UniqueBBID, BBClusterInfo> &FuncClusterInfo) { assert(MF.hasBBSections() && "BB Sections is not set for function."); + // With function splitting disabled, the profile only decides the layout and + // the function is emitted as a single section. + if (MF.getTarget().getBBSectionsType() == BasicBlockSection::List && + MF.getTarget().Options.FunctionSplitting == FunctionSplittingMode::None) + return; // This variable stores the section ID of the cluster containing eh_pads (if // all eh_pads are one cluster). If more than one cluster contain eh_pads, we // set it equal to ExceptionSectionID. @@ -441,11 +447,21 @@ bool BasicBlockSections::handleBBSections(MachineFunction &MF) { // Make sure that the entry block is placed at the beginning. if (&X == &EntryBB || &Y == &EntryBB) return &X == &EntryBB; - // If the two basic block are in the same section, the order is decided by - // their position within the section. - if (XSectionID.Type == MBBSectionID::SectionType::Default) - return FuncClusterInfo.lookup(*X.getBBID()).PositionInCluster < - FuncClusterInfo.lookup(*Y.getBBID()).PositionInCluster; + // If the two basic blocks are in the same section, the order is decided by + // their cluster and their position within the cluster. A section holds more + // than one cluster only with -function-splitting=none. Basic blocks which + // are not in any cluster come last, in their original order. + if (XSectionID.Type == MBBSectionID::SectionType::Default) { + auto XI = FuncClusterInfo.find(*X.getBBID()); + auto YI = FuncClusterInfo.find(*Y.getBBID()); + bool XInCluster = XI != FuncClusterInfo.end(); + bool YInCluster = YI != FuncClusterInfo.end(); + if (XInCluster != YInCluster) + return XInCluster; + if (XInCluster) + return std::tie(XI->second.ClusterID, XI->second.PositionInCluster) < + std::tie(YI->second.ClusterID, YI->second.PositionInCluster); + } return X.getNumber() < Y.getNumber(); }; diff --git a/llvm/test/CodeGen/Generic/machine-function-splitter.ll b/llvm/test/CodeGen/Generic/machine-function-splitter.ll index cf8abe3a536b29..47b4ec2aa8d730 100644 --- a/llvm/test/CodeGen/Generic/machine-function-splitter.ll +++ b/llvm/test/CodeGen/Generic/machine-function-splitter.ll @@ -17,6 +17,7 @@ ; RUN: echo 'c2' >> %t ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -basic-block-sections=%t -function-splitting=all | FileCheck %s --check-prefixes=MFS-BBSECTIONS,MFS-BBSECTIONS-ALL ; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -basic-block-sections=%t -function-splitting=bbsections | FileCheck %s --check-prefixes=MFS-BBSECTIONS,MFS-BBSECTIONS-ONLY +; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -basic-block-sections=%t -function-splitting=none | FileCheck %s --check-prefixes=MFS-BBSECTIONS-NONE ; RUN: llc < %s -mtriple=aarch64-unknown-linux-gnu -aarch64-min-jump-table-entries=4 -function-splitting=all | FileCheck %s -check-prefixes=MFS-DEFAULTS,MFS-DEFAULTS-AARCH64 ; RUN: llc < %s -mtriple=aarch64-unknown-linux-gnu -aarch64-min-jump-table-entries=4 -function-splitting=all -mfs-psi-cutoff=0 -mfs-count-threshold=2000 | FileCheck %s --dump-input=always -check-prefixes=MFS-OPTS1,MFS-OPTS1-AARCH64 @@ -638,6 +639,11 @@ define void @foo21(i1 zeroext %0) { ; MFS-NOBBSECTIONS-NOT: foo21.cold: ; MFS-BBSECTIONS: .section .text.split.foo21 ; MFS-BBSECTIONS: foo21.cold +;; Check that with -function-splitting=none, the function is not split. +; MFS-BBSECTIONS-NONE: .section .text.hot.foo21 +; MFS-BBSECTIONS-NONE-LABEL: foo21: +; MFS-BBSECTIONS-NONE-NOT: .section +; MFS-BBSECTIONS-NONE: .size foo21, %2 = alloca i8, align 1 %3 = zext i1 %0 to i8 store i8 %3, ptr %2, align 1 @@ -669,6 +675,15 @@ define void @foo22(i1 zeroext %0) nounwind !prof !14 !section_prefix !15 { ; MFS-BBSECTIONS-NEXT: foo22.__part.1: ; MFS-BBSECTIONS: callq baz ; MFS-BBSECTIONS-NOT: .section .text.split.foo22 +;; Check that with -function-splitting=none, both clusters are emitted in a +;; single section. +; MFS-BBSECTIONS-NONE: .section .text.hot.foo22 +; MFS-BBSECTIONS-NONE-LABEL: foo22: +; MFS-BBSECTIONS-NONE: callq bar +; MFS-BBSECTIONS-NONE-NOT: .section +; MFS-BBSECTIONS-NONE: callq baz +; MFS-BBSECTIONS-NONE-NOT: .section +; MFS-BBSECTIONS-NONE: .size foo22, br i1 %0, label %2, label %4, !prof !17 2: ; preds = %1 diff --git a/llvm/test/CodeGen/X86/basic-block-sections-clusters.ll b/llvm/test/CodeGen/X86/basic-block-sections-clusters.ll index a2ea84ff885928..5b007e992a5cbd 100644 --- a/llvm/test/CodeGen/X86/basic-block-sections-clusters.ll +++ b/llvm/test/CodeGen/X86/basic-block-sections-clusters.ll @@ -27,6 +27,10 @@ ; RUN: echo 'c 1 3' >> %t4 ; RUN: llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -basic-block-sections=%t3 | FileCheck %s -check-prefix=LINUX-SECTIONS2 ; RUN: llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -basic-block-sections=%t4 | FileCheck %s -check-prefix=LINUX-SECTIONS2 +; +; Test3: With -function-splitting=none, the Test1 profile still decides the +; layout, but all basic blocks are placed in the function's section. +; RUN: llc < %s -O0 -mtriple=x86_64-pc-linux -function-sections -basic-block-sections=%t2 -function-splitting=none | FileCheck %s -check-prefix=LINUX-NOSPLIT define void @foo(i1 zeroext) nounwind { %2 = alloca i8, align 1 @@ -90,3 +94,14 @@ declare i32 @baz() #1 ; LINUX-SECTIONS2-NOT: .LBB_END0_{{0-9}}+ ; LINUX-SECTIONS2-LABEL: .Lfunc_end0: ; LINUX-SECTIONS2-NEXT: .size foo, .Lfunc_end0-foo + +; LINUX-NOSPLIT: .section .text.foo,"ax",@progbits +; LINUX-NOSPLIT-LABEL: foo: +; LINUX-NOSPLIT-NOT: .section +; LINUX-NOSPLIT: # %bb.2: +; LINUX-NOSPLIT-NOT: .section +; LINUX-NOSPLIT: .LBB0_1: +; LINUX-NOSPLIT-NOT: .section +; LINUX-NOSPLIT: .LBB0_3: +; LINUX-NOSPLIT-NOT: .section +; LINUX-NOSPLIT-LABEL: .Lfunc_end0: _______________________________________________ cfe-commits mailing list [email protected] https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
