modimo updated this revision to Diff 288445.
modimo added a comment.
only use BFI when profile is enabled, have LICM mark BFI as preserved
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D86156/new/
https://reviews.llvm.org/D86156
Files:
clang/test/CodeGen/thinlto-distributed-newpm.ll
llvm/include/llvm/Analysis/BlockFrequencyInfo.h
llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h
llvm/include/llvm/Analysis/LoopAnalysisManager.h
llvm/include/llvm/Transforms/Scalar/LoopPassManager.h
llvm/lib/Analysis/BlockFrequencyInfo.cpp
llvm/lib/Passes/PassBuilder.cpp
llvm/lib/Transforms/Scalar/LICM.cpp
llvm/lib/Transforms/Scalar/LoopDistribute.cpp
llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp
llvm/lib/Transforms/Scalar/LoopUnswitch.cpp
llvm/lib/Transforms/Utils/LoopVersioning.cpp
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
llvm/test/Other/new-pm-defaults.ll
llvm/test/Other/new-pm-thinlto-defaults.ll
llvm/test/Other/opt-O2-pipeline.ll
llvm/test/Other/opt-O3-pipeline-enable-matrix.ll
llvm/test/Other/opt-O3-pipeline.ll
llvm/test/Other/opt-Os-pipeline.ll
llvm/unittests/Transforms/Scalar/LoopPassManagerTest.cpp
Index: llvm/unittests/Transforms/Scalar/LoopPassManagerTest.cpp
===================================================================
--- llvm/unittests/Transforms/Scalar/LoopPassManagerTest.cpp
+++ llvm/unittests/Transforms/Scalar/LoopPassManagerTest.cpp
@@ -9,7 +9,10 @@
#include "llvm/Transforms/Scalar/LoopPassManager.h"
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/Analysis/AssumptionCache.h"
+#include "llvm/Analysis/BlockFrequencyInfo.h"
+#include "llvm/Analysis/BranchProbabilityInfo.h"
#include "llvm/Analysis/MemorySSA.h"
+#include "llvm/Analysis/PostDominators.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/TargetTransformInfo.h"
@@ -294,6 +297,9 @@
// those.
FAM.registerPass([&] { return AAManager(); });
FAM.registerPass([&] { return AssumptionAnalysis(); });
+ FAM.registerPass([&] { return BlockFrequencyAnalysis(); });
+ FAM.registerPass([&] { return BranchProbabilityAnalysis(); });
+ FAM.registerPass([&] { return PostDominatorTreeAnalysis(); });
FAM.registerPass([&] { return MemorySSAAnalysis(); });
FAM.registerPass([&] { return ScalarEvolutionAnalysis(); });
FAM.registerPass([&] { return TargetLibraryAnalysis(); });
Index: llvm/test/Other/opt-Os-pipeline.ll
===================================================================
--- llvm/test/Other/opt-Os-pipeline.ll
+++ llvm/test/Other/opt-Os-pipeline.ll
@@ -97,6 +97,8 @@
; CHECK-NEXT: Loop Pass Manager
; CHECK-NEXT: Rotate Loops
; CHECK-NEXT: Memory SSA
+; CHECK-NEXT: Lazy Branch Probability Analysis
+; CHECK-NEXT: Lazy Block Frequency Analysis
; CHECK-NEXT: Loop Pass Manager
; CHECK-NEXT: Loop Invariant Code Motion
; CHECK-NEXT: Unswitch loops
@@ -154,6 +156,8 @@
; CHECK-NEXT: LCSSA Verifier
; CHECK-NEXT: Loop-Closed SSA Form Pass
; CHECK-NEXT: Scalar Evolution Analysis
+; CHECK-NEXT: Lazy Branch Probability Analysis
+; CHECK-NEXT: Lazy Block Frequency Analysis
; CHECK-NEXT: Loop Pass Manager
; CHECK-NEXT: Loop Invariant Code Motion
; CHECK-NEXT: Post-Dominator Tree Construction
@@ -256,10 +260,10 @@
; CHECK-NEXT: LCSSA Verifier
; CHECK-NEXT: Loop-Closed SSA Form Pass
; CHECK-NEXT: Scalar Evolution Analysis
-; CHECK-NEXT: Loop Pass Manager
-; CHECK-NEXT: Loop Invariant Code Motion
; CHECK-NEXT: Lazy Branch Probability Analysis
; CHECK-NEXT: Lazy Block Frequency Analysis
+; CHECK-NEXT: Loop Pass Manager
+; CHECK-NEXT: Loop Invariant Code Motion
; CHECK-NEXT: Optimization Remark Emitter
; CHECK-NEXT: Warn about non-applied transformations
; CHECK-NEXT: Alignment from assumptions
Index: llvm/test/Other/opt-O3-pipeline.ll
===================================================================
--- llvm/test/Other/opt-O3-pipeline.ll
+++ llvm/test/Other/opt-O3-pipeline.ll
@@ -116,6 +116,8 @@
; CHECK-NEXT: Loop Pass Manager
; CHECK-NEXT: Rotate Loops
; CHECK-NEXT: Memory SSA
+; CHECK-NEXT: Lazy Branch Probability Analysis
+; CHECK-NEXT: Lazy Block Frequency Analysis
; CHECK-NEXT: Loop Pass Manager
; CHECK-NEXT: Loop Invariant Code Motion
; CHECK-NEXT: Unswitch loops
@@ -173,6 +175,8 @@
; CHECK-NEXT: LCSSA Verifier
; CHECK-NEXT: Loop-Closed SSA Form Pass
; CHECK-NEXT: Scalar Evolution Analysis
+; CHECK-NEXT: Lazy Branch Probability Analysis
+; CHECK-NEXT: Lazy Block Frequency Analysis
; CHECK-NEXT: Loop Pass Manager
; CHECK-NEXT: Loop Invariant Code Motion
; CHECK-NEXT: Post-Dominator Tree Construction
@@ -275,10 +279,10 @@
; CHECK-NEXT: LCSSA Verifier
; CHECK-NEXT: Loop-Closed SSA Form Pass
; CHECK-NEXT: Scalar Evolution Analysis
-; CHECK-NEXT: Loop Pass Manager
-; CHECK-NEXT: Loop Invariant Code Motion
; CHECK-NEXT: Lazy Branch Probability Analysis
; CHECK-NEXT: Lazy Block Frequency Analysis
+; CHECK-NEXT: Loop Pass Manager
+; CHECK-NEXT: Loop Invariant Code Motion
; CHECK-NEXT: Optimization Remark Emitter
; CHECK-NEXT: Warn about non-applied transformations
; CHECK-NEXT: Alignment from assumptions
Index: llvm/test/Other/opt-O3-pipeline-enable-matrix.ll
===================================================================
--- llvm/test/Other/opt-O3-pipeline-enable-matrix.ll
+++ llvm/test/Other/opt-O3-pipeline-enable-matrix.ll
@@ -116,6 +116,8 @@
; CHECK-NEXT: Loop Pass Manager
; CHECK-NEXT: Rotate Loops
; CHECK-NEXT: Memory SSA
+; CHECK-NEXT: Lazy Branch Probability Analysis
+; CHECK-NEXT: Lazy Block Frequency Analysis
; CHECK-NEXT: Loop Pass Manager
; CHECK-NEXT: Loop Invariant Code Motion
; CHECK-NEXT: Unswitch loops
@@ -173,6 +175,8 @@
; CHECK-NEXT: LCSSA Verifier
; CHECK-NEXT: Loop-Closed SSA Form Pass
; CHECK-NEXT: Scalar Evolution Analysis
+; CHECK-NEXT: Lazy Branch Probability Analysis
+; CHECK-NEXT: Lazy Block Frequency Analysis
; CHECK-NEXT: Loop Pass Manager
; CHECK-NEXT: Loop Invariant Code Motion
; CHECK-NEXT: Post-Dominator Tree Construction
@@ -282,10 +286,10 @@
; CHECK-NEXT: LCSSA Verifier
; CHECK-NEXT: Loop-Closed SSA Form Pass
; CHECK-NEXT: Scalar Evolution Analysis
-; CHECK-NEXT: Loop Pass Manager
-; CHECK-NEXT: Loop Invariant Code Motion
; CHECK-NEXT: Lazy Branch Probability Analysis
; CHECK-NEXT: Lazy Block Frequency Analysis
+; CHECK-NEXT: Loop Pass Manager
+; CHECK-NEXT: Loop Invariant Code Motion
; CHECK-NEXT: Optimization Remark Emitter
; CHECK-NEXT: Warn about non-applied transformations
; CHECK-NEXT: Alignment from assumptions
Index: llvm/test/Other/opt-O2-pipeline.ll
===================================================================
--- llvm/test/Other/opt-O2-pipeline.ll
+++ llvm/test/Other/opt-O2-pipeline.ll
@@ -111,6 +111,8 @@
; CHECK-NEXT: Loop Pass Manager
; CHECK-NEXT: Rotate Loops
; CHECK-NEXT: Memory SSA
+; CHECK-NEXT: Lazy Branch Probability Analysis
+; CHECK-NEXT: Lazy Block Frequency Analysis
; CHECK-NEXT: Loop Pass Manager
; CHECK-NEXT: Loop Invariant Code Motion
; CHECK-NEXT: Unswitch loops
@@ -168,6 +170,8 @@
; CHECK-NEXT: LCSSA Verifier
; CHECK-NEXT: Loop-Closed SSA Form Pass
; CHECK-NEXT: Scalar Evolution Analysis
+; CHECK-NEXT: Lazy Branch Probability Analysis
+; CHECK-NEXT: Lazy Block Frequency Analysis
; CHECK-NEXT: Loop Pass Manager
; CHECK-NEXT: Loop Invariant Code Motion
; CHECK-NEXT: Post-Dominator Tree Construction
@@ -270,10 +274,10 @@
; CHECK-NEXT: LCSSA Verifier
; CHECK-NEXT: Loop-Closed SSA Form Pass
; CHECK-NEXT: Scalar Evolution Analysis
-; CHECK-NEXT: Loop Pass Manager
-; CHECK-NEXT: Loop Invariant Code Motion
; CHECK-NEXT: Lazy Branch Probability Analysis
; CHECK-NEXT: Lazy Block Frequency Analysis
+; CHECK-NEXT: Loop Pass Manager
+; CHECK-NEXT: Loop Invariant Code Motion
; CHECK-NEXT: Optimization Remark Emitter
; CHECK-NEXT: Warn about non-applied transformations
; CHECK-NEXT: Alignment from assumptions
Index: llvm/test/Other/new-pm-thinlto-defaults.ll
===================================================================
--- llvm/test/Other/new-pm-thinlto-defaults.ll
+++ llvm/test/Other/new-pm-thinlto-defaults.ll
@@ -128,6 +128,9 @@
; CHECK-O-NEXT: Running analysis: LoopAnalysis
; CHECK-O-NEXT: Running pass: LCSSAPass
; CHECK-O-NEXT: Finished llvm::Function pass manager run
+; CHECK-O-NEXT: Running analysis: BlockFrequencyAnalysis
+; CHECK-O-NEXT: Running analysis: BranchProbabilityAnalysis
+; CHECK-O-NEXT: Running analysis: PostDominatorTreeAnalysis
; CHECK-O-NEXT: Running analysis: ScalarEvolutionAnalysis
; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy
; CHECK-O-NEXT: Starting Loop pass manager run.
@@ -184,7 +187,6 @@
; CHECK-O23SZ-NEXT: Finished llvm::Function pass manager run
; CHECK-O23SZ-NEXT: Running pass: LICMPass on Loop at depth 1 containing: %loop
; CHECK-O-NEXT: Running pass: ADCEPass
-; CHECK-O-NEXT: Running analysis: PostDominatorTreeAnalysis
; CHECK-O-NEXT: Running pass: SimplifyCFGPass
; CHECK-O-NEXT: Running pass: InstCombinePass
; CHECK-O-NEXT: Finished llvm::Function pass manager run.
@@ -210,8 +212,6 @@
; CHECK-POSTLINK-O-NEXT: Running pass: LoopDistributePass
; CHECK-POSTLINK-O-NEXT: Running pass: InjectTLIMappings
; CHECK-POSTLINK-O-NEXT: Running pass: LoopVectorizePass
-; CHECK-POSTLINK-O-NEXT: Running analysis: BlockFrequencyAnalysis
-; CHECK-POSTLINK-O-NEXT: Running analysis: BranchProbabilityAnalysis
; CHECK-POSTLINK-O-NEXT: Running pass: LoopLoadEliminationPass
; CHECK-POSTLINK-O-NEXT: Running analysis: LoopAccessAnalysis
; CHECK-POSTLINK-O-NEXT: Running pass: InstCombinePass
Index: llvm/test/Other/new-pm-defaults.ll
===================================================================
--- llvm/test/Other/new-pm-defaults.ll
+++ llvm/test/Other/new-pm-defaults.ll
@@ -164,6 +164,9 @@
; CHECK-O-NEXT: Running analysis: LoopAnalysis
; CHECK-O-NEXT: Running pass: LCSSAPass
; CHECK-O-NEXT: Finished llvm::Function pass manager run.
+; CHECK-O-NEXT: Running analysis: BlockFrequencyAnalysis
+; CHECK-O-NEXT: Running analysis: BranchProbabilityAnalysis
+; CHECK-O-NEXT: Running analysis: PostDominatorTreeAnalysis
; CHECK-O-NEXT: Running analysis: ScalarEvolutionAnalysis
; CHECK-O-NEXT: Running analysis: InnerAnalysisManagerProxy
; CHECK-O-NEXT: Starting Loop pass manager run.
@@ -212,7 +215,6 @@
; CHECK-O23SZ-NEXT: Running pass: LICMPass
; CHECK-EP-SCALAR-LATE-NEXT: Running pass: NoOpFunctionPass
; CHECK-O-NEXT: Running pass: ADCEPass
-; CHECK-O-NEXT: Running analysis: PostDominatorTreeAnalysis
; CHECK-O-NEXT: Running pass: SimplifyCFGPass
; CHECK-O-NEXT: Running pass: InstCombinePass
; CHECK-EP-PEEPHOLE-NEXT: Running pass: NoOpFunctionPass
@@ -240,8 +242,6 @@
; CHECK-O-NEXT: Running pass: LoopDistributePass
; CHECK-O-NEXT: Running pass: InjectTLIMappings
; CHECK-O-NEXT: Running pass: LoopVectorizePass
-; CHECK-O-NEXT: Running analysis: BlockFrequencyAnalysis
-; CHECK-O-NEXT: Running analysis: BranchProbabilityAnalysis
; CHECK-O-NEXT: Running pass: LoopLoadEliminationPass
; CHECK-O-NEXT: Running analysis: LoopAccessAnalysis
; CHECK-O-NEXT: Running pass: InstCombinePass
Index: llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
===================================================================
--- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -8558,7 +8558,8 @@
auto &LAM = AM.getResult<LoopAnalysisManagerFunctionProxy>(F).getManager();
std::function<const LoopAccessInfo &(Loop &)> GetLAA =
[&](Loop &L) -> const LoopAccessInfo & {
- LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE, TLI, TTI, MSSA};
+ LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE,
+ TLI, TTI, nullptr, MSSA};
return LAM.getResult<LoopAccessAnalysis>(L, AR);
};
auto &MAMProxy = AM.getResult<ModuleAnalysisManagerFunctionProxy>(F);
Index: llvm/lib/Transforms/Utils/LoopVersioning.cpp
===================================================================
--- llvm/lib/Transforms/Utils/LoopVersioning.cpp
+++ llvm/lib/Transforms/Utils/LoopVersioning.cpp
@@ -357,7 +357,8 @@
auto &LAM = AM.getResult<LoopAnalysisManagerFunctionProxy>(F).getManager();
auto GetLAA = [&](Loop &L) -> const LoopAccessInfo & {
- LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE, TLI, TTI, MSSA};
+ LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE,
+ TLI, TTI, nullptr, MSSA};
return LAM.getResult<LoopAccessAnalysis>(L, AR);
};
Index: llvm/lib/Transforms/Scalar/LoopUnswitch.cpp
===================================================================
--- llvm/lib/Transforms/Scalar/LoopUnswitch.cpp
+++ llvm/lib/Transforms/Scalar/LoopUnswitch.cpp
@@ -32,6 +32,7 @@
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/CodeMetrics.h"
#include "llvm/Analysis/InstructionSimplify.h"
+#include "llvm/Analysis/LazyBlockFrequencyInfo.h"
#include "llvm/Analysis/LegacyDivergenceAnalysis.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/LoopIterator.h"
@@ -217,6 +218,10 @@
/// loop preheaders be inserted into the CFG.
///
void getAnalysisUsage(AnalysisUsage &AU) const override {
+ // Lazy BFI and BPI are marked as preserved here so Loop Unswitching
+ // can remain part of the same loop pass as LICM
+ AU.addPreserved<LazyBlockFrequencyInfoPass>();
+ AU.addPreserved<LazyBranchProbabilityInfoPass>();
AU.addRequired<AssumptionCacheTracker>();
AU.addRequired<TargetTransformInfoWrapperPass>();
if (EnableMSSALoopDependency) {
Index: llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp
===================================================================
--- llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp
+++ llvm/lib/Transforms/Scalar/LoopLoadElimination.cpp
@@ -705,7 +705,8 @@
auto &LAM = AM.getResult<LoopAnalysisManagerFunctionProxy>(F).getManager();
bool Changed = eliminateLoadsAcrossLoops(
F, LI, DT, BFI, PSI, [&](Loop &L) -> const LoopAccessInfo & {
- LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE, TLI, TTI, MSSA};
+ LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE,
+ TLI, TTI, nullptr, MSSA};
return LAM.getResult<LoopAccessAnalysis>(L, AR);
});
Index: llvm/lib/Transforms/Scalar/LoopDistribute.cpp
===================================================================
--- llvm/lib/Transforms/Scalar/LoopDistribute.cpp
+++ llvm/lib/Transforms/Scalar/LoopDistribute.cpp
@@ -1058,7 +1058,8 @@
auto &LAM = AM.getResult<LoopAnalysisManagerFunctionProxy>(F).getManager();
std::function<const LoopAccessInfo &(Loop &)> GetLAA =
[&](Loop &L) -> const LoopAccessInfo & {
- LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE, TLI, TTI, nullptr};
+ LoopStandardAnalysisResults AR = {AA, AC, DT, LI, SE,
+ TLI, TTI, nullptr, nullptr};
return LAM.getResult<LoopAccessAnalysis>(L, AR);
};
Index: llvm/lib/Transforms/Scalar/LICM.cpp
===================================================================
--- llvm/lib/Transforms/Scalar/LICM.cpp
+++ llvm/lib/Transforms/Scalar/LICM.cpp
@@ -39,6 +39,7 @@
#include "llvm/Analysis/ConstantFolding.h"
#include "llvm/Analysis/GlobalsModRef.h"
#include "llvm/Analysis/GuardUtils.h"
+#include "llvm/Analysis/LazyBlockFrequencyInfo.h"
#include "llvm/Analysis/Loads.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/LoopIterator.h"
@@ -171,8 +172,8 @@
namespace {
struct LoopInvariantCodeMotion {
bool runOnLoop(Loop *L, AAResults *AA, LoopInfo *LI, DominatorTree *DT,
- TargetLibraryInfo *TLI, TargetTransformInfo *TTI,
- ScalarEvolution *SE, MemorySSA *MSSA,
+ BlockFrequencyInfo *BFI, TargetLibraryInfo *TLI,
+ TargetTransformInfo *TTI, ScalarEvolution *SE, MemorySSA *MSSA,
OptimizationRemarkEmitter *ORE);
LoopInvariantCodeMotion(unsigned LicmMssaOptCap,
@@ -208,19 +209,23 @@
MemorySSA *MSSA = EnableMSSALoopDependency
? (&getAnalysis<MemorySSAWrapperPass>().getMSSA())
: nullptr;
+ bool hasProfileData = L->getHeader()->getParent()->hasProfileData();
+ BlockFrequencyInfo *BFI =
+ hasProfileData ? &getAnalysis<LazyBlockFrequencyInfoPass>().getBFI()
+ : nullptr;
// For the old PM, we can't use OptimizationRemarkEmitter as an analysis
- // pass. Function analyses need to be preserved across loop transformations
+ // pass. Function analyses need to be preserved across loop transformations
// but ORE cannot be preserved (see comment before the pass definition).
OptimizationRemarkEmitter ORE(L->getHeader()->getParent());
- return LICM.runOnLoop(L,
- &getAnalysis<AAResultsWrapperPass>().getAAResults(),
- &getAnalysis<LoopInfoWrapperPass>().getLoopInfo(),
- &getAnalysis<DominatorTreeWrapperPass>().getDomTree(),
- &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(
- *L->getHeader()->getParent()),
- &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(
- *L->getHeader()->getParent()),
- SE ? &SE->getSE() : nullptr, MSSA, &ORE);
+ return LICM.runOnLoop(
+ L, &getAnalysis<AAResultsWrapperPass>().getAAResults(),
+ &getAnalysis<LoopInfoWrapperPass>().getLoopInfo(),
+ &getAnalysis<DominatorTreeWrapperPass>().getDomTree(), BFI,
+ &getAnalysis<TargetLibraryInfoWrapperPass>().getTLI(
+ *L->getHeader()->getParent()),
+ &getAnalysis<TargetTransformInfoWrapperPass>().getTTI(
+ *L->getHeader()->getParent()),
+ SE ? &SE->getSE() : nullptr, MSSA, &ORE);
}
/// This transformation requires natural loop information & requires that
@@ -236,6 +241,9 @@
}
AU.addRequired<TargetTransformInfoWrapperPass>();
getLoopAnalysisUsage(AU);
+ LazyBlockFrequencyInfoPass::getLazyBFIAnalysisUsage(AU);
+ AU.addPreserved<LazyBlockFrequencyInfoPass>();
+ AU.addPreserved<LazyBranchProbabilityInfoPass>();
}
private:
@@ -251,8 +259,8 @@
OptimizationRemarkEmitter ORE(L.getHeader()->getParent());
LoopInvariantCodeMotion LICM(LicmMssaOptCap, LicmMssaNoAccForPromotionCap);
- if (!LICM.runOnLoop(&L, &AR.AA, &AR.LI, &AR.DT, &AR.TLI, &AR.TTI, &AR.SE,
- AR.MSSA, &ORE))
+ if (!LICM.runOnLoop(&L, &AR.AA, &AR.LI, &AR.DT, AR.BFI, &AR.TLI, &AR.TTI,
+ &AR.SE, AR.MSSA, &ORE))
return PreservedAnalyses::all();
auto PA = getLoopPassPreservedAnalyses();
@@ -272,6 +280,7 @@
INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass)
INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
INITIALIZE_PASS_DEPENDENCY(MemorySSAWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(LazyBFIPass)
INITIALIZE_PASS_END(LegacyLICMPass, "licm", "Loop Invariant Code Motion", false,
false)
@@ -286,8 +295,8 @@
/// times on one loop.
bool LoopInvariantCodeMotion::runOnLoop(
Loop *L, AAResults *AA, LoopInfo *LI, DominatorTree *DT,
- TargetLibraryInfo *TLI, TargetTransformInfo *TTI, ScalarEvolution *SE,
- MemorySSA *MSSA, OptimizationRemarkEmitter *ORE) {
+ BlockFrequencyInfo *BFI, TargetLibraryInfo *TLI, TargetTransformInfo *TTI,
+ ScalarEvolution *SE, MemorySSA *MSSA, OptimizationRemarkEmitter *ORE) {
bool Changed = false;
assert(L->isLCSSAForm(*DT) && "Loop is not in LCSSA form.");
Index: llvm/lib/Passes/PassBuilder.cpp
===================================================================
--- llvm/lib/Passes/PassBuilder.cpp
+++ llvm/lib/Passes/PassBuilder.cpp
@@ -512,13 +512,15 @@
FPM.addPass(
RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
FPM.addPass(createFunctionToLoopPassAdaptor(
- std::move(LPM1), EnableMSSALoopDependency, DebugLogging));
+ std::move(LPM1), EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true,
+ DebugLogging));
FPM.addPass(SimplifyCFGPass());
FPM.addPass(InstCombinePass());
// The loop passes in LPM2 (LoopFullUnrollPass) do not preserve MemorySSA.
// *All* loop passes must preserve it, in order to be able to use it.
FPM.addPass(createFunctionToLoopPassAdaptor(
- std::move(LPM2), /*UseMemorySSA=*/false, DebugLogging));
+ std::move(LPM2), /*UseMemorySSA=*/false, /*UseBlockFrequencyInfo=*/false,
+ DebugLogging));
// Delete small array after loop unroll.
FPM.addPass(SROA());
@@ -669,14 +671,16 @@
FPM.addPass(
RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
FPM.addPass(createFunctionToLoopPassAdaptor(
- std::move(LPM1), EnableMSSALoopDependency, DebugLogging));
+ std::move(LPM1), EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true,
+ DebugLogging));
FPM.addPass(SimplifyCFGPass());
FPM.addPass(InstCombinePass());
// The loop passes in LPM2 (IndVarSimplifyPass, LoopIdiomRecognizePass,
// LoopDeletionPass and LoopFullUnrollPass) do not preserve MemorySSA.
// *All* loop passes must preserve it, in order to be able to use it.
FPM.addPass(createFunctionToLoopPassAdaptor(
- std::move(LPM2), /*UseMemorySSA=*/false, DebugLogging));
+ std::move(LPM2), /*UseMemorySSA=*/false, /*UseBlockFrequencyInfo=*/false,
+ DebugLogging));
// Delete small array after loop unroll.
FPM.addPass(SROA());
@@ -713,7 +717,7 @@
FPM.addPass(DSEPass());
FPM.addPass(createFunctionToLoopPassAdaptor(
LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap),
- EnableMSSALoopDependency, DebugLogging));
+ EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true, DebugLogging));
if (PTO.Coroutines)
FPM.addPass(CoroElidePass());
@@ -791,7 +795,8 @@
FunctionPassManager FPM;
FPM.addPass(createFunctionToLoopPassAdaptor(
- LoopRotatePass(), EnableMSSALoopDependency, DebugLogging));
+ LoopRotatePass(), EnableMSSALoopDependency,
+ /*UseBlockFrequencyInfo=*/false, DebugLogging));
MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
// Add the profile lowering pass.
@@ -1115,7 +1120,8 @@
// First rotate loops that may have been un-rotated by prior passes.
OptimizePM.addPass(createFunctionToLoopPassAdaptor(
- LoopRotatePass(), EnableMSSALoopDependency, DebugLogging));
+ LoopRotatePass(), EnableMSSALoopDependency,
+ /*UseBlockFrequencyInfo=*/false, DebugLogging));
// Distribute loops to allow partial vectorization. I.e. isolate dependences
// into separate loop that would otherwise inhibit vectorization. This is
@@ -1179,7 +1185,7 @@
OptimizePM.addPass(RequireAnalysisPass<OptimizationRemarkEmitterAnalysis, Function>());
OptimizePM.addPass(createFunctionToLoopPassAdaptor(
LICMPass(PTO.LicmMssaOptCap, PTO.LicmMssaNoAccForPromotionCap),
- EnableMSSALoopDependency, DebugLogging));
+ EnableMSSALoopDependency, /*UseBlockFrequencyInfo=*/true, DebugLogging));
// Now that we've vectorized and unrolled loops, we may have more refined
// alignment information, try to re-derive it here.
@@ -2244,8 +2250,9 @@
}
#define LOOP_PASS(NAME, CREATE_PASS) \
if (Name == NAME) { \
- MPM.addPass(createModuleToFunctionPassAdaptor( \
- createFunctionToLoopPassAdaptor(CREATE_PASS, false, DebugLogging))); \
+ MPM.addPass( \
+ createModuleToFunctionPassAdaptor(createFunctionToLoopPassAdaptor( \
+ CREATE_PASS, false, false, DebugLogging))); \
return Error::success(); \
}
#define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
@@ -2255,7 +2262,7 @@
return Params.takeError(); \
MPM.addPass( \
createModuleToFunctionPassAdaptor(createFunctionToLoopPassAdaptor( \
- CREATE_PASS(Params.get()), false, DebugLogging))); \
+ CREATE_PASS(Params.get()), false, false, DebugLogging))); \
return Error::success(); \
}
#include "PassRegistry.def"
@@ -2356,8 +2363,9 @@
}
#define LOOP_PASS(NAME, CREATE_PASS) \
if (Name == NAME) { \
- CGPM.addPass(createCGSCCToFunctionPassAdaptor( \
- createFunctionToLoopPassAdaptor(CREATE_PASS, false, DebugLogging))); \
+ CGPM.addPass( \
+ createCGSCCToFunctionPassAdaptor(createFunctionToLoopPassAdaptor( \
+ CREATE_PASS, false, false, DebugLogging))); \
return Error::success(); \
}
#define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
@@ -2367,7 +2375,7 @@
return Params.takeError(); \
CGPM.addPass( \
createCGSCCToFunctionPassAdaptor(createFunctionToLoopPassAdaptor( \
- CREATE_PASS(Params.get()), false, DebugLogging))); \
+ CREATE_PASS(Params.get()), false, false, DebugLogging))); \
return Error::success(); \
}
#include "PassRegistry.def"
@@ -2404,8 +2412,9 @@
return Err;
// Add the nested pass manager with the appropriate adaptor.
bool UseMemorySSA = (Name == "loop-mssa");
- FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM), UseMemorySSA,
- DebugLogging));
+ FPM.addPass(createFunctionToLoopPassAdaptor(
+ std::move(LPM), UseMemorySSA, /*UseBlockFrequencyInfo=*/false,
+ DebugLogging));
return Error::success();
}
if (auto Count = parseRepeatPassName(Name)) {
@@ -2459,8 +2468,8 @@
// The risk is that it may become obsolete if we're not careful.
#define LOOP_PASS(NAME, CREATE_PASS) \
if (Name == NAME) { \
- FPM.addPass( \
- createFunctionToLoopPassAdaptor(CREATE_PASS, false, DebugLogging)); \
+ FPM.addPass(createFunctionToLoopPassAdaptor(CREATE_PASS, false, false, \
+ DebugLogging)); \
return Error::success(); \
}
#define LOOP_PASS_WITH_PARAMS(NAME, CREATE_PASS, PARSER) \
@@ -2469,7 +2478,7 @@
if (!Params) \
return Params.takeError(); \
FPM.addPass(createFunctionToLoopPassAdaptor(CREATE_PASS(Params.get()), \
- false, DebugLogging)); \
+ false, false, DebugLogging)); \
return Error::success(); \
}
#include "PassRegistry.def"
Index: llvm/lib/Analysis/BlockFrequencyInfo.cpp
===================================================================
--- llvm/lib/Analysis/BlockFrequencyInfo.cpp
+++ llvm/lib/Analysis/BlockFrequencyInfo.cpp
@@ -181,12 +181,36 @@
PAC.preservedSet<CFGAnalyses>());
}
+BlockFrequencyInfo::BFICallbackVH::BFICallbackVH(
+ std::shared_ptr<BlockFrequencyInfoImpl<BasicBlock>> BFI,
+ const BasicBlock *BB)
+ : CallbackVH(const_cast<BasicBlock *>(BB)), BFI(BFI), BB(BB) {}
+
+void BlockFrequencyInfo::BFICallbackVH::deleted() {
+ if (auto BFII = BFI.lock())
+ BFII->removeBlock(BB);
+ CallbackVH::deleted();
+}
+
+void BlockFrequencyInfo::BFICallbackVH::allUsesReplacedWith(Value *V) {
+ if (auto BFII = BFI.lock())
+ BFII->removeBlock(BB);
+ CallbackVH::allUsesReplacedWith(V);
+}
+
void BlockFrequencyInfo::calculate(const Function &F,
const BranchProbabilityInfo &BPI,
const LoopInfo &LI) {
if (!BFI)
BFI.reset(new ImplType);
BFI->calculate(F, BPI, LI);
+
+ BlockWatchers.clear();
+ auto RegisteredBlocks = BFI->getRegisteredBlocks();
+ BlockWatchers.reserve(RegisteredBlocks.size());
+ for (const auto *BB : RegisteredBlocks)
+ BlockWatchers.emplace_back(BFICallbackVH(BFI, BB));
+
if (ViewBlockFreqPropagationDAG != GVDT_None &&
(ViewBlockFreqFuncName.empty() ||
F.getName().equals(ViewBlockFreqFuncName))) {
Index: llvm/include/llvm/Transforms/Scalar/LoopPassManager.h
===================================================================
--- llvm/include/llvm/Transforms/Scalar/LoopPassManager.h
+++ llvm/include/llvm/Transforms/Scalar/LoopPassManager.h
@@ -41,6 +41,7 @@
#include "llvm/ADT/STLExtras.h"
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/Analysis/BasicAliasAnalysis.h"
+#include "llvm/Analysis/BlockFrequencyInfo.h"
#include "llvm/Analysis/GlobalsModRef.h"
#include "llvm/Analysis/LoopAnalysisManager.h"
#include "llvm/Analysis/LoopInfo.h"
@@ -233,9 +234,11 @@
: public PassInfoMixin<FunctionToLoopPassAdaptor<LoopPassT>> {
public:
explicit FunctionToLoopPassAdaptor(LoopPassT Pass, bool UseMemorySSA = false,
+ bool UseBlockFrequencyInfo = false,
bool DebugLogging = false)
: Pass(std::move(Pass)), LoopCanonicalizationFPM(DebugLogging),
- UseMemorySSA(UseMemorySSA) {
+ UseMemorySSA(UseMemorySSA),
+ UseBlockFrequencyInfo(UseBlockFrequencyInfo) {
LoopCanonicalizationFPM.addPass(LoopSimplifyPass());
LoopCanonicalizationFPM.addPass(LCSSAPass());
}
@@ -267,6 +270,9 @@
MemorySSA *MSSA = UseMemorySSA
? (&AM.getResult<MemorySSAAnalysis>(F).getMSSA())
: nullptr;
+ BlockFrequencyInfo *BFI = UseBlockFrequencyInfo
+ ? (&AM.getResult<BlockFrequencyAnalysis>(F))
+ : nullptr;
LoopStandardAnalysisResults LAR = {AM.getResult<AAManager>(F),
AM.getResult<AssumptionAnalysis>(F),
AM.getResult<DominatorTreeAnalysis>(F),
@@ -274,6 +280,7 @@
AM.getResult<ScalarEvolutionAnalysis>(F),
AM.getResult<TargetLibraryAnalysis>(F),
AM.getResult<TargetIRAnalysis>(F),
+ BFI,
MSSA};
// Setup the loop analysis manager from its proxy. It is important that
@@ -370,6 +377,8 @@
PA.preserve<DominatorTreeAnalysis>();
PA.preserve<LoopAnalysis>();
PA.preserve<ScalarEvolutionAnalysis>();
+ if (UseBlockFrequencyInfo)
+ PA.preserve<BlockFrequencyAnalysis>();
if (UseMemorySSA)
PA.preserve<MemorySSAAnalysis>();
// FIXME: What we really want to do here is preserve an AA category, but
@@ -389,6 +398,7 @@
FunctionPassManager LoopCanonicalizationFPM;
bool UseMemorySSA = false;
+ bool UseBlockFrequencyInfo = false;
};
/// A function to deduce a loop pass type and wrap it in the templated
@@ -396,9 +406,10 @@
template <typename LoopPassT>
FunctionToLoopPassAdaptor<LoopPassT>
createFunctionToLoopPassAdaptor(LoopPassT Pass, bool UseMemorySSA = false,
+ bool UseBlockFrequencyInfo = false,
bool DebugLogging = false) {
- return FunctionToLoopPassAdaptor<LoopPassT>(std::move(Pass), UseMemorySSA,
- DebugLogging);
+ return FunctionToLoopPassAdaptor<LoopPassT>(
+ std::move(Pass), UseMemorySSA, UseBlockFrequencyInfo, DebugLogging);
}
/// Pass for printing a loop's contents as textual IR.
Index: llvm/include/llvm/Analysis/LoopAnalysisManager.h
===================================================================
--- llvm/include/llvm/Analysis/LoopAnalysisManager.h
+++ llvm/include/llvm/Analysis/LoopAnalysisManager.h
@@ -57,6 +57,7 @@
ScalarEvolution &SE;
TargetLibraryInfo &TLI;
TargetTransformInfo &TTI;
+ BlockFrequencyInfo *BFI;
MemorySSA *MSSA;
};
Index: llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h
===================================================================
--- llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h
+++ llvm/include/llvm/Analysis/BlockFrequencyInfoImpl.h
@@ -847,10 +847,13 @@
/// (Running this until fixed point would "solve" the geometric
/// series by simulation.)
template <class BT> class BlockFrequencyInfoImpl : BlockFrequencyInfoImplBase {
+public:
+ using BlockT = typename bfi_detail::TypeMap<BT>::BlockT;
+
+private:
// This is part of a workaround for a GCC 4.7 crash on lambdas.
friend struct bfi_detail::BlockEdgesAdder<BT>;
- using BlockT = typename bfi_detail::TypeMap<BT>::BlockT;
using BlockKeyT = typename bfi_detail::TypeMap<BT>::BlockKeyT;
using FunctionT = typename bfi_detail::TypeMap<BT>::FunctionT;
using BranchProbabilityInfoT =
@@ -1015,6 +1018,13 @@
const BranchProbabilityInfoT &getBPI() const { return *BPI; }
+ /// Removes \p BB from internal datastructures.
+ /// \return false if the block did not exist.
+ bool removeBlock(const BlockT *BB);
+
+ /// \return all blocks known to this datastructure.
+ DenseSet<const BlockT *> getRegisteredBlocks() const;
+
/// Print the frequencies for the current function.
///
/// Prints the frequencies for the blocks in the current function.
@@ -1121,6 +1131,20 @@
}
}
+template <class BT>
+bool BlockFrequencyInfoImpl<BT>::removeBlock(const BlockT *BB) {
+ return Nodes.erase(BB);
+}
+
+template <class BT>
+DenseSet<const typename BlockFrequencyInfoImpl<BT>::BlockT *>
+BlockFrequencyInfoImpl<BT>::getRegisteredBlocks() const {
+ DenseSet<const BlockT *> Ret;
+ for (auto &BN : Nodes)
+ Ret.insert(BN.first);
+ return Ret;
+}
+
template <class BT> void BlockFrequencyInfoImpl<BT>::initializeRPOT() {
const BlockT *Entry = &F->front();
RPOT.reserve(F->size());
Index: llvm/include/llvm/Analysis/BlockFrequencyInfo.h
===================================================================
--- llvm/include/llvm/Analysis/BlockFrequencyInfo.h
+++ llvm/include/llvm/Analysis/BlockFrequencyInfo.h
@@ -15,10 +15,12 @@
#include "llvm/ADT/Optional.h"
#include "llvm/IR/PassManager.h"
+#include "llvm/IR/ValueHandle.h"
#include "llvm/Pass.h"
#include "llvm/Support/BlockFrequency.h"
#include <cstdint>
#include <memory>
+#include <vector>
namespace llvm {
@@ -37,7 +39,20 @@
class BlockFrequencyInfo {
using ImplType = BlockFrequencyInfoImpl<BasicBlock>;
- std::unique_ptr<ImplType> BFI;
+ class BFICallbackVH final : public CallbackVH {
+ std::weak_ptr<ImplType> BFI;
+ const BasicBlock *BB;
+
+ public:
+ BFICallbackVH(std::shared_ptr<ImplType> BFI, const BasicBlock *BB);
+ virtual ~BFICallbackVH() = default;
+
+ virtual void deleted() override;
+ virtual void allUsesReplacedWith(Value *V) override;
+ };
+ std::vector<BFICallbackVH> BlockWatchers;
+
+ std::shared_ptr<ImplType> BFI;
public:
BlockFrequencyInfo();
Index: clang/test/CodeGen/thinlto-distributed-newpm.ll
===================================================================
--- clang/test/CodeGen/thinlto-distributed-newpm.ll
+++ clang/test/CodeGen/thinlto-distributed-newpm.ll
@@ -112,6 +112,9 @@
; CHECK-O: Running analysis: LoopAnalysis on main
; CHECK-O: Running pass: LCSSAPass on main
; CHECK-O: Finished {{.*}}Function pass manager run.
+; CHECK-O: Running analysis: BlockFrequencyAnalysis on main
+; CHECK-O: Running analysis: BranchProbabilityAnalysis on main
+; CHECK-O: Running analysis: PostDominatorTreeAnalysis on main
; CHECK-O: Running pass: SimplifyCFGPass on main
; CHECK-O: Running pass: InstCombinePass on main
; CHECK-O: Starting {{.*}}Function pass manager run.
@@ -136,7 +139,6 @@
; CHECK-O: Running pass: LCSSAPass on main
; CHECK-O: Finished {{.*}}Function pass manager run.
; CHECK-O: Running pass: ADCEPass on main
-; CHECK-O: Running analysis: PostDominatorTreeAnalysis on main
; CHECK-O: Running pass: SimplifyCFGPass on main
; CHECK-O: Running pass: InstCombinePass on main
; CHECK-O: Finished {{.*}}Function pass manager run.
@@ -146,10 +148,10 @@
; CHECK-O: Invalidating analysis: AAManager on main
; CHECK-O: Invalidating analysis: MemorySSAAnalysis on main
; CHECK-O: Invalidating analysis: LoopAnalysis on main
+; CHECK-O: Invalidating analysis: PostDominatorTreeAnalysis on main
; CHECK-O: Invalidating analysis: PhiValuesAnalysis on main
; CHECK-O: Invalidating analysis: MemoryDependenceAnalysis on main
; CHECK-O: Invalidating analysis: DemandedBitsAnalysis on main
-; CHECK-O: Invalidating analysis: PostDominatorTreeAnalysis on main
; CHECK-O: Invalidating analysis: CallGraphAnalysis
; CHECK-O: Finished {{.*}}Module pass manager run.
; CHECK-O: Starting {{.*}}Module pass manager run.
_______________________________________________
cfe-commits mailing list
[email protected]
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
