[llvm-branch-commits] [clang] 54e03d0 - [PGO] Verify BFI counts after loading profile data
Author: Rong Xu
Date: 2020-12-14T15:56:10-08:00
New Revision: 54e03d03a7a4d47f09d40bcbcfe484066a52a077
URL:
https://github.com/llvm/llvm-project/commit/54e03d03a7a4d47f09d40bcbcfe484066a52a077
DIFF:
https://github.com/llvm/llvm-project/commit/54e03d03a7a4d47f09d40bcbcfe484066a52a077.diff
LOG: [PGO] Verify BFI counts after loading profile data
This patch adds the functionality to compare BFI counts with real
profile
counts right after reading the profile. It will print remarks under
-Rpass-analysis=pgo, or the internal option -pass-remarks-analysis=pgo.
Differential Revision: https://reviews.llvm.org/D91813
Added:
llvm/test/Transforms/PGOProfile/Inputs/bfi_verification.proftext
llvm/test/Transforms/PGOProfile/bfi_verification.ll
Modified:
clang/lib/CodeGen/CGCall.cpp
clang/lib/CodeGen/CodeGenModule.cpp
llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
Removed:
diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index 73194be922dd..ced287643c28 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -1943,6 +1943,11 @@ void CodeGenModule::ConstructAttributeList(
FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
if (TargetDecl->hasAttr())
FuncAttrs.addAttribute(llvm::Attribute::Cold);
+if (TargetDecl->hasAttr()) {
+ // xur
+ fprintf(stderr, "hihi 2\n");
+ // FuncAttrs.addAttribute(llvm::Attribute::Hot);
+}
if (TargetDecl->hasAttr())
FuncAttrs.addAttribute(llvm::Attribute::NoDuplicate);
if (TargetDecl->hasAttr())
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp
b/clang/lib/CodeGen/CodeGenModule.cpp
index 0bb9c91f2434..dec0cba84343 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -1744,6 +1744,13 @@ void
CodeGenModule::SetLLVMFunctionAttributesForDefinition(const Decl *D,
B.addAttribute(llvm::Attribute::OptimizeForSize);
B.addAttribute(llvm::Attribute::Cold);
}
+if (D->hasAttr()) {
+ if (!ShouldAddOptNone)
+B.addAttribute(llvm::Attribute::OptimizeForSize);
+ // xur
+ // B.addAttribute(llvm::Attribute::Hot);
+ fprintf(stderr, "hihi 1\n");
+}
if (D->hasAttr())
B.addAttribute(llvm::Attribute::MinSize);
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index 0228c8a8ef14..eba8d9e9c3c3 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -252,6 +252,30 @@ static cl::opt PGOInstrumentEntry(
"pgo-instrument-entry", cl::init(false), cl::Hidden,
cl::desc("Force to instrument function entry basicblock."));
+static cl::opt PGOVerifyHotBFI(
+"pgo-verify-hot-bfi", cl::init(false), cl::Hidden,
+cl::desc("Print out the non-match BFI count if a hot raw profile count "
+ "becomes non-hot, or a cold raw profile count becomes hot. "
+ "The print is enabled under -Rpass-analysis=pgo, or "
+ "internal option -pass-remakrs-analysis=pgo."));
+
+static cl::opt PGOVerifyBFI(
+"pgo-verify-bfi", cl::init(false), cl::Hidden,
+cl::desc("Print out mismatched BFI counts after setting profile metadata "
+ "The print is enabled under -Rpass-analysis=pgo, or "
+ "internal option -pass-remakrs-analysis=pgo."));
+
+static cl::opt PGOVerifyBFIRatio(
+"pgo-verify-bfi-ratio", cl::init(5), cl::Hidden,
+cl::desc("Set the threshold for pgo-verify-big -- only print out "
+ "mismatched BFI if the
diff erence percentage is greater than "
+ "this value (in percentage)."));
+
+static cl::opt PGOVerifyBFICutoff(
+"pgo-verify-bfi-cutoff", cl::init(1), cl::Hidden,
+cl::desc("Set the threshold for pgo-verify-bfi -- skip the counts whose "
+ "profile count value is below."));
+
// Command line option to turn on CFG dot dump after profile annotation.
// Defined in Analysis/BlockFrequencyInfo.cpp: -pgo-view-counts
extern cl::opt PGOViewCounts;
@@ -1616,6 +1640,82 @@ PreservedAnalyses PGOInstrumentationGen::run(Module &M,
return PreservedAnalyses::none();
}
+// Compare the profile count values with BFI count values, and print out
+// the non-matching ones.
+static void verifyFuncBFI(PGOUseFunc &Func, LoopInfo &LI,
+ BranchProbabilityInfo &NBPI,
+ uint64_t HotCountThreshold,
+ uint64_t ColdCountThreshold) {
+ Function &F = Func.getFunc();
+ BlockFrequencyInfo NBFI(F, NBPI, LI);
+ // bool PrintFunc = false;
+ bool HotBBOnly = PGOVerifyHotBFI;
+ std::string Msg;
+ OptimizationRemarkEmitter ORE(&F);
+
+ unsigned BBNum = 0, BBMisMatchNum = 0, NonZeroBBNum = 0;
+ for (auto &BBI : F) {
+uint64_t CountValue =
[llvm-branch-commits] [clang] c36f31c - [PGO] remove unintentional code in early commit
Author: Rong Xu
Date: 2020-12-14T18:41:49-08:00
New Revision: c36f31c4db065e987160a776749f5da81efc24c5
URL:
https://github.com/llvm/llvm-project/commit/c36f31c4db065e987160a776749f5da81efc24c5
DIFF:
https://github.com/llvm/llvm-project/commit/c36f31c4db065e987160a776749f5da81efc24c5.diff
LOG: [PGO] remove unintentional code in early commit
Remove unintentional code in
commit 54e03d [PGO] Verify BFI counts after loading profile data.
Added:
Modified:
clang/lib/CodeGen/CGCall.cpp
clang/lib/CodeGen/CodeGenModule.cpp
Removed:
diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index ced287643c28..73194be922dd 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -1943,11 +1943,6 @@ void CodeGenModule::ConstructAttributeList(
FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
if (TargetDecl->hasAttr())
FuncAttrs.addAttribute(llvm::Attribute::Cold);
-if (TargetDecl->hasAttr()) {
- // xur
- fprintf(stderr, "hihi 2\n");
- // FuncAttrs.addAttribute(llvm::Attribute::Hot);
-}
if (TargetDecl->hasAttr())
FuncAttrs.addAttribute(llvm::Attribute::NoDuplicate);
if (TargetDecl->hasAttr())
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp
b/clang/lib/CodeGen/CodeGenModule.cpp
index dec0cba84343..0bb9c91f2434 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -1744,13 +1744,6 @@ void
CodeGenModule::SetLLVMFunctionAttributesForDefinition(const Decl *D,
B.addAttribute(llvm::Attribute::OptimizeForSize);
B.addAttribute(llvm::Attribute::Cold);
}
-if (D->hasAttr()) {
- if (!ShouldAddOptNone)
-B.addAttribute(llvm::Attribute::OptimizeForSize);
- // xur
- // B.addAttribute(llvm::Attribute::Hot);
- fprintf(stderr, "hihi 1\n");
-}
if (D->hasAttr())
B.addAttribute(llvm::Attribute::MinSize);
___
llvm-branch-commits mailing list
llvm-branch-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
[llvm-branch-commits] [llvm] 0abd744 - [PGO] Use the sum of profile counts to fix the function entry count
Author: Rong Xu
Date: 2020-12-16T13:37:43-08:00
New Revision: 0abd744597ee502b6424e5a99fb940ca0f866fe9
URL:
https://github.com/llvm/llvm-project/commit/0abd744597ee502b6424e5a99fb940ca0f866fe9
DIFF:
https://github.com/llvm/llvm-project/commit/0abd744597ee502b6424e5a99fb940ca0f866fe9.diff
LOG: [PGO] Use the sum of profile counts to fix the function entry count
Raw profile count values for each BB are not kept after profile
annotation. We record function entry count and branch weights
and use them to compute the count when needed. This mechanism
works well in a perfect world, but often breaks in real programs,
because of number prevision, inconsistent profile, or bugs in
BFI). This patch uses sum of profile count values to fix
function entry count to make the BFI count close to real profile
counts.
Differential Revision: https://reviews.llvm.org/D61540
Added:
llvm/test/Transforms/PGOProfile/Inputs/fix_bfi.proftext
llvm/test/Transforms/PGOProfile/fix_bfi.ll
Modified:
llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
llvm/test/Transforms/PGOProfile/bfi_verification.ll
Removed:
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index eba8d9e9c3c3..8627e8239b2e 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -252,6 +252,10 @@ static cl::opt PGOInstrumentEntry(
"pgo-instrument-entry", cl::init(false), cl::Hidden,
cl::desc("Force to instrument function entry basicblock."));
+static cl::opt
+PGOFixEntryCount("pgo-fix-entry-count", cl::init(true), cl::Hidden,
+ cl::desc("Fix function entry count in profile use."));
+
static cl::opt PGOVerifyHotBFI(
"pgo-verify-hot-bfi", cl::init(false), cl::Hidden,
cl::desc("Print out the non-match BFI count if a hot raw profile count "
@@ -1640,6 +1644,53 @@ PreservedAnalyses PGOInstrumentationGen::run(Module &M,
return PreservedAnalyses::none();
}
+// Using the ratio b/w sums of profile count values and BFI count values to
+// adjust the func entry count.
+static void fixFuncEntryCount(PGOUseFunc &Func, LoopInfo &LI,
+ BranchProbabilityInfo &NBPI) {
+ Function &F = Func.getFunc();
+ BlockFrequencyInfo NBFI(F, NBPI, LI);
+#ifndef NDEBUG
+ auto BFIEntryCount = F.getEntryCount();
+ assert(BFIEntryCount.hasValue() && (BFIEntryCount.getCount() > 0) &&
+ "Invalid BFI Entrycount");
+#endif
+ auto SumCount = APFloat::getZero(APFloat::IEEEdouble());
+ auto SumBFICount = APFloat::getZero(APFloat::IEEEdouble());
+ for (auto &BBI : F) {
+uint64_t CountValue = 0;
+uint64_t BFICountValue = 0;
+if (!Func.findBBInfo(&BBI))
+ continue;
+auto BFICount = NBFI.getBlockProfileCount(&BBI);
+CountValue = Func.getBBInfo(&BBI).CountValue;
+BFICountValue = BFICount.getValue();
+SumCount.add(APFloat(CountValue * 1.0), APFloat::rmNearestTiesToEven);
+SumBFICount.add(APFloat(BFICountValue * 1.0),
APFloat::rmNearestTiesToEven);
+ }
+ if (SumCount.isZero())
+return;
+
+ assert(SumBFICount.compare(APFloat(0.0)) == APFloat::cmpGreaterThan &&
+ "Incorrect sum of BFI counts");
+ if (SumBFICount.compare(SumCount) == APFloat::cmpEqual)
+return;
+ double Scale = (SumCount / SumBFICount).convertToDouble();
+ if (Scale < 1.001 && Scale > 0.999)
+return;
+
+ uint64_t FuncEntryCount = Func.getBBInfo(&*F.begin()).CountValue;
+ uint64_t NewEntryCount = 0.5 + FuncEntryCount * Scale;
+ if (NewEntryCount == 0)
+NewEntryCount = 1;
+ if (NewEntryCount != FuncEntryCount) {
+F.setEntryCount(ProfileCount(NewEntryCount, Function::PCT_Real));
+LLVM_DEBUG(dbgs() << "FixFuncEntryCount: in " << F.getName()
+ << ", entry_count " << FuncEntryCount << " --> "
+ << NewEntryCount << "\n");
+ }
+}
+
// Compare the profile count values with BFI count values, and print out
// the non-matching ones.
static void verifyFuncBFI(PGOUseFunc &Func, LoopInfo &LI,
@@ -1842,10 +1893,15 @@ static bool annotateAllFunctions(
}
}
-// Verify BlockFrequency information.
-if (PGOVerifyBFI || PGOVerifyHotBFI) {
+if (PGOVerifyBFI || PGOVerifyHotBFI || PGOFixEntryCount) {
LoopInfo LI{DominatorTree(F)};
BranchProbabilityInfo NBPI(F, LI);
+
+ // Fix func entry count.
+ if (PGOFixEntryCount)
+fixFuncEntryCount(Func, LI, NBPI);
+
+ // Verify BlockFrequency information.
uint64_t HotCountThreshold = 0, ColdCountThreshold = 0;
if (PGOVerifyHotBFI) {
HotCountThreshold = PSI->getOrCompHotCountThreshold();
diff --git a/llvm/test/Transforms/PGOProfile/Inputs/fix_bfi.proftext
b/llvm/test/Transforms/PGOProfile/Inputs/fix_bfi.proftext
new file mode 100644
index .
[llvm-branch-commits] [llvm] 3733463 - [IR][PGO] Add hot func attribute and use hot/cold attribute in func section
Author: Rong Xu
Date: 2020-12-17T18:41:12-08:00
New Revision: 3733463dbb58a29892be3872bd32f93cb9af492c
URL:
https://github.com/llvm/llvm-project/commit/3733463dbb58a29892be3872bd32f93cb9af492c
DIFF:
https://github.com/llvm/llvm-project/commit/3733463dbb58a29892be3872bd32f93cb9af492c.diff
LOG: [IR][PGO] Add hot func attribute and use hot/cold attribute in func section
Clang FE currently has hot/cold function attribute. But we only have
cold function attribute in LLVM IR.
This patch adds support of hot function attribute to LLVM IR. This
attribute will be used in setting function section prefix/suffix.
Currently .hot and .unlikely suffix only are added in PGO (Sample PGO)
compilation (through isFunctionHotInCallGraph and
isFunctionColdInCallGraph).
This patch changes the behavior. The new behavior is:
(1) If the user annotates a function as hot or isFunctionHotInCallGraph
is true, this function will be marked as hot. Otherwise,
(2) If the user annotates a function as cold or
isFunctionColdInCallGraph is true, this function will be marked as
cold.
The changes are:
(1) user annotated function attribute will used in setting function
section prefix/suffix.
(2) hot attribute overwrites profile count based hotness.
(3) profile count based hotness overwrite user annotated cold attribute.
The intention for these changes is to provide the user a way to mark
certain function as hot in cases where training input is hard to cover
all the hot functions.
Differential Revision: https://reviews.llvm.org/D92493
Added:
llvm/test/CodeGen/X86/hot-unlikely-section-prefix.ll
llvm/test/MC/AsmParser/function_hot_attr.ll
Modified:
clang/lib/CodeGen/CGCall.cpp
clang/lib/CodeGen/CodeGenModule.cpp
clang/test/CodeGen/attributes.c
llvm/docs/LangRef.rst
llvm/include/llvm/Bitcode/LLVMBitCodes.h
llvm/include/llvm/IR/Attributes.td
llvm/lib/AsmParser/LLParser.cpp
llvm/lib/Bitcode/Reader/BitcodeReader.cpp
llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
llvm/lib/CodeGen/CodeGenPrepare.cpp
llvm/lib/IR/Attributes.cpp
llvm/lib/IR/Verifier.cpp
llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
llvm/test/Bitcode/attributes.ll
Removed:
diff --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index bfc7b8e74d8f..e28736bd3d2f 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -1944,6 +1944,8 @@ void CodeGenModule::ConstructAttributeList(
FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
if (TargetDecl->hasAttr())
FuncAttrs.addAttribute(llvm::Attribute::Cold);
+if (TargetDecl->hasAttr())
+ FuncAttrs.addAttribute(llvm::Attribute::Hot);
if (TargetDecl->hasAttr())
FuncAttrs.addAttribute(llvm::Attribute::NoDuplicate);
if (TargetDecl->hasAttr())
diff --git a/clang/lib/CodeGen/CodeGenModule.cpp
b/clang/lib/CodeGen/CodeGenModule.cpp
index 7dd343dbcc16..93916e85a461 100644
--- a/clang/lib/CodeGen/CodeGenModule.cpp
+++ b/clang/lib/CodeGen/CodeGenModule.cpp
@@ -1744,7 +1744,8 @@ void
CodeGenModule::SetLLVMFunctionAttributesForDefinition(const Decl *D,
B.addAttribute(llvm::Attribute::OptimizeForSize);
B.addAttribute(llvm::Attribute::Cold);
}
-
+if (D->hasAttr())
+ B.addAttribute(llvm::Attribute::Hot);
if (D->hasAttr())
B.addAttribute(llvm::Attribute::MinSize);
}
diff --git a/clang/test/CodeGen/attributes.c b/clang/test/CodeGen/attributes.c
index 0c1455d3c612..5ef176c138af 100644
--- a/clang/test/CodeGen/attributes.c
+++ b/clang/test/CodeGen/attributes.c
@@ -63,6 +63,13 @@ void t72() { t71(); }
// CHECK: call void @t71() [[COLDSITE:#[0-9]+]]
// CHECK: declare void @t71() [[COLDDECL:#[0-9]+]]
+// CHECK: define void @t82() [[HOTDEF:#[0-9]+]] {
+void t81(void) __attribute__((hot));
+void t82() __attribute__((hot));
+void t82() { t81(); }
+// CHECK: call void @t81() [[HOTSITE:#[0-9]+]]
+// CHECK: declare void @t81() [[HOTDECL:#[0-9]+]]
+
// CHECK: define void @t10() [[NUW]] section "xSECT" {
void t10(void) __attribute__((section("xSECT")));
void t10(void) {}
@@ -72,6 +79,9 @@ void __attribute__((section("xSECT"))) t11(void) {}
// CHECK: define i32 @t19() [[NUW]] {
extern int t19(void) __attribute__((weak_import));
int t19(void) {
+// RUN: %clang_cc1 -emit-llvm -fcf-protection=branch -triple i386-linux-gnu -o
%t %s
+// RUN: %clang_cc1 -emit-llvm -fcf-protection=branch -triple i386-linux-gnu -o
%t %s
+// RUN: %clang_cc1 -emit-llvm -fcf-protection=branch -triple i386-linux-gnu -o
%t %s
return 10;
}
@@ -111,6 +121,9 @@ void t24(f_t f1) {
// CHECK: attributes [[NR]] = { noinline noreturn nounwind{{.*}} }
// CHECK: attributes [[COLDDEF]] = { cold {{.*}}}
// CHECK: attributes [[COLDDECL]] = { cold {{.*}}}
+// CHECK: attributes [[HOTDEF]] = { hot {{.*}}}
+// CHECK: attributes [[HOTDECL]] = { hot {{.*}}}
// CHECK: attributes [[NOC
[llvm-branch-commits] [llvm] 31c0b87 - Fix clang-ppc64le-rhel buildbot build error
Author: Rong Xu Date: 2020-12-17T19:14:43-08:00 New Revision: 31c0b8700b4f531b2353fbb0e72d6934f65814f8 URL: https://github.com/llvm/llvm-project/commit/31c0b8700b4f531b2353fbb0e72d6934f65814f8 DIFF: https://github.com/llvm/llvm-project/commit/31c0b8700b4f531b2353fbb0e72d6934f65814f8.diff LOG: Fix clang-ppc64le-rhel buildbot build error ix buildbot build error due to commit 3733463d: [IR][PGO] Add hot func attribute and use hot/cold attribute in func section Added: Modified: llvm/lib/Transforms/Utils/CodeExtractor.cpp Removed: diff --git a/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/llvm/lib/Transforms/Utils/CodeExtractor.cpp index 91297b8dea19..e30812935426 100644 --- a/llvm/lib/Transforms/Utils/CodeExtractor.cpp +++ b/llvm/lib/Transforms/Utils/CodeExtractor.cpp @@ -943,6 +943,7 @@ Function *CodeExtractor::constructFunction(const ValueSet &inputs, // Those attributes should be safe to propagate to the extracted function. case Attribute::AlwaysInline: case Attribute::Cold: + case Attribute::Hot: case Attribute::NoRecurse: case Attribute::InlineHint: case Attribute::MinSize: ___ llvm-branch-commits mailing list llvm-branch-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-branch-commits
