This revision was automatically updated to reflect the committed changes.
Closed by commit rG3733463dbb58: [IR][PGO] Add hot func attribute and use
hot/cold attribute in func section (authored by xur).
Herald added a project: clang.
Herald added a subscriber: cfe-commits.
Changed prior to commit:
https://reviews.llvm.org/D92493?vs=312273&id=312664#toc
Repository:
rG LLVM Github Monorepo
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D92493/new/
https://reviews.llvm.org/D92493
Files:
clang/lib/CodeGen/CGCall.cpp
clang/lib/CodeGen/CodeGenModule.cpp
clang/test/CodeGen/attributes.c
llvm/docs/LangRef.rst
llvm/include/llvm/Bitcode/LLVMBitCodes.h
llvm/include/llvm/IR/Attributes.td
llvm/lib/AsmParser/LLParser.cpp
llvm/lib/Bitcode/Reader/BitcodeReader.cpp
llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
llvm/lib/CodeGen/CodeGenPrepare.cpp
llvm/lib/IR/Attributes.cpp
llvm/lib/IR/Verifier.cpp
llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
llvm/test/Bitcode/attributes.ll
llvm/test/CodeGen/X86/hot-unlikely-section-prefix.ll
llvm/test/MC/AsmParser/function_hot_attr.ll
Index: llvm/test/MC/AsmParser/function_hot_attr.ll
===================================================================
--- /dev/null
+++ llvm/test/MC/AsmParser/function_hot_attr.ll
@@ -0,0 +1,13 @@
+; Test hot function attribute
+; RUN: llc < %s | FileCheck %s
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: hot noinline norecurse nounwind readnone uwtable
+define dso_local i32 @hot4() #0 {
+entry:
+ ret i32 1
+}
+; CHECK: .section .text.hot.,"ax",@progbits
+; CHECK: .globl hot4
+
+attributes #0 = { hot noinline norecurse nounwind readnone uwtable }
Index: llvm/test/CodeGen/X86/hot-unlikely-section-prefix.ll
===================================================================
--- /dev/null
+++ llvm/test/CodeGen/X86/hot-unlikely-section-prefix.ll
@@ -0,0 +1,101 @@
+; Test hot or unlikely section postfix based on profile and user annotation.
+; RUN: llc < %s | FileCheck %s
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: inlinehint norecurse nounwind readnone uwtable
+define dso_local i32 @hot1() #0 !prof !31 {
+entry:
+ ret i32 1
+}
+; CHECK: .section .text.hot.,"ax",@progbits
+; CHECK: .globl hot1
+
+; Function Attrs: cold norecurse nounwind readnone uwtable
+define dso_local i32 @cold1() #1 !prof !32 {
+entry:
+ ret i32 1
+}
+; CHECK: .section .text.unlikely.,"ax",@progbits
+; CHECK: .globl cold1
+
+; Function Attrs: cold inlinehint noinline norecurse nounwind optsize readnone uwtable
+define dso_local i32 @hot2() #2 !prof !31 {
+entry:
+ ret i32 1
+}
+; CHECK: .section .text.hot.,"ax",@progbits
+; CHECK: .globl hot2
+
+define dso_local i32 @normal() {
+entry:
+ ret i32 1
+}
+; CHECK: text
+; CHECK: .globl normal
+
+; Function Attrs: hot noinline norecurse nounwind readnone uwtable
+define dso_local i32 @hot3() #3 !prof !32 {
+entry:
+ ret i32 1
+}
+; CHECK: .section .text.hot.,"ax",@progbits
+; CHECK: .globl hot3
+
+; Function Attrs: cold noinline norecurse nounwind optsize readnone uwtable
+define dso_local i32 @cold2() #4 {
+entry:
+ ret i32 1
+}
+; CHECK: .section .text.unlikely.,"ax",@progbits
+; CHECK: .globl cold2
+
+; Function Attrs: hot noinline norecurse nounwind readnone uwtable
+define dso_local i32 @hot4() #3 {
+entry:
+ ret i32 1
+}
+; CHECK: .section .text.hot.,"ax",@progbits
+; CHECK: .globl hot4
+
+attributes #0 = { inlinehint norecurse nounwind readnone uwtable }
+attributes #1 = { cold norecurse nounwind readnone uwtable }
+attributes #2 = { cold inlinehint noinline norecurse nounwind optsize readnone uwtable }
+attributes #3 = { hot noinline norecurse nounwind readnone uwtable }
+attributes #4 = { cold noinline norecurse nounwind optsize readnone uwtable }
+
+!llvm.module.flags = !{!0, !1}
+!llvm.ident = !{!30}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 1, !"ProfileSummary", !2}
+!2 = !{!3, !4, !5, !6, !7, !8, !9, !10, !11, !12}
+!3 = !{!"ProfileFormat", !"InstrProf"}
+!4 = !{!"TotalCount", i64 402020}
+!5 = !{!"MaxCount", i64 200000}
+!6 = !{!"MaxInternalCount", i64 2000}
+!7 = !{!"MaxFunctionCount", i64 200000}
+!8 = !{!"NumCounts", i64 7}
+!9 = !{!"NumFunctions", i64 5}
+!10 = !{!"IsPartialProfile", i64 0}
+!11 = !{!"PartialProfileRatio", double 0.000000e+00}
+!12 = !{!"DetailedSummary", !13}
+!13 = !{!14, !15, !16, !17, !18, !19, !20, !21, !22, !23, !24, !25, !26, !27, !28, !29}
+!14 = !{i32 10000, i64 200000, i32 1}
+!15 = !{i32 100000, i64 200000, i32 1}
+!16 = !{i32 200000, i64 200000, i32 1}
+!17 = !{i32 300000, i64 200000, i32 1}
+!18 = !{i32 400000, i64 200000, i32 1}
+!19 = !{i32 500000, i64 100000, i32 3}
+!20 = !{i32 600000, i64 100000, i32 3}
+!21 = !{i32 700000, i64 100000, i32 3}
+!22 = !{i32 800000, i64 100000, i32 3}
+!23 = !{i32 900000, i64 100000, i32 3}
+!24 = !{i32 950000, i64 100000, i32 3}
+!25 = !{i32 990000, i64 100000, i32 3}
+!26 = !{i32 999000, i64 2000, i32 4}
+!27 = !{i32 999900, i64 2000, i32 4}
+!28 = !{i32 999990, i64 10, i32 6}
+!29 = !{i32 999999, i64 10, i32 6}
+!30 = !{!"clang version 12.0.0 (https://github.com/llvm/llvm-project.git 53c5fdd59a5cf7fbb4dcb7a7e84c9c4a40d32a84)"}
+!31 = !{!"function_entry_count", i64 100000}
+!32 = !{!"function_entry_count", i64 10}
Index: llvm/test/Bitcode/attributes.ll
===================================================================
--- llvm/test/Bitcode/attributes.ll
+++ llvm/test/Bitcode/attributes.ll
@@ -410,6 +410,18 @@
ret void
}
+; CHECK: define void @f70() #43
+define void @f70() cold
+{
+ ret void
+}
+
+; CHECK: define void @f71() #44
+define void @f71() hot
+{
+ ret void
+}
+
; CHECK: attributes #0 = { noreturn }
; CHECK: attributes #1 = { nounwind }
; CHECK: attributes #2 = { readnone }
@@ -453,4 +465,6 @@
; CHECK: attributes #40 = { null_pointer_is_valid }
; CHECK: attributes #41 = { mustprogress }
; CHECK: attributes #42 = { nocallback }
+; CHECK: attributes #43 = { cold }
+; CHECK: attributes #44 = { hot }
; CHECK: attributes #[[NOBUILTIN]] = { nobuiltin }
Index: llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
===================================================================
--- llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -1921,6 +1921,17 @@
<< "\n");
}
for (auto &F : ColdFunctions) {
+ // Only set when there is no Attribute::Hot set by the user. For Hot
+ // attribute, user's annotation has the precedence over the profile.
+ if (F->hasFnAttribute(Attribute::Hot)) {
+ auto &Ctx = M.getContext();
+ std::string Msg = std::string("Function ") + F->getName().str() +
+ std::string(" is annotated as a hot function but"
+ " the profile is cold");
+ Ctx.diagnose(
+ DiagnosticInfoPGOProfile(M.getName().data(), Msg, DS_Warning));
+ continue;
+ }
F->addFnAttr(Attribute::Cold);
LLVM_DEBUG(dbgs() << "Set cold attribute to function: " << F->getName()
<< "\n");
Index: llvm/lib/IR/Verifier.cpp
===================================================================
--- llvm/lib/IR/Verifier.cpp
+++ llvm/lib/IR/Verifier.cpp
@@ -1626,6 +1626,7 @@
case Attribute::Builtin:
case Attribute::NoBuiltin:
case Attribute::Cold:
+ case Attribute::Hot:
case Attribute::OptForFuzzing:
case Attribute::OptimizeNone:
case Attribute::JumpTable:
Index: llvm/lib/IR/Attributes.cpp
===================================================================
--- llvm/lib/IR/Attributes.cpp
+++ llvm/lib/IR/Attributes.cpp
@@ -449,6 +449,8 @@
return "zeroext";
if (hasAttribute(Attribute::Cold))
return "cold";
+ if (hasAttribute(Attribute::Hot))
+ return "hot";
if (hasAttribute(Attribute::ImmArg))
return "immarg";
if (hasAttribute(Attribute::NoUndef))
Index: llvm/lib/CodeGen/CodeGenPrepare.cpp
===================================================================
--- llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -472,9 +472,17 @@
PSI = &getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI();
OptSize = F.hasOptSize();
if (ProfileGuidedSectionPrefix) {
- if (PSI->isFunctionHotInCallGraph(&F, *BFI))
+ // The hot attribute overwrites profile count based hotness while profile
+ // counts based hotness overwrite the cold attribute.
+ // This is a conservative behabvior.
+ if (F.hasFnAttribute(Attribute::Hot) ||
+ PSI->isFunctionHotInCallGraph(&F, *BFI))
F.setSectionPrefix("hot");
- else if (PSI->isFunctionColdInCallGraph(&F, *BFI))
+ // If PSI shows this function is not hot, we will placed the function
+ // into unlikely section if (1) PSI shows this is a cold function, or
+ // (2) the function has a attribute of cold.
+ else if (PSI->isFunctionColdInCallGraph(&F, *BFI) ||
+ F.hasFnAttribute(Attribute::Cold))
F.setSectionPrefix("unlikely");
else if (ProfileUnknownInSpecialSection && PSI->hasPartialSampleProfile() &&
PSI->isFunctionHotnessUnknown(F))
Index: llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
===================================================================
--- llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -626,6 +626,8 @@
return bitc::ATTR_KIND_IN_ALLOCA;
case Attribute::Cold:
return bitc::ATTR_KIND_COLD;
+ case Attribute::Hot:
+ return bitc::ATTR_KIND_HOT;
case Attribute::InaccessibleMemOnly:
return bitc::ATTR_KIND_INACCESSIBLEMEM_ONLY;
case Attribute::InaccessibleMemOrArgMemOnly:
Index: llvm/lib/Bitcode/Reader/BitcodeReader.cpp
===================================================================
--- llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -1539,6 +1539,8 @@
return Attribute::ByRef;
case bitc::ATTR_KIND_MUSTPROGRESS:
return Attribute::MustProgress;
+ case bitc::ATTR_KIND_HOT:
+ return Attribute::Hot;
}
}
Index: llvm/lib/AsmParser/LLParser.cpp
===================================================================
--- llvm/lib/AsmParser/LLParser.cpp
+++ llvm/lib/AsmParser/LLParser.cpp
@@ -1340,6 +1340,7 @@
case lltok::kw_argmemonly: B.addAttribute(Attribute::ArgMemOnly); break;
case lltok::kw_builtin: B.addAttribute(Attribute::Builtin); break;
case lltok::kw_cold: B.addAttribute(Attribute::Cold); break;
+ case lltok::kw_hot: B.addAttribute(Attribute::Hot); break;
case lltok::kw_convergent: B.addAttribute(Attribute::Convergent); break;
case lltok::kw_inaccessiblememonly:
B.addAttribute(Attribute::InaccessibleMemOnly); break;
Index: llvm/include/llvm/IR/Attributes.td
===================================================================
--- llvm/include/llvm/IR/Attributes.td
+++ llvm/include/llvm/IR/Attributes.td
@@ -63,6 +63,9 @@
/// Can only be moved to control-equivalent blocks.
def Convergent : EnumAttr<"convergent">;
+/// Marks function as being in a hot path and frequently called.
+def Hot: EnumAttr<"hot">;
+
/// Pointer is known to be dereferenceable.
def Dereferenceable : IntAttr<"dereferenceable">;
Index: llvm/include/llvm/Bitcode/LLVMBitCodes.h
===================================================================
--- llvm/include/llvm/Bitcode/LLVMBitCodes.h
+++ llvm/include/llvm/Bitcode/LLVMBitCodes.h
@@ -654,6 +654,7 @@
ATTR_KIND_BYREF = 69,
ATTR_KIND_MUSTPROGRESS = 70,
ATTR_KIND_NO_CALLBACK = 71,
+ ATTR_KIND_HOT = 72,
};
enum ComdatSelectionKindCodes {
Index: llvm/docs/LangRef.rst
===================================================================
--- llvm/docs/LangRef.rst
+++ llvm/docs/LangRef.rst
@@ -1496,6 +1496,15 @@
can prove that the function does not execute any convergent operations.
Similarly, the optimizer may remove ``convergent`` on calls/invokes when it
can prove that the call/invoke cannot call a convergent function.
+``hot``
+ This attribute indicates that this function is a hot spot of the program
+ execution. The function will be optimized more aggressively and will be
+ placed into special subsection of the text section to improving locality.
+
+ When profile feedback is enabled, this attribute has the precedence over
+ the profile information. By marking a function ``hot``, users can work
+ around the cases where the training input does not have good coverage
+ on all the hot functions.
``inaccessiblememonly``
This attribute indicates that the function may only access memory that
is not accessible by the module being compiled. This is a weaker form
Index: clang/test/CodeGen/attributes.c
===================================================================
--- clang/test/CodeGen/attributes.c
+++ clang/test/CodeGen/attributes.c
@@ -63,6 +63,13 @@
// CHECK: call void @t71() [[COLDSITE:#[0-9]+]]
// CHECK: declare void @t71() [[COLDDECL:#[0-9]+]]
+// CHECK: define void @t82() [[HOTDEF:#[0-9]+]] {
+void t81(void) __attribute__((hot));
+void t82() __attribute__((hot));
+void t82() { t81(); }
+// CHECK: call void @t81() [[HOTSITE:#[0-9]+]]
+// CHECK: declare void @t81() [[HOTDECL:#[0-9]+]]
+
// CHECK: define void @t10() [[NUW]] section "xSECT" {
void t10(void) __attribute__((section("xSECT")));
void t10(void) {}
@@ -72,6 +79,9 @@
// CHECK: define i32 @t19() [[NUW]] {
extern int t19(void) __attribute__((weak_import));
int t19(void) {
+// RUN: %clang_cc1 -emit-llvm -fcf-protection=branch -triple i386-linux-gnu -o %t %s
+// RUN: %clang_cc1 -emit-llvm -fcf-protection=branch -triple i386-linux-gnu -o %t %s
+// RUN: %clang_cc1 -emit-llvm -fcf-protection=branch -triple i386-linux-gnu -o %t %s
return 10;
}
@@ -111,6 +121,9 @@
// CHECK: attributes [[NR]] = { noinline noreturn nounwind{{.*}} }
// CHECK: attributes [[COLDDEF]] = { cold {{.*}}}
// CHECK: attributes [[COLDDECL]] = { cold {{.*}}}
+// CHECK: attributes [[HOTDEF]] = { hot {{.*}}}
+// CHECK: attributes [[HOTDECL]] = { hot {{.*}}}
// CHECK: attributes [[NOCF_CHECK_FUNC]] = { nocf_check {{.*}}}
// CHECK: attributes [[COLDSITE]] = { cold {{.*}}}
+// CHECK: attributes [[HOTSITE]] = { hot {{.*}}}
// CHECK: attributes [[NOCF_CHECK_CALL]] = { nocf_check }
Index: clang/lib/CodeGen/CodeGenModule.cpp
===================================================================
--- clang/lib/CodeGen/CodeGenModule.cpp
+++ clang/lib/CodeGen/CodeGenModule.cpp
@@ -1744,7 +1744,8 @@
B.addAttribute(llvm::Attribute::OptimizeForSize);
B.addAttribute(llvm::Attribute::Cold);
}
-
+ if (D->hasAttr<HotAttr>())
+ B.addAttribute(llvm::Attribute::Hot);
if (D->hasAttr<MinSizeAttr>())
B.addAttribute(llvm::Attribute::MinSize);
}
Index: clang/lib/CodeGen/CGCall.cpp
===================================================================
--- clang/lib/CodeGen/CGCall.cpp
+++ clang/lib/CodeGen/CGCall.cpp
@@ -1944,6 +1944,8 @@
FuncAttrs.addAttribute(llvm::Attribute::NoReturn);
if (TargetDecl->hasAttr<ColdAttr>())
FuncAttrs.addAttribute(llvm::Attribute::Cold);
+ if (TargetDecl->hasAttr<HotAttr>())
+ FuncAttrs.addAttribute(llvm::Attribute::Hot);
if (TargetDecl->hasAttr<NoDuplicateAttr>())
FuncAttrs.addAttribute(llvm::Attribute::NoDuplicate);
if (TargetDecl->hasAttr<ConvergentAttr>())
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
