LevitatingLion updated this revision to Diff 254217.
LevitatingLion added a comment.
Herald added a reviewer: sstefan1.
I rebased my changes onto 49d00824bbb
<https://reviews.llvm.org/rG49d00824bbbb8945b92c0f592c6951a881a6242f>, renamed
the attribute to 'alwaysinline_recursively', and added some more tests. The
testcase 'highLevelStructure.3.2.ll' does not fail anymore, all regression
tests pass.
Are there any more places where changes are required? I looked at the changes
when other attributes were introduced and grep'd for 'Attribute::AlwaysInline'
to find places which need handling of the new attribute.
CHANGES SINCE LAST ACTION
https://reviews.llvm.org/D70366/new/
https://reviews.llvm.org/D70366
Files:
clang/lib/CodeGen/CGCall.cpp
llvm/docs/BitCodeFormat.rst
llvm/docs/LangRef.rst
llvm/include/llvm/Bitcode/LLVMBitCodes.h
llvm/include/llvm/IR/Attributes.td
llvm/lib/Analysis/InlineCost.cpp
llvm/lib/AsmParser/LLLexer.cpp
llvm/lib/AsmParser/LLParser.cpp
llvm/lib/AsmParser/LLToken.h
llvm/lib/Bitcode/Reader/BitcodeReader.cpp
llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
llvm/lib/CodeGen/SafeStack.cpp
llvm/lib/IR/Attributes.cpp
llvm/lib/IR/Verifier.cpp
llvm/lib/Target/AMDGPU/AMDGPUInline.cpp
llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp
llvm/lib/Transforms/IPO/AlwaysInliner.cpp
llvm/lib/Transforms/IPO/Attributor.cpp
llvm/lib/Transforms/IPO/ForceFunctionAttrs.cpp
llvm/lib/Transforms/IPO/HotColdSplitting.cpp
llvm/lib/Transforms/IPO/Inliner.cpp
llvm/lib/Transforms/IPO/PartialInlining.cpp
llvm/lib/Transforms/IPO/SyntheticCountsPropagation.cpp
llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
llvm/lib/Transforms/Utils/CodeExtractor.cpp
llvm/test/Transforms/Inline/always-inline-recursively.ll
Index: llvm/test/Transforms/Inline/always-inline-recursively.ll
===================================================================
--- /dev/null
+++ llvm/test/Transforms/Inline/always-inline-recursively.ll
@@ -0,0 +1,267 @@
+; RUN: opt < %s -inline-threshold=0 -inline -S | FileCheck %s
+; RUN: opt < %s -inline-threshold=0 -always-inline -S | FileCheck %s
+;
+; Ensure the threshold has no impact on these decisions.
+; RUN: opt < %s -inline-threshold=20000000 -inline -S | FileCheck %s
+; RUN: opt < %s -inline-threshold=20000000 -always-inline -S | FileCheck %s
+; RUN: opt < %s -inline-threshold=-20000000 -inline -S | FileCheck %s
+; RUN: opt < %s -inline-threshold=-20000000 -always-inline -S | FileCheck %s
+
+; In the tests involving recursive functions we call the external function and
+; continue recursion depending on the external variable, so that any recursion
+; conditions are opaque to the optimizer
+
+; Following tests are conducted, for annotating call-sites and function declarations:
+; Test that a simple tree call-graph is inlined
+; Test that functions marked noinline are not inlined
+; Test that a recursive call is not inlined
+; Test that an indirectly recursive call is inlined until a directly recursive call remains
+
+; External funcion not visible to the optimizer
+declare void @ext_func()
+; External variable not visible to the optimizer
+@ext_var = external global i32
+
+; Test that a simple tree call-graph is inlined
+; when annotating call-sites
+
+define void @test_calls_tree() {
+; CHECK-LABEL: @test_calls_tree() {
+ call void @test_calls_tree_1() alwaysinline_recursively
+; CHECK-NEXT: call void @ext_func() #1
+ call void @test_calls_tree_2() alwaysinline_recursively
+; CHECK-NEXT: call void @ext_func() #1
+; CHECK-NEXT: call void @ext_func() #1
+; CHECK-NEXT: call void @ext_func() #1
+ ret void
+; CHECK-NEXT: ret void
+}
+
+define void @test_calls_tree_1() {
+ call void @ext_func()
+ ret void
+}
+
+define void @test_calls_tree_2() {
+ call void @test_calls_tree_2_1()
+ call void @test_calls_tree_2_2()
+ call void @test_calls_tree_2_3()
+ ret void
+}
+
+define void @test_calls_tree_2_1() {
+ call void @ext_func()
+ ret void
+}
+
+define void @test_calls_tree_2_2() {
+ call void @ext_func()
+ ret void
+}
+
+define void @test_calls_tree_2_3() {
+ call void @ext_func()
+ ret void
+}
+
+; Test that functions marked noinline are not inlined
+; when annotating call-sites
+
+define void @test_calls_noinline() {
+; CHECK-LABEL: @test_calls_noinline() {
+ call void @test_calls_noinline_inlined() alwaysinline_recursively
+; CHECK-NEXT: call void @ext_func() #1
+ call void @test_calls_noinline_inlined2() alwaysinline_recursively
+; CHECK-NEXT: call void @test_calls_noinline_notinlined()
+ ret void
+; CHECK-NEXT: ret void
+}
+
+define void @test_calls_noinline_inlined() {
+ call void @ext_func()
+ ret void
+}
+
+define void @test_calls_noinline_inlined2() {
+ call void @test_calls_noinline_notinlined()
+ ret void
+}
+
+define void @test_calls_noinline_notinlined() noinline {
+ call void @ext_func()
+ ret void
+}
+
+; Test that a recursive call is not inlined
+; when annotating call-sites
+
+define void @test_calls_rec() {
+; CHECK-LABEL: @test_calls_rec() {
+ call void @test_calls_rec_func() alwaysinline_recursively
+; CHECK-NEXT: call void @test_calls_rec_func() #1
+ ret void
+; CHECK-NEXT: ret void
+}
+
+define void @test_calls_rec_func() {
+ call void @ext_func()
+ %1 = load i32, i32* @ext_var
+ %2 = icmp ne i32 %1, 0
+ br i1 %2, label %3, label %4
+
+3:
+ call void @test_calls_rec_func()
+ br label %4
+
+4:
+ ret void
+}
+
+; Test that an indirectly recursive call is inlined
+; until a directly recursive call remains
+; when annotating call-sites
+
+define void @test_calls_irec() {
+; CHECK-LABEL: @test_calls_irec() {
+ call void @test_calls_irec_func() alwaysinline_recursively
+; CHECK: call void @test_calls_irec() #1
+ ret void
+; CHECK: ret void
+}
+
+define void @test_calls_irec_func() {
+ call void @ext_func()
+ %1 = load i32, i32* @ext_var
+ %2 = icmp ne i32 %1, 0
+ br i1 %2, label %3, label %4
+
+3:
+ call void @test_calls_irec()
+ br label %4
+
+4:
+ ret void
+}
+
+; Test that a simple tree call-graph is inlined
+; when annotating function definitions
+
+define void @test_defs_tree() {
+; CHECK-LABEL: @test_defs_tree() {
+ call void @test_defs_tree_1()
+; CHECK-NEXT: call void @ext_func() #1
+ call void @test_defs_tree_2()
+; CHECK-NEXT: call void @ext_func() #1
+; CHECK-NEXT: call void @ext_func() #1
+; CHECK-NEXT: call void @ext_func() #1
+ ret void
+; CHECK-NEXT: ret void
+}
+
+define void @test_defs_tree_1() alwaysinline_recursively {
+ call void @ext_func()
+ ret void
+}
+
+define void @test_defs_tree_2() alwaysinline_recursively {
+ call void @test_defs_tree_2_1()
+ call void @test_defs_tree_2_2()
+ call void @test_defs_tree_2_3()
+ ret void
+}
+
+define void @test_defs_tree_2_1() {
+ call void @ext_func()
+ ret void
+}
+
+define void @test_defs_tree_2_2() {
+ call void @ext_func()
+ ret void
+}
+
+define void @test_defs_tree_2_3() {
+ call void @ext_func()
+ ret void
+}
+
+; Test that functions marked noinline are not inlined
+; when annotating function definitions
+
+define void @test_defs_noinline() {
+; CHECK-LABEL: @test_defs_noinline() {
+ call void @test_defs_noinline_inlined()
+; CHECK-NEXT: call void @ext_func() #1
+ call void @test_defs_noinline_inlined2()
+; CHECK-NEXT: call void @test_defs_noinline_notinlined()
+ ret void
+; CHECK-NEXT: ret void
+}
+
+define void @test_defs_noinline_inlined() alwaysinline_recursively {
+ call void @ext_func()
+ ret void
+}
+
+define void @test_defs_noinline_inlined2() alwaysinline_recursively {
+ call void @test_defs_noinline_notinlined()
+ ret void
+}
+
+define void @test_defs_noinline_notinlined() noinline {
+ call void @ext_func()
+ ret void
+}
+
+; Test that a recursive call is not inlined
+; when annotating function definitions
+
+define void @test_defs_rec() {
+; CHECK-LABEL: @test_defs_rec() {
+ call void @test_defs_rec_func()
+; CHECK-NEXT: call void @test_defs_rec_func()
+ ret void
+; CHECK-NEXT: ret void
+}
+
+define void @test_defs_rec_func() alwaysinline_recursively {
+ call void @ext_func()
+ %1 = load i32, i32* @ext_var
+ %2 = icmp ne i32 %1, 0
+ br i1 %2, label %3, label %4
+
+3:
+ call void @test_defs_rec_func()
+ br label %4
+
+4:
+ ret void
+}
+
+; Test that an indirectly recursive call is inlined
+; until a directly recursive call remains
+; when annotating function definitions
+
+define void @test_defs_irec() {
+; CHECK-LABEL: @test_defs_irec() {
+ call void @test_defs_irec_func()
+; CHECK: call void @test_defs_irec() #1
+ ret void
+; CHECK: ret void
+}
+
+define void @test_defs_irec_func() alwaysinline_recursively {
+ call void @ext_func()
+ %1 = load i32, i32* @ext_var
+ %2 = icmp ne i32 %1, 0
+ br i1 %2, label %3, label %4
+
+3:
+ call void @test_defs_irec()
+ br label %4
+
+4:
+ ret void
+}
+
+; CHECK: attributes #1 = { alwaysinline_recursively }
Index: llvm/lib/Transforms/Utils/CodeExtractor.cpp
===================================================================
--- llvm/lib/Transforms/Utils/CodeExtractor.cpp
+++ llvm/lib/Transforms/Utils/CodeExtractor.cpp
@@ -892,6 +892,7 @@
continue;
// Those attributes should be safe to propagate to the extracted function.
case Attribute::AlwaysInline:
+ case Attribute::AlwaysInlineRecursively:
case Attribute::Cold:
case Attribute::NoRecurse:
case Attribute::InlineHint:
Index: llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
===================================================================
--- llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -751,7 +751,8 @@
// have its address taken. Doing so would create an undefined external ref to
// the function, which would fail to link.
if (HasAvailableExternallyLinkage &&
- F->hasFnAttribute(Attribute::AlwaysInline))
+ (F->hasFnAttribute(Attribute::AlwaysInline) ||
+ F->hasFnAttribute(Attribute::AlwaysInlineRecursively)))
return false;
// Prohibit function address recording if the function is both internal and
Index: llvm/lib/Transforms/IPO/SyntheticCountsPropagation.cpp
===================================================================
--- llvm/lib/Transforms/IPO/SyntheticCountsPropagation.cpp
+++ llvm/lib/Transforms/IPO/SyntheticCountsPropagation.cpp
@@ -77,6 +77,7 @@
if (F.isDeclaration())
continue;
if (F.hasFnAttribute(Attribute::AlwaysInline) ||
+ F.hasFnAttribute(Attribute::AlwaysInlineRecursively) ||
F.hasFnAttribute(Attribute::InlineHint)) {
// Use a higher value for inline functions to account for the fact that
// these are usually beneficial to inline.
Index: llvm/lib/Transforms/IPO/PartialInlining.cpp
===================================================================
--- llvm/lib/Transforms/IPO/PartialInlining.cpp
+++ llvm/lib/Transforms/IPO/PartialInlining.cpp
@@ -1271,6 +1271,9 @@
if (F->hasFnAttribute(Attribute::AlwaysInline))
return {false, nullptr};
+ if (F->hasFnAttribute(Attribute::AlwaysInlineRecursively))
+ return {false, nullptr};
+
if (F->hasFnAttribute(Attribute::NoInline))
return {false, nullptr};
Index: llvm/lib/Transforms/IPO/Inliner.cpp
===================================================================
--- llvm/lib/Transforms/IPO/Inliner.cpp
+++ llvm/lib/Transforms/IPO/Inliner.cpp
@@ -682,6 +682,9 @@
DebugLoc DLoc = CS->getDebugLoc();
BasicBlock *Block = CS.getParent();
+ bool AlwaysInlineRecursively =
+ CS.hasFnAttr(Attribute::AlwaysInlineRecursively);
+
// Attempt to inline the function.
using namespace ore;
@@ -712,8 +715,16 @@
int NewHistoryID = InlineHistory.size();
InlineHistory.push_back(std::make_pair(Callee, InlineHistoryID));
- for (Value *Ptr : InlineInfo.InlinedCalls)
- CallSites.push_back(std::make_pair(CallSite(Ptr), NewHistoryID));
+ for (Value *Ptr : InlineInfo.InlinedCalls) {
+ CallSite NewCS(Ptr);
+ // Propagate alwaysinline_recursively attribute to all inlined call
+ // sites which are not marked noinline
+ if (AlwaysInlineRecursively &&
+ !NewCS.hasFnAttr(Attribute::NoInline))
+ NewCS.addAttribute(AttributeList::FunctionIndex,
+ Attribute::AlwaysInlineRecursively);
+ CallSites.push_back(std::make_pair(NewCS, NewHistoryID));
+ }
}
}
@@ -813,7 +824,8 @@
// Handle the case when this function is called and we only want to care
// about always-inline functions. This is a bit of a hack to share code
// between here and the InlineAlways pass.
- if (AlwaysInlineOnly && !F->hasFnAttribute(Attribute::AlwaysInline))
+ if (AlwaysInlineOnly && !F->hasFnAttribute(Attribute::AlwaysInline) &&
+ !F->hasFnAttribute(Attribute::AlwaysInlineRecursively))
continue;
// If the only remaining users of the function are dead constants, remove
@@ -1079,6 +1091,9 @@
DebugLoc DLoc = CS->getDebugLoc();
BasicBlock *Block = CS.getParent();
+ bool AlwaysInlineRecursively =
+ CS.hasFnAttr(Attribute::AlwaysInlineRecursively);
+
using namespace ore;
InlineResult IR = InlineFunction(CS, IFI);
@@ -1114,9 +1129,16 @@
if (tryPromoteCall(CS))
NewCallee = CS.getCalledFunction();
}
- if (NewCallee)
- if (!NewCallee->isDeclaration())
+ if (NewCallee) {
+ if (!NewCallee->isDeclaration()) {
+ // Propagate alwaysinline_recursively attribute to all inlined
+ // call sites, which are not marked noinline
+ if (AlwaysInlineRecursively && !CS.hasFnAttr(Attribute::NoInline))
+ CS.addAttribute(AttributeList::FunctionIndex,
+ Attribute::AlwaysInlineRecursively);
Calls.push_back({CS, NewHistoryID});
+ }
+ }
}
}
Index: llvm/lib/Transforms/IPO/HotColdSplitting.cpp
===================================================================
--- llvm/lib/Transforms/IPO/HotColdSplitting.cpp
+++ llvm/lib/Transforms/IPO/HotColdSplitting.cpp
@@ -203,6 +203,8 @@
bool HotColdSplitting::shouldOutlineFrom(const Function &F) const {
if (F.hasFnAttribute(Attribute::AlwaysInline))
return false;
+ if (F.hasFnAttribute(Attribute::AlwaysInlineRecursively))
+ return false;
if (F.hasFnAttribute(Attribute::NoInline))
return false;
Index: llvm/lib/Transforms/IPO/ForceFunctionAttrs.cpp
===================================================================
--- llvm/lib/Transforms/IPO/ForceFunctionAttrs.cpp
+++ llvm/lib/Transforms/IPO/ForceFunctionAttrs.cpp
@@ -29,6 +29,7 @@
static Attribute::AttrKind parseAttrKind(StringRef Kind) {
return StringSwitch<Attribute::AttrKind>(Kind)
.Case("alwaysinline", Attribute::AlwaysInline)
+ .Case("alwaysinline_recursively", Attribute::AlwaysInlineRecursively)
.Case("builtin", Attribute::Builtin)
.Case("cold", Attribute::Cold)
.Case("convergent", Attribute::Convergent)
Index: llvm/lib/Transforms/IPO/Attributor.cpp
===================================================================
--- llvm/lib/Transforms/IPO/Attributor.cpp
+++ llvm/lib/Transforms/IPO/Attributor.cpp
@@ -8380,7 +8380,8 @@
ReadOrWriteInsts.push_back(&I);
}
- if (F.hasFnAttribute(Attribute::AlwaysInline) &&
+ if ((F.hasFnAttribute(Attribute::AlwaysInline) ||
+ F.hasFnAttribute(Attribute::AlwaysInlineRecursively)) &&
isInlineViable(F).isSuccess())
InfoCache.InlineableFunctions.insert(&F);
}
Index: llvm/lib/Transforms/IPO/AlwaysInliner.cpp
===================================================================
--- llvm/lib/Transforms/IPO/AlwaysInliner.cpp
+++ llvm/lib/Transforms/IPO/AlwaysInliner.cpp
@@ -47,7 +47,9 @@
bool Changed = false;
SmallVector<Function *, 16> InlinedFunctions;
for (Function &F : M)
- if (!F.isDeclaration() && F.hasFnAttribute(Attribute::AlwaysInline) &&
+ if (!F.isDeclaration() &&
+ (F.hasFnAttribute(Attribute::AlwaysInline) ||
+ F.hasFnAttribute(Attribute::AlwaysInlineRecursively)) &&
isInlineViable(F).isSuccess()) {
Calls.clear();
@@ -164,7 +166,8 @@
if (Callee->isDeclaration())
return InlineCost::getNever("no definition");
- if (!CS.hasFnAttr(Attribute::AlwaysInline))
+ if (!CS.hasFnAttr(Attribute::AlwaysInline) &&
+ !CS.hasFnAttr(Attribute::AlwaysInlineRecursively))
return InlineCost::getNever("no alwaysinline attribute");
auto IsViable = isInlineViable(*Callee);
Index: llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp
===================================================================
--- llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp
+++ llvm/lib/Target/Hexagon/HexagonLoopIdiomRecognition.cpp
@@ -2090,7 +2090,8 @@
// Don't generate memmove if this function will be inlined. This is
// because the caller will undergo this transformation after inlining.
Function *Func = CurLoop->getHeader()->getParent();
- if (Func->hasFnAttribute(Attribute::AlwaysInline))
+ if ((Func->hasFnAttribute(Attribute::AlwaysInline) ||
+ Func->hasFnAttribute(Attribute::AlwaysInlineRecursively)))
goto CleanupAndExit;
// In case of a memmove, the call to memmove will be executed instead
Index: llvm/lib/Target/AMDGPU/AMDGPUInline.cpp
===================================================================
--- llvm/lib/Target/AMDGPU/AMDGPUInline.cpp
+++ llvm/lib/Target/AMDGPU/AMDGPUInline.cpp
@@ -188,7 +188,8 @@
if (!TTI.areInlineCompatible(Caller, Callee))
return llvm::InlineCost::getNever("incompatible");
- if (CS.hasFnAttr(Attribute::AlwaysInline)) {
+ if (CS.hasFnAttr(Attribute::AlwaysInline) ||
+ CS.hasFnAttr(Attribute::AlwaysInlineRecursively)) {
auto IsViable = isInlineViable(*Callee);
if (IsViable.isSuccess())
return llvm::InlineCost::getAlways("alwaysinline viable");
Index: llvm/lib/IR/Verifier.cpp
===================================================================
--- llvm/lib/IR/Verifier.cpp
+++ llvm/lib/IR/Verifier.cpp
@@ -1552,6 +1552,7 @@
case Attribute::SpeculativeLoadHardening:
case Attribute::Speculatable:
case Attribute::StrictFP:
+ case Attribute::AlwaysInlineRecursively:
return true;
default:
break;
@@ -1663,6 +1664,12 @@
"'noinline and alwaysinline' are incompatible!",
V);
+ Assert(!(Attrs.hasAttribute(Attribute::NoInline) &&
+ Attrs.hasAttribute(Attribute::AlwaysInlineRecursively)),
+ "Attributes "
+ "'noinline and alwaysinline_recursively' are incompatible!",
+ V);
+
if (Attrs.hasAttribute(Attribute::ByVal) && Attrs.getByValType()) {
Assert(Attrs.getByValType() == cast<PointerType>(Ty)->getElementType(),
"Attribute 'byval' type does not match parameter!", V);
@@ -1815,6 +1822,11 @@
Attrs.hasFnAttribute(Attribute::AlwaysInline)),
"Attributes 'noinline and alwaysinline' are incompatible!", V);
+ Assert(!(Attrs.hasFnAttribute(Attribute::NoInline) &&
+ Attrs.hasFnAttribute(Attribute::AlwaysInlineRecursively)),
+ "Attributes 'noinline and alwaysinline_recursively' are incompatible!",
+ V);
+
if (Attrs.hasFnAttribute(Attribute::OptimizeNone)) {
Assert(Attrs.hasFnAttribute(Attribute::NoInline),
"Attribute 'optnone' requires 'noinline'!", V);
Index: llvm/lib/IR/Attributes.cpp
===================================================================
--- llvm/lib/IR/Attributes.cpp
+++ llvm/lib/IR/Attributes.cpp
@@ -324,6 +324,8 @@
return "sanitize_memtag";
if (hasAttribute(Attribute::AlwaysInline))
return "alwaysinline";
+ if (hasAttribute(Attribute::AlwaysInlineRecursively))
+ return "alwaysinline_recursively";
if (hasAttribute(Attribute::ArgMemOnly))
return "argmemonly";
if (hasAttribute(Attribute::Builtin))
Index: llvm/lib/CodeGen/SafeStack.cpp
===================================================================
--- llvm/lib/CodeGen/SafeStack.cpp
+++ llvm/lib/CodeGen/SafeStack.cpp
@@ -707,7 +707,8 @@
bool SafeStack::ShouldInlinePointerAddress(CallSite &CS) {
Function *Callee = CS.getCalledFunction();
- if (CS.hasFnAttr(Attribute::AlwaysInline) &&
+ if ((CS.hasFnAttr(Attribute::AlwaysInline) ||
+ CS.hasFnAttr(Attribute::AlwaysInlineRecursively)) &&
isInlineViable(*Callee).isSuccess())
return true;
if (Callee->isInterposable() || Callee->hasFnAttribute(Attribute::NoInline) ||
Index: llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
===================================================================
--- llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -604,6 +604,8 @@
return bitc::ATTR_KIND_ALLOC_SIZE;
case Attribute::AlwaysInline:
return bitc::ATTR_KIND_ALWAYS_INLINE;
+ case Attribute::AlwaysInlineRecursively:
+ return bitc::ATTR_KIND_ALWAYS_INLINE_RECURSIVELY;
case Attribute::ArgMemOnly:
return bitc::ATTR_KIND_ARGMEMONLY;
case Attribute::Builtin:
Index: llvm/lib/Bitcode/Reader/BitcodeReader.cpp
===================================================================
--- llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -1304,6 +1304,10 @@
case Attribute::SanitizeMemTag:
llvm_unreachable("sanitize_memtag attribute not supported in raw format");
break;
+ case Attribute::AlwaysInlineRecursively:
+ llvm_unreachable(
+ "alwaysinline_recursively attribute not supported in raw format");
+ break;
}
llvm_unreachable("Unsupported attribute type");
}
@@ -1313,12 +1317,10 @@
for (Attribute::AttrKind I = Attribute::None; I != Attribute::EndAttrKinds;
I = Attribute::AttrKind(I + 1)) {
- if (I == Attribute::SanitizeMemTag ||
- I == Attribute::Dereferenceable ||
- I == Attribute::DereferenceableOrNull ||
- I == Attribute::ArgMemOnly ||
- I == Attribute::AllocSize ||
- I == Attribute::NoSync)
+ if (I == Attribute::SanitizeMemTag || I == Attribute::Dereferenceable ||
+ I == Attribute::DereferenceableOrNull || I == Attribute::ArgMemOnly ||
+ I == Attribute::AllocSize || I == Attribute::NoSync ||
+ I == Attribute::AlwaysInlineRecursively)
continue;
if (uint64_t A = (Val & getRawAttributeMask(I))) {
if (I == Attribute::Alignment)
Index: llvm/lib/AsmParser/LLToken.h
===================================================================
--- llvm/lib/AsmParser/LLToken.h
+++ llvm/lib/AsmParser/LLToken.h
@@ -176,6 +176,7 @@
kw_attributes,
kw_allocsize,
kw_alwaysinline,
+ kw_alwaysinline_recursively,
kw_argmemonly,
kw_sanitize_address,
kw_sanitize_hwaddress,
Index: llvm/lib/AsmParser/LLParser.cpp
===================================================================
--- llvm/lib/AsmParser/LLParser.cpp
+++ llvm/lib/AsmParser/LLParser.cpp
@@ -1270,6 +1270,9 @@
continue;
}
case lltok::kw_alwaysinline: B.addAttribute(Attribute::AlwaysInline); break;
+ case lltok::kw_alwaysinline_recursively:
+ B.addAttribute(Attribute::AlwaysInlineRecursively);
+ break;
case lltok::kw_argmemonly: B.addAttribute(Attribute::ArgMemOnly); break;
case lltok::kw_builtin: B.addAttribute(Attribute::Builtin); break;
case lltok::kw_cold: B.addAttribute(Attribute::Cold); break;
@@ -1654,6 +1657,7 @@
case lltok::kw_alignstack:
case lltok::kw_alwaysinline:
+ case lltok::kw_alwaysinline_recursively:
case lltok::kw_argmemonly:
case lltok::kw_builtin:
case lltok::kw_inlinehint:
@@ -1752,6 +1756,7 @@
case lltok::kw_alignstack:
case lltok::kw_alwaysinline:
+ case lltok::kw_alwaysinline_recursively:
case lltok::kw_argmemonly:
case lltok::kw_builtin:
case lltok::kw_cold:
Index: llvm/lib/AsmParser/LLLexer.cpp
===================================================================
--- llvm/lib/AsmParser/LLLexer.cpp
+++ llvm/lib/AsmParser/LLLexer.cpp
@@ -632,6 +632,7 @@
KEYWORD(attributes);
KEYWORD(alwaysinline);
+ KEYWORD(alwaysinline_recursively);
KEYWORD(allocsize);
KEYWORD(argmemonly);
KEYWORD(builtin);
Index: llvm/lib/Analysis/InlineCost.cpp
===================================================================
--- llvm/lib/Analysis/InlineCost.cpp
+++ llvm/lib/Analysis/InlineCost.cpp
@@ -2245,6 +2245,14 @@
return llvm::InlineCost::getNever(IsViable.getFailureReason());
}
+ // Inline call sites marked alwaysinline_recursively
+ if (Call.hasFnAttr(Attribute::AlwaysInlineRecursively)) {
+ auto IsViable = isInlineViable(*Callee);
+ if (IsViable.isSuccess())
+ return llvm::InlineCost::getAlways("alwaysinline_recursively attribute");
+ return llvm::InlineCost::getNever(IsViable.getFailureReason());
+ }
+
// Never inline functions with conflicting attributes (unless callee has
// always-inline attribute).
Function *Caller = Call.getCaller();
Index: llvm/include/llvm/IR/Attributes.td
===================================================================
--- llvm/include/llvm/IR/Attributes.td
+++ llvm/include/llvm/IR/Attributes.td
@@ -23,6 +23,9 @@
/// inline=always.
def AlwaysInline : EnumAttr<"alwaysinline">;
+/// Like AlwaysInline, but applies recursively
+def AlwaysInlineRecursively : EnumAttr<"alwaysinline_recursively">;
+
/// Function can access memory only using pointers based on its arguments.
def ArgMemOnly : EnumAttr<"argmemonly">;
Index: llvm/include/llvm/Bitcode/LLVMBitCodes.h
===================================================================
--- llvm/include/llvm/Bitcode/LLVMBitCodes.h
+++ llvm/include/llvm/Bitcode/LLVMBitCodes.h
@@ -633,6 +633,7 @@
ATTR_KIND_NOFREE = 62,
ATTR_KIND_NOSYNC = 63,
ATTR_KIND_SANITIZE_MEMTAG = 64,
+ ATTR_KIND_ALWAYS_INLINE_RECURSIVELY = 65,
};
enum ComdatSelectionKindCodes {
Index: llvm/docs/LangRef.rst
===================================================================
--- llvm/docs/LangRef.rst
+++ llvm/docs/LangRef.rst
@@ -1393,6 +1393,9 @@
This attribute indicates that the inliner should attempt to inline
this function into callers whenever possible, ignoring any active
inlining size threshold for this caller.
+``alwaysinline_recursively``
+ This attribute is similar to ``alwaysinline``, but also applies recursively to
+ all inlined function calls.
``builtin``
This indicates that the callee function at a call site should be
recognized as a built-in function, even though the function's declaration
Index: llvm/docs/BitCodeFormat.rst
===================================================================
--- llvm/docs/BitCodeFormat.rst
+++ llvm/docs/BitCodeFormat.rst
@@ -1059,6 +1059,7 @@
* code 57: ``optforfuzzing``
* code 58: ``shadowcallstack``
* code 64: ``sanitize_memtag``
+* code 65: ``alwaysinline_recursively``
.. note::
The ``allocsize`` attribute has a special encoding for its arguments. Its two
Index: clang/lib/CodeGen/CGCall.cpp
===================================================================
--- clang/lib/CodeGen/CGCall.cpp
+++ clang/lib/CodeGen/CGCall.cpp
@@ -4526,13 +4526,13 @@
// Apply some call-site-specific attributes.
// TODO: work this into building the attribute set.
- // Apply always_inline to all calls within flatten functions.
+ // Apply alwaysinline_recursively to all calls within flatten functions.
// FIXME: should this really take priority over __try, below?
if (CurCodeDecl && CurCodeDecl->hasAttr<FlattenAttr>() &&
!(TargetDecl && TargetDecl->hasAttr<NoInlineAttr>())) {
Attrs =
Attrs.addAttribute(getLLVMContext(), llvm::AttributeList::FunctionIndex,
- llvm::Attribute::AlwaysInline);
+ llvm::Attribute::AlwaysInlineRecursively);
}
// Disable inlining inside SEH __try blocks.
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
