eugenis created this revision.
eugenis added reviewers: chandlerc, rsmith.
eugenis added a subscriber: cfe-commits.
eugenis set the repository for this revision to rL LLVM.
Currently always_inline definitions are emitted as (in most cases) an
available_externally llvm function with an alwaysinline attribute. This is not
exactly right: always_inline functions are NOT available externally, and, for
example, libc++ uses this semantics to preserve ABI stability.
Emitting an undefined symbol for an always_inline function is always a bug. The
current code can still do it in certain cases.
a. Inliner is an SCC pass. It traverses the graph starting from the roots,
which are either main() function, or all externally-visible functions. Inlining
does not happen in functions that are not reachable.
b. Dead code elimination is not perfect. There are cases where a function will
become unreachable due to some late optimizations and will still be emitted
into the binary.
This patch changes the way always_inline functions are emitted in the Clang
codegen to ensure this never happens. A function F is emitted as a pair of
a. internal F.inlinefunction() alwaysinline { **original function body** }
and, depending on the function visibility, either
b1. declare F()
or
b2. define external F() { musttail call F.inlinefunction() }
Frontend ensures that all direct calls go to F.inlinefunction().
This provides a simple invariant that all alwaysinline functions are internal,
which can be checked in the IR verifier. Another invariant would be that
alwaysinline functions never reach the backend.
This patch is based on ideas by Chandler Carruth and Richard Smith.
Repository:
rL LLVM
http://reviews.llvm.org/D12087
Files:
lib/CodeGen/CGCXX.cpp
lib/CodeGen/CodeGenModule.cpp
lib/CodeGen/CodeGenModule.h
lib/CodeGen/ItaniumCXXABI.cpp
test/CodeGen/2008-05-19-AlwaysInline.c
test/CodeGen/always-inline.c
test/CodeGen/always_inline.c
test/CodeGen/alwaysinline.c
test/CodeGen/dllimport.c
test/CodeGen/function-attributes.c
test/CodeGen/pr9614.c
test/CodeGenCXX/alwaysinline.cpp
test/CodeGenCXX/dllimport.cpp
test/Frontend/optimization-remark-line-directive.c
test/Frontend/optimization-remark.c
test/Modules/cxx-irgen.cpp
Index: test/Modules/cxx-irgen.cpp
===================================================================
--- test/Modules/cxx-irgen.cpp
+++ test/Modules/cxx-irgen.cpp
@@ -26,14 +26,17 @@
};
}
-// CHECK-DAG: define available_externally hidden {{.*}}{{signext i32|i32}} @_ZN1SIiE1gEv({{.*}} #[[ALWAYS_INLINE:.*]] align
+// CHECK-DAG: define internal i32 @_ZN1SIiE1gEv.inlinefunction() #[[ALWAYS_INLINE:.*]] align
+// CHECK-DAG: declare hidden i32 @_ZN1SIiE1gEv()
int a = S<int>::g();
int b = h();
// CHECK-DAG: define linkonce_odr {{.*}}{{signext i32|i32}} @_Z3minIiET_S0_S0_(i32
int c = min(1, 2);
-// CHECK: define available_externally {{.*}}{{signext i32|i32}} @_ZN1SIiE1fEv({{.*}} #[[ALWAYS_INLINE]] align
+// CHECK-DAG: define internal {{.*}}{{signext i32|i32}} @_ZN1SIiE1fEv.inlinefunction() #[[ALWAYS_INLINE]] align
+// CHECK-DAG: declare {{.*}}{{signext i32|i32}} @_ZN1SIiE1fEv()
+
namespace ImplicitSpecialMembers {
// CHECK-LABEL: define {{.*}} @_ZN22ImplicitSpecialMembers1BC2ERKS0_(
Index: test/Frontend/optimization-remark.c
===================================================================
--- test/Frontend/optimization-remark.c
+++ test/Frontend/optimization-remark.c
@@ -32,6 +32,8 @@
// CHECK-NOT: !llvm.dbg.cu = !{
int foo(int x, int y) __attribute__((always_inline));
+// expected-remark@+2 {{foo.inlinefunction should always be inlined}}
+// expected-remark@+1 {{foo.inlinefunction inlined into foo}}
int foo(int x, int y) { return x + y; }
float foz(int x, int y) __attribute__((noinline));
@@ -45,7 +47,7 @@
// expected-remark@+5 {{foz will not be inlined into bar}}
// expected-remark@+4 {{foz should never be inlined}}
// expected-remark@+3 {{foz will not be inlined into bar}}
-// expected-remark@+2 {{foo should always be inlined}}
-// expected-remark@+1 {{foo inlined into bar}}
+// expected-remark@+2 {{foo.inlinefunction should always be inlined}}
+// expected-remark@+1 {{foo.inlinefunction inlined into bar}}
return foo(j, j - 2) * foz(j - 2, j);
}
Index: test/Frontend/optimization-remark-line-directive.c
===================================================================
--- test/Frontend/optimization-remark-line-directive.c
+++ test/Frontend/optimization-remark-line-directive.c
@@ -5,8 +5,9 @@
// RUN: %clang_cc1 %s -Rpass=inline -gline-tables-only -dwarf-column-info -emit-llvm-only -verify
int foo(int x, int y) __attribute__((always_inline));
+// expected-remark@+1 {{foo.inlinefunction inlined into foo}}
int foo(int x, int y) { return x + y; }
-// expected-remark@+2 {{foo inlined into bar}} expected-note@+2 {{could not determine the original source location for /bad/path/to/original.c:1230:25}}
+// expected-remark@+2 {{foo.inlinefunction inlined into bar}} expected-note@+2 {{could not determine the original source location for /bad/path/to/original.c:1230:25}}
#line 1230 "/bad/path/to/original.c"
int bar(int j) { return foo(j, j - 2); }
Index: test/CodeGenCXX/dllimport.cpp
===================================================================
--- test/CodeGenCXX/dllimport.cpp
+++ test/CodeGenCXX/dllimport.cpp
@@ -244,6 +244,11 @@
USE(noinline)
// MSC2-NOT: @"\01?alwaysInline@@YAXXZ"()
+// MSC2: declare dllimport void @"\01?alwaysInline@@YAXXZ"()
+// MSC2-NOT: @"\01?alwaysInline@@YAXXZ"()
+
+// GNU2-NOT: @_Z12alwaysInlinev()
+// GNU2: define linkonce_odr void @_Z12alwaysInlinev()
// GNU2-NOT: @_Z12alwaysInlinev()
__declspec(dllimport) __attribute__((always_inline)) inline void alwaysInline() {}
USE(alwaysInline)
Index: test/CodeGenCXX/alwaysinline.cpp
===================================================================
--- test/CodeGenCXX/alwaysinline.cpp
+++ test/CodeGenCXX/alwaysinline.cpp
@@ -0,0 +1,83 @@
+// Test different kinds of alwaysinline *structor definitions.
+
+// RUN: %clang_cc1 -disable-llvm-optzns -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK
+// RUN: %clang_cc1 -disable-llvm-optzns -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK-CALL
+
+// RUN: %clang_cc1 -mconstructor-aliases -disable-llvm-optzns -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK
+// RUN: %clang_cc1 -mconstructor-aliases -disable-llvm-optzns -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK-CALL
+
+struct A1 {
+ __attribute__((__always_inline__)) A1() {}
+ __attribute__((__always_inline__)) ~A1() {}
+};
+
+void g1() {
+ A1 a1;
+}
+
+struct A2 {
+ inline __attribute__((__always_inline__)) A2() {}
+ inline __attribute__((__always_inline__)) ~A2() {}
+};
+
+void g2() {
+ A2 a2;
+}
+
+struct A3 {
+ inline __attribute__((gnu_inline, __always_inline__)) A3() {}
+ inline __attribute__((gnu_inline, __always_inline__)) ~A3() {}
+};
+
+void g3() {
+ A3 a3;
+}
+
+// CHECK-DAG: define linkonce_odr void @_ZN2A1C1Ev(%struct.A1*) unnamed_addr #[[NOAI:[01-9]+]]
+// CHECK-DAG: define linkonce_odr void @_ZN2A1C2Ev(%struct.A1*) unnamed_addr #[[NOAI:[01-9]+]]
+// CHECK-DAG: define linkonce_odr void @_ZN2A1D1Ev(%struct.A1*) unnamed_addr #[[NOAI:[01-9]+]]
+// CHECK-DAG: define linkonce_odr void @_ZN2A1D2Ev(%struct.A1*) unnamed_addr #[[NOAI:[01-9]+]]
+
+// CHECK-DAG: define linkonce_odr void @_ZN2A2C1Ev(%struct.A2*) unnamed_addr #[[NOAI:[01-9]+]]
+// CHECK-DAG: define linkonce_odr void @_ZN2A2C2Ev(%struct.A2*) unnamed_addr #[[NOAI:[01-9]+]]
+// CHECK-DAG: define linkonce_odr void @_ZN2A2D1Ev(%struct.A2*) unnamed_addr #[[NOAI:[01-9]+]]
+// CHECK-DAG: define linkonce_odr void @_ZN2A2D2Ev(%struct.A2*) unnamed_addr #[[NOAI:[01-9]+]]
+
+// CHECK-DAG: define void @_ZN2A3C1Ev(%struct.A3*) unnamed_addr #[[NOAI:[01-9]+]]
+// CHECK-DAG: define void @_ZN2A3C2Ev(%struct.A3*) unnamed_addr #[[NOAI:[01-9]+]]
+// CHECK-DAG: define void @_ZN2A3D1Ev(%struct.A3*) unnamed_addr #[[NOAI:[01-9]+]]
+// CHECK-DAG: define void @_ZN2A3D2Ev(%struct.A3*) unnamed_addr #[[NOAI:[01-9]+]]
+
+// CHECK-DAG: define internal void @_ZN2A1C1Ev.inlinefunction(%struct.A1* %this) unnamed_addr #[[AI:[01-9]+]]
+// CHECK-DAG: define internal void @_ZN2A1C2Ev.inlinefunction(%struct.A1* %this) unnamed_addr #[[AI]]
+// CHECK-DAG: define internal void @_ZN2A1D1Ev.inlinefunction(%struct.A1* %this) unnamed_addr #[[AI]]
+// CHECK-DAG: define internal void @_ZN2A1D2Ev.inlinefunction(%struct.A1* %this) unnamed_addr #[[AI]]
+
+// CHECK-DAG: define internal void @_ZN2A2C1Ev.inlinefunction(%struct.A2* %this) unnamed_addr #[[AIIH:[01-9]+]]
+// CHECK-DAG: define internal void @_ZN2A2C2Ev.inlinefunction(%struct.A2* %this) unnamed_addr #[[AIIH]]
+// CHECK-DAG: define internal void @_ZN2A2D1Ev.inlinefunction(%struct.A2* %this) unnamed_addr #[[AIIH]]
+// CHECK-DAG: define internal void @_ZN2A2D2Ev.inlinefunction(%struct.A2* %this) unnamed_addr #[[AIIH]]
+
+// CHECK-DAG: define internal void @_ZN2A3C1Ev.inlinefunction(%struct.A3* %this) unnamed_addr #[[AIIH]]
+// CHECK-DAG: define internal void @_ZN2A3C2Ev.inlinefunction(%struct.A3* %this) unnamed_addr #[[AIIH]]
+// CHECK-DAG: define internal void @_ZN2A3D1Ev.inlinefunction(%struct.A3* %this) unnamed_addr #[[AIIH]]
+// CHECK-DAG: define internal void @_ZN2A3D2Ev.inlinefunction(%struct.A3* %this) unnamed_addr #[[AIIH]]
+
+// CHECK-DAG: attributes #[[AI]] = {{.*alwaysinline.*}}
+// CHECK-DAG: attributes #[[AIIH]] = {{.*alwaysinline.*inlinehint.*}}
+// CHECK-NOT: attributes #[[NOAI]] = {{.*alwaysinline.*}}
+
+// CHECK-CALL-LABEL: define void @_Z2g1v()
+// CHECK-CALL: call void @_ZN2A1C1Ev.inlinefunction
+// CHECK-CALL: call void @_ZN2A1D1Ev.inlinefunction
+// CHECK-CALL: ret void
+
+// CHECK-CALL-LABEL: define void @_Z2g2v()
+// CHECK-CALL: call void @_ZN2A2C1Ev.inlinefunction
+// CHECK-CALL: call void @_ZN2A2D1Ev.inlinefunction
+// CHECK-CALL: ret void
+
+// CHECK-CALL-LABEL: define void @_Z2g3v()
+// CHECK-CALL: call void @_ZN2A3C1Ev.inlinefunction
+// CHECK-CALL: call void @_ZN2A3D1Ev.inlinefunction
+// CHECK-CALL: ret void
Index: test/CodeGen/pr9614.c
===================================================================
--- test/CodeGen/pr9614.c
+++ test/CodeGen/pr9614.c
@@ -1,4 +1,5 @@
-// RUN: %clang_cc1 -triple x86_64-pc-linux -emit-llvm %s -o - | FileCheck %s
+// RUN: %clang_cc1 -triple x86_64-pc-linux -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK0
+// RUN: %clang_cc1 -triple x86_64-pc-linux -O1 -disable-llvm-optzns -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK --check-prefix=CHECK1
extern void foo_alias (void) __asm ("foo");
inline void foo (void) {
@@ -24,17 +25,18 @@
void f(void) {
foo();
- abs(0);
+ int x = abs(0);
strrchr_foo("", '.');
prefetch();
memchr("", '.', 0);
}
// CHECK-LABEL: define void @f()
// CHECK: call void @foo()
-// CHECK: call i32 @abs(i32 0)
+// CHECK: call i32 @abs(
// CHECK: call i8* @strrchr(
-// CHECK: call void @llvm.prefetch(
+// CHECK0: call void @llvm.prefetch(
+// CHECK1: call void @prefetch.inlinefunction(
// CHECK: call i8* @memchr(
// CHECK: ret void
Index: test/CodeGen/function-attributes.c
===================================================================
--- test/CodeGen/function-attributes.c
+++ test/CodeGen/function-attributes.c
@@ -25,8 +25,11 @@
void f7(unsigned short x) { }
+// ((always_inline)) function produces an
+// (internal, alwaysinline) f8.inlinefunction() and an external stub f8().
+// At -Os, the .inlinefunction is optimized out.
// CHECK-LABEL: define void @f8()
-// CHECK: [[AI:#[0-9]+]]
+// CHECK: [[NUW:#[0-9]+]]
// CHECK: {
void __attribute__((always_inline)) f8(void) { }
@@ -129,7 +132,6 @@
}
// CHECK: attributes [[NUW]] = { nounwind optsize readnone{{.*}} }
-// CHECK: attributes [[AI]] = { alwaysinline nounwind optsize readnone{{.*}} }
// CHECK: attributes [[ALIGN]] = { nounwind optsize readnone alignstack=16{{.*}} }
// CHECK: attributes [[RT]] = { nounwind optsize returns_twice{{.*}} }
// CHECK: attributes [[NR]] = { noreturn nounwind optsize }
Index: test/CodeGen/dllimport.c
===================================================================
--- test/CodeGen/dllimport.c
+++ test/CodeGen/dllimport.c
@@ -92,6 +92,8 @@
// GNU-DAG: declare void @noinline()
// GO1-DAG: define available_externally void @noinline()
// CHECK-NOT: @alwaysInline()
+// CHECK: declare{{( | dllimport )}}void @alwaysInline()
+// CHECK-NOT: @alwaysInline()
// O1-NOT: @alwaysInline()
__declspec(dllimport) __attribute__((noinline)) inline void noinline(void) {}
__declspec(dllimport) __attribute__((always_inline)) inline void alwaysInline(void) {}
Index: test/CodeGen/alwaysinline.c
===================================================================
--- test/CodeGen/alwaysinline.c
+++ test/CodeGen/alwaysinline.c
@@ -0,0 +1,88 @@
+// Test different kinds of alwaysinline definitions.
+
+// RUN: %clang_cc1 -disable-llvm-optzns -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK
+// RUN: %clang_cc1 -disable-llvm-optzns -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK-DIRECT
+// RUN: %clang_cc1 -disable-llvm-optzns -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK-INDIRECT
+// RUN: %clang_cc1 -disable-llvm-optzns -fno-inline -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK
+// RUN: %clang_cc1 -disable-llvm-optzns -fno-inline -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK-DIRECT
+// RUN: %clang_cc1 -disable-llvm-optzns -fno-inline -emit-llvm %s -o - | FileCheck %s --check-prefix=CHECK-INDIRECT
+
+void __attribute__((__always_inline__)) f1() {}
+inline void __attribute__((__always_inline__)) f2() {}
+static inline void __attribute__((__always_inline__)) f3() {}
+inline void __attribute__((gnu_inline, __always_inline__)) f4() {}
+static inline void __attribute__((gnu_inline, __always_inline__)) f5() {}
+inline void __attribute__((visibility("hidden"), __always_inline__)) f6() {}
+inline void __attribute__((visibility("hidden"), gnu_inline, __always_inline__)) f7() {}
+
+void g() {
+ f1();
+ f2();
+ f3();
+ f4();
+ f5();
+ f6();
+ f7();
+}
+
+void (*p)(void);
+void h() {
+ p = f1;
+ p = f2;
+ p = f3;
+ p = f4;
+ p = f5;
+ p = f6;
+ p = f7;
+}
+
+void (*const cp1)(void) = f1;
+void (*p1)(void) = f1;
+void (*p2)(int) = (void (*)(int))f1;
+
+// CHECK-DAG: define internal void @f1.inlinefunction() #[[AI:[01-9]+]]
+// CHECK-DAG: define internal void @f2.inlinefunction() #[[AI:[01-9]+]]
+// CHECK-DAG: define internal void @f3.inlinefunction() #[[AI:[01-9]+]]
+// CHECK-DAG: define internal void @f4.inlinefunction() #[[AI:[01-9]+]]
+// CHECK-DAG: define internal void @f5.inlinefunction() #[[AI:[01-9]+]]
+// CHECK-DAG: define internal void @f6.inlinefunction() #[[AI:[01-9]+]]
+// CHECK-DAG: define internal void @f7.inlinefunction() #[[AI:[01-9]+]]
+
+// CHECK-DAG: musttail call void @f1.inlinefunction()
+// CHECK-DAG: musttail call void @f3.inlinefunction()
+// CHECK-DAG: musttail call void @f4.inlinefunction()
+// CHECK-DAG: musttail call void @f5.inlinefunction()
+// CHECK-DAG: musttail call void @f7.inlinefunction()
+
+// CHECK-DAG: define void @f1() #[[NOAI:[01-9]+]]
+// CHECK-DAG: declare void @f2() #[[NOAI:[01-9]+]]
+// CHECK-DAG: define internal void @f3() #[[NOAI:[01-9]+]]
+// CHECK-DAG: define void @f4() #[[NOAI:[01-9]+]]
+// CHECK-DAG: define internal void @f5() #[[NOAI:[01-9]+]]
+// CHECK-DAG: declare hidden void @f6() #[[NOAI:[01-9]+]]
+// CHECK-DAG: define hidden void @f7() #[[NOAI:[01-9]+]]
+
+// CHECK: attributes #[[AI]] = {{.*alwaysinline.*}}
+// CHECK-NOT: attributes #[[NOAI]] = {{.*alwaysinline.*}}
+
+// CHECK-DIRECT: define void @g()
+// CHECK-DIRECT-NEXT: entry:
+// CHECK-DIRECT-NEXT: call void @f1.inlinefunction()
+// CHECK-DIRECT-NEXT: call void @f2.inlinefunction()
+// CHECK-DIRECT-NEXT: call void @f3.inlinefunction()
+// CHECK-DIRECT-NEXT: call void @f4.inlinefunction()
+// CHECK-DIRECT-NEXT: call void @f5.inlinefunction()
+// CHECK-DIRECT-NEXT: call void @f6.inlinefunction()
+// CHECK-DIRECT-NEXT: call void @f7.inlinefunction()
+// CHECK-DIRECT-NEXT: ret void
+
+// CHECK-INDIRECT: define void @h()
+// CHECK-INDIRECT-NEXT: entry:
+// CHECK-INDIRECT-NEXT: store void ()* @f1,
+// CHECK-INDIRECT-NEXT: store void ()* @f2,
+// CHECK-INDIRECT-NEXT: store void ()* @f3,
+// CHECK-INDIRECT-NEXT: store void ()* @f4,
+// CHECK-INDIRECT-NEXT: store void ()* @f5,
+// CHECK-INDIRECT-NEXT: store void ()* @f6,
+// CHECK-INDIRECT-NEXT: store void ()* @f7,
+// CHECK-INDIRECT-NEXT: ret void
Index: test/CodeGen/always_inline.c
===================================================================
--- test/CodeGen/always_inline.c
+++ test/CodeGen/always_inline.c
@@ -1,8 +1,5 @@
-// RUN: %clang -emit-llvm -S -o %t %s
-// RUN: not grep '@f0' %t
-// RUN: not grep 'call ' %t
-// RUN: %clang -mllvm -disable-llvm-optzns -emit-llvm -S -o %t %s
-// RUN: grep '@f0' %t | count 2
+// RUN: %clang -emit-llvm -S -o - %s | FileCheck %s
+// RUN: %clang -mllvm -disable-llvm-optzns -emit-llvm -S -o - %s | FileCheck %s --check-prefix=CHECK-NO-OPTZNS
//static int f0() {
static int __attribute__((always_inline)) f0() {
@@ -18,3 +15,14 @@
int f2() { return 7; }
int f3(void) { return f2(); }
+// CHECK-LABEL: define i32 @f1()
+// CHECK: ret i32 1
+// CHECK-LABEL: define i32 @f2()
+// CHECK: ret i32 7
+// CHECK-LABEL: define i32 @f3()
+// CHECK: ret i32 7
+
+// CHECK-NO-OPTZNS: define i32 @f3()
+// CHECK-NO-OPTZNS-NEXT: entry:
+// CHECK-NO-OPTZNS-NEXT: call i32 @f2.inlinefunction()
+// CHECK-NO-OPTZNS-NEXT: ret i32
Index: test/CodeGen/always-inline.c
===================================================================
--- test/CodeGen/always-inline.c
+++ test/CodeGen/always-inline.c
@@ -1,7 +1,9 @@
// RUN: %clang_cc1 -emit-llvm %s -o - | FileCheck %s
// RUN: %clang_cc1 -fno-inline -emit-llvm %s -o - | FileCheck %s
+// CHECK-LABEL: define void @i_want_bar()
// CHECK-NOT: foo
+// CHECK: ret void
void bar() {
}
Index: test/CodeGen/2008-05-19-AlwaysInline.c
===================================================================
--- test/CodeGen/2008-05-19-AlwaysInline.c
+++ test/CodeGen/2008-05-19-AlwaysInline.c
@@ -1,4 +1,4 @@
-// RUN: %clang_cc1 %s -emit-llvm -o - | not grep sabrina
+// RUN: %clang_cc1 %s -emit-llvm -o - | not grep 'call.*sabrina('
static inline int sabrina (void) __attribute__((always_inline));
static inline int sabrina (void)
Index: lib/CodeGen/ItaniumCXXABI.cpp
===================================================================
--- lib/CodeGen/ItaniumCXXABI.cpp
+++ lib/CodeGen/ItaniumCXXABI.cpp
@@ -3224,6 +3224,9 @@
if (MD->getParent()->getNumVBases())
return StructorCodegen::Emit;
+ if (MD->hasAttr<AlwaysInlineAttr>())
+ return StructorCodegen::Emit;
+
GlobalDecl AliasDecl;
if (const auto *DD = dyn_cast<CXXDestructorDecl>(MD)) {
AliasDecl = GlobalDecl(DD, Dtor_Complete);
Index: lib/CodeGen/CodeGenModule.h
===================================================================
--- lib/CodeGen/CodeGenModule.h
+++ lib/CodeGen/CodeGenModule.h
@@ -1233,6 +1233,17 @@
/// Emits target specific Metadata for global declarations.
void EmitTargetMetadata();
+ /// Replaces alwaysinline functions with a pair of internal xxx.inlinefunction
+ /// for direct calls, and a stub for indirect calls, and rewrites all uses of
+ /// those.
+ void RewriteAlwaysInlineFunctions();
+ void RewriteAlwaysInlineFunction(llvm::Function *Fn);
+
+ // Replaces all uses of GV other than direct calls with IndirectReplacement.
+ void
+ RewriteUsesOfAlwaysInlineFunction(llvm::GlobalValue *GV,
+ llvm::GlobalValue *IndirectReplacement);
+
/// Emit the llvm.gcov metadata used to tell LLVM where to emit the .gcno and
/// .gcda files in a way that persists in .bc files.
void EmitCoverageFile();
Index: lib/CodeGen/CodeGenModule.cpp
===================================================================
--- lib/CodeGen/CodeGenModule.cpp
+++ lib/CodeGen/CodeGenModule.cpp
@@ -429,6 +429,84 @@
EmitVersionIdentMetadata();
EmitTargetMetadata();
+
+ RewriteAlwaysInlineFunctions();
+}
+
+/// Replace all uses of GV that are not direct calls (or invokes) with
+/// IndirectReplacement.
+void CodeGenModule::RewriteUsesOfAlwaysInlineFunction(
+ llvm::GlobalValue *GV, llvm::GlobalValue *IndirectReplacement) {
+ llvm::GlobalValue::use_iterator UI = GV->use_begin(), E = GV->use_end();
+ for (; UI != E;) {
+ llvm::Use &U = *UI;
+ ++UI;
+ auto *C = dyn_cast<llvm::Constant>(U.getUser());
+ if (C && !isa<llvm::GlobalValue>(C)) {
+ C->handleOperandChange(GV, IndirectReplacement, &U);
+ } else if (!isa<llvm::InvokeInst>(U.getUser()) &&
+ !isa<llvm::CallInst>(U.getUser())) {
+ U.set(IndirectReplacement);
+ }
+ }
+}
+
+void CodeGenModule::RewriteAlwaysInlineFunction(llvm::Function *Fn) {
+ std::string Name = Fn->getName();
+ std::string InlineName = Name + ".inlinefunction";
+ Fn->setName(InlineName);
+
+ llvm::FunctionType *FT = Fn->getFunctionType();
+ llvm::LLVMContext &Ctx = getModule().getContext();
+ llvm::Function *StubFn =
+ llvm::Function::Create(FT, Fn->getLinkage(), Name, &getModule());
+ assert(StubFn->getName() == Name && "name was uniqued!");
+
+ // Insert the stub immediately after the original function. Helps with the
+ // fragile tests, among other things.
+ StubFn->removeFromParent();
+ TheModule.getFunctionList().insertAfter(Fn, StubFn);
+
+ StubFn->copyAttributesFrom(Fn);
+ StubFn->removeFnAttr(llvm::Attribute::AlwaysInline);
+ StubFn->setPersonalityFn(nullptr);
+
+ // AvailableExternally functions are replaced with a declaration.
+ // Everyone else gets a wrapper that musttail-calls the original function.
+ if (Fn->getLinkage() == llvm::GlobalValue::AvailableExternallyLinkage) {
+ StubFn->setLinkage(llvm::GlobalValue::ExternalLinkage);
+ } else {
+ llvm::BasicBlock *BB = llvm::BasicBlock::Create(Ctx, "entry", StubFn);
+ std::vector<llvm::Value *> Args;
+ for (llvm::Function::arg_iterator ai = StubFn->arg_begin();
+ ai != StubFn->arg_end(); ++ai)
+ Args.push_back(&*ai);
+ llvm::CallInst *CI = llvm::CallInst::Create(Fn, Args, "", BB);
+ CI->setCallingConv(Fn->getCallingConv());
+ CI->setTailCallKind(llvm::CallInst::TCK_MustTail);
+ CI->setAttributes(Fn->getAttributes());
+ if (FT->getReturnType()->isVoidTy())
+ llvm::ReturnInst::Create(Ctx, BB);
+ else
+ llvm::ReturnInst::Create(Ctx, CI, BB);
+ }
+
+ if (Fn->hasComdat())
+ StubFn->setComdat(Fn->getComdat());
+
+ Fn->setLinkage(llvm::GlobalValue::InternalLinkage);
+ Fn->setDLLStorageClass(llvm::GlobalVariable::DefaultStorageClass);
+ Fn->setVisibility(llvm::GlobalValue::DefaultVisibility);
+
+ RewriteUsesOfAlwaysInlineFunction(Fn, StubFn);
+}
+
+void CodeGenModule::RewriteAlwaysInlineFunctions() {
+ for (llvm::Function &Fn : TheModule.functions()) {
+ if (Fn.hasFnAttribute(llvm::Attribute::AlwaysInline)) {
+ RewriteAlwaysInlineFunction(&Fn);
+ }
+ }
}
void CodeGenModule::UpdateCompletedType(const TagDecl *TD) {
Index: lib/CodeGen/CGCXX.cpp
===================================================================
--- lib/CodeGen/CGCXX.cpp
+++ lib/CodeGen/CGCXX.cpp
@@ -102,6 +102,9 @@
D->getType()->getAs<FunctionType>()->getCallConv())
return true;
+ if (BaseD->hasAttr<AlwaysInlineAttr>())
+ return true;
+
return TryEmitDefinitionAsAlias(GlobalDecl(D, Dtor_Base),
GlobalDecl(BaseD, Dtor_Base),
false);
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
