vitalybuka updated this revision to Diff 134657.
vitalybuka marked 6 inline comments as done.
vitalybuka added a comment.
addressed comments
https://reviews.llvm.org/D42995
Files:
clang/lib/CodeGen/BackendUtil.cpp
clang/test/CodeGen/Inputs/thinlto-distributed-backend-skip.bc
clang/test/CodeGen/thinlto-distributed-backend-skip.ll
llvm/include/llvm/IR/ModuleSummaryIndex.h
llvm/lib/Bitcode/Reader/BitcodeReader.cpp
llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
llvm/test/tools/gold/X86/v1.12/thinlto_emit_linked_objects.ll
llvm/tools/gold/gold-plugin.cpp
Index: llvm/tools/gold/gold-plugin.cpp
===================================================================
--- llvm/tools/gold/gold-plugin.cpp
+++ llvm/tools/gold/gold-plugin.cpp
@@ -811,9 +811,14 @@
// final link. Frequently the distributed build system will want to
// confirm that all expected outputs are created based on all of the
// modules provided to the linker.
+// If SkipModule is true then .thinlto.bc should contain just
+// SkipModuleByDistributedBackend flag which requests distributed backend
+// to skip the compilation of the corresponding module and produce an empty
+// object file.
static void writeEmptyDistributedBuildOutputs(const std::string &ModulePath,
const std::string &OldPrefix,
- const std::string &NewPrefix) {
+ const std::string &NewPrefix,
+ bool SkipModule) {
std::string NewModulePath =
getThinLTOOutputFile(ModulePath, OldPrefix, NewPrefix);
std::error_code EC;
@@ -823,6 +828,12 @@
if (EC)
message(LDPL_FATAL, "Failed to write '%s': %s",
(NewModulePath + ".thinlto.bc").c_str(), EC.message().c_str());
+
+ if (SkipModule) {
+ ModuleSummaryIndex Index(false);
+ Index.setSkipModuleByDistributedBackend();
+ WriteIndexToFile(Index, OS, nullptr);
+ }
}
if (options::thinlto_emit_imports_files) {
raw_fd_ostream OS(NewModulePath + ".imports", EC,
@@ -878,6 +889,11 @@
assert(ObjFilename.second);
if (const void *View = getSymbolsAndView(F))
addModule(*Lto, F, View, ObjFilename.first->first());
+ else if (options::thinlto_index_only) {
+ ObjFilename.first->second = true;
+ writeEmptyDistributedBuildOutputs(Identifier, OldPrefix, NewPrefix,
+ /* SkipModule */ true);
+ }
}
SmallString<128> Filename;
@@ -895,7 +911,7 @@
auto AddStream =
[&](size_t Task) -> std::unique_ptr<lto::NativeObjectStream> {
IsTemporary[Task] = !SaveTemps;
- int FD = getOutputFileName(Filename, /*TempOutFile=*/!SaveTemps,
+ int FD = getOutputFileName(Filename, /* TempOutFile */ !SaveTemps,
Filenames[Task], Task);
return llvm::make_unique<lto::NativeObjectStream>(
llvm::make_unique<llvm::raw_fd_ostream>(FD, true));
@@ -920,7 +936,7 @@
for (auto &Identifier : ObjectToIndexFileState)
if (!Identifier.getValue())
writeEmptyDistributedBuildOutputs(Identifier.getKey(), OldPrefix,
- NewPrefix);
+ NewPrefix, /* SkipModule */ false);
if (options::TheOutputType == options::OT_DISABLE ||
options::TheOutputType == options::OT_BC_ONLY)
Index: llvm/test/tools/gold/X86/v1.12/thinlto_emit_linked_objects.ll
===================================================================
--- llvm/test/tools/gold/X86/v1.12/thinlto_emit_linked_objects.ll
+++ llvm/test/tools/gold/X86/v1.12/thinlto_emit_linked_objects.ll
@@ -31,6 +31,24 @@
; RUN: ls %t2.o.imports
; RUN: ls %t3.o.imports
+; Regular *thinlto.bc file. "SkipModuleByDistributedBackend" flag (0x2)
+; should not be set.
+; RUN: llvm-bcanalyzer --dump %t1.o.thinlto.bc | FileCheck %s -check-prefixes=CHECK-BC1
+; CHECK-BC1: <GLOBALVAL_SUMMARY_BLOCK
+; CHECK-BC1: <FLAGS op0=1/>
+; CHECK-BC1: </GLOBALVAL_SUMMARY_BLOCK
+
+; Nothing interesting in the corresponding object file, so
+; "SkipModuleByDistributedBackend" flag (0x2) should be set.
+; RUN: llvm-bcanalyzer --dump %t2.o.thinlto.bc | FileCheck %s -check-prefixes=CHECK-BC2
+; CHECK-BC2: <GLOBALVAL_SUMMARY_BLOCK
+; CHECK-BC2: <FLAGS op0=2/>
+; CHECK-BC2: </GLOBALVAL_SUMMARY_BLOCK
+
+; Empty as the corresponding object file is not ThinTLO.
+; RUN: not llvm-bcanalyzer --dump %t3.o.thinlto.bc 2>&1 | FileCheck %s -check-prefixes=CHECK-BC3
+; CHECK-BC3: LLVM ERROR: Unexpected end of file
+
; RUN: cat %t.index | FileCheck %s
; CHECK: thinlto_emit_linked_objects.ll.tmp1.o
; CHECK-NOT: thinlto_emit_linked_objects.ll.tmp2.o
Index: llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
===================================================================
--- llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -3604,10 +3604,13 @@
Stream.EnterSubblock(bitc::GLOBALVAL_SUMMARY_BLOCK_ID, 3);
Stream.EmitRecord(bitc::FS_VERSION, ArrayRef<uint64_t>{INDEX_VERSION});
- // Write the index flags. Currently we only write a single flag, the value of
- // withGlobalValueDeadStripping, which only applies to the combined index.
- Stream.EmitRecord(bitc::FS_FLAGS,
- ArrayRef<uint64_t>{Index.withGlobalValueDeadStripping()});
+ // Write the index flags.
+ uint64_t Flags = 0;
+ if (Index.withGlobalValueDeadStripping())
+ Flags |= 0x1;
+ if (Index.skipModuleByDistributedBackend())
+ Flags |= 0x2;
+ Stream.EmitRecord(bitc::FS_FLAGS, ArrayRef<uint64_t>{Flags});
for (const auto &GVI : valueIds()) {
Stream.EmitRecord(bitc::FS_VALUE_GUID,
Index: llvm/lib/Bitcode/Reader/BitcodeReader.cpp
===================================================================
--- llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -5186,11 +5186,14 @@
case bitc::FS_FLAGS: { // [flags]
uint64_t Flags = Record[0];
// Scan flags (set only on the combined index).
- assert(Flags <= 1 && "Unexpected bits in flag");
+ assert(Flags <= 0x3 && "Unexpected bits in flag");
// 1 bit: WithGlobalValueDeadStripping flag.
if (Flags & 0x1)
TheIndex.setWithGlobalValueDeadStripping();
+ // 1 bit: SkipModuleByDistributedBackend flag.
+ if (Flags & 0x2)
+ TheIndex.setSkipModuleByDistributedBackend();
break;
}
case bitc::FS_VALUE_GUID: { // [valueid, refguid]
Index: llvm/include/llvm/IR/ModuleSummaryIndex.h
===================================================================
--- llvm/include/llvm/IR/ModuleSummaryIndex.h
+++ llvm/include/llvm/IR/ModuleSummaryIndex.h
@@ -682,6 +682,13 @@
/// considered live.
bool WithGlobalValueDeadStripping = false;
+ /// Indicates that distributed backend should skip compilation of the
+ /// module. Flag is suppose to be set by distributed ThinLTO indexing
+ /// when it detected that the module is not needed during the final
+ /// linking. As result distributed backend should just output a minimal
+ /// valid object file.
+ bool SkipModuleByDistributedBackend = false;
+
/// If true then we're performing analysis of IR module, filling summary
/// accordingly. The value of 'false' means we're reading summary from
/// BC or YAML source. Affects the type of value stored in NameOrGV union
@@ -718,6 +725,13 @@
WithGlobalValueDeadStripping = true;
}
+ bool skipModuleByDistributedBackend() const {
+ return SkipModuleByDistributedBackend;
+ }
+ void setSkipModuleByDistributedBackend() {
+ SkipModuleByDistributedBackend = true;
+ }
+
bool isGlobalValueLive(const GlobalValueSummary *GVS) const {
return !WithGlobalValueDeadStripping || GVS->isLive();
}
Index: clang/test/CodeGen/thinlto-distributed-backend-skip.ll
===================================================================
--- /dev/null
+++ clang/test/CodeGen/thinlto-distributed-backend-skip.ll
@@ -0,0 +1,21 @@
+; REQUIRES: x86-registered-target
+
+; Check that ThinLTO backend respects "SkipModuleByDistributedBackend"
+; flag which can be set by indexing.
+
+; RUN: opt -thinlto-bc -o %t.o %s
+
+; RUN: %clang_cc1 -triple x86_64-grtev4-linux-gnu \
+; RUN: -fthinlto-index=%S/Inputs/thinlto-distributed-backend-skip.bc \
+; RUN: -emit-llvm -o - -x ir %t.o | FileCheck %s
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-grtev4-linux-gnu"
+
+; CHECK: "empty"
+; CHECK: target triple =
+; CHECK-NOT: @main
+define i32 @main() {
+entry:
+ ret i32 0
+}
Index: clang/lib/CodeGen/BackendUtil.cpp
===================================================================
--- clang/lib/CodeGen/BackendUtil.cpp
+++ clang/lib/CodeGen/BackendUtil.cpp
@@ -1147,6 +1147,7 @@
const llvm::DataLayout &TDesc, Module *M,
BackendAction Action,
std::unique_ptr<raw_pwrite_stream> OS) {
+ std::unique_ptr<llvm::Module> EmptyModule;
if (!CGOpts.ThinLTOIndexFile.empty()) {
// If we are performing a ThinLTO importing compile, load the function index
// into memory and pass it into runThinLTOBackend, which will run the
@@ -1164,11 +1165,22 @@
// A null CombinedIndex means we should skip ThinLTO compilation
// (LLVM will optionally ignore empty index files, returning null instead
// of an error).
- bool DoThinLTOBackend = CombinedIndex != nullptr;
- if (DoThinLTOBackend) {
- runThinLTOBackend(CombinedIndex.get(), M, HeaderOpts, CGOpts, TOpts,
- LOpts, std::move(OS), CGOpts.SampleProfileFile, Action);
- return;
+ if (CombinedIndex) {
+ if (!CombinedIndex->skipModuleByDistributedBackend()) {
+ runThinLTOBackend(CombinedIndex.get(), M, HeaderOpts, CGOpts, TOpts,
+ LOpts, std::move(OS), CGOpts.SampleProfileFile,
+ Action);
+ return;
+ }
+ // Distributed indexing detected that nothing from the module is needed
+ // for the final linking. So we can skip the compilation. We sill need to
+ // output an empty object file to make sure that a linker does not fail
+ // trying to read it. Also for some features, like CFI, we must skip
+ // the compilation as CombinedIndex does not contain all required
+ // information.
+ EmptyModule = llvm::make_unique<llvm::Module>("empty", M->getContext());
+ EmptyModule->setTargetTriple(M->getTargetTriple());
+ M = EmptyModule.get();
}
}
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
