sfantao updated this revision to Diff 62550.
sfantao marked 6 inline comments as done.
sfantao added a comment.
- Remove \brief and return errors instead of crashing when temporary tiles
can't be created.
http://reviews.llvm.org/D21851
Files:
test/Driver/clang-offload-bundler.c
tools/clang-offload-bundler/ClangOffloadBundler.cpp
Index: tools/clang-offload-bundler/ClangOffloadBundler.cpp
===================================================================
--- tools/clang-offload-bundler/ClangOffloadBundler.cpp
+++ tools/clang-offload-bundler/ClangOffloadBundler.cpp
@@ -75,14 +75,23 @@
/// Magic string that marks the existence of offloading data.
#define OFFLOAD_BUNDLER_MAGIC_STR "__CLANG_OFFLOAD_BUNDLE__"
+/// Path to the current binary.
+static std::string BundlerExecutable;
+
/// Obtain the offload kind and real machine triple out of the target
/// information specified by the user.
static void getOffloadKindAndTriple(StringRef Target, StringRef &OffloadKind,
StringRef &Triple) {
auto KindTriplePair = Target.split('-');
OffloadKind = KindTriplePair.first;
Triple = KindTriplePair.second;
}
+static StringRef getTriple(StringRef Target) {
+ StringRef OffloadKind;
+ StringRef Triple;
+ getOffloadKindAndTriple(Target, OffloadKind, Triple);
+ return Triple;
+}
static bool hasHostKind(StringRef Target) {
StringRef OffloadKind;
StringRef Triple;
@@ -113,37 +122,37 @@
/// \a OS.
virtual void WriteBundleStart(raw_fd_ostream &OS, StringRef TargetTriple) = 0;
/// Write the marker that closes a bundle for the triple \a TargetTriple to \a
- /// OS.
- virtual void WriteBundleEnd(raw_fd_ostream &OS, StringRef TargetTriple) = 0;
+ /// OS. Return true if any error was found.
+ virtual bool WriteBundleEnd(raw_fd_ostream &OS, StringRef TargetTriple) = 0;
/// Write the bundle from \a Input into \a OS.
virtual void WriteBundle(raw_fd_ostream &OS, MemoryBuffer &Input) = 0;
FileHandler() {}
virtual ~FileHandler() {}
};
-// Handler for binary files. The bundled file will have the following format
-// (all integers are stored in little-endian format):
-//
-// "OFFLOAD_BUNDLER_MAGIC_STR" (ASCII encoding of the string)
-//
-// NumberOfOffloadBundles (8-byte integer)
-//
-// OffsetOfBundle1 (8-byte integer)
-// SizeOfBundle1 (8-byte integer)
-// NumberOfBytesInTripleOfBundle1 (8-byte integer)
-// TripleOfBundle1 (byte length defined before)
-//
-// ...
-//
-// OffsetOfBundleN (8-byte integer)
-// SizeOfBundleN (8-byte integer)
-// NumberOfBytesInTripleOfBundleN (8-byte integer)
-// TripleOfBundleN (byte length defined before)
-//
-// Bundle1
-// ...
-// BundleN
+/// Handler for binary files. The bundled file will have the following format
+/// (all integers are stored in little-endian format):
+///
+/// "OFFLOAD_BUNDLER_MAGIC_STR" (ASCII encoding of the string)
+///
+/// NumberOfOffloadBundles (8-byte integer)
+///
+/// OffsetOfBundle1 (8-byte integer)
+/// SizeOfBundle1 (8-byte integer)
+/// NumberOfBytesInTripleOfBundle1 (8-byte integer)
+/// TripleOfBundle1 (byte length defined before)
+///
+/// ...
+///
+/// OffsetOfBundleN (8-byte integer)
+/// SizeOfBundleN (8-byte integer)
+/// NumberOfBytesInTripleOfBundleN (8-byte integer)
+/// TripleOfBundleN (byte length defined before)
+///
+/// Bundle1
+/// ...
+/// BundleN
/// Read 8-byte integers to/from a buffer in little-endian format.
static uint64_t Read8byteIntegerFromBuffer(StringRef Buffer, size_t pos) {
@@ -300,24 +309,241 @@
}
}
void WriteBundleStart(raw_fd_ostream &OS, StringRef TargetTriple) {}
- void WriteBundleEnd(raw_fd_ostream &OS, StringRef TargetTriple) {}
+ bool WriteBundleEnd(raw_fd_ostream &OS, StringRef TargetTriple) {
+ return false;
+ }
void WriteBundle(raw_fd_ostream &OS, MemoryBuffer &Input) {
OS.write(Input.getBufferStart(), Input.getBufferSize());
}
BinaryFileHandler() : FileHandler() {}
~BinaryFileHandler() {}
};
-// Handler for text files. The bundled file will have the following format.
-//
-// "Comment OFFLOAD_BUNDLER_MAGIC_STR__START__ triple"
-// Bundle 1
-// "Comment OFFLOAD_BUNDLER_MAGIC_STR__END__ triple"
-// ...
-// "Comment OFFLOAD_BUNDLER_MAGIC_STR__START__ triple"
-// Bundle N
-// "Comment OFFLOAD_BUNDLER_MAGIC_STR__END__ triple"
+/// Handler for object files. The bundles are organized by sections with a
+/// designated name.
+///
+/// In order to bundle we create an IR file with the content of each section and
+/// use incremental linking to produce the resulting object. We also add section
+/// with a single byte to state the name of the component the main object file
+/// (the one we are bundling into) refers to.
+///
+/// To unbundle, we use just copy the contents of the designated section. If the
+/// requested bundle refer to the main object file, we just copy it with no
+/// changes.
+class ObjectFileHandler final : public FileHandler {
+
+ /// The object file we are currently dealing with.
+ ObjectFile &Obj;
+
+ /// Return the input file contents.
+ StringRef getInputFileContents() const { return Obj.getData(); }
+
+ /// Return true if the provided section is an offload section and return the
+ /// triple by reference.
+ static bool isOffloadSection(SectionRef CurSection,
+ StringRef &OffloadTriple) {
+ StringRef SectionName;
+ CurSection.getName(SectionName);
+
+ if (SectionName.empty())
+ return false;
+
+ // If it does not start with the reserved suffix, just skip this section.
+ if (!SectionName.startswith(OFFLOAD_BUNDLER_MAGIC_STR))
+ return false;
+
+ // Return the triple that is right after the reserved prefix.
+ OffloadTriple = SectionName.substr(sizeof(OFFLOAD_BUNDLER_MAGIC_STR) - 1);
+ return true;
+ }
+
+ /// Total number of inputs.
+ unsigned NumberOfInputs = 0;
+
+ /// Total number of processed inputs, i.e, inputs that were already
+ /// read from the buffers.
+ unsigned NumberOfProcessedInputs = 0;
+
+ /// LLVM context used to to create the auxiliar modules.
+ LLVMContext VMContext;
+
+ /// LLVM module used to create an object with all the bundle
+ /// components.
+ std::unique_ptr<Module> AuxModule;
+
+ /// The current triple we are working with.
+ StringRef CurrentTriple;
+
+ /// The name of the main input file.
+ StringRef MainInputFileName;
+
+ /// Iterator of the current and next section.
+ section_iterator CurrentSection;
+ section_iterator NextSection;
+
+public:
+ void ReadHeader(MemoryBuffer &Input) {}
+ StringRef ReadBundleStart(MemoryBuffer &Input) {
+
+ while (NextSection != Obj.section_end()) {
+ CurrentSection = NextSection;
+ ++NextSection;
+
+ StringRef OffloadTriple;
+ // Check if the current section name starts with the reserved prefix. If
+ // so, return the triple.
+ if (isOffloadSection(*CurrentSection, OffloadTriple))
+ return OffloadTriple;
+ }
+ return StringRef();
+ }
+ void ReadBundleEnd(MemoryBuffer &Input) {}
+ void ReadBundle(raw_fd_ostream &OS, MemoryBuffer &Input) {
+ // If the current section has size one, that means that the content we are
+ // interested in is the file itself. Otherwise it is the content of the
+ // section.
+ //
+ // TODO: Instead of copying the input file as is, deactivate the section
+ // that are no longer needed.
+
+ StringRef Content;
+ CurrentSection->getContents(Content);
+
+ if (Content.size() < 2)
+ OS.write(Input.getBufferStart(), Input.getBufferSize());
+ else
+ OS.write(Content.data(), Content.size());
+
+ return;
+ }
+
+ void WriteHeader(raw_fd_ostream &OS,
+ ArrayRef<std::unique_ptr<MemoryBuffer>> Inputs) {
+ // Record number of inputs.
+ NumberOfInputs = Inputs.size();
+
+ // Create an LLVM module to have the content we need to bundle.
+ auto *M = new Module("clang-offload-bundle", VMContext);
+ M->setTargetTriple(getTriple(TargetNames.front()));
+ AuxModule.reset(M);
+ }
+ void WriteBundleStart(raw_fd_ostream &OS, StringRef TargetTriple) {
+ ++NumberOfProcessedInputs;
+
+ // Record the triple we are using, that will be used to name the section we
+ // will create.
+ CurrentTriple = TargetTriple;
+ }
+ bool WriteBundleEnd(raw_fd_ostream &OS, StringRef TargetTriple) {
+ assert(NumberOfProcessedInputs <= NumberOfInputs &&
+ "Processing more inputs that actually exist!");
+
+ // If this is not the last output, we don't have to do anything.
+ if (NumberOfProcessedInputs != NumberOfInputs)
+ return false;
+
+ // Create the bitcode file name to write the resulting code to.
+ SmallString<128> BitcodeFileName;
+ if (sys::fs::createTemporaryFile("clang-offload-bundler", "bc",
+ BitcodeFileName)) {
+ llvm::errs() << "error: unable to create temporary file.\n";
+ return true;
+ }
+
+ // Write the bitcode to the temporary file.
+ {
+ std::error_code EC;
+ raw_fd_ostream BitcodeFile(BitcodeFileName, EC, sys::fs::F_None);
+ if (EC) {
+ llvm::errs() << "error: unable to open temporary file.\n";
+ return true;
+ }
+ WriteBitcodeToFile(AuxModule.get(), BitcodeFile);
+ }
+
+ // Find clang in order to create the bundle binary.
+ StringRef Dir = llvm::sys::path::parent_path(BundlerExecutable);
+
+ auto ClangBinary = sys::findProgramByName("clang", Dir);
+ if (ClangBinary.getError()) {
+ // Remove bitcode file.
+ sys::fs::remove(BitcodeFileName);
+
+ llvm::errs() << "error: unable to find 'clang' in path.\n";
+ return true;
+ }
+
+ // Do the incremental linking. We write to the output file directly. So, we
+ // close it and use the name to pass down to clang.
+ OS.close();
+ SmallString<128> TargetName = getTriple(TargetNames.front());
+ const char *ClangArgs[] = {"clang",
+ "-r",
+ "-target",
+ TargetName.c_str(),
+ "-o",
+ OutputFileNames.front().c_str(),
+ InputFileNames.front().c_str(),
+ BitcodeFileName.c_str(),
+ "-nostdlib",
+ nullptr};
+
+ if (sys::ExecuteAndWait(ClangBinary.get(), ClangArgs)) {
+ // Remove bitcode file.
+ sys::fs::remove(BitcodeFileName);
+
+ llvm::errs() << "error: incremental linking by external tool failed.\n";
+ return true;
+ }
+
+ // Remove bitcode file.
+ sys::fs::remove(BitcodeFileName);
+ return false;
+ }
+ void WriteBundle(raw_fd_ostream &OS, MemoryBuffer &Input) {
+ Module *M = AuxModule.get();
+
+ // Create the new section name, it will consist of the reserved prefix
+ // concatenated with the triple.
+ std::string SectionName = OFFLOAD_BUNDLER_MAGIC_STR;
+ SectionName += CurrentTriple;
+
+ // Create the constant with the content of the section. For the input we are
+ // bundling into (the first input), this is just a place-holder, so a single
+ // byte is sufficient.
+ Constant *Content;
+ if (NumberOfProcessedInputs == 1) {
+ uint8_t Byte[] = {0};
+ Content = ConstantDataArray::get(VMContext, Byte);
+ } else
+ Content = ConstantDataArray::get(
+ VMContext, ArrayRef<uint8_t>(reinterpret_cast<const uint8_t *>(
+ Input.getBufferStart()),
+ Input.getBufferSize()));
+
+ // Create the global in the desired section. We don't want these globals in
+ // the symbol table, so we mark them private.
+ auto *GV = new GlobalVariable(*M, Content->getType(), /*IsConstant=*/true,
+ GlobalVariable::PrivateLinkage, Content);
+ GV->setSection(SectionName);
+ }
+
+ ObjectFileHandler(ObjectFile &Obj)
+ : FileHandler(), Obj(Obj), CurrentSection(Obj.section_begin()),
+ NextSection(Obj.section_begin()) {}
+ ~ObjectFileHandler() {}
+};
+
+/// Handler for text files. The bundled file will have the following format.
+///
+/// "Comment OFFLOAD_BUNDLER_MAGIC_STR__START__ triple"
+/// Bundle 1
+/// "Comment OFFLOAD_BUNDLER_MAGIC_STR__END__ triple"
+/// ...
+/// "Comment OFFLOAD_BUNDLER_MAGIC_STR__START__ triple"
+/// Bundle N
+/// "Comment OFFLOAD_BUNDLER_MAGIC_STR__END__ triple"
class TextFileHandler final : public FileHandler {
/// String that begins a line comment.
StringRef Comment;
@@ -383,8 +609,9 @@
void WriteBundleStart(raw_fd_ostream &OS, StringRef TargetTriple) {
OS << BundleStartString << TargetTriple << "\n";
}
- void WriteBundleEnd(raw_fd_ostream &OS, StringRef TargetTriple) {
+ bool WriteBundleEnd(raw_fd_ostream &OS, StringRef TargetTriple) {
OS << BundleEndString << TargetTriple << "\n";
+ return false;
}
void WriteBundle(raw_fd_ostream &OS, MemoryBuffer &Input) {
OS << Input.getBuffer();
@@ -400,6 +627,28 @@
}
};
+/// Return an appropriate object file handler. We use the specific object
+/// handler if we know how to deal with that format, otherwise we use a default
+/// binary file handler.
+static FileHandler *CreateObjectFileHandler(MemoryBuffer &FirstInput) {
+ // Check if the input file format is one that we know how to deal with.
+ Expected<std::unique_ptr<Binary>> BinaryOrErr = createBinary(FirstInput);
+
+ // Failed to open the input as a known binary. Use the default binary handler.
+ if (!BinaryOrErr)
+ return new BinaryFileHandler();
+
+ // We only support regular object files. If this is not an object file,
+ // default to the binary handler. The handler will be owned by the client of
+ // this function.
+ ObjectFile *Obj = dyn_cast<ObjectFile>(BinaryOrErr.get().release());
+
+ if (!Obj)
+ return new BinaryFileHandler();
+
+ return new ObjectFileHandler(*Obj);
+}
+
/// Return an appropriate handler given the input files and options.
static FileHandler *CreateFileHandler(MemoryBuffer &FirstInput) {
if (FilesType == "i")
@@ -413,7 +662,7 @@
if (FilesType == "s")
return new TextFileHandler(/*Comment=*/"#");
if (FilesType == "o")
- return new BinaryFileHandler();
+ return CreateObjectFileHandler(FirstInput);
if (FilesType == "gch")
return new BinaryFileHandler();
if (FilesType == "ast")
@@ -463,12 +712,14 @@
// Write header.
FH.get()->WriteHeader(OutputFile, InputBuffers);
- // Write all bundles along with the start/end markers.
+ // Write all bundles along with the start/end markers. If an error was found
+ // writing the end of the bundle component, abort the bundle writing.
auto Input = InputBuffers.begin();
for (auto &Triple : TargetNames) {
FH.get()->WriteBundleStart(OutputFile, Triple);
FH.get()->WriteBundle(OutputFile, *Input->get());
- FH.get()->WriteBundleEnd(OutputFile, Triple);
+ if (FH.get()->WriteBundleEnd(OutputFile, Triple))
+ return true;
++Input;
}
return false;
@@ -666,5 +917,10 @@
if (Error)
return 1;
+ // Save the current executable directory as it will be useful to find other
+ // tools.
+ BundlerExecutable =
+ llvm::sys::fs::getMainExecutable(argv[0], &BundlerExecutable);
+
return Unbundle ? UnbundleFiles() : BundleFiles();
}
Index: test/Driver/clang-offload-bundler.c
===================================================================
--- test/Driver/clang-offload-bundler.c
+++ test/Driver/clang-offload-bundler.c
@@ -6,6 +6,7 @@
// RUN: %clang -O0 -target powerpc64le-ibm-linux-gnu %s -S -emit-llvm -o %t.ll
// RUN: %clang -O0 -target powerpc64le-ibm-linux-gnu %s -c -emit-llvm -o %t.bc
// RUN: %clang -O0 -target powerpc64le-ibm-linux-gnu %s -S -o %t.s
+// RUN: %clang -O0 -target powerpc64le-ibm-linux-gnu %s -c -o %t.o
// RUN: %clang -O0 -target powerpc64le-ibm-linux-gnu %s -emit-ast -o %t.ast
//
@@ -181,6 +182,67 @@
// RUN: diff %t.empty %t.res.tgt1
// RUN: diff %t.empty %t.res.tgt2
+//
+// Check object bundle/unbundle. The content should be bundled into an ELF section (we are using a PowerPC little-endian host which uses ELF).
+//
+// RUN: clang-offload-bundler -type=o -targets=host-powerpc64le-ibm-linux-gnu,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -inputs=%t.o,%t.tgt1,%t.tgt2 -outputs=%t.bundle3.o
+// RUN: llvm-readobj -sections %t.bundle3.o | FileCheck %s --check-prefix CK-OBJ
+// RUN: clang-offload-bundler -type=o -targets=host-powerpc64le-ibm-linux-gnu,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -outputs=%t.res.o,%t.res.tgt1,%t.res.tgt2 -inputs=%t.bundle3.o -unbundle
+// RUN: diff %t.bundle3.o %t.res.o
+// RUN: diff %t.tgt1 %t.res.tgt1
+// RUN: diff %t.tgt2 %t.res.tgt2
+// CK-OBJ: Section {
+// CK-OBJ: Index:
+// CK-OBJ: Name: __CLANG_OFFLOAD_BUNDLE__host-powerpc64le-ibm-linux-gnu (
+// CK-OBJ: Type: SHT_PROGBITS (0x1)
+// CK-OBJ: Flags [ (0x2)
+// CK-OBJ: SHF_ALLOC (0x2)
+// CK-OBJ: ]
+// CK-OBJ: Address: 0x0
+// CK-OBJ: Offset: 0x
+// CK-OBJ: Size: 1
+// CK-OBJ: Link: 0
+// CK-OBJ: Info: 0
+// CK-OBJ: AddressAlignment:
+// CK-OBJ: EntrySize: 0
+// CK-OBJ: }
+// CK-OBJ: Section {
+// CK-OBJ: Index:
+// CK-OBJ: Name: __CLANG_OFFLOAD_BUNDLE__openmp-powerpc64le-ibm-linux-gnu (
+// CK-OBJ: Type: SHT_PROGBITS (0x1)
+// CK-OBJ: Flags [ (0x2)
+// CK-OBJ: SHF_ALLOC (0x2)
+// CK-OBJ: ]
+// CK-OBJ: Address: 0x0
+// CK-OBJ: Offset: 0x
+// CK-OBJ: Size: 25
+// CK-OBJ: Link: 0
+// CK-OBJ: Info: 0
+// CK-OBJ: AddressAlignment:
+// CK-OBJ: EntrySize: 0
+// CK-OBJ: }
+// CK-OBJ: Section {
+// CK-OBJ: Index:
+// CK-OBJ: Name: __CLANG_OFFLOAD_BUNDLE__openmp-x86_64-pc-linux-gnu (
+// CK-OBJ: Type: SHT_PROGBITS (0x1)
+// CK-OBJ: Flags [ (0x2)
+// CK-OBJ: SHF_ALLOC (0x2)
+// CK-OBJ: ]
+// CK-OBJ: Address: 0x0
+// CK-OBJ: Offset: 0x
+// CK-OBJ: Size: 25
+// CK-OBJ: Link: 0
+// CK-OBJ: Info: 0
+// CK-OBJ: AddressAlignment:
+// CK-OBJ: EntrySize: 0
+// CK-OBJ: }
+
+// Check if we can unbundle a file with no magic strings.
+// RUN: clang-offload-bundler -type=o -targets=host-powerpc64le-ibm-linux-gnu,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -outputs=%t.res.o,%t.res.tgt1,%t.res.tgt2 -inputs=%t.o -unbundle
+// RUN: diff %t.o %t.res.o
+// RUN: diff %t.empty %t.res.tgt1
+// RUN: diff %t.empty %t.res.tgt2
+
// Some code so that we can create a binary out of this file.
int A = 0;
void test_func(void) {
_______________________________________________
cfe-commits mailing list
[email protected]
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
