sdmitriev created this revision.
sdmitriev added a reviewer: ABataev.
Herald added a reviewer: alexshap.
Herald added a reviewer: jdoerfert.
Herald added a project: clang.
Herald added a subscriber: cfe-commits.
Fat object size has significantly increased after D65819
<https://reviews.llvm.org/D65819> which changed bundler tool to add host object
as a normal bundle to the fat output which almost doubled its size. That patch
was fixing the following issues
1. Problems associated with the partial linking - global constructors were not
called for partially linking objects which clearly resulted in incorrect
behavior.
2. Eliminating "junk" target object sections from the linked binary on the host
side.
The first problem is no longer relevant because we do not use partial linking
for creating fat objects anymore. Target objects sections are now inserted into
the resulting fat object with a help of llvm-objcopy tool.
The second issue, "junk" sections in the linked host binary, has been fixed in
D73408 <https://reviews.llvm.org/D73408> by adding "exclude" flag to the fat
object's sections which contain target objects. This flag tells linker to drop
section from the inputs when linking executable or shared library, therefore
these sections will not be propagated in the linked binary.
Since both problems have been solved, we can revert D65819
<https://reviews.llvm.org/D65819> changes to reduce fat object size and this
patch essentially is doing that.
Repository:
rG LLVM Github Monorepo
https://reviews.llvm.org/D73642
Files:
clang/test/Driver/clang-offload-bundler.c
clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp
Index: clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp
===================================================================
--- clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp
+++ clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp
@@ -30,7 +30,6 @@
#include "llvm/Support/Error.h"
#include "llvm/Support/ErrorOr.h"
#include "llvm/Support/FileSystem.h"
-#include "llvm/Support/FileUtilities.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/Program.h"
@@ -433,11 +432,16 @@
Error ReadBundleEnd(MemoryBuffer &Input) final { return Error::success(); }
Error ReadBundle(raw_fd_ostream &OS, MemoryBuffer &Input) final {
- Expected<StringRef> Content = CurrentSection->getContents();
- if (!Content)
- return Content.takeError();
+ Expected<StringRef> ContentOrErr = CurrentSection->getContents();
+ if (!ContentOrErr)
+ return ContentOrErr.takeError();
+ StringRef Content = *ContentOrErr;
- OS.write(Content->data(), Content->size());
+ // Copy fat object contents to the output when extracting host bundle.
+ if (Content.size() == 1u && Content.front() == 0)
+ Content = StringRef(Input.getBufferStart(), Input.getBufferSize());
+
+ OS.write(Content.data(), Content.size());
return Error::success();
}
@@ -486,22 +490,60 @@
// to pass down to llvm-objcopy.
OS.close();
+ // Temporary files that need to be removed.
+ struct TempFileList : public SmallVector<SmallString<128u>, 2u> {
+ ~TempFileList() {
+ for (const auto &File : *this)
+ sys::fs::remove(File);
+ clear();
+ }
+
+ Expected<StringRef> Create(Optional<ArrayRef<char>> Contents) {
+ SmallString<128u> FileName;
+ if (std::error_code EC = sys::fs::createTemporaryFile(
+ "clang-offload-bundler", "tmp", FileName))
+ return createFileError(FileName, EC);
+ push_back(FileName);
+
+ if (Contents) {
+ std::error_code EC;
+ raw_fd_ostream HostFile(FileName, EC);
+ if (EC)
+ return createFileError(FileName, EC);
+ HostFile.write(Contents->data(), Contents->size());
+ }
+ return back();
+ }
+ } TempFiles;
+
// Create an intermediate temporary file to save object after the first
// llvm-objcopy run.
- SmallString<128u> IntermediateObj;
- if (std::error_code EC = sys::fs::createTemporaryFile(
- "clang-offload-bundler", "tmp", IntermediateObj))
- return createFileError(IntermediateObj, EC);
- FileRemover IntermediateObjRemover(IntermediateObj);
+ Expected<SmallString<128u>> IntermediateObjOrErr = TempFiles.Create(None);
+ if (!IntermediateObjOrErr)
+ return IntermediateObjOrErr.takeError();
+ const SmallString<128u> &IntermediateObj = *IntermediateObjOrErr;
// Compose llvm-objcopy command line for add target objects' sections.
BumpPtrAllocator Alloc;
StringSaver SS{Alloc};
SmallVector<StringRef, 8u> ObjcopyArgs{"llvm-objcopy"};
- for (unsigned I = 0; I < NumberOfInputs; ++I)
+ for (unsigned I = 0; I < NumberOfInputs; ++I) {
+ StringRef InputFile = InputFileNames[I];
+ if (I == HostInputIndex) {
+ // Special handling for the host bundle. We do not need to add a
+ // standard bundle for the host object since we are going to use fat
+ // object as a host object. Therefore use dummy contents (one zero byte)
+ // when creating section for the host bundle.
+ Expected<StringRef> TempFileOrErr = TempFiles.Create(ArrayRef<char>(0));
+ if (!TempFileOrErr)
+ return TempFileOrErr.takeError();
+ InputFile = *TempFileOrErr;
+ }
+
ObjcopyArgs.push_back(SS.save(Twine("--add-section=") +
OFFLOAD_BUNDLER_MAGIC_STR + TargetNames[I] +
- "=" + InputFileNames[I]));
+ "=" + InputFile));
+ }
ObjcopyArgs.push_back(InputFileNames[HostInputIndex]);
ObjcopyArgs.push_back(IntermediateObj);
Index: clang/test/Driver/clang-offload-bundler.c
===================================================================
--- clang/test/Driver/clang-offload-bundler.c
+++ clang/test/Driver/clang-offload-bundler.c
@@ -253,16 +253,16 @@
// RUN: clang-offload-bundler -type=o -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -inputs=%t.o,%t.tgt1,%t.tgt2 -outputs=%t.bundle3.o -### 2>&1 \
// RUN: | FileCheck %s -DHOST=%itanium_abi_triple -DINOBJ1=%t.o -DINOBJ2=%t.tgt1 -DINOBJ3=%t.tgt2 -DOUTOBJ=%t.bundle3.o --check-prefix CK-OBJ-CMD
-// CK-OBJ-CMD: llvm-objcopy{{(.exe)?}}" "--add-section=__CLANG_OFFLOAD_BUNDLE__host-[[HOST]]=[[INOBJ1]]" "--add-section=__CLANG_OFFLOAD_BUNDLE__openmp-powerpc64le-ibm-linux-gnu=[[INOBJ2]]" "--add-section=__CLANG_OFFLOAD_BUNDLE__openmp-x86_64-pc-linux-gnu=[[INOBJ3]]" "[[INOBJ1]]" "[[TEMPOBJ:.*]]"
+// CK-OBJ-CMD: llvm-objcopy{{(.exe)?}}" "--add-section=__CLANG_OFFLOAD_BUNDLE__host-[[HOST]]={{.*}}" "--add-section=__CLANG_OFFLOAD_BUNDLE__openmp-powerpc64le-ibm-linux-gnu=[[INOBJ2]]" "--add-section=__CLANG_OFFLOAD_BUNDLE__openmp-x86_64-pc-linux-gnu=[[INOBJ3]]" "[[INOBJ1]]" "[[TEMPOBJ:.*]]"
// CK-OBJ-CMD: llvm-objcopy{{(.exe)?}}" "--set-section-flags=__CLANG_OFFLOAD_BUNDLE__host-[[HOST]]=readonly,exclude" "--set-section-flags=__CLANG_OFFLOAD_BUNDLE__openmp-powerpc64le-ibm-linux-gnu=readonly,exclude" "--set-section-flags=__CLANG_OFFLOAD_BUNDLE__openmp-x86_64-pc-linux-gnu=readonly,exclude" "[[TEMPOBJ]]" "[[OUTOBJ]]"
// RUN: clang-offload-bundler -type=o -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -inputs=%t.o,%t.tgt1,%t.tgt2 -outputs=%t.bundle3.o
// RUN: clang-offload-bundler -type=o -targets=host-%itanium_abi_triple,openmp-powerpc64le-ibm-linux-gnu,openmp-x86_64-pc-linux-gnu -outputs=%t.res.o,%t.res.tgt1,%t.res.tgt2 -inputs=%t.bundle3.o -unbundle
-// RUN: diff %t.o %t.res.o
+// RUN: diff %t.bundle3.o %t.res.o
// RUN: diff %t.tgt1 %t.res.tgt1
// RUN: diff %t.tgt2 %t.res.tgt2
// RUN: clang-offload-bundler -type=o -targets=openmp-powerpc64le-ibm-linux-gnu,host-%itanium_abi_triple,openmp-x86_64-pc-linux-gnu -outputs=%t.res.tgt1,%t.res.o,%t.res.tgt2 -inputs=%t.bundle3.o -unbundle
-// RUN: diff %t.o %t.res.o
+// RUN: diff %t.bundle3.o %t.res.o
// RUN: diff %t.tgt1 %t.res.tgt1
// RUN: diff %t.tgt2 %t.res.tgt2
// RUN: clang-offload-bundler -type=o -targets=openmp-powerpc64le-ibm-linux-gnu -outputs=%t.res.tgt1 -inputs=%t.bundle3.o -unbundle
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
