noajshu created this revision.
Herald added subscribers: ormris, dexonsmith, steven_wu, hiraditya,
arichardson, sbc100, emaste.
Herald added a reviewer: MaskRay.
Herald added a reviewer: gkm.
Herald added a project: lld-macho.
Herald added a reviewer: lld-macho.
noajshu requested review of this revision.
Herald added subscribers: llvm-commits, cfe-commits, aheejin.
Herald added projects: clang, LLVM.
This diff makes several amendments to the local file caching mechanism which
was migrated from ThinLTO to Support in D112751
<https://reviews.llvm.org/D112751> in response to follow-up discussion on that
commit.
Repository:
rG LLVM Github Monorepo
https://reviews.llvm.org/D113080
Files:
clang/lib/CodeGen/BackendUtil.cpp
lld/COFF/LTO.cpp
lld/ELF/LTO.cpp
lld/MachO/LTO.cpp
lld/wasm/LTO.cpp
llvm/include/llvm/LTO/LTO.h
llvm/include/llvm/Support/Caching.h
llvm/lib/LTO/LTO.cpp
llvm/lib/LTO/LTOCodeGenerator.cpp
llvm/lib/Support/Caching.cpp
llvm/tools/gold/gold-plugin.cpp
llvm/tools/llvm-lto/llvm-lto.cpp
llvm/tools/llvm-lto2/llvm-lto2.cpp
Index: llvm/tools/llvm-lto2/llvm-lto2.cpp
===================================================================
--- llvm/tools/llvm-lto2/llvm-lto2.cpp
+++ llvm/tools/llvm-lto2/llvm-lto2.cpp
@@ -362,20 +362,20 @@
if (HasErrors)
return 1;
- auto AddStream = [&](size_t Task) -> std::unique_ptr<NativeObjectStream> {
+ auto AddStream = [&](size_t Task) -> std::unique_ptr<CachedFileStream> {
std::string Path = OutputFilename + "." + utostr(Task);
std::error_code EC;
auto S = std::make_unique<raw_fd_ostream>(Path, EC, sys::fs::OF_None);
check(EC, Path);
- return std::make_unique<NativeObjectStream>(std::move(S));
+ return std::make_unique<CachedFileStream>(std::move(S));
};
auto AddBuffer = [&](size_t Task, std::unique_ptr<MemoryBuffer> MB) {
*AddStream(Task)->OS << MB->getBuffer();
};
- NativeObjectCache Cache;
+ FileCache Cache;
if (!CacheDir.empty())
Cache = check(localCache("ThinLTO", "Thin", CacheDir, AddBuffer),
"failed to create cache");
Index: llvm/tools/llvm-lto/llvm-lto.cpp
===================================================================
--- llvm/tools/llvm-lto/llvm-lto.cpp
+++ llvm/tools/llvm-lto/llvm-lto.cpp
@@ -1097,7 +1097,7 @@
error("writing merged module failed.");
}
- auto AddStream = [&](size_t Task) -> std::unique_ptr<NativeObjectStream> {
+ auto AddStream = [&](size_t Task) -> std::unique_ptr<CachedFileStream> {
std::string PartFilename = OutputFilename;
if (Parallelism != 1)
PartFilename += "." + utostr(Task);
@@ -1107,7 +1107,7 @@
std::make_unique<raw_fd_ostream>(PartFilename, EC, sys::fs::OF_None);
if (EC)
error("error opening the file '" + PartFilename + "': " + EC.message());
- return std::make_unique<NativeObjectStream>(std::move(S));
+ return std::make_unique<CachedFileStream>(std::move(S));
};
if (!CodeGen.compileOptimized(AddStream, Parallelism))
Index: llvm/tools/gold/gold-plugin.cpp
===================================================================
--- llvm/tools/gold/gold-plugin.cpp
+++ llvm/tools/gold/gold-plugin.cpp
@@ -1081,11 +1081,11 @@
size_t MaxTasks = Lto->getMaxTasks();
std::vector<std::pair<SmallString<128>, bool>> Files(MaxTasks);
- auto AddStream = [&](size_t Task) -> std::unique_ptr<NativeObjectStream> {
+ auto AddStream = [&](size_t Task) -> std::unique_ptr<CachedFileStream> {
Files[Task].second = !SaveTemps;
int FD = getOutputFileName(Filename, /* TempOutFile */ !SaveTemps,
Files[Task].first, Task);
- return std::make_unique<NativeObjectStream>(
+ return std::make_unique<CachedFileStream>(
std::make_unique<llvm::raw_fd_ostream>(FD, true));
};
@@ -1093,7 +1093,7 @@
*AddStream(Task)->OS << MB->getBuffer();
};
- NativeObjectCache Cache;
+ FileCache Cache;
if (!options::cache_dir.empty())
Cache = check(localCache("ThinLTO", "Thin", options::cache_dir, AddBuffer));
Index: llvm/lib/Support/Caching.cpp
===================================================================
--- llvm/lib/Support/Caching.cpp
+++ llvm/lib/Support/Caching.cpp
@@ -1,4 +1,4 @@
-//===-Caching.cpp - LLVM File Cache Handling ------------------------------===//
+//===-Caching.cpp - LLVM Local File Cache ---------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
@@ -6,18 +6,17 @@
//
//===----------------------------------------------------------------------===//
//
-// This file implements the Caching used by ThinLTO.
+// This file implements the localCache function, which simplifies creating,
+// adding to, and querying a local file system cache. localCache takes care of
+// periodically pruning older files from the cache using a CachePruningPolicy.
//
//===----------------------------------------------------------------------===//
#include "llvm/Support/Caching.h"
-#include "llvm/ADT/StringExtras.h"
#include "llvm/Support/Errc.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/Path.h"
-#include "llvm/Support/Process.h"
-#include "llvm/Support/raw_ostream.h"
#if !defined(_MSC_VER) && !defined(__MINGW32__)
#include <unistd.h>
@@ -27,10 +26,10 @@
using namespace llvm;
-Expected<NativeObjectCache> llvm::localCache(Twine CacheNameRef,
- Twine TempFilePrefixRef,
- Twine CacheDirectoryPathRef,
- AddBufferFn AddBuffer) {
+Expected<FileCache> llvm::localCache(Twine CacheNameRef,
+ Twine TempFilePrefixRef,
+ Twine CacheDirectoryPathRef,
+ AddBufferFn AddBuffer) {
if (std::error_code EC = sys::fs::create_directories(CacheDirectoryPathRef))
return errorCodeToError(EC);
@@ -75,9 +74,9 @@
report_fatal_error(Twine("Failed to open cache file ") + EntryPath +
": " + EC.message() + "\n");
- // This native object stream is responsible for commiting the resulting
- // file to the cache and calling AddBuffer to add it to the link.
- struct CacheStream : NativeObjectStream {
+ // This file stream is responsible for commiting the resulting file to the
+ // cache and calling AddBuffer to add it to the link.
+ struct CacheStream : CachedFileStream {
AddBufferFn AddBuffer;
sys::fs::TempFile TempFile;
std::string EntryPath;
@@ -86,7 +85,7 @@
CacheStream(std::unique_ptr<raw_pwrite_stream> OS, AddBufferFn AddBuffer,
sys::fs::TempFile TempFile, std::string EntryPath,
unsigned Task)
- : NativeObjectStream(std::move(OS)), AddBuffer(std::move(AddBuffer)),
+ : CachedFileStream(std::move(OS)), AddBuffer(std::move(AddBuffer)),
TempFile(std::move(TempFile)), EntryPath(std::move(EntryPath)),
Task(Task) {}
@@ -138,7 +137,7 @@
}
};
- return [=](size_t Task) -> std::unique_ptr<NativeObjectStream> {
+ return [=](size_t Task) -> std::unique_ptr<CachedFileStream> {
// Write to a temporary to avoid race condition
SmallString<64> TempFilenameModel;
sys::path::append(TempFilenameModel, CacheDirectoryPath,
Index: llvm/lib/LTO/LTOCodeGenerator.cpp
===================================================================
--- llvm/lib/LTO/LTOCodeGenerator.cpp
+++ llvm/lib/LTO/LTOCodeGenerator.cpp
@@ -245,7 +245,7 @@
// make unique temp output file to put generated code
SmallString<128> Filename;
- auto AddStream = [&](size_t Task) -> std::unique_ptr<NativeObjectStream> {
+ auto AddStream = [&](size_t Task) -> std::unique_ptr<CachedFileStream> {
StringRef Extension(Config.CGFileType == CGFT_AssemblyFile ? "s" : "o");
int FD;
@@ -254,7 +254,7 @@
if (EC)
emitError(EC.message());
- return std::make_unique<NativeObjectStream>(
+ return std::make_unique<CachedFileStream>(
std::make_unique<llvm::raw_fd_ostream>(FD, true));
};
Index: llvm/lib/LTO/LTO.cpp
===================================================================
--- llvm/lib/LTO/LTO.cpp
+++ llvm/lib/LTO/LTO.cpp
@@ -997,7 +997,7 @@
return Error::success();
}
-Error LTO::run(AddStreamFn AddStream, NativeObjectCache Cache) {
+Error LTO::run(AddStreamFn AddStream, FileCache Cache) {
// Compute "dead" symbols, we don't want to import/export these!
DenseSet<GlobalValue::GUID> GUIDPreservedSymbols;
DenseMap<GlobalValue::GUID, PrevailingType> GUIDPrevailingResolutions;
@@ -1183,7 +1183,7 @@
class InProcessThinBackend : public ThinBackendProc {
ThreadPool BackendThreadPool;
AddStreamFn AddStream;
- NativeObjectCache Cache;
+ FileCache Cache;
std::set<GlobalValue::GUID> CfiFunctionDefs;
std::set<GlobalValue::GUID> CfiFunctionDecls;
@@ -1195,7 +1195,7 @@
const Config &Conf, ModuleSummaryIndex &CombinedIndex,
ThreadPoolStrategy ThinLTOParallelism,
const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
- AddStreamFn AddStream, NativeObjectCache Cache)
+ AddStreamFn AddStream, FileCache Cache)
: ThinBackendProc(Conf, CombinedIndex, ModuleToDefinedGVSummaries),
BackendThreadPool(ThinLTOParallelism), AddStream(std::move(AddStream)),
Cache(std::move(Cache)) {
@@ -1208,8 +1208,8 @@
}
Error runThinLTOBackendThread(
- AddStreamFn AddStream, NativeObjectCache Cache, unsigned Task,
- BitcodeModule BM, ModuleSummaryIndex &CombinedIndex,
+ AddStreamFn AddStream, FileCache Cache, unsigned Task, BitcodeModule BM,
+ ModuleSummaryIndex &CombinedIndex,
const FunctionImporter::ImportMapTy &ImportList,
const FunctionImporter::ExportSetTy &ExportList,
const std::map<GlobalValue::GUID, GlobalValue::LinkageTypes> &ResolvedODR,
@@ -1301,7 +1301,7 @@
ThinBackend lto::createInProcessThinBackend(ThreadPoolStrategy Parallelism) {
return [=](const Config &Conf, ModuleSummaryIndex &CombinedIndex,
const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
- AddStreamFn AddStream, NativeObjectCache Cache) {
+ AddStreamFn AddStream, FileCache Cache) {
return std::make_unique<InProcessThinBackend>(
Conf, CombinedIndex, Parallelism, ModuleToDefinedGVSummaries, AddStream,
Cache);
@@ -1395,14 +1395,14 @@
raw_fd_ostream *LinkedObjectsFile, IndexWriteCallback OnWrite) {
return [=](const Config &Conf, ModuleSummaryIndex &CombinedIndex,
const StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
- AddStreamFn AddStream, NativeObjectCache Cache) {
+ AddStreamFn AddStream, FileCache Cache) {
return std::make_unique<WriteIndexesThinBackend>(
Conf, CombinedIndex, ModuleToDefinedGVSummaries, OldPrefix, NewPrefix,
ShouldEmitImportsFiles, LinkedObjectsFile, OnWrite);
};
}
-Error LTO::runThinLTO(AddStreamFn AddStream, NativeObjectCache Cache,
+Error LTO::runThinLTO(AddStreamFn AddStream, FileCache Cache,
const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols) {
timeTraceProfilerBegin("ThinLink", StringRef(""));
auto TimeTraceScopeExit = llvm::make_scope_exit([]() {
Index: llvm/include/llvm/Support/Caching.h
===================================================================
--- llvm/include/llvm/Support/Caching.h
+++ llvm/include/llvm/Support/Caching.h
@@ -1,4 +1,4 @@
-//===- Caching.h - LLVM File Cache Handling Configuration -------*- C++ -*-===//
+//===- Caching.h - LLVM Local File Cache ------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
@@ -6,8 +6,9 @@
//
//===----------------------------------------------------------------------===//
//
-// This file defines the localCache function, which allows clients to add a
-// filesystem cache. This is used by ThinLTO.
+// This file defines the CachedFileStream and the localCache function, which
+// simplifies caching files on the local filesystem in a directory whose
+// contents are managed by a CachePruningPolicy.
//
//===----------------------------------------------------------------------===//
@@ -15,48 +16,42 @@
#define LLVM_SUPPORT_CACHING_H
#include "llvm/Support/Error.h"
-#include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/thread.h"
namespace llvm {
-/// This class wraps an output stream for a native object. Most clients should
-/// just be able to return an instance of this base class from the stream
-/// callback, but if a client needs to perform some action after the stream is
-/// written to, that can be done by deriving from this class and overriding the
-/// destructor.
-class NativeObjectStream {
+class MemoryBuffer;
+
+/// This class wraps an output stream for a file. Most clients should just be
+/// able to return an instance of this base class from the stream callback, but
+/// if a client needs to perform some action after the stream is written to,
+/// that can be done by deriving from this class and overriding the destructor.
+class CachedFileStream {
public:
- NativeObjectStream(std::unique_ptr<raw_pwrite_stream> OS)
- : OS(std::move(OS)) {}
+ CachedFileStream(std::unique_ptr<raw_pwrite_stream> OS) : OS(std::move(OS)) {}
std::unique_ptr<raw_pwrite_stream> OS;
- virtual ~NativeObjectStream() = default;
+ virtual ~CachedFileStream() = default;
};
-/// This type defines the callback to add a native object that is generated on
-/// the fly.
+/// This type defines the callback to add a file that is generated on the fly.
///
/// Stream callbacks must be thread safe.
using AddStreamFn =
- std::function<std::unique_ptr<NativeObjectStream>(unsigned Task)>;
+ std::function<std::unique_ptr<CachedFileStream>(unsigned Task)>;
-/// This is the type of a native object cache. To request an item from the
-/// cache, pass a unique string as the Key. For hits, the cached file will be
-/// added to the link and this function will return AddStreamFn(). For misses,
-/// the cache will return a stream callback which must be called at most once to
-/// produce content for the stream. The native object stream produced by the
-/// stream callback will add the file to the link after the stream is written
-/// to.
+/// This is the type of a file cache. To request an item from the cache, pass a
+/// unique string as the Key. For hits, the cached file will be added to the
+/// link and this function will return AddStreamFn(). For misses, the cache will
+/// return a stream callback which must be called at most once to produce
+/// content for the stream. The file stream produced by the stream callback will
+/// add the file to the link after the stream is written to.
///
/// Clients generally look like this:
///
/// if (AddStreamFn AddStream = Cache(Task, Key))
/// ProduceContent(AddStream);
-using NativeObjectCache =
- std::function<AddStreamFn(unsigned Task, StringRef Key)>;
+using FileCache = std::function<AddStreamFn(unsigned Task, StringRef Key)>;
-/// This type defines the callback to add a pre-existing native object file
-/// (e.g. in a cache).
+/// This type defines the callback to add a pre-existing file (e.g. in a cache).
///
/// Buffer callbacks must be thread safe.
using AddBufferFn =
@@ -67,10 +62,9 @@
/// the cache directory if it does not already exist. The cache name appears in
/// error messages for errors during caching. The temporary file prefix is used
/// in the temporary file naming scheme used when writing files atomically.
-Expected<NativeObjectCache> localCache(Twine CacheNameRef,
- Twine TempFilePrefixRef,
- Twine CacheDirectoryPathRef,
- AddBufferFn AddBuffer);
+Expected<FileCache> localCache(Twine CacheNameRef, Twine TempFilePrefixRef,
+ Twine CacheDirectoryPathRef,
+ AddBufferFn AddBuffer);
} // namespace llvm
#endif
Index: llvm/include/llvm/LTO/LTO.h
===================================================================
--- llvm/include/llvm/LTO/LTO.h
+++ llvm/include/llvm/LTO/LTO.h
@@ -194,7 +194,7 @@
using ThinBackend = std::function<std::unique_ptr<ThinBackendProc>(
const Config &C, ModuleSummaryIndex &CombinedIndex,
StringMap<GVSummaryMapTy> &ModuleToDefinedGVSummaries,
- AddStreamFn AddStream, NativeObjectCache Cache)>;
+ AddStreamFn AddStream, FileCache Cache)>;
/// This ThinBackend runs the individual backend jobs in-process.
/// The default value means to use one job per hardware core (not hyper-thread).
@@ -267,7 +267,7 @@
///
/// The client will receive at most one callback (via either AddStream or
/// Cache) for each task identifier.
- Error run(AddStreamFn AddStream, NativeObjectCache Cache = nullptr);
+ Error run(AddStreamFn AddStream, FileCache Cache = nullptr);
/// Static method that returns a list of libcall symbols that can be generated
/// by LTO but might not be visible from bitcode symbol table.
@@ -399,7 +399,7 @@
const SymbolResolution *&ResI, const SymbolResolution *ResE);
Error runRegularLTO(AddStreamFn AddStream);
- Error runThinLTO(AddStreamFn AddStream, NativeObjectCache Cache,
+ Error runThinLTO(AddStreamFn AddStream, FileCache Cache,
const DenseSet<GlobalValue::GUID> &GUIDPreservedSymbols);
Error checkPartiallySplit();
Index: lld/wasm/LTO.cpp
===================================================================
--- lld/wasm/LTO.cpp
+++ lld/wasm/LTO.cpp
@@ -127,7 +127,7 @@
// The --thinlto-cache-dir option specifies the path to a directory in which
// to cache native object files for ThinLTO incremental builds. If a path was
// specified, configure LTO to use it as the cache directory.
- NativeObjectCache cache;
+ FileCache cache;
if (!config->thinLTOCacheDir.empty())
cache =
check(localCache("ThinLTO", "Thin", config->thinLTOCacheDir,
@@ -137,7 +137,7 @@
checkError(ltoObj->run(
[&](size_t task) {
- return std::make_unique<NativeObjectStream>(
+ return std::make_unique<CachedFileStream>(
std::make_unique<raw_svector_ostream>(buf[task]));
},
cache));
Index: lld/MachO/LTO.cpp
===================================================================
--- lld/MachO/LTO.cpp
+++ lld/MachO/LTO.cpp
@@ -105,7 +105,7 @@
// The -cache_path_lto option specifies the path to a directory in which
// to cache native object files for ThinLTO incremental builds. If a path was
// specified, configure LTO to use it as the cache directory.
- NativeObjectCache cache;
+ FileCache cache;
if (!config->thinLTOCacheDir.empty())
cache =
check(localCache("ThinLTO", "Thin", config->thinLTOCacheDir,
@@ -115,7 +115,7 @@
checkError(ltoObj->run(
[&](size_t task) {
- return std::make_unique<NativeObjectStream>(
+ return std::make_unique<CachedFileStream>(
std::make_unique<raw_svector_ostream>(buf[task]));
},
cache));
Index: lld/ELF/LTO.cpp
===================================================================
--- lld/ELF/LTO.cpp
+++ lld/ELF/LTO.cpp
@@ -304,7 +304,7 @@
// The --thinlto-cache-dir option specifies the path to a directory in which
// to cache native object files for ThinLTO incremental builds. If a path was
// specified, configure LTO to use it as the cache directory.
- NativeObjectCache cache;
+ FileCache cache;
if (!config->thinLTOCacheDir.empty())
cache =
check(localCache("ThinLTO", "Thin", config->thinLTOCacheDir,
@@ -315,7 +315,7 @@
if (!bitcodeFiles.empty())
checkError(ltoObj->run(
[&](size_t task) {
- return std::make_unique<NativeObjectStream>(
+ return std::make_unique<CachedFileStream>(
std::make_unique<raw_svector_ostream>(buf[task]));
},
cache));
Index: lld/COFF/LTO.cpp
===================================================================
--- lld/COFF/LTO.cpp
+++ lld/COFF/LTO.cpp
@@ -164,7 +164,7 @@
// The /lldltocache option specifies the path to a directory in which to cache
// native object files for ThinLTO incremental builds. If a path was
// specified, configure LTO to use it as the cache directory.
- NativeObjectCache cache;
+ FileCache cache;
if (!config->ltoCache.empty())
cache =
check(localCache("ThinLTO", "Thin", config->ltoCache,
@@ -174,7 +174,7 @@
checkError(ltoObj->run(
[&](size_t task) {
- return std::make_unique<NativeObjectStream>(
+ return std::make_unique<CachedFileStream>(
std::make_unique<raw_svector_ostream>(buf[task]));
},
cache));
Index: clang/lib/CodeGen/BackendUtil.cpp
===================================================================
--- clang/lib/CodeGen/BackendUtil.cpp
+++ clang/lib/CodeGen/BackendUtil.cpp
@@ -1561,7 +1561,7 @@
return;
auto AddStream = [&](size_t Task) {
- return std::make_unique<NativeObjectStream>(std::move(OS));
+ return std::make_unique<CachedFileStream>(std::move(OS));
};
lto::Config Conf;
if (CGOpts.SaveTempsFilePrefix != "") {
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
