dexonsmith created this revision.
dexonsmith added reviewers: Bigcheese, arphaman, sammccall.
Herald added subscribers: ributzka, tschuett.
dexonsmith requested review of this revision.
Herald added a project: clang.
Modules dependendency scanning builds lightweight modules (based on
minimized preprocess-only sources) in order to construct the module
dependency graph. Add `-intercept-module-outputs` (on-by-default), which
keeps these lightweight modules in memory instead of writing to and
reading from disk, leveraging llvm::vfs::OutputManager.
This patch uses a separate cache per worker, but these would ideally be
kept in a shared result cache (could be an in-memory CAS).
This is a WIP because it's based on the OutputManager RFC (see patches
https://reviews.llvm.org/D95501 / https://reviews.llvm.org/D95502) but I'm
posting this as well since it's a simple example usage.
I haven't benchmarked (yet); it's motivated by profiling from a couple of years
ago that showed some slowdowns here.
Repository:
rG LLVM Github Monorepo
https://reviews.llvm.org/D95628
Files:
clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h
clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h
clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp
clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
clang/tools/clang-scan-deps/ClangScanDeps.cpp
Index: clang/tools/clang-scan-deps/ClangScanDeps.cpp
===================================================================
--- clang/tools/clang-scan-deps/ClangScanDeps.cpp
+++ clang/tools/clang-scan-deps/ClangScanDeps.cpp
@@ -160,6 +160,12 @@
"until reaching the end directive."),
llvm::cl::init(true), llvm::cl::cat(DependencyScannerCategory));
+llvm::cl::opt<bool> InterceptModuleOutputs(
+ "intercept-module-outputs",
+ llvm::cl::desc(
+ "Keep the module outputs in-memory only, skipping the on-disk cache."),
+ llvm::cl::init(true), llvm::cl::cat(DependencyScannerCategory));
+
llvm::cl::opt<bool> Verbose("v", llvm::cl::Optional,
llvm::cl::desc("Use verbose output."),
llvm::cl::init(false),
@@ -485,7 +491,8 @@
SharedStream DependencyOS(llvm::outs());
DependencyScanningService Service(ScanMode, Format, ReuseFileManager,
- SkipExcludedPPRanges);
+ SkipExcludedPPRanges,
+ InterceptModuleOutputs);
llvm::ThreadPool Pool(llvm::hardware_concurrency(NumThreads));
std::vector<std::unique_ptr<DependencyScanningTool>> WorkerTools;
for (unsigned I = 0; I < Pool.getThreadCount(); ++I)
Index: clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
===================================================================
--- clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
+++ clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
@@ -50,12 +50,13 @@
public:
DependencyScanningAction(
StringRef WorkingDirectory, DependencyConsumer &Consumer,
+ std::shared_ptr<llvm::vfs::OutputManager> Outputs,
llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS,
ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings,
ScanningOutputFormat Format)
: WorkingDirectory(WorkingDirectory), Consumer(Consumer),
- DepFS(std::move(DepFS)), PPSkipMappings(PPSkipMappings),
- Format(Format) {}
+ Outputs(std::move(Outputs)), DepFS(std::move(DepFS)),
+ PPSkipMappings(PPSkipMappings), Format(Format) {}
bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation,
FileManager *FileMgr,
@@ -65,6 +66,14 @@
CompilerInstance Compiler(std::move(PCHContainerOps));
Compiler.setInvocation(std::move(Invocation));
+ // Configure the output manager.
+ if (Outputs) {
+ Compiler.setOutputManager(std::move(Outputs));
+
+ // FIXME: Is this the best way to set the flag?
+ Compiler.getFrontendOpts().BuildingImplicitModuleUsesLock = false;
+ }
+
// Don't print 'X warnings and Y errors generated'.
Compiler.getDiagnosticOpts().ShowCarets = false;
// Create the compiler's actual diagnostics engine.
@@ -142,6 +151,7 @@
private:
StringRef WorkingDirectory;
DependencyConsumer &Consumer;
+ std::shared_ptr<llvm::vfs::OutputManager> Outputs;
llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS;
ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings;
ScanningOutputFormat Format;
@@ -155,6 +165,35 @@
DiagOpts = new DiagnosticOptions();
PCHContainerOps = std::make_shared<PCHContainerOperations>();
RealFS = llvm::vfs::createPhysicalFileSystem();
+
+ if (Service.interceptModuleOutputs()) {
+ using namespace llvm;
+ using namespace llvm::vfs;
+
+ // Create an in-memory filesystem for storing modules and overlay it on top
+ // of RealFS.
+ auto InMemoryFS = makeIntrusiveRefCnt<InMemoryFileSystem>();
+ {
+ auto OverlayFS =
+ makeIntrusiveRefCnt<OverlayFileSystem>(createPhysicalFileSystem());
+ OverlayFS->pushOverlay(InMemoryFS);
+ RealFS = std::move(OverlayFS);
+ }
+
+ // Capture outputs with a ".pcm" extension and save them in ModulesCache.
+ // Don't reject attempts to overwrite (it'll still reject if the content
+ // doesn't match).
+ //
+ // FIXME: The workers could share a single cache using an in-memory CAS.
+ auto InMemoryOutputs =
+ makeIntrusiveRefCnt<InMemoryOutputBackend>(std::move(InMemoryFS));
+ InMemoryOutputs->setShouldRejectExistingFiles(false);
+ Outputs = std::make_shared<OutputManager>(makeFilteringOutputBackend(
+ std::move(InMemoryOutputs), [](StringRef OutputPath, OutputConfig) {
+ return llvm::sys::path::extension(OutputPath) == ".pcm";
+ }));
+ }
+
if (Service.canSkipExcludedPPRanges())
PPSkipMappings =
std::make_unique<ExcludedPreprocessorDirectiveSkipMapping>();
@@ -193,7 +232,7 @@
Tool.setRestoreWorkingDir(false);
Tool.setPrintErrorMessage(false);
Tool.setDiagnosticConsumer(&DC);
- DependencyScanningAction Action(WorkingDirectory, Consumer, DepFS,
+ DependencyScanningAction Action(WorkingDirectory, Consumer, Outputs, DepFS,
PPSkipMappings.get(), Format);
return !Tool.run(&Action);
});
Index: clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp
===================================================================
--- clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp
+++ clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp
@@ -14,6 +14,7 @@
DependencyScanningService::DependencyScanningService(
ScanningMode Mode, ScanningOutputFormat Format, bool ReuseFileManager,
- bool SkipExcludedPPRanges)
+ bool SkipExcludedPPRanges, bool InterceptModuleOutputs)
: Mode(Mode), Format(Format), ReuseFileManager(ReuseFileManager),
- SkipExcludedPPRanges(SkipExcludedPPRanges) {}
+ SkipExcludedPPRanges(SkipExcludedPPRanges),
+ InterceptModuleOutputs(InterceptModuleOutputs) {}
Index: clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h
===================================================================
--- clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h
+++ clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h
@@ -19,6 +19,7 @@
#include "clang/Tooling/DependencyScanning/ModuleDepCollector.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/OutputManager.h"
#include <string>
namespace clang {
@@ -68,6 +69,9 @@
std::shared_ptr<PCHContainerOperations> PCHContainerOps;
std::unique_ptr<ExcludedPreprocessorDirectiveSkipMapping> PPSkipMappings;
+ /// Output manager to use.
+ std::shared_ptr<llvm::vfs::OutputManager> Outputs;
+
llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> RealFS;
/// The file system that is used by each worker when scanning for
/// dependencies. This filesystem persists accross multiple compiler
Index: clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h
===================================================================
--- clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h
+++ clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h
@@ -48,7 +48,8 @@
public:
DependencyScanningService(ScanningMode Mode, ScanningOutputFormat Format,
bool ReuseFileManager = true,
- bool SkipExcludedPPRanges = true);
+ bool SkipExcludedPPRanges = true,
+ bool InterceptModuleOutputs = true);
ScanningMode getMode() const { return Mode; }
@@ -56,6 +57,8 @@
bool canReuseFileManager() const { return ReuseFileManager; }
+ bool interceptModuleOutputs() const { return InterceptModuleOutputs; }
+
bool canSkipExcludedPPRanges() const { return SkipExcludedPPRanges; }
DependencyScanningFilesystemSharedCache &getSharedCache() {
@@ -70,6 +73,8 @@
/// ranges by bumping the buffer pointer in the lexer instead of lexing the
/// tokens in the range until reaching the corresponding directive.
const bool SkipExcludedPPRanges;
+ /// Whether to intercept modules in-memory to avoid disk traffic.
+ const bool InterceptModuleOutputs;
/// The global file system cache.
DependencyScanningFilesystemSharedCache SharedCache;
};
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
