arphaman created this revision.
arphaman added a reviewer: Bigcheese.
Herald added subscribers: jfb, dexonsmith, jkorous, mgorny.
Herald added a project: clang.
This patch introduces an outline for the `clang-scan-deps` tool that will be
used to implement fast dependency discovery phase using implicit modules for
explicit module builds.
The initial version of the tool works by computing non-modular header
dependencies for files in the compilation database without any optimizations
(i.e. without source minimization from https://reviews.llvm.org/D55463). The
tool spawns a number of worker threads to run the clang compiler workers in
parallel.
The immediate goal for `clang-scan-deps` is to create a `ClangScanDeps` library
which will be used to build up this tool to use the source minimization and
caching multi-threaded filesystem to implement the optimized non-incremental
dependency scanning phase for a non-modular build. This will allow us to do
benchmarks and comparisons for performance that the minimization and caching
give us, and maybe setup CI to track that performance. @Bigcheese will then
extend the tool to have the support for computing modular dependencies for
Clang modules. The tool could possibly interact with build systems in the
future if desired, but it's not our immediate goal.
This tool is based on code that was included in the original WIP patch I posted
before the dev meeting last October: https://reviews.llvm.org/D53354 .
Repository:
rC Clang
https://reviews.llvm.org/D60233
Files:
test/ClangScanDeps/Inputs/header.h
test/ClangScanDeps/Inputs/regular_cdb.json
test/ClangScanDeps/regular_cdb.cpp
tools/CMakeLists.txt
tools/clang-scan-deps/CMakeLists.txt
tools/clang-scan-deps/ClangScanDeps.cpp
Index: tools/clang-scan-deps/ClangScanDeps.cpp
===================================================================
--- /dev/null
+++ tools/clang-scan-deps/ClangScanDeps.cpp
@@ -0,0 +1,211 @@
+//===-- ClangScanDeps.cpp - Implementation of clang-scan-deps -------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Frontend/CompilerInstance.h"
+#include "clang/Frontend/CompilerInvocation.h"
+#include "clang/Frontend/FrontendActions.h"
+#include "clang/Frontend/PCHContainerOperations.h"
+#include "clang/FrontendTool/Utils.h"
+#include "clang/Tooling/JSONCompilationDatabase.h"
+#include "clang/Tooling/Tooling.h"
+#include "clang/Tooling/CommonOptionsParser.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/JSON.h"
+#include "llvm/Support/Options.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Program.h"
+#include "llvm/Support/Signals.h"
+#include "llvm/Support/Threading.h"
+#include <thread>
+
+using namespace clang;
+
+namespace {
+
+/// A clang tool that runs the preprocessor only for the given compiler
+/// invocation.
+class PreprocessorOnlyTool : public tooling::ToolAction {
+public:
+ PreprocessorOnlyTool(StringRef WorkingDirectory)
+ : WorkingDirectory(WorkingDirectory) {}
+
+ bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation,
+ FileManager *Files,
+ std::shared_ptr<PCHContainerOperations> PCHContainerOps,
+ DiagnosticConsumer *DiagConsumer) override {
+ // Create a compiler instance to handle the actual work.
+ CompilerInstance Compiler(std::move(PCHContainerOps));
+ Compiler.setInvocation(std::move(Invocation));
+ Files->getFileSystemOpts().WorkingDir = WorkingDirectory;
+ Compiler.setFileManager(Files);
+
+ // Create the compiler's actual diagnostics engine.
+ Compiler.createDiagnostics(DiagConsumer, /*ShouldOwnClient=*/false);
+ if (!Compiler.hasDiagnostics())
+ return false;
+
+ Compiler.createSourceManager(*Files);
+
+ auto Action = llvm::make_unique<PreprocessOnlyAction>();
+ const bool Success = Compiler.ExecuteAction(*Action);
+ Files->clearStatCache();
+ return Success;
+ }
+
+private:
+ StringRef WorkingDirectory;
+};
+
+/// A proxy file system that doesn't call `chdir` when changing the working
+/// directory of a clang tool.
+class ProxyFileSystemWithoutChdir : public llvm::vfs::ProxyFileSystem {
+public:
+ ProxyFileSystemWithoutChdir(
+ llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS)
+ : ProxyFileSystem(std::move(FS)) {}
+
+ llvm::ErrorOr<std::string> getCurrentWorkingDirectory() const override {
+ assert(!CWD.empty() && "empty CWD");
+ return CWD;
+ }
+
+ std::error_code setCurrentWorkingDirectory(const Twine &Path) override {
+ CWD = Path.str();
+ return {};
+ }
+
+private:
+ std::string CWD;
+};
+
+/// The high-level implementation of the dependency discovery tool that runs on
+/// an individual worker thread.
+class DependencyScanningTool {
+public:
+ DependencyScanningTool(const tooling::CompilationDatabase &Compilations)
+ : Compilations(Compilations) {
+ PCHContainerOps = std::make_shared<PCHContainerOperations>();
+ BaseFS = new ProxyFileSystemWithoutChdir(llvm::vfs::getRealFileSystem());
+ }
+
+ /// Computes the dependencies for the given file.
+ ///
+ /// \returns True on error.
+ bool runOnFile(const std::string &Input, StringRef CWD) {
+ BaseFS->setCurrentWorkingDirectory(CWD);
+ tooling::ClangTool Tool(Compilations, Input, PCHContainerOps, BaseFS);
+ Tool.clearArgumentsAdjusters();
+ Tool.setRestoreWorkingDir(false);
+ PreprocessorOnlyTool Action(CWD);
+ return Tool.run(&Action);
+ }
+
+private:
+ const tooling::CompilationDatabase &Compilations;
+ std::shared_ptr<PCHContainerOperations> PCHContainerOps;
+ /// The real filesystem used a base for all the operations performed by the
+ /// tool.
+ llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS;
+};
+
+llvm::cl::opt<bool> Help("h", llvm::cl::desc("Alias for -help"),
+ llvm::cl::Hidden);
+
+llvm::cl::OptionCategory DependencyScannerCategory("Tool options");
+
+llvm::cl::opt<unsigned>
+ NumThreads("j", llvm::cl::Optional,
+ llvm::cl::desc("Number of worker threads to use"),
+ llvm::cl::init(0));
+
+llvm::cl::opt<std::string>
+ CompilationDB("compilation-database",
+ llvm::cl::desc("Compilation database"), llvm::cl::Required,
+ llvm::cl::cat(DependencyScannerCategory));
+
+} // end anonymous namespace
+
+int main(int argc, const char **argv) {
+ llvm::cl::ResetAllOptionOccurrences();
+ llvm::cl::HideUnrelatedOptions(DependencyScannerCategory);
+ if (!llvm::cl::ParseCommandLineOptions(argc, argv))
+ return 1;
+
+ std::string ErrorMessage;
+ std::unique_ptr<tooling::JSONCompilationDatabase> Compilations =
+ tooling::JSONCompilationDatabase::loadFromFile(
+ CompilationDB, ErrorMessage,
+ tooling::JSONCommandLineSyntax::AutoDetect);
+ if (!Compilations) {
+ llvm::errs() << "error: " << ErrorMessage << "\n";
+ return 1;
+ }
+
+ llvm::cl::PrintOptionValues();
+
+ // By default the tool runs on all inputs in the CDB.
+ std::vector<std::pair<std::string, std::string>> Inputs;
+ for (const auto &Command : Compilations->getAllCompileCommands())
+ Inputs.emplace_back(Command.Filename, Command.Directory);
+
+ // The command options are rewritten to run Clang in preprocessor only mode.
+ auto AdjustingCompilations =
+ llvm::make_unique<tooling::ArgumentsAdjustingCompilations>(
+ std::move(Compilations));
+ AdjustingCompilations->appendArgumentsAdjuster(
+ [](const tooling::CommandLineArguments &Args, StringRef /*unused*/) {
+ tooling::CommandLineArguments AdjustedArgs = Args;
+ AdjustedArgs.push_back("-o");
+ AdjustedArgs.push_back("/dev/null");
+ AdjustedArgs.push_back("-Xclang");
+ AdjustedArgs.push_back("-Eonly");
+ AdjustedArgs.push_back("-Xclang");
+ AdjustedArgs.push_back("-sys-header-deps");
+ return AdjustedArgs;
+ });
+
+ unsigned NumWorkers =
+ NumThreads == 0 ? llvm::hardware_concurrency() : NumThreads;
+ std::vector<std::unique_ptr<DependencyScanningTool>> WorkerTools;
+ for (unsigned I = 0; I < NumWorkers; ++I)
+ WorkerTools.push_back(
+ llvm::make_unique<DependencyScanningTool>(*AdjustingCompilations));
+
+ std::vector<std::thread> WorkerThreads;
+ std::atomic<bool> HadErrors(false);
+ std::mutex Lock;
+ size_t Index = 0;
+
+ llvm::outs() << "Running clang-scan-deps on " << Inputs.size()
+ << " files using " << NumWorkers << " workers\n";
+ for (unsigned I = 0; I < NumWorkers; ++I) {
+ WorkerThreads.emplace_back(
+ [I, &Lock, &Index, &Inputs, &HadErrors, &WorkerTools]() {
+ while (true) {
+ std::string Input;
+ StringRef CWD;
+ // Take the next input.
+ {
+ std::unique_lock<std::mutex> LockGuard(Lock);
+ if (Index >= Inputs.size())
+ return;
+ const auto &Compilation = Inputs[Index++];
+ Input = Compilation.first;
+ CWD = Compilation.second;
+ }
+ // Run the tool on it.
+ HadErrors = WorkerTools[I]->runOnFile(Input, CWD);
+ }
+ });
+ }
+ for (auto &W : WorkerThreads)
+ W.join();
+
+ return HadErrors;
+}
Index: tools/clang-scan-deps/CMakeLists.txt
===================================================================
--- /dev/null
+++ tools/clang-scan-deps/CMakeLists.txt
@@ -0,0 +1,26 @@
+set(LLVM_LINK_COMPONENTS
+ Core
+ Support
+)
+
+add_clang_tool(clang-scan-deps
+ ClangScanDeps.cpp
+ )
+
+set(CLANG_SCAN_DEPS_LIB_DEPS
+ clangAST
+ clangBasic
+ clangCodeGen
+ clangDriver
+ clangFrontend
+ clangFrontendTool
+ clangLex
+ clangParse
+ clangTooling
+ )
+
+target_link_libraries(clang-scan-deps
+ PRIVATE
+ ${CLANG_SCAN_DEPS_LIB_DEPS}
+ )
+
Index: tools/CMakeLists.txt
===================================================================
--- tools/CMakeLists.txt
+++ tools/CMakeLists.txt
@@ -8,6 +8,7 @@
add_clang_subdirectory(clang-fuzzer)
add_clang_subdirectory(clang-import-test)
add_clang_subdirectory(clang-offload-bundler)
+add_clang_subdirectory(clang-scan-deps)
add_clang_subdirectory(c-index-test)
Index: test/ClangScanDeps/regular_cdb.cpp
===================================================================
--- /dev/null
+++ test/ClangScanDeps/regular_cdb.cpp
@@ -0,0 +1,14 @@
+// RUN: rm -rf %t.dir
+// RUN: rm -rf %t.cdb
+// RUN: mkdir -p %t.dir
+// RUN: cp %s %t.dir/regular_cdb.cpp
+// RUN: mkdir %t.dir/Inputs
+// RUN: cp %S/Inputs/header.h %t.dir/Inputs/header.h
+// RUN: sed -e "s|DIR|%/t.dir|g" %S/Inputs/regular_cdb.json > %t.cdb
+// RUN: clang-scan-deps -compilation-database %t.cdb -j 1
+// RUN: cat %t.dir/regular_cdb.d | FileCheck %s
+
+#include "header.h"
+
+// CHECK: regular_cdb.cpp
+// CHECK-NEXT: Inputs/header.h
Index: test/ClangScanDeps/Inputs/regular_cdb.json
===================================================================
--- /dev/null
+++ test/ClangScanDeps/Inputs/regular_cdb.json
@@ -0,0 +1,7 @@
+[
+{
+ "directory": "DIR",
+ "command": "clang -c DIR/regular_cdb.cpp -IInputs -MD -MF DIR/regular_cdb.d",
+ "file": "DIR/regular_cdb.cpp"
+}
+]
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
