On Wed, 12 Jun 2019 at 14:29, Alex Lorenz via cfe-commits < cfe-commits@lists.llvm.org> wrote:
> Author: arphaman > Date: Wed Jun 12 14:32:49 2019 > New Revision: 363204 > > URL: http://llvm.org/viewvc/llvm-project?rev=363204&view=rev > Log: > [clang-scan-deps] initial outline of the tool that runs preprocessor to > find > dependencies over a JSON compilation database > > This commit introduces an outline for the clang-scan-deps tool that will be > used to implement fast dependency discovery phase using implicit modules > for > explicit module builds. > > The initial version of the tool works by computing non-modular header > dependencies > for files in the compilation database without any optimizations > (i.e. without source minimization from r362459). > The tool spawns a number of worker threads to run the clang compiler > workers in parallel. > > The immediate goal for clang-scan-deps is to create a ClangScanDeps library > which will be used to build up this tool to use the source minimization and > caching multi-threaded filesystem to implement the optimized > non-incremental > dependency scanning phase for a non-modular build. This will allow us to do > benchmarks and comparisons for performance that the minimization and > caching give us > > Differential Revision: https://reviews.llvm.org/D60233 > > Added: > cfe/trunk/test/ClangScanDeps/ > cfe/trunk/test/ClangScanDeps/Inputs/ > cfe/trunk/test/ClangScanDeps/Inputs/header.h > cfe/trunk/test/ClangScanDeps/Inputs/header2.h > cfe/trunk/test/ClangScanDeps/Inputs/regular_cdb.json > cfe/trunk/test/ClangScanDeps/regular_cdb.cpp > cfe/trunk/tools/clang-scan-deps/ > cfe/trunk/tools/clang-scan-deps/CMakeLists.txt > cfe/trunk/tools/clang-scan-deps/ClangScanDeps.cpp > Modified: > cfe/trunk/test/CMakeLists.txt > cfe/trunk/tools/CMakeLists.txt > > Modified: cfe/trunk/test/CMakeLists.txt > URL: > http://llvm.org/viewvc/llvm-project/cfe/trunk/test/CMakeLists.txt?rev=363204&r1=363203&r2=363204&view=diff > > ============================================================================== > --- cfe/trunk/test/CMakeLists.txt (original) > +++ cfe/trunk/test/CMakeLists.txt Wed Jun 12 14:32:49 2019 > @@ -57,6 +57,7 @@ list(APPEND CLANG_TEST_DEPS > clang-rename > clang-refactor > clang-diff > + clang-scan-deps > diagtool > hmaptool > ) > > Added: cfe/trunk/test/ClangScanDeps/Inputs/header.h > URL: > http://llvm.org/viewvc/llvm-project/cfe/trunk/test/ClangScanDeps/Inputs/header.h?rev=363204&view=auto > > ============================================================================== > --- cfe/trunk/test/ClangScanDeps/Inputs/header.h (added) > +++ cfe/trunk/test/ClangScanDeps/Inputs/header.h Wed Jun 12 14:32:49 2019 > @@ -0,0 +1,3 @@ > +#ifdef INCLUDE_HEADER2 > +#include "header2.h" > +#endif > > Added: cfe/trunk/test/ClangScanDeps/Inputs/header2.h > URL: > http://llvm.org/viewvc/llvm-project/cfe/trunk/test/ClangScanDeps/Inputs/header2.h?rev=363204&view=auto > > ============================================================================== > --- cfe/trunk/test/ClangScanDeps/Inputs/header2.h (added) > +++ cfe/trunk/test/ClangScanDeps/Inputs/header2.h Wed Jun 12 14:32:49 2019 > @@ -0,0 +1 @@ > +// header 2. > > Added: cfe/trunk/test/ClangScanDeps/Inputs/regular_cdb.json > URL: > http://llvm.org/viewvc/llvm-project/cfe/trunk/test/ClangScanDeps/Inputs/regular_cdb.json?rev=363204&view=auto > > ============================================================================== > --- cfe/trunk/test/ClangScanDeps/Inputs/regular_cdb.json (added) > +++ cfe/trunk/test/ClangScanDeps/Inputs/regular_cdb.json Wed Jun 12 > 14:32:49 2019 > @@ -0,0 +1,12 @@ > +[ > +{ > + "directory": "DIR", > + "command": "clang -c DIR/regular_cdb.cpp -IInputs -MD -MF > DIR/regular_cdb.d", > + "file": "DIR/regular_cdb.cpp" > +}, > +{ > + "directory": "DIR", > + "command": "clang -c DIR/regular_cdb.cpp -IInputs -D INCLUDE_HEADER2 > -MD -MF DIR/regular_cdb2.d", > + "file": "DIR/regular_cdb.cpp" > +} > +] > > Added: cfe/trunk/test/ClangScanDeps/regular_cdb.cpp > URL: > http://llvm.org/viewvc/llvm-project/cfe/trunk/test/ClangScanDeps/regular_cdb.cpp?rev=363204&view=auto > > ============================================================================== > --- cfe/trunk/test/ClangScanDeps/regular_cdb.cpp (added) > +++ cfe/trunk/test/ClangScanDeps/regular_cdb.cpp Wed Jun 12 14:32:49 2019 > @@ -0,0 +1,27 @@ > +// RUN: rm -rf %t.dir > +// RUN: rm -rf %t.cdb > +// RUN: mkdir -p %t.dir > +// RUN: cp %s %t.dir/regular_cdb.cpp > +// RUN: mkdir %t.dir/Inputs > +// RUN: cp %S/Inputs/header.h %t.dir/Inputs/header.h > +// RUN: cp %S/Inputs/header2.h %t.dir/Inputs/header2.h > +// RUN: sed -e "s|DIR|%/t.dir|g" %S/Inputs/regular_cdb.json > %t.cdb > +// > +// RUN: clang-scan-deps -compilation-database %t.cdb -j 1 > +// RUN: cat %t.dir/regular_cdb.d | FileCheck %s > +// RUN: cat %t.dir/regular_cdb2.d | FileCheck --check-prefix=CHECK2 %s > +// RUN: rm -rf %t.dir/regular_cdb.d %t.dir/regular_cdb2.d > +// > +// RUN: clang-scan-deps -compilation-database %t.cdb -j 2 > +// RUN: cat %t.dir/regular_cdb.d | FileCheck %s > +// RUN: cat %t.dir/regular_cdb2.d | FileCheck --check-prefix=CHECK2 %s > + > +#include "header.h" > + > +// CHECK: regular_cdb.cpp > +// CHECK-NEXT: Inputs{{/|\\}}header.h > +// CHECK-NOT: header2 > + > +// CHECK2: regular_cdb.cpp > +// CHECK2-NEXT: Inputs{{/|\\}}header.h > +// CHECK2-NEXT: Inputs{{/|\\}}header2.h > > Modified: cfe/trunk/tools/CMakeLists.txt > URL: > http://llvm.org/viewvc/llvm-project/cfe/trunk/tools/CMakeLists.txt?rev=363204&r1=363203&r2=363204&view=diff > > ============================================================================== > --- cfe/trunk/tools/CMakeLists.txt (original) > +++ cfe/trunk/tools/CMakeLists.txt Wed Jun 12 14:32:49 2019 > @@ -8,6 +8,7 @@ add_clang_subdirectory(clang-format-vs) > add_clang_subdirectory(clang-fuzzer) > add_clang_subdirectory(clang-import-test) > add_clang_subdirectory(clang-offload-bundler) > +add_clang_subdirectory(clang-scan-deps) > > add_clang_subdirectory(c-index-test) > > > Added: cfe/trunk/tools/clang-scan-deps/CMakeLists.txt > URL: > http://llvm.org/viewvc/llvm-project/cfe/trunk/tools/clang-scan-deps/CMakeLists.txt?rev=363204&view=auto > > ============================================================================== > --- cfe/trunk/tools/clang-scan-deps/CMakeLists.txt (added) > +++ cfe/trunk/tools/clang-scan-deps/CMakeLists.txt Wed Jun 12 14:32:49 2019 > @@ -0,0 +1,26 @@ > +set(LLVM_LINK_COMPONENTS > + Core > + Support > +) > + > +add_clang_tool(clang-scan-deps > + ClangScanDeps.cpp > + ) > + > +set(CLANG_SCAN_DEPS_LIB_DEPS > + clangAST > + clangBasic > + clangCodeGen > + clangDriver > + clangFrontend > + clangFrontendTool > + clangLex > + clangParse > + clangTooling > + ) > + > +target_link_libraries(clang-scan-deps > + PRIVATE > + ${CLANG_SCAN_DEPS_LIB_DEPS} > + ) > + > > Added: cfe/trunk/tools/clang-scan-deps/ClangScanDeps.cpp > URL: > http://llvm.org/viewvc/llvm-project/cfe/trunk/tools/clang-scan-deps/ClangScanDeps.cpp?rev=363204&view=auto > > ============================================================================== > --- cfe/trunk/tools/clang-scan-deps/ClangScanDeps.cpp (added) > +++ cfe/trunk/tools/clang-scan-deps/ClangScanDeps.cpp Wed Jun 12 14:32:49 > 2019 > @@ -0,0 +1,218 @@ > +//===-- ClangScanDeps.cpp - Implementation of clang-scan-deps > -------------===// > +// > +// The LLVM Compiler Infrastructure > +// > +// This file is distributed under the University of Illinois Open Source > +// License. See LICENSE.TXT for details. > +// > > +//===----------------------------------------------------------------------===// > Sorry, I forgot to update the old license comment in this file. I updated it in r363207. Thanks, Alex > + > +#include "clang/Frontend/CompilerInstance.h" > +#include "clang/Frontend/CompilerInvocation.h" > +#include "clang/Frontend/FrontendActions.h" > +#include "clang/Frontend/PCHContainerOperations.h" > +#include "clang/FrontendTool/Utils.h" > +#include "clang/Tooling/CommonOptionsParser.h" > +#include "clang/Tooling/JSONCompilationDatabase.h" > +#include "clang/Tooling/Tooling.h" > +#include "llvm/Support/FileSystem.h" > +#include "llvm/Support/InitLLVM.h" > +#include "llvm/Support/JSON.h" > +#include "llvm/Support/Options.h" > +#include "llvm/Support/Path.h" > +#include "llvm/Support/Program.h" > +#include "llvm/Support/Signals.h" > +#include "llvm/Support/Threading.h" > +#include <thread> > + > +using namespace clang; > + > +namespace { > + > +/// A clang tool that runs the preprocessor only for the given compiler > +/// invocation. > +class PreprocessorOnlyTool : public tooling::ToolAction { > +public: > + PreprocessorOnlyTool(StringRef WorkingDirectory) > + : WorkingDirectory(WorkingDirectory) {} > + > + bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation, > + FileManager *FileMgr, > + std::shared_ptr<PCHContainerOperations> > PCHContainerOps, > + DiagnosticConsumer *DiagConsumer) override { > + // Create a compiler instance to handle the actual work. > + CompilerInstance Compiler(std::move(PCHContainerOps)); > + Compiler.setInvocation(std::move(Invocation)); > + FileMgr->getFileSystemOpts().WorkingDir = WorkingDirectory; > + Compiler.setFileManager(FileMgr); > + > + // Create the compiler's actual diagnostics engine. > + Compiler.createDiagnostics(DiagConsumer, /*ShouldOwnClient=*/false); > + if (!Compiler.hasDiagnostics()) > + return false; > + > + Compiler.createSourceManager(*FileMgr); > + > + auto Action = llvm::make_unique<PreprocessOnlyAction>(); > + const bool Result = Compiler.ExecuteAction(*Action); > + FileMgr->clearStatCache(); > + return Result; > + } > + > +private: > + StringRef WorkingDirectory; > +}; > + > +/// A proxy file system that doesn't call `chdir` when changing the > working > +/// directory of a clang tool. > +class ProxyFileSystemWithoutChdir : public llvm::vfs::ProxyFileSystem { > +public: > + ProxyFileSystemWithoutChdir( > + llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS) > + : ProxyFileSystem(std::move(FS)) {} > + > + llvm::ErrorOr<std::string> getCurrentWorkingDirectory() const override { > + assert(!CWD.empty() && "empty CWD"); > + return CWD; > + } > + > + std::error_code setCurrentWorkingDirectory(const Twine &Path) override { > + CWD = Path.str(); > + return {}; > + } > + > +private: > + std::string CWD; > +}; > + > +/// The high-level implementation of the dependency discovery tool that > runs on > +/// an individual worker thread. > +class DependencyScanningTool { > +public: > + /// Construct a dependency scanning tool. > + /// > + /// \param Compilations The reference to the compilation database > that's > + /// used by the clang tool. > + DependencyScanningTool(const tooling::CompilationDatabase &Compilations) > + : Compilations(Compilations) { > + PCHContainerOps = std::make_shared<PCHContainerOperations>(); > + BaseFS = new > ProxyFileSystemWithoutChdir(llvm::vfs::getRealFileSystem()); > + } > + > + /// Computes the dependencies for the given file. > + /// > + /// \returns True on error. > + bool runOnFile(const std::string &Input, StringRef CWD) { > + BaseFS->setCurrentWorkingDirectory(CWD); > + tooling::ClangTool Tool(Compilations, Input, PCHContainerOps, BaseFS); > + Tool.clearArgumentsAdjusters(); > + Tool.setRestoreWorkingDir(false); > + PreprocessorOnlyTool Action(CWD); > + return Tool.run(&Action); > + } > + > +private: > + const tooling::CompilationDatabase &Compilations; > + std::shared_ptr<PCHContainerOperations> PCHContainerOps; > + /// The real filesystem used as a base for all the operations performed > by the > + /// tool. > + llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS; > +}; > + > +llvm::cl::opt<bool> Help("h", llvm::cl::desc("Alias for -help"), > + llvm::cl::Hidden); > + > +llvm::cl::OptionCategory DependencyScannerCategory("Tool options"); > + > +llvm::cl::opt<unsigned> > + NumThreads("j", llvm::cl::Optional, > + llvm::cl::desc("Number of worker threads to use (default: > use " > + "all concurrent threads)"), > + llvm::cl::init(0)); > + > +llvm::cl::opt<std::string> > + CompilationDB("compilation-database", > + llvm::cl::desc("Compilation database"), > llvm::cl::Required, > + llvm::cl::cat(DependencyScannerCategory)); > + > +} // end anonymous namespace > + > +int main(int argc, const char **argv) { > + llvm::InitLLVM X(argc, argv); > + llvm::cl::HideUnrelatedOptions(DependencyScannerCategory); > + if (!llvm::cl::ParseCommandLineOptions(argc, argv)) > + return 1; > + > + std::string ErrorMessage; > + std::unique_ptr<tooling::JSONCompilationDatabase> Compilations = > + tooling::JSONCompilationDatabase::loadFromFile( > + CompilationDB, ErrorMessage, > + tooling::JSONCommandLineSyntax::AutoDetect); > + if (!Compilations) { > + llvm::errs() << "error: " << ErrorMessage << "\n"; > + return 1; > + } > + > + llvm::cl::PrintOptionValues(); > + > + // By default the tool runs on all inputs in the CDB. > + std::vector<std::pair<std::string, std::string>> Inputs; > + for (const auto &Command : Compilations->getAllCompileCommands()) > + Inputs.emplace_back(Command.Filename, Command.Directory); > + > + // The command options are rewritten to run Clang in preprocessor only > mode. > + auto AdjustingCompilations = > + llvm::make_unique<tooling::ArgumentsAdjustingCompilations>( > + std::move(Compilations)); > + AdjustingCompilations->appendArgumentsAdjuster( > + [](const tooling::CommandLineArguments &Args, StringRef /*unused*/) > { > + tooling::CommandLineArguments AdjustedArgs = Args; > + AdjustedArgs.push_back("-o"); > + AdjustedArgs.push_back("/dev/null"); > + AdjustedArgs.push_back("-Xclang"); > + AdjustedArgs.push_back("-Eonly"); > + AdjustedArgs.push_back("-Xclang"); > + AdjustedArgs.push_back("-sys-header-deps"); > + return AdjustedArgs; > + }); > + > + unsigned NumWorkers = > + NumThreads == 0 ? llvm::hardware_concurrency() : NumThreads; > + std::vector<std::unique_ptr<DependencyScanningTool>> WorkerTools; > + for (unsigned I = 0; I < NumWorkers; ++I) > + WorkerTools.push_back( > + > llvm::make_unique<DependencyScanningTool>(*AdjustingCompilations)); > + > + std::vector<std::thread> WorkerThreads; > + std::atomic<bool> HadErrors(false); > + std::mutex Lock; > + size_t Index = 0; > + > + llvm::outs() << "Running clang-scan-deps on " << Inputs.size() > + << " files using " << NumWorkers << " workers\n"; > + for (unsigned I = 0; I < NumWorkers; ++I) { > + WorkerThreads.emplace_back( > + [I, &Lock, &Index, &Inputs, &HadErrors, &WorkerTools]() { > + while (true) { > + std::string Input; > + StringRef CWD; > + // Take the next input. > + { > + std::unique_lock<std::mutex> LockGuard(Lock); > + if (Index >= Inputs.size()) > + return; > + const auto &Compilation = Inputs[Index++]; > + Input = Compilation.first; > + CWD = Compilation.second; > + } > + // Run the tool on it. > + if (WorkerTools[I]->runOnFile(Input, CWD)) > + HadErrors = true; > + } > + }); > + } > + for (auto &W : WorkerThreads) > + W.join(); > + > + return HadErrors; > +} > > > _______________________________________________ > cfe-commits mailing list > cfe-commits@lists.llvm.org > https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits >
_______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
