ChuanqiXu updated this revision to Diff 532056. CHANGES SINCE LAST ACTION https://reviews.llvm.org/D153114/new/
https://reviews.llvm.org/D153114 Files: clang-tools-extra/clangd/CMakeLists.txt clang-tools-extra/clangd/ClangdLSPServer.cpp clang-tools-extra/clangd/ClangdLSPServer.h clang-tools-extra/clangd/GlobalCompilationDatabase.cpp clang-tools-extra/clangd/GlobalCompilationDatabase.h clang-tools-extra/clangd/ModulesManager.cpp clang-tools-extra/clangd/ModulesManager.h clang-tools-extra/clangd/TUScheduler.cpp clang-tools-extra/clangd/test/CMakeLists.txt clang-tools-extra/clangd/test/modules.test clang-tools-extra/clangd/tool/ClangdMain.cpp clang-tools-extra/clangd/unittests/CMakeLists.txt clang-tools-extra/clangd/unittests/ModulesManagerTests.cpp clang-tools-extra/docs/ReleaseNotes.rst
Index: clang-tools-extra/docs/ReleaseNotes.rst =================================================================== --- clang-tools-extra/docs/ReleaseNotes.rst +++ clang-tools-extra/docs/ReleaseNotes.rst @@ -48,6 +48,9 @@ Improvements to clangd ---------------------- +- Implemented the experimental support for C++20 modules. This can be enabled by + `-experimental-modules-support` option. + Inlay hints ^^^^^^^^^^^ Index: clang-tools-extra/clangd/unittests/ModulesManagerTests.cpp =================================================================== --- /dev/null +++ clang-tools-extra/clangd/unittests/ModulesManagerTests.cpp @@ -0,0 +1,353 @@ +//===-- ModulesManagerTests.cpp ---------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "Config.h" +#include "ModulesManager.h" + +#include "llvm/ADT/STLExtras.h" +#include "llvm/Support/FileSystem.h" +#include "llvm/Support/raw_ostream.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using namespace clang; +using namespace clang::clangd; +using namespace llvm; + +namespace { +class ModulesManagerTest : public ::testing::Test { + void SetUp() override { + ASSERT_FALSE(sys::fs::createUniqueDirectory("modules-test", TestDir)); + llvm::errs() << "Created TestDir: " << TestDir << "\n"; + } + + void TearDown() override { + // sys::fs::remove_directories(TestDir); + } + +public: + SmallString<256> TestDir; + + // Add files to the working testing directory and repalce all the + // `__DIR__` to TestDir. + void addFile(StringRef Path, StringRef Contents) { + ASSERT_FALSE(sys::path::is_absolute(Path)); + + SmallString<256> AbsPath(TestDir); + sys::path::append(AbsPath, Path); + + ASSERT_FALSE( + sys::fs::create_directories(llvm::sys::path::parent_path(AbsPath))); + + std::error_code EC; + llvm::raw_fd_ostream OS(AbsPath, EC); + ASSERT_FALSE(EC); + + std::size_t Pos = Contents.find("__DIR__"); + while (Pos != llvm::StringRef::npos) { + OS << Contents.take_front(Pos); + OS << TestDir; + Contents = Contents.drop_front(Pos + sizeof("__DIR__") - 1); + Pos = Contents.find("__DIR__"); + } + + OS << Contents; + } + + // Get the absolute path for file specified by Path under testing working + // directory. + std::string getFullPath(StringRef Path) { + SmallString<128> Result(TestDir); + sys::path::append(Result, Path); + return Result.str().str(); + } +}; + +TEST_F(ModulesManagerTest, ReplaceCommandsTest) { + addFile("build/compile_commands.json", R"cpp( +[ +{ + "directory": "__DIR__", + "command": "clang++ -std=c++20 __DIR__/M.cppm -c -o __DIR__/M.o -fmodule-file=D=__DIR__/D.pcm", + "file": "__DIR__/M.cppm", + "output": "__DIR__/M.o" +} +] + )cpp"); + + addFile("M.cppm", R"cpp( +export module M; +import D; + )cpp"); + + RealThreadsafeFS TFS; + DirectoryBasedGlobalCompilationDatabase::Options Opts(TFS); + DirectoryBasedModulesGlobalCompilationDatabase MCDB(Opts, + /*AsyncThreadsCount*/ 4); + + std::optional<tooling::CompileCommand> Cmd = + MCDB.getCompileCommand(getFullPath("M.cppm")); + EXPECT_TRUE(Cmd); + // Since the graph is not built yet. We don't expect to see the mutated + // command line for modules. + EXPECT_FALSE(any_of(Cmd->CommandLine, [](StringRef Arg) { + return Arg.count("-fprebuilt-module-path"); + })); + EXPECT_TRUE(any_of(Cmd->CommandLine, [](StringRef Arg) { + return Arg.count("-fmodule-file="); + })); + + ModulesManager *MMgr = MCDB.getModulesManager(); + EXPECT_TRUE(MMgr); + MMgr->UpdateNode(getFullPath("M.cppm")); + + MMgr->waitUntilInitialized(); + + Cmd = MCDB.getCompileCommand(getFullPath("M.cppm")); + EXPECT_TRUE(Cmd); + // Since the graph has been built. We expect to see the mutated command line + // for modules. + EXPECT_TRUE(any_of(Cmd->CommandLine, [](StringRef Arg) { + return Arg.count("-fprebuilt-module-path"); + })); + EXPECT_FALSE(any_of(Cmd->CommandLine, [](StringRef Arg) { + return Arg.count("-fmodule-file="); + })); +} + +void AddHelloWorldExample(ModulesManagerTest *Test) { + assert(Test); + + Test->addFile("build/compile_commands.json", R"cpp( +[ +{ + "directory": "__DIR__", + "command": "clang++ -std=c++20 __DIR__/M.cppm -c -o __DIR__/M.o", + "file": "__DIR__/M.cppm", + "output": "__DIR__/M.o" +}, +{ + "directory": "__DIR__", + "command": "clang++ -std=c++20 __DIR__/Impl.cpp -c -o __DIR__/Impl.o", + "file": "__DIR__/Impl.cpp", + "output": "__DIR__/Impl.o" +}, +{ + "directory": "__DIR__", + "command": "clang++ -std=c++20 __DIR__/impl_part.cppm -c -o __DIR__/impl_part.o", + "file": "__DIR__/impl_part.cppm", + "output": "__DIR__/impl_part.o" +}, +{ + "directory": "__DIR__", + "command": "clang++ -std=c++20 __DIR__/interface_part.cppm -c -o __DIR__/interface_part.o", + "file": "__DIR__/interface_part.cppm", + "output": "__DIR__/interface_part.o" +}, +{ + "directory": "__DIR__", + "command": "clang++ -std=c++20 __DIR__/User.cpp -c -o __DIR__/User.o", + "file": "__DIR__/User.cpp", + "output": "__DIR__/User.o" +} +] +)cpp"); + + Test->addFile("M.cppm", R"cpp( +export module M; +export import :interface_part; +import :impl_part; +export void Hello(); + )cpp"); + + Test->addFile("Impl.cpp", R"cpp( +module; +#include "header.mock" +module M; +void Hello() { +} + )cpp"); + + Test->addFile("impl_part.cppm", R"cpp( +module; +#include "header.mock" +module M:impl_part; +import :interface_part; + +void World() { + +} + )cpp"); + + Test->addFile("header.mock", ""); + + Test->addFile("interface_part.cppm", R"cpp( +export module M:interface_part; +export void World(); + )cpp"); + + Test->addFile("User.cpp", R"cpp( +import M; +import third_party_module; +int main() { + Hello(); + World(); + return 0; +} + )cpp"); +} + +TEST_F(ModulesManagerTest, BuildGraphTest) { + AddHelloWorldExample(this); + + RealThreadsafeFS TFS; + DirectoryBasedGlobalCompilationDatabase::Options Opts(TFS); + DirectoryBasedModulesGlobalCompilationDatabase MCDB(Opts, + /*AsyncThreadsCount*/ 4); + + ModulesManager *MMgr = MCDB.getModulesManager(); + EXPECT_TRUE(MMgr); + EXPECT_FALSE(MMgr->HasGraph()); + MMgr->UpdateNode(getFullPath("M.cppm")); + + MMgr->waitUntilInitialized(); + + EXPECT_TRUE(MMgr->HasGraph()); + EXPECT_EQ(MMgr->GraphSize(), 5u); + EXPECT_TRUE(MMgr->IsDirectlyDependent(getFullPath("M.cppm"), + getFullPath("impl_part.cppm"))); + EXPECT_TRUE(MMgr->IsDirectlyDependent(getFullPath("M.cppm"), + getFullPath("interface_part.cppm"))); + EXPECT_TRUE(MMgr->IsDirectlyDependent(getFullPath("impl_part.cppm"), + getFullPath("interface_part.cppm"))); + EXPECT_TRUE(MMgr->IsDirectlyDependent(getFullPath("Impl.cpp"), + getFullPath("M.cppm"))); + EXPECT_TRUE(MMgr->IsDirectlyDependent(getFullPath("User.cpp"), + getFullPath("M.cppm"))); + EXPECT_TRUE(MMgr->HasThirdpartyDependencies(getFullPath("User.cpp"))); +} + +TEST_F(ModulesManagerTest, GenerateModuleInterfaceAndUpdateTest) { + AddHelloWorldExample(this); + + RealThreadsafeFS TFS; + DirectoryBasedGlobalCompilationDatabase::Options Opts(TFS); + Opts.CompileCommandsDir = getFullPath("build"); + DirectoryBasedModulesGlobalCompilationDatabase MCDB(Opts, + /*AsyncThreadsCount*/ 4); + + ModulesManager *MMgr = MCDB.getModulesManager(); + EXPECT_TRUE(MMgr); + + MMgr->UpdateNode(getFullPath("M.cppm")); + + MMgr->waitUntilInitialized(); + + EXPECT_FALSE(MMgr->IsReadyToCompile(getFullPath("User.cpp"))); + + std::condition_variable ReadyCompileCV; + + MMgr->addCallbackAfterReady(getFullPath("User.cpp"), [&ReadyCompileCV]() { + ReadyCompileCV.notify_all(); + }); + MMgr->GenerateModuleInterfacesFor(getFullPath("User.cpp")); + + std::mutex Mu; + std::unique_lock<std::mutex> Lock(Mu); + ReadyCompileCV.wait(Lock, [MMgr, this]() { + return MMgr->IsReadyToCompile(getFullPath("User.cpp")); + }); + + EXPECT_TRUE(MMgr->IsReadyToCompile(getFullPath("User.cpp"))); + EXPECT_TRUE( + sys::fs::exists(getFullPath("build/.cache/clangd/module_files/M.pcm"))); + EXPECT_TRUE(sys::fs::exists( + getFullPath("build/.cache/clangd/module_files/M-impl_part.pcm"))); + EXPECT_TRUE(sys::fs::exists( + getFullPath("build/.cache/clangd/module_files/M-interface_part.pcm"))); + + MMgr->UpdateNode(getFullPath("User.cpp")); + EXPECT_TRUE(MMgr->IsReadyToCompile(getFullPath("User.cpp"))); + + MMgr->UpdateNode(getFullPath("Impl.cpp")); + EXPECT_TRUE(MMgr->IsReadyToCompile(getFullPath("User.cpp"))); + + MMgr->UpdateNode(getFullPath("M.cppm")); + EXPECT_FALSE(MMgr->IsReadyToCompile(getFullPath("User.cpp"))); +} + +TEST_F(ModulesManagerTest, InvalidBMITest) { + addFile("build/compile_commands.json", R"cpp( +[ +{ + "directory": "__DIR__", + "command": "clang++ -std=c++20 __DIR__/M.cppm -c -o __DIR__/M.o", + "file": "__DIR__/M.cppm", + "output": "__DIR__/M.o" +}, +{ + "directory": "__DIR__", + "command": "clang++ -std=c++20 __DIR__/User.cpp -c -o __DIR__/User.o", + "file": "__DIR__/User.cpp", + "output": "__DIR__/User.o" +} +] + )cpp"); + + addFile("M.cppm", R"cpp( +export module M; +export void Func() { + wlajdliajlwdjawdjlaw // invalid program +} + )cpp"); + + addFile("User.cpp", R"cpp( +import M; +void foo() { + Func(); +} + )cpp"); + + RealThreadsafeFS TFS; + DirectoryBasedGlobalCompilationDatabase::Options Opts(TFS); + DirectoryBasedModulesGlobalCompilationDatabase MCDB(Opts, + /*AsyncThreadsCount*/ 4); + ModulesManager *MMgr = MCDB.getModulesManager(); + EXPECT_TRUE(MMgr); + MMgr->UpdateNode(getFullPath("User.cpp")); + MMgr->waitUntilInitialized(); + + EXPECT_FALSE(MMgr->IsReadyToCompile(getFullPath("User.cpp"))); + EXPECT_FALSE(MMgr->HasInvalidDependencies(getFullPath("User.cpp"))); + + std::condition_variable ReadyCompileCV; + bool Failed = false; + + MMgr->addCallbackAfterReady( + getFullPath("User.cpp"), + /*ReadyCallback*/ [&ReadyCompileCV]() { ReadyCompileCV.notify_all(); }, + /*Failed Callback*/ + [&ReadyCompileCV, &Failed]() { + ReadyCompileCV.notify_all(); + Failed = true; + }); + MMgr->GenerateModuleInterfacesFor(getFullPath("User.cpp")); + + std::mutex Mu; + std::unique_lock<std::mutex> Lock(Mu); + ReadyCompileCV.wait(Lock, [MMgr, this]() { + return MMgr->HasInvalidDependencies(getFullPath("User.cpp")); + }); + + EXPECT_TRUE(MMgr->HasInvalidDependencies(getFullPath("User.cpp"))); + // Make sure that the failure callback are called. + EXPECT_TRUE(Failed); +} + +} // anonymous namespace Index: clang-tools-extra/clangd/unittests/CMakeLists.txt =================================================================== --- clang-tools-extra/clangd/unittests/CMakeLists.txt +++ clang-tools-extra/clangd/unittests/CMakeLists.txt @@ -72,6 +72,7 @@ LoggerTests.cpp LSPBinderTests.cpp LSPClient.cpp + ModulesManagerTests.cpp ModulesTests.cpp ParsedASTTests.cpp PathMappingTests.cpp Index: clang-tools-extra/clangd/tool/ClangdMain.cpp =================================================================== --- clang-tools-extra/clangd/tool/ClangdMain.cpp +++ clang-tools-extra/clangd/tool/ClangdMain.cpp @@ -557,6 +557,13 @@ }; #endif +opt<bool> ExperimentalModulesSupport{ + "experimental-modules-support", + cat(Features), + desc("Experimental support for standard c++ modules"), + init(false), +}; + /// Supports a test URI scheme with relaxed constraints for lit tests. /// The path in a test URI will be combined with a platform-specific fake /// directory to form an absolute path. For example, test:///a.cpp is resolved @@ -870,6 +877,7 @@ ClangdLSPServer::Options Opts; Opts.UseDirBasedCDB = (CompileArgsFrom == FilesystemCompileArgs); + Opts.ExperimentalModulesSupport = ExperimentalModulesSupport; switch (PCHStorage) { case PCHStorageFlag::Memory: Index: clang-tools-extra/clangd/test/modules.test =================================================================== --- /dev/null +++ clang-tools-extra/clangd/test/modules.test @@ -0,0 +1,79 @@ +# A smoke test to check the modules can work basically. +# +# RUN: rm -fr %t +# RUN: mkdir -p %t +# RUN: split-file %s %t +# +# RUN: sed -e "s|DIR|%/t|g" %t/compile_commands.json.tmpl > %t/compile_commands.json.tmp +# RUN: sed -e "s|CLANG_CC|%clang|g" %t/compile_commands.json.tmp > %t/compile_commands.json +# RUN: sed -e "s|DIR|%/t|g" %t/definition.jsonrpc.tmpl > %t/definition.jsonrpc +# +# RUN: clangd -experimental-modules-support -lit-test < %t/definition.jsonrpc \ +# RUN: | FileCheck -strict-whitespace %t/definition.jsonrpc + +#--- A.cppm +export module A; +export void printA() {} + +#--- Use.cpp +import A; +void foo() { + print +} + +#--- compile_commands.json.tmpl +[ + { + "directory": "DIR", + "command": "CLANG_CC -fprebuilt-module-path=DIR -std=c++20 -o DIR/main.cpp.o -c DIR/Use.cpp", + "file": "DIR/Use.cpp" + }, + { + "directory": "DIR", + "command": "CLANG_CC -std=c++20 DIR/A.cppm --precompile -o DIR/A.pcm", + "file": "DIR/A.cppm" + } +] + +#--- definition.jsonrpc.tmpl +{ + "jsonrpc": "2.0", + "id": 0, + "method": "initialize", + "params": { + "processId": 123, + "rootPath": "clangd", + "capabilities": { + "textDocument": { + "completion": { + "completionItem": { + "snippetSupport": true + } + } + } + }, + "trace": "off" + } +} +--- +{ + "jsonrpc": "2.0", + "method": "textDocument/didOpen", + "params": { + "textDocument": { + "uri": "file://DIR/Use.cpp", + "languageId": "cpp", + "version": 1, + "text": "import A;\nvoid foo() {\n print\n}\n" + } + } +} + +# CHECK: "message"{{.*}}printA{{.*}}(fix available) + +--- +{"jsonrpc":"2.0","id":1,"method":"textDocument/completion","params":{"textDocument":{"uri":"file://DIR/Use.cpp"},"context":{"triggerKind":1},"position":{"line":2,"character":6}}} +--- +{"jsonrpc":"2.0","id":2,"method":"shutdown"} +--- +{"jsonrpc":"2.0","method":"exit"} Index: clang-tools-extra/clangd/test/CMakeLists.txt =================================================================== --- clang-tools-extra/clangd/test/CMakeLists.txt +++ clang-tools-extra/clangd/test/CMakeLists.txt @@ -4,6 +4,7 @@ clangd-indexer # No tests for it, but we should still make sure they build. dexp + split-file ) if(CLANGD_BUILD_XPC) Index: clang-tools-extra/clangd/TUScheduler.cpp =================================================================== --- clang-tools-extra/clangd/TUScheduler.cpp +++ clang-tools-extra/clangd/TUScheduler.cpp @@ -52,6 +52,7 @@ #include "Config.h" #include "Diagnostics.h" #include "GlobalCompilationDatabase.h" +#include "ModulesManager.h" #include "ParsedAST.h" #include "Preamble.h" #include "index/CanonicalIncludes.h" @@ -647,6 +648,9 @@ TUScheduler::FileStats stats() const; bool isASTCached() const; + void waitForModulesBuilt() const; + void notifyModulesBuilt() const; + private: // Details of an update request that are relevant to scheduling. struct UpdateType { @@ -758,6 +762,10 @@ SynchronizedTUStatus Status; PreambleThread PreamblePeer; + + mutable std::condition_variable ModulesCV; + // mutable std::condition_variable FileInputsCV; + // std::mutex InitMu; }; /// A smart-pointer-like class that points to an active ASTWorker. @@ -855,6 +863,17 @@ bool ContentChanged) { llvm::StringLiteral TaskName = "Update"; auto Task = [=]() mutable { + if (auto *ModuleMgr = CDB.getModulesManager(); + ModuleMgr && !ModuleMgr->IsReadyToCompile(FileName) && + !ModuleMgr->HasInvalidDependencies(FileName)) { + log("{0} is not ready. wait for all the modules built.", FileName); + + ModuleMgr->addCallbackAfterReady(FileName, + [this]() { notifyModulesBuilt(); }); + ModuleMgr->GenerateModuleInterfacesFor(FileName); + waitForModulesBuilt(); + } + // Get the actual command as `Inputs` does not have a command. // FIXME: some build systems like Bazel will take time to preparing // environment to build the file, it would be nice if we could emit a @@ -1278,6 +1297,18 @@ PreambleCV.wait(Lock, [this] { return LatestPreamble || Done; }); } +void ASTWorker::waitForModulesBuilt() const { + auto *ModuleMgr = CDB.getModulesManager(); + assert(ModuleMgr); + std::unique_lock<std::mutex> Lock(Mutex); + ModulesCV.wait(Lock, [ModuleMgr, this]() { + return ModuleMgr->IsReadyToCompile(FileName) || + ModuleMgr->HasInvalidDependencies(FileName); + }); +} + +void ASTWorker::notifyModulesBuilt() const { ModulesCV.notify_all(); } + tooling::CompileCommand ASTWorker::getCurrentCompileCommand() const { std::unique_lock<std::mutex> Lock(Mutex); return FileInputs.CompileCommand; @@ -1684,7 +1715,13 @@ ContentChanged = true; FD->Contents = Inputs.Contents; } + + if (auto *ModuleMgr = CDB.getModulesManager()) + // TODO: Update all the nodes which are dependent on this. + ModuleMgr->UpdateNode(File.str()); + FD->Worker->update(std::move(Inputs), WantDiags, ContentChanged); + // There might be synthetic update requests, don't change the LastActiveFile // in such cases. if (ContentChanged) Index: clang-tools-extra/clangd/ModulesManager.h =================================================================== --- /dev/null +++ clang-tools-extra/clangd/ModulesManager.h @@ -0,0 +1,326 @@ +//===--- ModulesManager.h ----------------------------------------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "GlobalCompilationDatabase.h" +#include "support/Path.h" +#include "support/Threading.h" + +#include "clang/Frontend/FrontendAction.h" +#include "clang/Tooling/DependencyScanning/DependencyScanningService.h" +#include "clang/Tooling/DependencyScanning/DependencyScanningTool.h" +#include "llvm/ADT/ConcurrentHashtable.h" +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/StringSet.h" + +#include <map> +#include <optional> +#include <set> +#include <string> +#include <vector> + +namespace clang::clangd { + +namespace detail { +/// A simple wrapper for AsyncTaskRunner and Semaphore to run modules related +/// task - Scanning, Generating Module Interfaces - concurrently. +class ModulesTaskRunner { + ModulesTaskRunner(unsigned AsyncThreadsCount) : Barrier(AsyncThreadsCount) { + if (AsyncThreadsCount > 0) + Runner.emplace(); + } + + friend class clang::clangd::ModulesManager; + +public: + void RunTask(llvm::unique_function<void()> Task, + const llvm::Twine &Name = ""); + + void wait(); + +private: + std::optional<AsyncTaskRunner> Runner; + Semaphore Barrier; +}; + +/// A data structure to describe the C++20 named modules dependency information +/// of the project. So that a file which doesn't declare a module name nor uses +/// a modules won't be recorded in the graph. +/// +/// Note that this graph is not thread safe. The thread safety is guaranteed by +/// its owner. +class ModulesDependencyGraph { + struct ModulesDependencyNode { + ModulesDependencyNode(PathRef Name) : Name(Name.str()) {} + + /// The corresponding filename of the node. + std::string Name; + /// The module unit name of the node (if it provides). + std::optional<std::string> Provided; + /// The set of names that the node directly requires. The transitively + /// required names are not recorded here. + llvm::StringSet<> Requires; + + /// Update Provided and Requires information by provided P1689 rule. + /// + /// @return false if the node doesn't change. Return true otherwise. + bool + UpdateProvidedAndRequires(const tooling::dependencies::P1689Rule &Rule); + + /// The users of the current node. The node should be in the Deps for the + /// users. + std::set<ModulesDependencyNode *> Users; + /// The dependent nodes of the current node. Note that it is possible that + /// `Requires.size() > Deps.size()` since there are possibly third party + /// modules for which we can't its source code. But the size of Deps should + /// never be larger than Requires. + std::set<ModulesDependencyNode *> Deps; + + /// The corresponding BMI path of the current Node. This is only meaningful + /// if the current node is a module interface. + /// + /// When the BMIPath is not set, it implies that we know nothing about its + /// BMI. When one node has empty BMIPath (""), it implies that the node is + /// not able to be compiled. When one node has non-empty BMIPath, it implies + /// that the node has a meaningfull BMI and all the (transitively) dependent + /// nodes have meaningful BMI. + std::optional<std::string> BMIPath; + }; + +public: + ModulesDependencyGraph() = default; + ModulesDependencyGraph(const ModulesDependencyGraph &) = delete; + ModulesDependencyGraph(ModulesDependencyGraph &&) = delete; + ModulesDependencyGraph operator=(const ModulesDependencyGraph &) = delete; + ModulesDependencyGraph operator=(ModulesDependencyGraph &&) = delete; + ~ModulesDependencyGraph() = default; + + bool empty() const { return Nodes.empty(); } + size_t size() const { return Nodes.size(); } + + bool IsInGraph(PathRef Path) const { return Nodes.count(Path.str()); } + bool IsModuleInterface(PathRef Path) const { + return IsInGraph(Path) && Nodes.at(Path.str())->Provided; + } + std::string GetModuleInterfaceName(PathRef Path) const { + assert(IsModuleInterface(Path)); + return *Nodes.at(Path.str())->Provided; + } + + /// Whether if all the (transitive) dependencies have valid BMI. + /// + /// Note that this doesn't include third party modules for which the source + /// codes can't be found. + bool IsReadyToCompile(PathRef Filename) const; + /// Whether there is any dependencies has invalid BMI. + bool HasInvalidDependencies(PathRef Filename) const; + + const ModulesDependencyNode *getNode(PathRef Path) const { + if (!Nodes.count(Path.str())) + return nullptr; + + return Nodes.at(Path.str()).get(); + } + + /// Get the direct users. + std::set<ModulesDependencyNode *> getNodeUsers(PathRef Path) const { + if (!Nodes.count(Path.str())) + return {}; + + return Nodes.at(Path.str())->Users; + } + + /// Get all transitive users. + std::set<ModulesDependencyNode *> getAllUsers(PathRef Path) const; + + void clear() { + Nodes.clear(); + ModuleMapper.clear(); + } + + bool SetBMIPath(PathRef Filename, PathRef BMIPath); + + /// Update/Insert a node in/to the graph. When all of \p AllowInsertNewNode , + /// \p AllowUpdateModuleMapper and \p AllowUpdateDependency are false, it is + /// thread safe to run `UpdateSingleNode` parallelly. If the caller want to + /// run `UpdateSingleNode` parallelly, it is the responsibility of the caller + /// to call `InsertNewNode` ahead of time and call `reconstructModuleMapper` + /// and `UpdateDependencies` later to maintain the validability of the graph. + void UpdateSingleNode(PathRef Filename, + const tooling::dependencies::P1689Rule &Rule, + bool AllowInsertNewNode = true, + bool AllowUpdateModuleMapper = true, + bool AllowUpdateDependency = true); + /// Insert new nodes to the graph. This shouldn't be called if the caller + /// don't want to run `UpdateSingleNode` parallelly. + void InsertNewNode(PathRef Filename); + /// Update the dependency information after updating node(s). This shouldn't + /// be called if the caller don't want to run `UpdateSingleNode` parallelly. + void UpdateDependencies(const llvm::StringSet<> &Files, + bool RecalculateEdges = true); + + /// Construct the module map from module name to ModulesDependencyNode* after + /// updating node(s). This shouldn't be called if the caller don't want to run + /// `UpdateSingleNode` parallelly. + void reconstructModuleMapper(); + + /// If the specified Path has (non-transitively) third party dependencies. + /// This is only called by testing purpose. + bool HasThirdpartyDependencies(PathRef Path) const; + +private: + void UpdateModuleMapper(PathRef Filename, + const tooling::dependencies::P1689Rule &Rule); + + /// Map from source file path to ModulesDependencyNode*. + llvm::StringMap<std::unique_ptr<ModulesDependencyNode>> Nodes; + /// Map from the module name to ModulesDependencyNode*. This is a helper + /// strucutre to build the depenendcy graph faster. So it doesn't own the + /// nodes. It is OK for this to get ready before calling UpdateDependencies. + llvm::StringMap<ModulesDependencyNode *> ModuleMapper; +}; +} // namespace detail + +/// A manager to manage the dependency information of modules and module files +/// states. The users can use `IsReadyToCompile(Path)` to get if it is Ok to +/// compile the file specified by `Path`. In case it is not ready, the user can +/// use the following pattern to wait for it to gets ready: +/// +/// ModuleMgr.addCallbackAfterReady(Path, [] () { +/// notify it is ready; +/// }); +/// ModuleMgr.GenerateModuleInterfacesFor(Path); +/// wait for it to get ready. +/// +/// Every time we meet a new file or a file get changed, we should call +/// `UpdateNode(PathRef)` to try to update its status in the graph no matter if +/// it was related to modules before. Since it is possible that the change of +/// the file is to introduce module related things. So it should be job of +/// ModulesManager to decide whether or not it is related to modules. +class ModulesManager { +public: + ModulesManager(const DirectoryBasedModulesGlobalCompilationDatabase &CDB, + unsigned AsyncThreadsCount); + ~ModulesManager(); + + /// If Filename is not related to moduls, e.g, not a module unit nor import + /// any modules, nothing will happen. Otherwise the modules manager will try + /// to update the modules graph responsitively. + void UpdateNode(PathRef Filename); + /// Update a lot of files at the same time. This should only be called by + /// DirectoryBasedModulesGlobalCompilationDatabase as a callback when finding + /// new compilation database. + void UpdateBunchFiles(const std::vector<std::string> &Files); + + /// Whether we have atleast one node in the modules graph. + bool HasGraph() const; + /// If the specified file is in the graph. + bool IsInGraph(PathRef Path) const; + /// If the specified file is a module interface unit. + bool IsModuleInterface(PathRef Path) const; + std::string GetModuleInterfaceName(PathRef Path) const; + + /// Helper functions for testing + bool IsDirectlyDependent(PathRef A, PathRef B) const; + /// @return the number of nodes in the graph. + size_t GraphSize() const; + bool HasThirdpartyDependencies(PathRef A) const; + + /// We'll initialize the graph asynchronously. It is necessary for tests + /// to wait for the graph get initialized. + void waitUntilInitialized() const; + + /// @return If the BMIs of the dependencies for the specified file are valid + /// already. + bool IsReadyToCompile(PathRef Filename) const; + /// @return true if there is any (transitive) dependencies are invalid. False + /// otherwise, this doesn't include third party modules. + bool HasInvalidDependencies(PathRef Filename) const; + + /// Add a callback which will be called when the corresponding source file + /// gets ready. This requires `!IsReadyToCompile(Filename)` and + /// `!HasInvalidDependencies(Filename)`. + /// + /// @param ReadyCallback the callback get called when the file specified by + /// Filename gets ready. + /// @param FailureCallback the callback get called when file specified by + /// Filename are known not to be ready. It is possible that its dependencies + /// are known not to be able to be compiled. If not specified, it will be the + /// same with ReadyCallback. + void addCallbackAfterReady(PathRef Filename, + std::function<void()> ReadyCallback, + std::function<void()> FailureCallback = nullptr); + + /// Generate module files for file specified by Path. This will generate + /// module files for the transitively dependencies. It requires there is no + /// invalid dependencies. + void GenerateModuleInterfacesFor(PathRef Path); + +private: + void GenerateModuleInterface(PathRef Path); + + /// Invoke ClangScanDeps to get P1689 rule for the specified file. + std::optional<tooling::dependencies::P1689Rule> + getRequiresAndProvide(PathRef Path); + + detail::ModulesDependencyGraph Graph; + + /// A mutex for the graph. Any operation to the graph should be guarded by the + /// mutex. + mutable std::mutex Mutex; + + /// A helper class to track the scheduled tasks to generate module files to + /// not schedule duplicated task to generate module files for the same file. + llvm::StringSet<> ScheduledInterfaces; + + /// A map from path of source files to the callbacks when the file gets ready + /// or failed to get ready. + llvm::StringMap<llvm::SmallVector< + std::pair<std::function<void()>, std::function<void()>>, 8>> + WaitingCallables; + /// All the operations for WaitingCallables should be guarded by the mutex. + mutable std::mutex WaitingCallablesMutex; + /// Set BMIPath to the node specified by Filename and run the corresponding + /// callbacks (if any). If the BMIPath is empty, it implies that the node is + /// invalid to compile. + void setBMIPath(PathRef Filename, PathRef BMIPath); + + const DirectoryBasedModulesGlobalCompilationDatabase &CDB; + clang::tooling::dependencies::DependencyScanningService Service; + + enum GraphInitState { Uninitialized, Initializing, Initialized }; + std::atomic<unsigned> InitializeState = Uninitialized; + mutable std::mutex InitializationMutex; + mutable std::condition_variable InitCV; + + /// Atomatically test if the graph is uninitialized. And if yes, change its + /// value to Initializing atomatically. This should only be called before + /// InitializeGraph + bool IsUnitialized() { + unsigned UninitializedState = Uninitialized; + return InitializeState.compare_exchange_strong(UninitializedState, + Initializing); + } + + /// Lock the whole graph until is initialized. This is an abbrevation for: + /// + /// waitUntilInitialized(); + /// std::lock_guard<std::mutex> Lock(Mutex); + [[nodiscard]] std::lock_guard<std::mutex> lockGraph() const { + waitUntilInitialized(); + return std::lock_guard(Mutex); + } + + /// Initialize the graph. Modules manager will try to do it asynchronously if + /// we have the resources. + void InitializeGraph(PathRef Filename); + + unsigned AsyncThreadsCount = 0; + detail::ModulesTaskRunner Runner; +}; + +} // namespace clang::clangd Index: clang-tools-extra/clangd/ModulesManager.cpp =================================================================== --- /dev/null +++ clang-tools-extra/clangd/ModulesManager.cpp @@ -0,0 +1,639 @@ +//===--- ModulesManager.cpp --------------------------------------*- C++-*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "ModulesManager.h" +#include "Compiler.h" +#include "support/Logger.h" + +#include "clang/Frontend/FrontendActions.h" +#include "clang/Tooling/Tooling.h" + +#include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/ScopeExit.h" + +using namespace clang; +using namespace clang::clangd; +using namespace clang::tooling::dependencies; + +ModulesManager::ModulesManager( + const DirectoryBasedModulesGlobalCompilationDatabase &CDB, + unsigned AsyncThreadsCount) + : CDB(CDB), Service(ScanningMode::CanonicalPreprocessing, + ScanningOutputFormat::P1689), + AsyncThreadsCount(AsyncThreadsCount), Runner(AsyncThreadsCount) {} + +ModulesManager::~ModulesManager() { Runner.wait(); } + +bool clang::clangd::detail::ModulesDependencyGraph::SetBMIPath( + PathRef Filename, PathRef BMIPath) { + if (!Nodes.count(Filename)) + return false; + + Nodes[Filename]->BMIPath = BMIPath; + return true; +} + +std::optional<P1689Rule> ModulesManager::getRequiresAndProvide(PathRef Name) { + std::optional<tooling::CompileCommand> Cmd = + CDB.getOriginalCompileCommand(Name); + + if (!Cmd) + return std::nullopt; + + using namespace clang::tooling::dependencies; + + DependencyScanningTool ScanningTool(Service); + + std::string MakeformatOutput; + std::string MakeformatOutputPath; + llvm::Expected<P1689Rule> P1689Rule = + ScanningTool.getP1689ModuleDependencyFile( + *Cmd, Cmd->Directory, MakeformatOutput, MakeformatOutputPath); + + if (!P1689Rule) + return std::nullopt; + + return *P1689Rule; +} + +void ModulesManager::UpdateBunchFiles(const std::vector<std::string> &Files) { + if (IsUnitialized()) + return InitializeGraph(Files.front()); + + waitUntilInitialized(); + + // If the size of new files are not so many, maybe it is better + // to not re-initialize the whole graph. + // + // Maybe we need a better fine-tuned condition. + if (Files.size() < Graph.size() / 2) { + for (const std::string &File : Files) + UpdateNode(File); + + return; + } + + { + std::lock_guard<std::mutex> Lock(Mutex); + Graph.clear(); + InitializeState = Initializing; + } + InitializeGraph(Files.front()); +} + +void ModulesManager::UpdateNode(PathRef Filename) { + std::optional<P1689Rule> Rule = getRequiresAndProvide(Filename); + if (!Rule) + return; + + auto _ = lockGraph(); + Graph.UpdateSingleNode(Filename, *Rule); +} + +void ModulesManager::waitUntilInitialized() const { + if (InitializeState == Initializing) { + std::unique_lock<std::mutex> Lock(InitializationMutex); + InitCV.wait(Lock, [this]() { return InitializeState == Initialized; }); + } +} + +// TODO: Try to use the existing BMIs, which is created in last invocation of +// clangd. +void ModulesManager::InitializeGraph(PathRef Filename) { +#ifndef NDEBUG + assert(Graph.empty()); + assert(InitializeState == Initializing && + "InitializeState should be set before calling InitializeGraph."); +#endif + + // Initialization may be slow. Consider to do it asynchronously. + auto Task = [this, &CDB = CDB, &Graph = Graph, &Mutex = Mutex, + Filename = Filename.str(), &Runner = Runner]() { + std::vector<std::string> AllFiles = CDB.getAllFilesInProjectOf(Filename); + llvm::StringSet<> ModuleRelatedFiles; + + // In case we don't have enough resources, run it locally. + // Note that the current task may use async resources. + if (AsyncThreadsCount <= 1) { + for (const std::string &File : AllFiles) { + std::optional<P1689Rule> Rule = getRequiresAndProvide(File); + if (!Rule) + continue; + + std::lock_guard<std::mutex> Lock(Mutex); + ModuleRelatedFiles.insert(File); + Graph.UpdateSingleNode(File, *Rule, + /*InsertNewNode*/ true, + /*AllowUpdateModuleMapper*/ false, + /*UpdateDependency*/ false); + } + } else { + // We have threads. Let's try to scanning the graph pararrelly. + std::atomic<unsigned> UnfinishedTaskNum = AllFiles.size(); + std::mutex FinishedMu; + std::condition_variable FinishedCV; + + std::mutex Mu; + for (const auto &File : AllFiles) + Runner.RunTask([this, &Graph, File, &ModuleRelatedFiles, &FinishedCV, + &UnfinishedTaskNum, &Mu]() mutable { + std::optional<P1689Rule> Rule = getRequiresAndProvide(File); + if (!Rule) + return; + + // Try to make the critical section as short as possible. + { + std::lock_guard<std::mutex> Lock(Mu); + Graph.InsertNewNode(File); + ModuleRelatedFiles.insert(File); + } + + // It is thread safe now by design. + Graph.UpdateSingleNode(File, *Rule, + /*InsertNewNode*/ false, + /*AllowUpdateModuleMapper*/ false, + /*UpdateDependency*/ false); + + UnfinishedTaskNum--; + + if (!UnfinishedTaskNum) + FinishedCV.notify_all(); + }); + + std::unique_lock<std::mutex> Lock(FinishedMu); + FinishedCV.wait( + Lock, [&UnfinishedTaskNum]() { return UnfinishedTaskNum == 0; }); + } + + std::lock_guard<std::mutex> Lock(Mutex); + // Now construct the module mapper in one go. + Graph.reconstructModuleMapper(); + Graph.UpdateDependencies(ModuleRelatedFiles); + InitializeState = Initialized; + InitCV.notify_all(); + }; + + Runner.RunTask(std::move(Task)); +} + +void clang::clangd::detail::ModulesDependencyGraph::reconstructModuleMapper() { + ModuleMapper.clear(); + + for (auto &&Key : Nodes.keys()) + if (Nodes[Key]->Provided) + ModuleMapper[*Nodes[Key]->Provided] = Nodes[Key].get(); +} + +bool clang::clangd::detail::ModulesDependencyGraph::ModulesDependencyNode:: + UpdateProvidedAndRequires(const P1689Rule &Rule) { + bool DependencyChanged = + Rule.Provides ? (Provided != Rule.Provides->ModuleName) : !Provided; + + if (DependencyChanged) { + if (Rule.Provides) + Provided = Rule.Provides->ModuleName; + else + Provided = std::nullopt; + } + + DependencyChanged |= (Requires.size() != Rule.Requires.size()); + llvm::StringSet<> RequiresTmp = Requires; + for (const P1689ModuleInfo &Required : Rule.Requires) { + auto [_, Inserted] = Requires.insert(Required.ModuleName); + DependencyChanged |= Inserted; + RequiresTmp.erase(Required.ModuleName); + } + + DependencyChanged |= !RequiresTmp.empty(); + for (auto &NotEliminated : RequiresTmp) + if (Requires.count(NotEliminated.getKey())) + Requires.erase(NotEliminated.getKey()); + + return DependencyChanged; +} + +void clang::clangd::detail::ModulesDependencyGraph::InsertNewNode( + PathRef Filename) { + if (!Nodes.count(Filename)) + Nodes.insert({Filename, std::make_unique<ModulesDependencyNode>(Filename)}); +} + +void clang::clangd::detail::ModulesDependencyGraph::UpdateModuleMapper( + PathRef Filename, const P1689Rule &Rule) { + ModulesDependencyNode *Node = Nodes[Filename].get(); + assert(Node); + + bool ModuleNameChanged = Rule.Provides + ? (Node->Provided != Rule.Provides->ModuleName) + : !Node->Provided; + + if (ModuleNameChanged) { + for (auto *User : Node->Users) + User->Deps.erase(Node); + + Node->Users.clear(); + } + + if (Node->Provided) { + llvm::StringRef OriginalModuleName = *Node->Provided; + assert(ModuleMapper.count(OriginalModuleName)); + ModuleMapper.erase(OriginalModuleName); + } + + if (!Rule.Provides) + return; + + llvm::StringRef NewModuleName = Rule.Provides->ModuleName; + ModuleMapper.insert({NewModuleName, Node}); +} + +void clang::clangd::detail::ModulesDependencyGraph::UpdateSingleNode( + PathRef Filename, const P1689Rule &Rule, bool AllowInsertNewNode, + bool AllowUpdateModuleMapper, bool UpdateDependency) { + if (AllowInsertNewNode) + InsertNewNode(Filename); + + if (AllowUpdateModuleMapper) + UpdateModuleMapper(Filename, Rule); + + ModulesDependencyNode *UpdatingNode = Nodes[Filename].get(); + assert(UpdatingNode); + bool DependencyChanged = UpdatingNode->UpdateProvidedAndRequires(Rule); + if (!UpdateDependency) + return; + + UpdateDependencies({Filename}, DependencyChanged); +} + +/// UpdateDependencies does 2 things: +/// - Add edges to the dependency graph. +/// - For all the changed files and the (transitively) users, reset their state +/// of BMIPath. +void clang::clangd::detail::ModulesDependencyGraph::UpdateDependencies( + const llvm::StringSet<> &Files, bool RecalculateEdges) { + if (RecalculateEdges) + for (const auto &FileIter : Files) { + llvm::StringRef File = FileIter.getKey(); + ModulesDependencyNode *Node = Nodes[File].get(); + assert(Node); + + for (auto *Dep : Node->Deps) + Dep->Users.erase(Node); + + Node->Deps.clear(); + + for (auto &Required : Node->Requires) { + llvm::StringRef RequiredModuleName = Required.getKey(); + + // It is possible that we're importing a third party module. + if (!ModuleMapper.count(RequiredModuleName)) + continue; + + ModulesDependencyNode *RequiredNode = ModuleMapper[RequiredModuleName]; + Node->Deps.insert(RequiredNode); + RequiredNode->Users.insert(Node); + } + } + + /// Reset the states of BMIPath. + llvm::SmallSetVector<StringRef, 32> Worklist; + + // Due to the defect in llvm::StringSet, we can't use + // + // Worklist(Files.begin(), Files.end()); + // + // to initialize the worklist. + for (const auto &FileIter : Files) + Worklist.insert(FileIter.getKey()); + + // There shouldn't be circular modules dependency informations in a valid + // project. But we can't assume the project is valid. + llvm::StringSet<> NoDepCircleChecker; + + while (!Worklist.empty()) { + llvm::StringRef Filename = Worklist.pop_back_val(); + Nodes[Filename.str()]->BMIPath.reset(); + + if (NoDepCircleChecker.count(Filename)) { + elog("found circular modules dependency information in the project."); + break; + } + + NoDepCircleChecker.insert(Filename); + + for (auto *User : Nodes[Filename.str()]->Users) + if (Nodes[User->Name]->BMIPath) + Worklist.insert(User->Name); + } +} + +bool ModulesManager::HasGraph() const { + auto _ = lockGraph(); + return !Graph.empty(); +} + +bool ModulesManager::IsInGraph(PathRef Path) const { + auto _ = lockGraph(); + return Graph.IsInGraph(Path); +} + +bool ModulesManager::IsModuleInterface(PathRef Path) const { + auto _ = lockGraph(); + return Graph.IsModuleInterface(Path); +} + +std::string ModulesManager::GetModuleInterfaceName(PathRef Path) const { + auto _ = lockGraph(); + return Graph.GetModuleInterfaceName(Path); +} + +bool ModulesManager::IsReadyToCompile(PathRef Filename) const { + auto _ = lockGraph(); + return Graph.IsReadyToCompile(Filename); +} + +bool clang::clangd::detail::ModulesDependencyGraph::IsReadyToCompile( + PathRef Filename) const { + if (!Nodes.count(Filename)) + return true; + + auto *Node = Nodes.at(Filename).get(); + return llvm::all_of(Node->Deps, [](auto *Dep) { + return Dep->BMIPath && !Dep->BMIPath->empty(); + }); +} + +bool ModulesManager::HasInvalidDependencies(PathRef Filename) const { + auto _ = lockGraph(); + return Graph.HasInvalidDependencies(Filename); +} + +bool clang::clangd::detail::ModulesDependencyGraph::HasInvalidDependencies( + PathRef Filename) const { + if (!Nodes.count(Filename)) + return true; + + auto *Node = Nodes.at(Filename).get(); + // It is ok to lookup for the direct dependencies since once a node has + // invalid BMI, all of its users will have invalid BMI. + return llvm::any_of(Node->Deps, [](auto *Dep) { + return Dep->BMIPath && Dep->BMIPath->empty(); + }); +} + +namespace { +llvm::SmallString<128> getAbsolutePath(const tooling::CompileCommand &Cmd) { + llvm::SmallString<128> AbsolutePath; + if (llvm::sys::path::is_absolute(Cmd.Filename)) { + AbsolutePath = Cmd.Filename; + } else { + AbsolutePath = Cmd.Directory; + llvm::sys::path::append(AbsolutePath, Cmd.Filename); + llvm::sys::path::remove_dots(AbsolutePath, true); + } + return AbsolutePath; +} +} // namespace + +void ModulesManager::addCallbackAfterReady( + PathRef Filename, std::function<void()> ReadyCallback, + std::function<void()> FailureCallback) { + std::lock_guard<std::mutex> Lock(WaitingCallablesMutex); + assert(!IsReadyToCompile(Filename)); + assert(!HasInvalidDependencies(Filename)); + if (FailureCallback) + WaitingCallables[Filename.str()].push_back( + {std::move(ReadyCallback), std::move(FailureCallback)}); + else + WaitingCallables[Filename.str()].push_back( + {std::move(ReadyCallback), std::move(ReadyCallback)}); +} + +std::set<clang::clangd::detail::ModulesDependencyGraph::ModulesDependencyNode *> +clang::clangd::detail::ModulesDependencyGraph::getAllUsers( + PathRef Filename) const { + std::set<ModulesDependencyNode *> Result; + + llvm::SmallSetVector<ModulesDependencyNode *, 16> Worklist; + Worklist.insert(Nodes.at(Filename).get()); + + while (!Worklist.empty()) { + auto *Node = Worklist.pop_back_val(); + assert(!Result.count(Node)); + Result.insert(Node); + + for (auto *User : Node->Users) + if (!Result.count(User)) + Worklist.insert(User); + } + + return Result; +} + +void ModulesManager::setBMIPath(PathRef Filename, PathRef BMIPath) { + auto _ = lockGraph(); + Graph.SetBMIPath(Filename, BMIPath); + bool Success = !BMIPath.empty(); + + for (auto *User : + Success ? Graph.getNodeUsers(Filename) : Graph.getAllUsers(Filename)) { + if (Success && !Graph.IsReadyToCompile(User->Name)) + continue; + + if (!Success) + Graph.SetBMIPath(User->Name, ""); + + std::lock_guard<std::mutex> Lock(WaitingCallablesMutex); + if (!WaitingCallables.count(User->Name)) + continue; + + for (const auto &[ReadyTask, FailureTask] : WaitingCallables[User->Name]) + Runner.RunTask(Success ? std::move(ReadyTask) : std::move(FailureTask)); + + WaitingCallables[User->Name].clear(); + } +} + +void ModulesManager::GenerateModuleInterface(PathRef Path) { + /// Return an empty string implies that a compilation failure. + /// Return a string to the compiled module interface file. + auto GenerateInterfaceTask = [this](PathRef Path) -> std::string { + auto _ = llvm::make_scope_exit([this, Path] { + auto _ = lockGraph(); + assert(ScheduledInterfaces.count(Path)); + ScheduledInterfaces.erase(Path); + }); + + auto Cmd = CDB.getCompileCommand(tooling::getAbsolutePath(Path)); + if (!Cmd) { + log("Failed to get the command for {0} when generating module interface", + Path); + return ""; + } + + ParseInputs Inputs; + Inputs.TFS = &CDB.getThreadsafeFS(); + Inputs.CompileCommand = std::move(*Cmd); + + IgnoreDiagnostics IgnoreDiags; + auto CI = buildCompilerInvocation(Inputs, IgnoreDiags); + if (!CI) { + log("Failed to build the compiler invocation for {0} when generating " + "module interface", + Path); + return ""; + } + + auto FS = Inputs.TFS->view(Inputs.CompileCommand.Directory); + auto AbsolutePath = getAbsolutePath(Inputs.CompileCommand); + auto Buf = FS->getBufferForFile(AbsolutePath); + if (!Buf) + return ""; + + log("Generating module file: {0}", Inputs.CompileCommand.Output); + CI->getFrontendOpts().OutputFile = Inputs.CompileCommand.Output; + + auto Clang = + prepareCompilerInstance(std::move(CI), /*Preamble=*/nullptr, + std::move(*Buf), std::move(FS), IgnoreDiags); + if (!Clang) + return ""; + + GenerateModuleInterfaceAction Action; + Clang->ExecuteAction(Action); + + if (Clang->getDiagnostics().hasErrorOccurred()) + return ""; + + return Inputs.CompileCommand.Output; + }; + + auto Task = [Path = Path.str(), this, + GenerateInterfaceTask = std::move(GenerateInterfaceTask)] { + { + auto _ = lockGraph(); + assert(Graph.IsModuleInterface(Path)); + if (ScheduledInterfaces.count(Path)) + return; + + ScheduledInterfaces.insert(Path); + } + + setBMIPath(Path, GenerateInterfaceTask(Path)); + }; + + assert(!HasInvalidDependencies(Path) && + "It is meaningless to require to generate module interface if there " + "is invalid dependencies."); + + if (!IsReadyToCompile(Path)) { + addCallbackAfterReady(Path, std::move(Task)); + GenerateModuleInterfacesFor(Path); + return; + } + + Runner.RunTask(std::move(Task)); +} + +void ModulesManager::GenerateModuleInterfacesFor(PathRef Path) { + // No graph implies that we're not in a modules repo. + if (!Graph.IsInGraph(Path)) + return; + + assert(!HasInvalidDependencies(Path) && + "It is meaningless to require to generate module interface if there " + "is invalid dependencies."); + + if (Graph.IsReadyToCompile(Path)) + return; + + llvm::SmallVector<std::string, 8> Names; + { + auto _ = lockGraph(); + for (auto *Dep : Graph.getNode(Path)->Deps) + if (!Dep->BMIPath) + Names.push_back(Dep->Name); + } + + for (const auto &Name : Names) + GenerateModuleInterface(Name); + + return; +} + +bool ModulesManager::IsDirectlyDependent(PathRef A, PathRef B) const { + auto _ = lockGraph(); + + auto *NodeA = Graph.getNode(A); + auto *NodeB = Graph.getNode(B); + + assert(NodeA); + assert(NodeB); + assert(NodeB->Provided); + + using namespace llvm; + return any_of(NodeA->Deps, [NodeA, NodeB](auto *Dep) { + // This is a little bit redudandant due to this is for testing. + return Dep == NodeB && + any_of(NodeB->Users, [NodeA](auto *User) { return User == NodeA; }); + }); +} + +size_t ModulesManager::GraphSize() const { + auto _ = lockGraph(); + return Graph.size(); +} + +bool clang::clangd::detail::ModulesDependencyGraph::HasThirdpartyDependencies( + PathRef Path) const { + auto *Node = getNode(Path); + + assert(Node); + assert(Node->Requires.size() >= Node->Users.size()); + + // This is redundant intentionally due to this function should be called + // in test. + bool Result = false; + for (const auto &Required : Node->Requires) { + llvm::StringRef RequiredModuleName = Required.getKey(); + + if (!ModuleMapper.count(RequiredModuleName)) { + Result = true; + continue; + } + + // Test that the required node is in the deps. + auto *RequiredNode = ModuleMapper.at(RequiredModuleName); + assert(llvm::any_of( + Node->Deps, [RequiredNode](auto *Dep) { return RequiredNode == Dep; })); + } + + return Result; +} + +bool ModulesManager::HasThirdpartyDependencies(PathRef Path) const { + auto _ = lockGraph(); + return Graph.HasThirdpartyDependencies(Path); +} + +void clang::clangd::detail::ModulesTaskRunner::RunTask( + llvm::unique_function<void()> Task, const llvm::Twine &Name) { + if (Runner) + Runner->runAsync( + Name, [Task = std::move(Task), &Barrier = this->Barrier]() mutable { + std::unique_lock<Semaphore> Lock(Barrier, std::try_to_lock); + Task(); + }); + else + Task(); +} + +void clang::clangd::detail::ModulesTaskRunner::wait() { + if (Runner) + Runner->wait(); +} Index: clang-tools-extra/clangd/GlobalCompilationDatabase.h =================================================================== --- clang-tools-extra/clangd/GlobalCompilationDatabase.h +++ clang-tools-extra/clangd/GlobalCompilationDatabase.h @@ -31,6 +31,8 @@ std::string SourceRoot; }; +class ModulesManager; + /// Provides compilation arguments used for parsing C and C++ files. class GlobalCompilationDatabase { public: @@ -45,6 +47,10 @@ return std::nullopt; } + virtual std::vector<std::string> getAllFilesInProjectOf(PathRef File) const { + return {}; + } + /// Makes a guess at how to build a file. /// The default implementation just runs clang on the file. /// Clangd should treat the results as unreliable. @@ -61,6 +67,8 @@ return OnCommandChanged.observe(std::move(L)); } + virtual ModulesManager *getModulesManager() const { return nullptr; } + protected: mutable CommandChanged OnCommandChanged; }; @@ -75,11 +83,15 @@ getCompileCommand(PathRef File) const override; std::optional<ProjectInfo> getProjectInfo(PathRef File) const override; + virtual std::vector<std::string> + getAllFilesInProjectOf(PathRef File) const override; tooling::CompileCommand getFallbackCommand(PathRef File) const override; bool blockUntilIdle(Deadline D) const override; + virtual ModulesManager *getModulesManager() const override; + private: const GlobalCompilationDatabase *Base; std::unique_ptr<GlobalCompilationDatabase> BaseOwner; @@ -121,10 +133,12 @@ /// Returns the path to first directory containing a compilation database in /// \p File's parents. std::optional<ProjectInfo> getProjectInfo(PathRef File) const override; + virtual std::vector<std::string> + getAllFilesInProjectOf(PathRef File) const override; bool blockUntilIdle(Deadline Timeout) const override; -private: +protected: Options Opts; class DirectoryCache; @@ -161,6 +175,40 @@ friend class DirectoryBasedGlobalCompilationDatabaseCacheTest; }; +/// DirectoryBasedModulesGlobalCompilationDatabase - owns the modules manager +/// and replace the module related options to refer to the modules built by +/// clangd it self. So that we can avoid depending the compiler in the user +/// space and avoid affecting user's build. +class DirectoryBasedModulesGlobalCompilationDatabase + : public DirectoryBasedGlobalCompilationDatabase { +public: + DirectoryBasedModulesGlobalCompilationDatabase( + const DirectoryBasedGlobalCompilationDatabase::Options &CDB, + unsigned AsyncThreadsCount); + + virtual ModulesManager *getModulesManager() const override { + return ModuleMgr.get(); + } + + virtual std::optional<tooling::CompileCommand> + getCompileCommand(PathRef File) const override; + + bool blockUntilIdle(Deadline D) const override; + + std::optional<tooling::CompileCommand> + getOriginalCompileCommand(PathRef File) const; + + const ThreadsafeFS &getThreadsafeFS() const { return Opts.TFS; } + +private: + llvm::SmallString<128> getModuleFilesCachePrefix(PathRef File) const; + llvm::SmallString<128> getMappedModuleFiles(PathRef) const; + + std::unique_ptr<ModulesManager> ModuleMgr; + + GlobalCompilationDatabase::CommandChanged::Subscription CommandsChanged; +}; + /// Extracts system include search path from drivers matching QueryDriverGlobs /// and adds them to the compile flags. /// Returns null when \p QueryDriverGlobs is empty. Index: clang-tools-extra/clangd/GlobalCompilationDatabase.cpp =================================================================== --- clang-tools-extra/clangd/GlobalCompilationDatabase.cpp +++ clang-tools-extra/clangd/GlobalCompilationDatabase.cpp @@ -9,6 +9,7 @@ #include "GlobalCompilationDatabase.h" #include "Config.h" #include "FS.h" +#include "ModulesManager.h" #include "SourceCode.h" #include "support/Logger.h" #include "support/Path.h" @@ -729,6 +730,22 @@ return Res->PI; } +std::vector<std::string> +DirectoryBasedGlobalCompilationDatabase::getAllFilesInProjectOf( + PathRef File) const { + CDBLookupRequest Req; + Req.FileName = File; + Req.ShouldBroadcast = false; + Req.FreshTime = Req.FreshTimeMissing = + std::chrono::steady_clock::time_point::min(); + auto Res = lookupCDB(Req); + if (!Res) { + log("Failed to get the Compilation Database?"); + return {}; + } + return Res->CDB->getAllFiles(); +} + OverlayCDB::OverlayCDB(const GlobalCompilationDatabase *Base, std::vector<std::string> FallbackFlags, CommandMangler Mangler) @@ -805,6 +822,13 @@ return Base->getProjectInfo(File); } +std::vector<std::string> +DelegatingCDB::getAllFilesInProjectOf(PathRef File) const { + if (!Base) + return {}; + return Base->getAllFilesInProjectOf(File); +} + tooling::CompileCommand DelegatingCDB::getFallbackCommand(PathRef File) const { if (!Base) return GlobalCompilationDatabase::getFallbackCommand(File); @@ -817,5 +841,107 @@ return Base->blockUntilIdle(D); } +ModulesManager *DelegatingCDB::getModulesManager() const { + if (!Base) + return nullptr; + return Base->getModulesManager(); +} + +DirectoryBasedModulesGlobalCompilationDatabase:: + DirectoryBasedModulesGlobalCompilationDatabase( + const DirectoryBasedGlobalCompilationDatabase::Options &options, + unsigned AsyncThreadsCount) + : DirectoryBasedGlobalCompilationDatabase(options) { + ModuleMgr = std::make_unique<ModulesManager>(*this, AsyncThreadsCount); + + CommandsChanged = watch([ModuleMgr = ModuleMgr.get()]( + const std::vector<std::string> &ChangedFiles) { + ModuleMgr->UpdateBunchFiles(ChangedFiles); + }); +} + +llvm::SmallString<128> +DirectoryBasedModulesGlobalCompilationDatabase::getModuleFilesCachePrefix( + PathRef File) const { + std::optional<ProjectInfo> PI = getProjectInfo(File); + if (!PI) + return {}; + + llvm::SmallString<128> Result(PI->SourceRoot); + llvm::sys::path::append(Result, ".cache"); + llvm::sys::path::append(Result, "clangd"); + llvm::sys::path::append(Result, "module_files"); + + llvm::sys::fs::create_directories(Result, /*IgnoreExisting*/ true); + return Result; +} + +llvm::SmallString<128> +DirectoryBasedModulesGlobalCompilationDatabase::getMappedModuleFiles( + PathRef File) const { + std::string ModuleFileName = ModuleMgr->GetModuleInterfaceName(File); + + // Replace ":" in the module name with "-" to follow clang's style. Since ":" + // is not a valid character in some file systems. + auto [PrimaryName, PartitionName] = + llvm::StringRef(ModuleFileName).split(":"); + if (!PartitionName.empty()) + ModuleFileName = PrimaryName.str() + "-" + PartitionName.str(); + + llvm::SmallString<128> Result = getModuleFilesCachePrefix(File); + llvm::sys::path::append(Result, ModuleFileName + ".pcm"); + return Result; +} + +std::optional<tooling::CompileCommand> +DirectoryBasedModulesGlobalCompilationDatabase::getOriginalCompileCommand( + PathRef File) const { + return DirectoryBasedGlobalCompilationDatabase::getCompileCommand(File); +} + +bool DirectoryBasedModulesGlobalCompilationDatabase::blockUntilIdle( + Deadline D) const { + ModuleMgr->waitUntilInitialized(); + return DirectoryBasedGlobalCompilationDatabase::blockUntilIdle(D); +} + +std::optional<tooling::CompileCommand> +DirectoryBasedModulesGlobalCompilationDatabase::getCompileCommand( + PathRef File) const { + auto Result = + DirectoryBasedGlobalCompilationDatabase::getCompileCommand(File); + if (!Result) + return std::nullopt; + + if (!ModuleMgr->HasGraph() || !ModuleMgr->IsInGraph(File)) + return Result; + + if (ModuleMgr->IsModuleInterface(File)) + Result->Output = getMappedModuleFiles(File).str(); + + std::vector<std::string> CommandLine; + CommandLine.reserve(Result->CommandLine.size() + 1); + + CommandLine.emplace_back(Result->CommandLine[0]); + llvm::SmallString<128> ModuleOutputPrefix = getModuleFilesCachePrefix(File); + CommandLine.emplace_back( + llvm::Twine("-fprebuilt-module-path=" + ModuleOutputPrefix).str()); + + for (std::size_t I = 1; I < Result->CommandLine.size(); I++) { + const std::string &Arg = Result->CommandLine[I]; + const auto &[Left, Right] = StringRef(Arg).split("="); + + // Remove original `-fmodule-file=<module-name>=<module-path>` form. + if (Left == "-fmodule-file" && Right.contains("=")) + continue; + + CommandLine.emplace_back(Arg); + } + + Result->CommandLine = CommandLine; + + return Result; +} + } // namespace clangd } // namespace clang Index: clang-tools-extra/clangd/ClangdLSPServer.h =================================================================== --- clang-tools-extra/clangd/ClangdLSPServer.h +++ clang-tools-extra/clangd/ClangdLSPServer.h @@ -43,6 +43,8 @@ /// Look for compilation databases, rather than using compile commands /// set via LSP (extensions) only. bool UseDirBasedCDB = true; + /// Enable experimental support for modules. + bool ExperimentalModulesSupport = false; /// The offset-encoding to use, or std::nullopt to negotiate it over LSP. std::optional<OffsetEncoding> Encoding; /// If set, periodically called to release memory. Index: clang-tools-extra/clangd/ClangdLSPServer.cpp =================================================================== --- clang-tools-extra/clangd/ClangdLSPServer.cpp +++ clang-tools-extra/clangd/ClangdLSPServer.cpp @@ -14,6 +14,7 @@ #include "Feature.h" #include "GlobalCompilationDatabase.h" #include "LSPBinder.h" +#include "ModulesManager.h" #include "Protocol.h" #include "SemanticHighlighting.h" #include "SourceCode.h" @@ -535,8 +536,14 @@ if (const auto &Dir = Params.initializationOptions.compilationDatabasePath) CDBOpts.CompileCommandsDir = Dir; CDBOpts.ContextProvider = Opts.ContextProvider; - BaseCDB = - std::make_unique<DirectoryBasedGlobalCompilationDatabase>(CDBOpts); + + if (Opts.ExperimentalModulesSupport) + BaseCDB = + std::make_unique<DirectoryBasedModulesGlobalCompilationDatabase>( + CDBOpts, Opts.AsyncThreadsCount); + else + BaseCDB = + std::make_unique<DirectoryBasedGlobalCompilationDatabase>(CDBOpts); } auto Mangler = CommandMangler::detect(); Mangler.SystemIncludeExtractor = Index: clang-tools-extra/clangd/CMakeLists.txt =================================================================== --- clang-tools-extra/clangd/CMakeLists.txt +++ clang-tools-extra/clangd/CMakeLists.txt @@ -97,6 +97,7 @@ IncludeFixer.cpp InlayHints.cpp JSONTransport.cpp + ModulesManager.cpp PathMapping.cpp Protocol.cpp Quality.cpp @@ -173,6 +174,7 @@ clangToolingInclusions clangToolingInclusionsStdlib clangToolingSyntax + clangDependencyScanning ) target_link_libraries(clangDaemon
_______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
