Author: ioeric Date: Fri Dec 15 04:25:02 2017 New Revision: 320807 URL: http://llvm.org/viewvc/llvm-project?rev=320807&view=rev Log: [clangd] Build in-memory index on symbols in files.
Reviewers: sammccall Reviewed By: sammccall Subscribers: klimek, mgorny, ilya-biryukov, cfe-commits Differential Revision: https://reviews.llvm.org/D41276 Added: clang-tools-extra/trunk/clangd/index/FileIndex.cpp - copied, changed from r320743, clang-tools-extra/trunk/clangd/index/FileSymbols.cpp clang-tools-extra/trunk/clangd/index/FileIndex.h - copied, changed from r320743, clang-tools-extra/trunk/clangd/index/FileSymbols.h clang-tools-extra/trunk/unittests/clangd/FileIndexTests.cpp Removed: clang-tools-extra/trunk/clangd/index/FileSymbols.cpp clang-tools-extra/trunk/clangd/index/FileSymbols.h clang-tools-extra/trunk/unittests/clangd/FileSymbolsTests.cpp Modified: clang-tools-extra/trunk/clangd/CMakeLists.txt clang-tools-extra/trunk/unittests/clangd/CMakeLists.txt Modified: clang-tools-extra/trunk/clangd/CMakeLists.txt URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/CMakeLists.txt?rev=320807&r1=320806&r2=320807&view=diff ============================================================================== --- clang-tools-extra/trunk/clangd/CMakeLists.txt (original) +++ clang-tools-extra/trunk/clangd/CMakeLists.txt Fri Dec 15 04:25:02 2017 @@ -19,7 +19,7 @@ add_clang_library(clangDaemon Protocol.cpp ProtocolHandlers.cpp Trace.cpp - index/FileSymbols.cpp + index/FileIndex.cpp index/Index.cpp index/MemIndex.cpp index/SymbolCollector.cpp Copied: clang-tools-extra/trunk/clangd/index/FileIndex.cpp (from r320743, clang-tools-extra/trunk/clangd/index/FileSymbols.cpp) URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/FileIndex.cpp?p2=clang-tools-extra/trunk/clangd/index/FileIndex.cpp&p1=clang-tools-extra/trunk/clangd/index/FileSymbols.cpp&r1=320743&r2=320807&rev=320807&view=diff ============================================================================== --- clang-tools-extra/trunk/clangd/index/FileSymbols.cpp (original) +++ clang-tools-extra/trunk/clangd/index/FileIndex.cpp Fri Dec 15 04:25:02 2017 @@ -1,4 +1,4 @@ -//===--- FileSymbols.cpp - Symbols from files. ------------------*- C++-*-===// +//===--- FileIndex.cpp - Indexes for files. ------------------------ C++-*-===// // // The LLVM Compiler Infrastructure // @@ -7,11 +7,30 @@ // //===----------------------------------------------------------------------===// -#include "FileSymbols.h" +#include "FileIndex.h" +#include "SymbolCollector.h" #include "clang/Index/IndexingAction.h" namespace clang { namespace clangd { +namespace { + +/// Retrieves namespace and class level symbols in \p Decls. +std::unique_ptr<SymbolSlab> indexAST(ASTContext &Ctx, + llvm::ArrayRef<const Decl *> Decls) { + auto Collector = std::make_shared<SymbolCollector>(); + index::IndexingOptions IndexOpts; + IndexOpts.SystemSymbolFilter = + index::IndexingOptions::SystemSymbolFilterKind::All; + IndexOpts.IndexFunctionLocals = false; + + index::indexTopLevelDecls(Ctx, Decls, Collector, IndexOpts); + auto Symbols = llvm::make_unique<SymbolSlab>(); + *Symbols = Collector->takeSymbols(); + return Symbols; +} + +} // namespace void FileSymbols::update(PathRef Path, std::unique_ptr<SymbolSlab> Slab) { std::lock_guard<std::mutex> Lock(Mutex); @@ -44,5 +63,21 @@ std::shared_ptr<std::vector<const Symbol return {std::move(Snap), Pointers}; } +void FileIndex::update(Context &Ctx, PathRef Path, ParsedAST *AST) { + if (!AST) { + FSymbols.update(Path, nullptr); + } else { + auto Slab = indexAST(AST->getASTContext(), AST->getTopLevelDecls()); + FSymbols.update(Path, std::move(Slab)); + } + auto Symbols = FSymbols.allSymbols(); + Index.build(std::move(Symbols)); +} + +bool FileIndex::fuzzyFind(Context &Ctx, const FuzzyFindRequest &Req, + std::function<void(const Symbol &)> Callback) const { + return Index.fuzzyFind(Ctx, Req, std::move(Callback)); +} + } // namespace clangd } // namespace clang Copied: clang-tools-extra/trunk/clangd/index/FileIndex.h (from r320743, clang-tools-extra/trunk/clangd/index/FileSymbols.h) URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/FileIndex.h?p2=clang-tools-extra/trunk/clangd/index/FileIndex.h&p1=clang-tools-extra/trunk/clangd/index/FileSymbols.h&r1=320743&r2=320807&rev=320807&view=diff ============================================================================== --- clang-tools-extra/trunk/clangd/index/FileSymbols.h (original) +++ clang-tools-extra/trunk/clangd/index/FileIndex.h Fri Dec 15 04:25:02 2017 @@ -1,4 +1,4 @@ -//===--- FileSymbols.h - Symbols from files. ---------------------*- C++-*-===// +//===--- FileIndex.h - Index for files. ---------------------------- C++-*-===// // // The LLVM Compiler Infrastructure // @@ -6,14 +6,20 @@ // License. See LICENSE.TXT for details. // //===----------------------------------------------------------------------===// +// +// FileIndex implements SymbolIndex for symbols from a set of files. Symbols are +// maintained at source-file granuality (e.g. with ASTs), and files can be +// updated dynamically. +// +//===---------------------------------------------------------------------===// -#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_FILESYMBOLS_H -#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_FILESYMBOLS_H +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_FILEINDEX_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_FILEINDEX_H -#include "../Path.h" +#include "../ClangdUnit.h" +#include "../Context.h" #include "Index.h" -#include "llvm/ADT/StringMap.h" -#include <mutex> +#include "MemIndex.h" namespace clang { namespace clangd { @@ -47,7 +53,22 @@ private: llvm::StringMap<std::shared_ptr<SymbolSlab>> FileToSlabs; }; +/// \brief This manages symbls from files and an in-memory index on all symbols. +class FileIndex : public SymbolIndex { +public: + /// \brief Update symbols in \p Path with symbols in \p AST. If \p AST is + /// nullptr, this removes all symbols in the file + void update(Context &Ctx, PathRef Path, ParsedAST *AST); + + bool fuzzyFind(Context &Ctx, const FuzzyFindRequest &Req, + std::function<void(const Symbol &)> Callback) const override; + +private: + FileSymbols FSymbols; + MemIndex Index; +}; + } // namespace clangd } // namespace clang -#endif // LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_FILESYMBOLS_H +#endif // LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_FILEINDEX_H Removed: clang-tools-extra/trunk/clangd/index/FileSymbols.cpp URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/FileSymbols.cpp?rev=320806&view=auto ============================================================================== --- clang-tools-extra/trunk/clangd/index/FileSymbols.cpp (original) +++ clang-tools-extra/trunk/clangd/index/FileSymbols.cpp (removed) @@ -1,48 +0,0 @@ -//===--- FileSymbols.cpp - Symbols from files. ------------------*- C++-*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// - -#include "FileSymbols.h" -#include "clang/Index/IndexingAction.h" - -namespace clang { -namespace clangd { - -void FileSymbols::update(PathRef Path, std::unique_ptr<SymbolSlab> Slab) { - std::lock_guard<std::mutex> Lock(Mutex); - if (!Slab) - FileToSlabs.erase(Path); - else - FileToSlabs[Path] = std::shared_ptr<SymbolSlab>(Slab.release()); -} - -std::shared_ptr<std::vector<const Symbol *>> FileSymbols::allSymbols() { - // The snapshot manages life time of symbol slabs and provides pointers of all - // symbols in all slabs. - struct Snapshot { - std::vector<const Symbol *> Pointers; - std::vector<std::shared_ptr<SymbolSlab>> KeepAlive; - }; - auto Snap = std::make_shared<Snapshot>(); - { - std::lock_guard<std::mutex> Lock(Mutex); - - for (const auto &FileAndSlab : FileToSlabs) { - Snap->KeepAlive.push_back(FileAndSlab.second); - for (const auto &Iter : *FileAndSlab.second) - Snap->Pointers.push_back(&Iter.second); - } - } - auto *Pointers = &Snap->Pointers; - // Use aliasing constructor to keep the snapshot alive along with the - // pointers. - return {std::move(Snap), Pointers}; -} - -} // namespace clangd -} // namespace clang Removed: clang-tools-extra/trunk/clangd/index/FileSymbols.h URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/FileSymbols.h?rev=320806&view=auto ============================================================================== --- clang-tools-extra/trunk/clangd/index/FileSymbols.h (original) +++ clang-tools-extra/trunk/clangd/index/FileSymbols.h (removed) @@ -1,53 +0,0 @@ -//===--- FileSymbols.h - Symbols from files. ---------------------*- C++-*-===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// - -#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_FILESYMBOLS_H -#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_FILESYMBOLS_H - -#include "../Path.h" -#include "Index.h" -#include "llvm/ADT/StringMap.h" -#include <mutex> - -namespace clang { -namespace clangd { - -/// \brief A container of Symbols from several source files. It can be updated -/// at source-file granularity, replacing all symbols from one file with a new -/// set. -/// -/// This implements a snapshot semantics for symbols in a file. Each update to a -/// file will create a new snapshot for all symbols in the file. Snapshots are -/// managed with shared pointers that are shared between this class and the -/// users. For each file, this class only stores a pointer pointing to the -/// newest snapshot, and an outdated snapshot is deleted by the last owner of -/// the snapshot, either this class or the symbol index. -/// -/// The snapshot semantics keeps critical sections minimal since we only need -/// locking when we swap or obtain refereces to snapshots. -class FileSymbols { -public: - /// \brief Updates all symbols in a file. If \p Slab is nullptr, symbols for - /// \p Path will be removed. - void update(PathRef Path, std::unique_ptr<SymbolSlab> Slab); - - // The shared_ptr keeps the symbols alive - std::shared_ptr<std::vector<const Symbol *>> allSymbols(); - -private: - mutable std::mutex Mutex; - - /// \brief Stores the latest snapshots for all active files. - llvm::StringMap<std::shared_ptr<SymbolSlab>> FileToSlabs; -}; - -} // namespace clangd -} // namespace clang - -#endif // LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_FILESYMBOLS_H Modified: clang-tools-extra/trunk/unittests/clangd/CMakeLists.txt URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/unittests/clangd/CMakeLists.txt?rev=320807&r1=320806&r2=320807&view=diff ============================================================================== --- clang-tools-extra/trunk/unittests/clangd/CMakeLists.txt (original) +++ clang-tools-extra/trunk/unittests/clangd/CMakeLists.txt Fri Dec 15 04:25:02 2017 @@ -12,7 +12,7 @@ add_extra_unittest(ClangdTests ClangdTests.cpp CodeCompleteTests.cpp ContextTests.cpp - FileSymbolsTests.cpp + FileIndexTests.cpp FuzzyMatchTests.cpp IndexTests.cpp JSONExprTests.cpp Added: clang-tools-extra/trunk/unittests/clangd/FileIndexTests.cpp URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/unittests/clangd/FileIndexTests.cpp?rev=320807&view=auto ============================================================================== --- clang-tools-extra/trunk/unittests/clangd/FileIndexTests.cpp (added) +++ clang-tools-extra/trunk/unittests/clangd/FileIndexTests.cpp Fri Dec 15 04:25:02 2017 @@ -0,0 +1,193 @@ +//===-- FileIndexTests.cpp ---------------------------*- C++ -*-----------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "index/FileIndex.h" +#include "clang/Frontend/CompilerInvocation.h" +#include "clang/Frontend/PCHContainerOperations.h" +#include "clang/Frontend/Utils.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::UnorderedElementsAre; + +namespace clang { +namespace clangd { + +namespace { + +Symbol symbol(llvm::StringRef ID) { + Symbol Sym; + Sym.ID = SymbolID(ID); + Sym.QualifiedName = ID; + return Sym; +} + +void addNumSymbolsToSlab(int Begin, int End, SymbolSlab *Slab) { + for (int i = Begin; i <= End; i++) + Slab->insert(symbol(std::to_string(i))); +} + +std::vector<std::string> +getSymbolNames(const std::vector<const Symbol *> &Symbols) { + std::vector<std::string> Names; + for (const Symbol *Sym : Symbols) + Names.push_back(Sym->QualifiedName); + return Names; +} + +TEST(FileSymbolsTest, UpdateAndGet) { + FileSymbols FS; + EXPECT_THAT(getSymbolNames(*FS.allSymbols()), UnorderedElementsAre()); + + auto Slab = llvm::make_unique<SymbolSlab>(); + addNumSymbolsToSlab(1, 3, Slab.get()); + + FS.update("f1", std::move(Slab)); + + EXPECT_THAT(getSymbolNames(*FS.allSymbols()), + UnorderedElementsAre("1", "2", "3")); +} + +TEST(FileSymbolsTest, Overlap) { + FileSymbols FS; + + auto Slab = llvm::make_unique<SymbolSlab>(); + addNumSymbolsToSlab(1, 3, Slab.get()); + + FS.update("f1", std::move(Slab)); + + Slab = llvm::make_unique<SymbolSlab>(); + addNumSymbolsToSlab(3, 5, Slab.get()); + + FS.update("f2", std::move(Slab)); + + EXPECT_THAT(getSymbolNames(*FS.allSymbols()), + UnorderedElementsAre("1", "2", "3", "3", "4", "5")); +} + +TEST(FileSymbolsTest, SnapshotAliveAfterRemove) { + FileSymbols FS; + + auto Slab = llvm::make_unique<SymbolSlab>(); + addNumSymbolsToSlab(1, 3, Slab.get()); + + FS.update("f1", std::move(Slab)); + + auto Symbols = FS.allSymbols(); + EXPECT_THAT(getSymbolNames(*Symbols), UnorderedElementsAre("1", "2", "3")); + + FS.update("f1", nullptr); + EXPECT_THAT(getSymbolNames(*FS.allSymbols()), UnorderedElementsAre()); + + EXPECT_THAT(getSymbolNames(*Symbols), UnorderedElementsAre("1", "2", "3")); +} + +std::vector<std::string> match(const SymbolIndex &I, + const FuzzyFindRequest &Req) { + std::vector<std::string> Matches; + auto Ctx = Context::empty(); + I.fuzzyFind(Ctx, Req, + [&](const Symbol &Sym) { Matches.push_back(Sym.QualifiedName); }); + return Matches; +} + +/// Create an ParsedAST for \p Code. Returns None if \p Code is empty. +llvm::Optional<ParsedAST> build(std::string Path, llvm::StringRef Code) { + Context Ctx = Context::empty(); + if (Code.empty()) + return llvm::None; + const char *Args[] = {"clang", "-xc++", Path.c_str()}; + + auto CI = createInvocationFromCommandLine(Args); + + auto Buf = llvm::MemoryBuffer::getMemBuffer(Code); + auto AST = ParsedAST::Build(Ctx, std::move(CI), nullptr, std::move(Buf), + std::make_shared<PCHContainerOperations>(), + vfs::getRealFileSystem()); + assert(AST.hasValue()); + return std::move(*AST); +} + +TEST(FileIndexTest, IndexAST) { + FileIndex M; + auto Ctx = Context::empty(); + M.update( + Ctx, "f1", + build("f1", "namespace ns { void f() {} class X {}; }").getPointer()); + + FuzzyFindRequest Req; + Req.Query = "ns::"; + EXPECT_THAT(match(M, Req), UnorderedElementsAre("ns::f", "ns::X")); +} + +TEST(FileIndexTest, NoLocal) { + FileIndex M; + auto Ctx = Context::empty(); + M.update( + Ctx, "f1", + build("f1", "namespace ns { void f() { int local = 0; } class X {}; }") + .getPointer()); + + FuzzyFindRequest Req; + Req.Query = ""; + EXPECT_THAT(match(M, Req), UnorderedElementsAre("ns", "ns::f", "ns::X")); +} + +TEST(FileIndexTest, IndexMultiASTAndDeduplicate) { + FileIndex M; + auto Ctx = Context::empty(); + M.update( + Ctx, "f1", + build("f1", "namespace ns { void f() {} class X {}; }").getPointer()); + M.update( + Ctx, "f2", + build("f2", "namespace ns { void ff() {} class X {}; }").getPointer()); + + FuzzyFindRequest Req; + Req.Query = "ns::"; + EXPECT_THAT(match(M, Req), + UnorderedElementsAre("ns::f", "ns::X", "ns::ff")); +} + +TEST(FileIndexTest, RemoveAST) { + FileIndex M; + auto Ctx = Context::empty(); + M.update( + Ctx, "f1", + build("f1", "namespace ns { void f() {} class X {}; }").getPointer()); + + FuzzyFindRequest Req; + Req.Query = "ns::"; + EXPECT_THAT(match(M, Req), UnorderedElementsAre("ns::f", "ns::X")); + + M.update(Ctx, "f1", nullptr); + EXPECT_THAT(match(M, Req), UnorderedElementsAre()); +} + +TEST(FileIndexTest, RemoveNonExisting) { + FileIndex M; + auto Ctx = Context::empty(); + M.update(Ctx, "no", nullptr); + EXPECT_THAT(match(M, FuzzyFindRequest()), UnorderedElementsAre()); +} + +TEST(FileIndexTest, ClassMembers) { + FileIndex M; + auto Ctx = Context::empty(); + M.update(Ctx, "f1", + build("f1", "class X { static int m1; int m2;};").getPointer()); + + FuzzyFindRequest Req; + Req.Query = ""; + EXPECT_THAT(match(M, Req), UnorderedElementsAre("X", "X::m1", "X::m2")); +} + +} // namespace +} // namespace clangd +} // namespace clang Removed: clang-tools-extra/trunk/unittests/clangd/FileSymbolsTests.cpp URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/unittests/clangd/FileSymbolsTests.cpp?rev=320806&view=auto ============================================================================== --- clang-tools-extra/trunk/unittests/clangd/FileSymbolsTests.cpp (original) +++ clang-tools-extra/trunk/unittests/clangd/FileSymbolsTests.cpp (removed) @@ -1,91 +0,0 @@ -//===-- FileSymbolsTests.cpp -------------------------*- C++ -*-----------===// -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// - -#include "index/FileSymbols.h" -#include "gmock/gmock.h" -#include "gtest/gtest.h" - -using testing::UnorderedElementsAre; - -namespace clang { -namespace clangd { - -namespace { - -Symbol symbol(llvm::StringRef ID) { - Symbol Sym; - Sym.ID = SymbolID(ID); - Sym.QualifiedName = ID; - return Sym; -} - -void addNumSymbolsToSlab(int Begin, int End, SymbolSlab *Slab) { - for (int i = Begin; i <= End; i++) - Slab->insert(symbol(std::to_string(i))); -} - -std::vector<std::string> -getSymbolNames(const std::vector<const Symbol *> &Symbols) { - std::vector<std::string> Names; - for (const Symbol *Sym : Symbols) - Names.push_back(Sym->QualifiedName); - return Names; -} - -TEST(FileSymbolsTest, UpdateAndGet) { - FileSymbols FS; - EXPECT_THAT(getSymbolNames(*FS.allSymbols()), UnorderedElementsAre()); - - auto Slab = llvm::make_unique<SymbolSlab>(); - addNumSymbolsToSlab(1, 3, Slab.get()); - - FS.update("f1", std::move(Slab)); - - EXPECT_THAT(getSymbolNames(*FS.allSymbols()), - UnorderedElementsAre("1", "2", "3")); -} - -TEST(FileSymbolsTest, Overlap) { - FileSymbols FS; - - auto Slab = llvm::make_unique<SymbolSlab>(); - addNumSymbolsToSlab(1, 3, Slab.get()); - - FS.update("f1", std::move(Slab)); - - Slab = llvm::make_unique<SymbolSlab>(); - addNumSymbolsToSlab(3, 5, Slab.get()); - - FS.update("f2", std::move(Slab)); - - EXPECT_THAT(getSymbolNames(*FS.allSymbols()), - UnorderedElementsAre("1", "2", "3", "3", "4", "5")); -} - -TEST(FileSymbolsTest, SnapshotAliveAfterRemove) { - FileSymbols FS; - - auto Slab = llvm::make_unique<SymbolSlab>(); - addNumSymbolsToSlab(1, 3, Slab.get()); - - FS.update("f1", std::move(Slab)); - - auto Symbols = FS.allSymbols(); - EXPECT_THAT(getSymbolNames(*Symbols), UnorderedElementsAre("1", "2", "3")); - - FS.update("f1", nullptr); - EXPECT_THAT(getSymbolNames(*FS.allSymbols()), UnorderedElementsAre()); - - EXPECT_THAT(getSymbolNames(*Symbols), UnorderedElementsAre("1", "2", "3")); -} - -} // namespace -} // namespace clangd -} // namespace clang - _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
