[clang-tools-extra] r339426 - [clangd] Allow consuming limited number of items

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Fri Aug 10 04:50:44 2018
New Revision: 339426

URL: http://llvm.org/viewvc/llvm-project?rev=339426&view=rev
Log:
[clangd] Allow consuming limited number of items

This patch modifies `consume` function to allow retrieval of limited
number of symbols. This is the "cheap" implementation of top-level
limiting iterator. In the future we would like to have a complete limit
iterator implementation to insert it into the query subtrees, but in the
meantime this version would be enough for a fully-functional
proof-of-concept Dex implementation.

Reviewers: ioeric, ilya-biryukov

Reviewed by: ioeric

Differential Revision: https://reviews.llvm.org/D50500

Modified:
clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp
clang-tools-extra/trunk/clangd/index/dex/Iterator.h
clang-tools-extra/trunk/unittests/clangd/DexIndexTests.cpp

Modified: clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp?rev=339426&r1=339425&r2=339426&view=diff
==
--- clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp Fri Aug 10 04:50:44 
2018
@@ -218,9 +218,10 @@ private:
 
 } // end namespace
 
-std::vector consume(Iterator &It) {
+std::vector consume(Iterator &It, size_t Limit) {
   std::vector Result;
-  for (; !It.reachedEnd(); It.advance())
+  for (size_t Retrieved = 0; !It.reachedEnd() && Retrieved < Limit;
+   It.advance(), ++Retrieved)
 Result.push_back(It.peek());
   return Result;
 }

Modified: clang-tools-extra/trunk/clangd/index/dex/Iterator.h
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/Iterator.h?rev=339426&r1=339425&r2=339426&view=diff
==
--- clang-tools-extra/trunk/clangd/index/dex/Iterator.h (original)
+++ clang-tools-extra/trunk/clangd/index/dex/Iterator.h Fri Aug 10 04:50:44 2018
@@ -101,9 +101,10 @@ private:
   virtual llvm::raw_ostream &dump(llvm::raw_ostream &OS) const = 0;
 };
 
-/// Exhausts given iterator and returns all processed DocIDs. The result
-/// contains sorted DocumentIDs.
-std::vector consume(Iterator &It);
+/// Advances the iterator until it is either exhausted or the number of
+/// requested items is reached. The result contains sorted DocumentIDs.
+std::vector consume(Iterator &It,
+   size_t Limit = std::numeric_limits::max());
 
 /// Returns a document iterator over given PostingList.
 std::unique_ptr create(PostingListRef Documents);

Modified: clang-tools-extra/trunk/unittests/clangd/DexIndexTests.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/unittests/clangd/DexIndexTests.cpp?rev=339426&r1=339425&r2=339426&view=diff
==
--- clang-tools-extra/trunk/unittests/clangd/DexIndexTests.cpp (original)
+++ clang-tools-extra/trunk/unittests/clangd/DexIndexTests.cpp Fri Aug 10 
04:50:44 2018
@@ -240,6 +240,27 @@ TEST(DexIndexIterators, StringRepresenta
 "(& (& [1, 3, 5, 8, 9] [1, 5, 7, 9]) (| [0, 5] [0, 1, 5] []))");
 }
 
+TEST(DexIndexIterators, Limit) {
+  const PostingList L0 = {4, 7, 8, 20, 42, 100};
+  const PostingList L1 = {1, 3, 5, 8, 9};
+  const PostingList L2 = {1, 5, 7, 9};
+  const PostingList L3 = {0, 5};
+  const PostingList L4 = {0, 1, 5};
+  const PostingList L5;
+
+  auto DocIterator = create(L0);
+  EXPECT_THAT(consume(*DocIterator, 42), ElementsAre(4, 7, 8, 20, 42, 100));
+
+  DocIterator = create(L0);
+  EXPECT_THAT(consume(*DocIterator), ElementsAre(4, 7, 8, 20, 42, 100));
+
+  DocIterator = create(L0);
+  EXPECT_THAT(consume(*DocIterator, 3), ElementsAre(4, 7, 8));
+
+  DocIterator = create(L0);
+  EXPECT_THAT(consume(*DocIterator, 0), ElementsAre());
+}
+
 testing::Matcher>
 trigramsAre(std::initializer_list Trigrams) {
   std::vector Tokens;


___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] r339548 - [clangd] Generate incomplete trigrams for the Dex index

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Mon Aug 13 01:57:06 2018
New Revision: 339548

URL: http://llvm.org/viewvc/llvm-project?rev=339548&view=rev
Log:
[clangd] Generate incomplete trigrams for the Dex index

This patch handles trigram generation "short" identifiers and queries.
Trigram generator produces incomplete trigrams for short names so that
the same query iterator API can be used to match symbols which don't
have enough symbols to form a trigram and correctly handle queries which
also are not sufficient for generating a full trigram.

Reviewed by: ioeric

Differential revision: https://reviews.llvm.org/D50517

Modified:
clang-tools-extra/trunk/clangd/index/dex/Iterator.h
clang-tools-extra/trunk/clangd/index/dex/Trigram.cpp
clang-tools-extra/trunk/clangd/index/dex/Trigram.h
clang-tools-extra/trunk/unittests/clangd/DexIndexTests.cpp

Modified: clang-tools-extra/trunk/clangd/index/dex/Iterator.h
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/Iterator.h?rev=339548&r1=339547&r2=339548&view=diff
==
--- clang-tools-extra/trunk/clangd/index/dex/Iterator.h (original)
+++ clang-tools-extra/trunk/clangd/index/dex/Iterator.h Mon Aug 13 01:57:06 2018
@@ -47,6 +47,14 @@ using DocID = uint32_t;
 /// Contains sorted sequence of DocIDs all of which belong to symbols matching
 /// certain criteria, i.e. containing a Search Token. PostingLists are values
 /// for the inverted index.
+// FIXME(kbobyrev): Posting lists for incomplete trigrams (one/two symbols) are
+// likely to be very dense and hence require special attention so that the 
index
+// doesn't use too much memory. Possible solution would be to construct
+// compressed posting lists which consist of ranges of DocIDs instead of
+// distinct DocIDs. A special case would be the empty query: for that case
+// TrueIterator should be implemented - an iterator which doesn't actually 
store
+// any PostingList within itself, but "contains" all DocIDs in range
+// [0, IndexSize).
 using PostingList = std::vector;
 /// Immutable reference to PostingList object.
 using PostingListRef = llvm::ArrayRef;

Modified: clang-tools-extra/trunk/clangd/index/dex/Trigram.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/Trigram.cpp?rev=339548&r1=339547&r2=339548&view=diff
==
--- clang-tools-extra/trunk/clangd/index/dex/Trigram.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/dex/Trigram.cpp Mon Aug 13 01:57:06 
2018
@@ -10,11 +10,9 @@
 #include "Trigram.h"
 #include "../../FuzzyMatch.h"
 #include "Token.h"
-
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/StringExtras.h"
-
 #include 
 #include 
 #include 
@@ -25,12 +23,11 @@ namespace clang {
 namespace clangd {
 namespace dex {
 
-// FIXME(kbobyrev): Deal with short symbol symbol names. A viable approach 
would
-// be generating unigrams and bigrams here, too. This would prevent symbol 
index
-// from applying fuzzy matching on a tremendous number of symbols and allow
-// supplementary retrieval for short queries.
-//
-// Short names (total segment length <3 characters) are currently ignored.
+/// This is used to mark unigrams and bigrams and distinct them from complete
+/// trigrams. Since '$' is not present in valid identifier names, it is safe to
+/// use it as the special symbol.
+static const char END_MARKER = '$';
+
 std::vector generateIdentifierTrigrams(llvm::StringRef Identifier) {
   // Apply fuzzy matching text segmentation.
   std::vector Roles(Identifier.size());
@@ -50,17 +47,45 @@ std::vector generateIdentifierTri
   // not available then 0 is stored.
   std::vector> Next(LowercaseIdentifier.size());
   unsigned NextTail = 0, NextHead = 0, NextNextHead = 0;
+  // Store two first HEAD characters in the identifier (if present).
+  std::deque TwoHeads;
   for (int I = LowercaseIdentifier.size() - 1; I >= 0; --I) {
 Next[I] = {{NextTail, NextHead, NextNextHead}};
 NextTail = Roles[I] == Tail ? I : 0;
 if (Roles[I] == Head) {
   NextNextHead = NextHead;
   NextHead = I;
+  TwoHeads.push_front(LowercaseIdentifier[I]);
+  if (TwoHeads.size() > 2)
+TwoHeads.pop_back();
 }
   }
 
   DenseSet UniqueTrigrams;
-  std::array Chars;
+
+  auto add = [&](std::string Chars) {
+UniqueTrigrams.insert(Token(Token::Kind::Trigram, Chars));
+  };
+
+  // FIXME(kbobyrev): Instead of producing empty trigram for each identifier,
+  // just use True Iterator on the query side when the query string is empty.
+  add({{END_MARKER, END_MARKER, END_MARKER}});
+
+  if (TwoHeads.size() == 2)
+add({{TwoHeads.front(), TwoHeads.back(), END_MARKER}});
+
+  if (!LowercaseIdentifier.empty())
+add({{LowercaseIdentifier.front(), END_MARKER, END_MARKER}});
+
+  if (LowercaseIdentifier.size() >= 2)
+add({{LowercaseIdentifier[0], Lowercase

[clang-tools-extra] r339673 - [clangd] NFC: Cleanup clangd help message

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Tue Aug 14 05:00:39 2018
New Revision: 339673

URL: http://llvm.org/viewvc/llvm-project?rev=339673&view=rev
Log:
[clangd] NFC: Cleanup clangd help message

Add missed space, fix a typo.

Reviewed by: ioeric

Differential Revision: https://reviews.llvm.org/D50702

Modified:
clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp

Modified: clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp?rev=339673&r1=339672&r2=339673&view=diff
==
--- clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp (original)
+++ clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp Tue Aug 14 05:00:39 2018
@@ -147,7 +147,7 @@ static llvm::cl::opt InputMirrorFi
 static llvm::cl::opt EnableIndex(
 "index",
 llvm::cl::desc("Enable index-based features such as global code completion 
"
-   "and searching for symbols."
+   "and searching for symbols. "
"Clang uses an index built from symbols in opened files"),
 llvm::cl::init(true));
 
@@ -160,7 +160,7 @@ static llvm::cl::opt
 static llvm::cl::opt HeaderInsertionDecorators(
 "header-insertion-decorators",
 llvm::cl::desc("Prepend a circular dot or space before the completion "
-   "label, depending on wether "
+   "label, depending on whether "
"an include line will be inserted or not."),
 llvm::cl::init(true));
 


___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] r339687 - NFC: Enforce good formatting across multiple clang-tools-extra files

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Tue Aug 14 09:03:32 2018
New Revision: 339687

URL: http://llvm.org/viewvc/llvm-project?rev=339687&view=rev
Log:
NFC: Enforce good formatting across multiple clang-tools-extra files

This patch improves readability of multiple files in clang-tools-extra
and enforces LLVM Coding Guidelines.

Reviewed by: ioeric

Differential Revision: https://reviews.llvm.org/D50707

Modified:
clang-tools-extra/trunk/clangd/ClangdLSPServer.cpp
clang-tools-extra/trunk/clangd/ClangdLSPServer.h
clang-tools-extra/trunk/clangd/ClangdUnit.cpp
clang-tools-extra/trunk/clangd/ClangdUnit.h
clang-tools-extra/trunk/clangd/CodeComplete.cpp
clang-tools-extra/trunk/clangd/CodeComplete.h
clang-tools-extra/trunk/clangd/CodeCompletionStrings.cpp
clang-tools-extra/trunk/clangd/CodeCompletionStrings.h
clang-tools-extra/trunk/clangd/Compiler.cpp
clang-tools-extra/trunk/clangd/Compiler.h
clang-tools-extra/trunk/clangd/Context.cpp
clang-tools-extra/trunk/clangd/Diagnostics.cpp
clang-tools-extra/trunk/clangd/Diagnostics.h
clang-tools-extra/trunk/clangd/GlobalCompilationDatabase.cpp
clang-tools-extra/trunk/clangd/GlobalCompilationDatabase.h
clang-tools-extra/trunk/clangd/Quality.cpp
clang-tools-extra/trunk/clangd/Quality.h
clang-tools-extra/trunk/clangd/XRefs.cpp
clang-tools-extra/trunk/clangd/XRefs.h

clang-tools-extra/trunk/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp
clang-tools-extra/trunk/clangd/index/CanonicalIncludes.h
clang-tools-extra/trunk/clangd/index/FileIndex.h
clang-tools-extra/trunk/clangd/index/Index.h
clang-tools-extra/trunk/clangd/index/Merge.cpp
clang-tools-extra/trunk/clangd/index/Merge.h
clang-tools-extra/trunk/clangd/index/SymbolYAML.h
clang-tools-extra/trunk/clangd/index/dex/Iterator.h
clang-tools-extra/trunk/clangd/index/dex/Token.h
clang-tools-extra/trunk/clangd/index/dex/Trigram.h
clang-tools-extra/trunk/modularize/ModuleAssistant.cpp
clang-tools-extra/trunk/unittests/clangd/Annotations.cpp
clang-tools-extra/trunk/unittests/clangd/Annotations.h
clang-tools-extra/trunk/unittests/clangd/SyncAPI.cpp
clang-tools-extra/trunk/unittests/clangd/SyncAPI.h
clang-tools-extra/trunk/unittests/clangd/TestTU.cpp
clang-tools-extra/trunk/unittests/clangd/TestTU.h

Modified: clang-tools-extra/trunk/clangd/ClangdLSPServer.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/ClangdLSPServer.cpp?rev=339687&r1=339686&r2=339687&view=diff
==
--- clang-tools-extra/trunk/clangd/ClangdLSPServer.cpp (original)
+++ clang-tools-extra/trunk/clangd/ClangdLSPServer.cpp Tue Aug 14 09:03:32 2018
@@ -5,7 +5,7 @@
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
-//===-===//
+//===--===//
 
 #include "ClangdLSPServer.h"
 #include "Diagnostics.h"

Modified: clang-tools-extra/trunk/clangd/ClangdLSPServer.h
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/ClangdLSPServer.h?rev=339687&r1=339686&r2=339687&view=diff
==
--- clang-tools-extra/trunk/clangd/ClangdLSPServer.h (original)
+++ clang-tools-extra/trunk/clangd/ClangdLSPServer.h Tue Aug 14 09:03:32 2018
@@ -5,7 +5,7 @@
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
-//===-===//
+//===--===//
 
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_CLANGDLSPSERVER_H
 #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_CLANGDLSPSERVER_H
@@ -172,4 +172,4 @@ private:
 } // namespace clangd
 } // namespace clang
 
-#endif
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANGD_CLANGDLSPSERVER_H

Modified: clang-tools-extra/trunk/clangd/ClangdUnit.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/ClangdUnit.cpp?rev=339687&r1=339686&r2=339687&view=diff
==
--- clang-tools-extra/trunk/clangd/ClangdUnit.cpp (original)
+++ clang-tools-extra/trunk/clangd/ClangdUnit.cpp Tue Aug 14 09:03:32 2018
@@ -1,11 +1,11 @@
-//===--- ClangdUnit.cpp -*- C++-*-===//
+//===--- ClangdUnit.cpp --*- 
C++-*-===//
 //
 // The LLVM Compiler Infrastructure
 //
 // This file is distributed under the University of Illinois Open Source
 // License. See LICENSE.TXT for details.
 //
-//===-===//
+//===-

[clang-tools-extra] r339877 - [clangd] NFC: Improve Dex Iterators debugging traits

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Thu Aug 16 06:19:43 2018
New Revision: 339877

URL: http://llvm.org/viewvc/llvm-project?rev=339877&view=rev
Log:
[clangd] NFC: Improve Dex Iterators debugging traits

This patch improves `dex::Iterator` string representation by
incorporating the information about the element which is currently being
pointed to by the `DocumentIterator`.

Reviewed by: ioeric

Differential Revision: https://reviews.llvm.org/D50689

Modified:
clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp
clang-tools-extra/trunk/clangd/index/dex/Iterator.h
clang-tools-extra/trunk/unittests/clangd/DexIndexTests.cpp

Modified: clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp?rev=339877&r1=339876&r2=339877&view=diff
==
--- clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp Thu Aug 16 06:19:43 
2018
@@ -49,10 +49,19 @@ public:
   llvm::raw_ostream &dump(llvm::raw_ostream &OS) const override {
 OS << '[';
 auto Separator = "";
-for (const auto &ID : Documents) {
-  OS << Separator << ID;
+for (auto It = std::begin(Documents); It != std::end(Documents); ++It) {
+  OS << Separator;
+  if (It == Index)
+OS << '{' << *It << '}';
+  else
+OS << *It;
   Separator = ", ";
 }
+OS << Separator;
+if (Index == std::end(Documents))
+  OS << "{END}";
+else
+  OS << "END";
 OS << ']';
 return OS;
   }

Modified: clang-tools-extra/trunk/clangd/index/dex/Iterator.h
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/Iterator.h?rev=339877&r1=339876&r2=339877&view=diff
==
--- clang-tools-extra/trunk/clangd/index/dex/Iterator.h (original)
+++ clang-tools-extra/trunk/clangd/index/dex/Iterator.h Thu Aug 16 06:19:43 2018
@@ -99,7 +99,9 @@ public:
   ///
   /// Where Type is the iterator type representation: "&" for And, "|" for Or,
   /// ChildN is N-th iterator child. Raw iterators over PostingList are
-  /// represented as "[ID1, ID2, ...]" where IDN is N-th PostingList entry.
+  /// represented as "[ID1, ID2, ..., {IDN}, ... END]" where IDN is N-th
+  /// PostingList entry and the element which is pointed to by the PostingList
+  /// iterator is enclosed in {} braces.
   friend llvm::raw_ostream &operator<<(llvm::raw_ostream &OS,
const Iterator &Iterator) {
 return Iterator.dump(OS);

Modified: clang-tools-extra/trunk/unittests/clangd/DexIndexTests.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/unittests/clangd/DexIndexTests.cpp?rev=339877&r1=339876&r2=339877&view=diff
==
--- clang-tools-extra/trunk/unittests/clangd/DexIndexTests.cpp (original)
+++ clang-tools-extra/trunk/unittests/clangd/DexIndexTests.cpp Thu Aug 16 
06:19:43 2018
@@ -231,13 +231,14 @@ TEST(DexIndexIterators, StringRepresenta
   const PostingList L4 = {0, 1, 5};
   const PostingList L5;
 
-  EXPECT_EQ(llvm::to_string(*(create(L0))), "[4, 7, 8, 20, 42, 100]");
+  EXPECT_EQ(llvm::to_string(*(create(L0))), "[{4}, 7, 8, 20, 42, 100, END]");
 
   auto Nested = createAnd(createAnd(create(L1), create(L2)),
   createOr(create(L3), create(L4), create(L5)));
 
   EXPECT_EQ(llvm::to_string(*Nested),
-"(& (& [1, 3, 5, 8, 9] [1, 5, 7, 9]) (| [0, 5] [0, 1, 5] []))");
+"(& (& [{1}, 3, 5, 8, 9, END] [{1}, 5, 7, 9, END]) (| [0, {5}, "
+"END] [0, {1}, 5, END] [{END}]))");
 }
 
 TEST(DexIndexIterators, Limit) {


___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] r340007 - [clangd] NFC: Mark Workspace Symbol feature complete in the documentation

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Fri Aug 17 03:14:31 2018
New Revision: 340007

URL: http://llvm.org/viewvc/llvm-project?rev=340007&view=rev
Log:
[clangd] NFC: Mark Workspace Symbol feature complete in the documentation

Workspace Symbol implementation was introduced in D44882 and should be
complete now.

Reviewed by: ioeric

Differential Revision: https://reviews.llvm.org/D50703

Modified:
clang-tools-extra/trunk/docs/clangd.rst

Modified: clang-tools-extra/trunk/docs/clangd.rst
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/docs/clangd.rst?rev=340007&r1=340006&r2=340007&view=diff
==
--- clang-tools-extra/trunk/docs/clangd.rst (original)
+++ clang-tools-extra/trunk/docs/clangd.rst Fri Aug 17 03:14:31 2018
@@ -64,7 +64,7 @@ extension to the protocol.
 | Completion  | Yes|   Yes|
 +-++--+
 | Diagnostics | Yes|   Yes|
-+-++--+ 
++-++--+
 | Fix-its | Yes|   Yes|
 +-++--+
 | Go to Definition| Yes|   Yes|
@@ -83,7 +83,7 @@ extension to the protocol.
 +-++--+
 | Document Symbols| Yes|   Yes|
 +-++--+
-| Workspace Symbols   | Yes|   No |
+| Workspace Symbols   | Yes|   Yes|
 +-++--+
 | Syntax and Semantic Coloring| No |   No |
 +-++--+


___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] r340155 - [clangd] Implement TRUE Iterator

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Mon Aug 20 01:47:30 2018
New Revision: 340155

URL: http://llvm.org/viewvc/llvm-project?rev=340155&view=rev
Log:
[clangd] Implement TRUE Iterator

This patch introduces TRUE Iterator which efficiently handles posting
lists containing all items within `[0, Size)` range.

Reviewed by: ioeric

Differential Revision: https://reviews.llvm.org/D50955

Modified:
clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp
clang-tools-extra/trunk/clangd/index/dex/Iterator.h
clang-tools-extra/trunk/unittests/clangd/DexIndexTests.cpp

Modified: clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp?rev=340155&r1=340154&r2=340155&view=diff
==
--- clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp Mon Aug 20 01:47:30 
2018
@@ -225,6 +225,41 @@ private:
   std::vector> Children;
 };
 
+/// TrueIterator handles PostingLists which contain all items of the index. It
+/// stores size of the virtual posting list, and all operations are performed
+/// in O(1).
+class TrueIterator : public Iterator {
+public:
+  TrueIterator(DocID Size) : Size(Size) {}
+
+  bool reachedEnd() const override { return Index >= Size; }
+
+  void advance() override {
+assert(!reachedEnd() && "Can't advance iterator after it reached the 
end.");
+++Index;
+  }
+
+  void advanceTo(DocID ID) override {
+assert(!reachedEnd() && "Can't advance iterator after it reached the 
end.");
+Index = std::min(ID, Size);
+  }
+
+  DocID peek() const override {
+assert(!reachedEnd() && "TrueIterator can't call peek() at the end.");
+return Index;
+  }
+
+private:
+  llvm::raw_ostream &dump(llvm::raw_ostream &OS) const override {
+OS << "(TRUE {" << Index << "} out of " << Size << ")";
+return OS;
+  }
+
+  DocID Index = 0;
+  /// Size of the underlying virtual PostingList.
+  DocID Size;
+};
+
 } // end namespace
 
 std::vector consume(Iterator &It, size_t Limit) {
@@ -249,6 +284,10 @@ createOr(std::vector(move(Children));
 }
 
+std::unique_ptr createTrue(DocID Size) {
+  return llvm::make_unique(Size);
+}
+
 } // namespace dex
 } // namespace clangd
 } // namespace clang

Modified: clang-tools-extra/trunk/clangd/index/dex/Iterator.h
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/Iterator.h?rev=340155&r1=340154&r2=340155&view=diff
==
--- clang-tools-extra/trunk/clangd/index/dex/Iterator.h (original)
+++ clang-tools-extra/trunk/clangd/index/dex/Iterator.h Mon Aug 20 01:47:30 2018
@@ -129,6 +129,10 @@ createAnd(std::vector
 createOr(std::vector> Children);
 
+/// Returns TRUE Iterator which iterates over "virtual" PostingList containing
+/// all items in range [0, Size) in an efficient manner.
+std::unique_ptr createTrue(DocID Size);
+
 /// This allows createAnd(create(...), create(...)) syntax.
 template  std::unique_ptr createAnd(Args... args) {
   std::vector> Children;

Modified: clang-tools-extra/trunk/unittests/clangd/DexIndexTests.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/unittests/clangd/DexIndexTests.cpp?rev=340155&r1=340154&r2=340155&view=diff
==
--- clang-tools-extra/trunk/unittests/clangd/DexIndexTests.cpp (original)
+++ clang-tools-extra/trunk/unittests/clangd/DexIndexTests.cpp Mon Aug 20 
01:47:30 2018
@@ -262,6 +262,19 @@ TEST(DexIndexIterators, Limit) {
   EXPECT_THAT(consume(*DocIterator, 0), ElementsAre());
 }
 
+TEST(DexIndexIterators, True) {
+  auto TrueIterator = createTrue(0U);
+  EXPECT_THAT(TrueIterator->reachedEnd(), true);
+  EXPECT_THAT(consume(*TrueIterator), ElementsAre());
+
+  PostingList L0 = {1, 2, 5, 7};
+  TrueIterator = createTrue(7U);
+  EXPECT_THAT(TrueIterator->peek(), 0);
+  auto AndIterator = createAnd(create(L0), move(TrueIterator));
+  EXPECT_THAT(AndIterator->reachedEnd(), false);
+  EXPECT_THAT(consume(*AndIterator), ElementsAre(1, 2, 5));
+}
+
 testing::Matcher>
 trigramsAre(std::initializer_list Trigrams) {
   std::vector Tokens;


___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] r340157 - [clangd] NFC: Cleanup Dex Iterator comments and simplify tests

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Mon Aug 20 02:16:14 2018
New Revision: 340157

URL: http://llvm.org/viewvc/llvm-project?rev=340157&view=rev
Log:
[clangd] NFC: Cleanup Dex Iterator comments and simplify tests

Proposed changes:

* Cleanup comments in `clangd/index/dex/Iterator.h`: Vim's `gq`
  formatting added redundant spaces instead of newlines in few
  places
* Few comments in `OrIterator` are wrong
* Use `EXPECT_TRUE(Condition)` instead of
  `EXPECT_THAT(Condition, true)` (same with `EXPECT_FALSE`)
* Don't expose `dump()` method to the public by misplacing
  `private:`

This patch does not affect functionality.

Reviewed by: ioeric

Differential Revision: https://reviews.llvm.org/D50956

Modified:
clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp
clang-tools-extra/trunk/clangd/index/dex/Iterator.h
clang-tools-extra/trunk/unittests/clangd/DexIndexTests.cpp

Modified: clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp?rev=340157&r1=340156&r2=340157&view=diff
==
--- clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp Mon Aug 20 02:16:14 
2018
@@ -46,6 +46,7 @@ public:
 return *Index;
   }
 
+private:
   llvm::raw_ostream &dump(llvm::raw_ostream &OS) const override {
 OS << '[';
 auto Separator = "";
@@ -66,7 +67,6 @@ public:
 return OS;
   }
 
-private:
   PostingListRef Documents;
   PostingListRef::const_iterator Index;
 };
@@ -103,6 +103,7 @@ public:
 
   DocID peek() const override { return Children.front()->peek(); }
 
+private:
   llvm::raw_ostream &dump(llvm::raw_ostream &OS) const override {
 OS << "(& ";
 auto Separator = "";
@@ -114,7 +115,6 @@ public:
 return OS;
   }
 
-private:
   /// Restores class invariants: each child will point to the same element 
after
   /// sync.
   void sync() {
@@ -180,7 +180,7 @@ public:
   /// Moves each child pointing to the smallest DocID to the next item.
   void advance() override {
 assert(!reachedEnd() &&
-   "OrIterator must have at least one child to advance().");
+   "OrIterator can't call advance() after it reached the end.");
 const auto SmallestID = peek();
 for (const auto &Child : Children)
   if (!Child->reachedEnd() && Child->peek() == SmallestID)
@@ -199,7 +199,7 @@ public:
   /// value.
   DocID peek() const override {
 assert(!reachedEnd() &&
-   "OrIterator must have at least one child to peek().");
+   "OrIterator can't peek() after it reached the end.");
 DocID Result = std::numeric_limits::max();
 
 for (const auto &Child : Children)
@@ -209,6 +209,7 @@ public:
 return Result;
   }
 
+private:
   llvm::raw_ostream &dump(llvm::raw_ostream &OS) const override {
 OS << "(| ";
 auto Separator = "";
@@ -220,7 +221,6 @@ public:
 return OS;
   }
 
-private:
   // FIXME(kbobyrev): Would storing Children in min-heap be faster?
   std::vector> Children;
 };

Modified: clang-tools-extra/trunk/clangd/index/dex/Iterator.h
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/Iterator.h?rev=340157&r1=340156&r2=340157&view=diff
==
--- clang-tools-extra/trunk/clangd/index/dex/Iterator.h (original)
+++ clang-tools-extra/trunk/clangd/index/dex/Iterator.h Mon Aug 20 02:16:14 2018
@@ -11,14 +11,14 @@
 // symbol, such as high fuzzy matching score, scope, type etc. The lists of all
 // symbols matching some criteria (e.g. belonging to "clang::clangd::" scope)
 // are expressed in a form of Search Tokens which are stored in the inverted
-// index.  Inverted index maps these tokens to the posting lists - sorted ( by
-// symbol quality) sequences of symbol IDs matching the token, e.g.  scope 
token
+// index. Inverted index maps these tokens to the posting lists - sorted (by
+// symbol quality) sequences of symbol IDs matching the token, e.g. scope token
 // "clangd::clangd::" is mapped to the list of IDs of all symbols which are
 // declared in this namespace. Search queries are build from a set of
 // requirements which can be combined with each other forming the query trees.
 // The leafs of such trees are posting lists, and the nodes are operations on
-// these posting lists, e.g. intersection or union.  Efficient processing of
-// these multi-level queries is handled by Iterators.  Iterators advance 
through
+// these posting lists, e.g. intersection or union. Efficient processing of
+// these multi-level queries is handled by Iterators. Iterators advance through
 // all leaf posting lists producing the result of search query, which preserves
 // the sorted order of IDs. Having the resulting IDs sorted is important,
 // because it allows receiving a certain number of the most valuable items 
(e.g.

Modified: 

[clang-tools-extra] r340175 - [clangd] DexIndex implementation prototype

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Mon Aug 20 07:39:32 2018
New Revision: 340175

URL: http://llvm.org/viewvc/llvm-project?rev=340175&view=rev
Log:
[clangd] DexIndex implementation prototype

This patch is a proof-of-concept Dex index implementation. It has
several flaws, which don't allow replacing static MemIndex yet, such as:

* Not being able to handle queries of small size (less than 3 symbols);
  a way to solve this is generating trigrams of smaller size and having
  such incomplete trigrams in the index structure.
* Speed measurements: while manually editing files in Vim and requesting
  autocompletion gives an impression that the performance is at least
  comparable with the current static index, having actual numbers is
  important because we don't want to hurt the users and roll out slow
  code. Eric (@ioeric) suggested that we should only replace MemIndex as
  soon as we have the evidence that this is not a regression in terms of
  performance. An approach which is likely to be successful here is to
  wait until we have benchmark library in the LLVM core repository, which
  is something I have suggested in the LLVM mailing lists, received
  positive feedback on and started working on. I will add a dependency as
  soon as the suggested patch is out for a review (currently there's at
  least one complication which is being addressed by
  https://github.com/google/benchmark/pull/649). Key performance
  improvements for iterators are sorting by cost and the limit iterator.
* Quality measurements: currently, boosting iterator and two-phase
  lookup stage are not implemented, without these the quality is likely to
  be worse than the current implementation can yield. Measuring quality is
  tricky, but another suggestion in the offline discussion was that the
  drop-in replacement should only happen after Boosting iterators
  implementation (and subsequent query enhancement).

The proposed changes do not affect Clangd functionality or performance,
`DexIndex` is only used in unit tests and not in production code.

Reviewed by: ioeric

Differential Revision: https://reviews.llvm.org/D50337

Added:
clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp
clang-tools-extra/trunk/clangd/index/dex/DexIndex.h
clang-tools-extra/trunk/unittests/clangd/TestIndex.cpp
clang-tools-extra/trunk/unittests/clangd/TestIndex.h
Modified:
clang-tools-extra/trunk/clangd/CMakeLists.txt
clang-tools-extra/trunk/clangd/index/dex/Token.h
clang-tools-extra/trunk/unittests/clangd/CMakeLists.txt
clang-tools-extra/trunk/unittests/clangd/DexIndexTests.cpp
clang-tools-extra/trunk/unittests/clangd/IndexTests.cpp

Modified: clang-tools-extra/trunk/clangd/CMakeLists.txt
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/CMakeLists.txt?rev=340175&r1=340174&r2=340175&view=diff
==
--- clang-tools-extra/trunk/clangd/CMakeLists.txt (original)
+++ clang-tools-extra/trunk/clangd/CMakeLists.txt Mon Aug 20 07:39:32 2018
@@ -43,6 +43,7 @@ add_clang_library(clangDaemon
   index/SymbolCollector.cpp
   index/SymbolYAML.cpp
 
+  index/dex/DexIndex.cpp
   index/dex/Iterator.cpp
   index/dex/Trigram.cpp
 

Added: clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp?rev=340175&view=auto
==
--- clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp (added)
+++ clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp Mon Aug 20 07:39:32 
2018
@@ -0,0 +1,167 @@
+//===--- DexIndex.cpp - Dex Symbol Index Implementation -*- C++ 
-*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===--===//
+
+#include "DexIndex.h"
+#include "../../FuzzyMatch.h"
+#include "../../Logger.h"
+#include 
+#include 
+
+namespace clang {
+namespace clangd {
+namespace dex {
+
+namespace {
+
+// Returns the tokens which are given symbol's characteristics. Currently, the
+// generated tokens only contain fuzzy matching trigrams and symbol's scope,
+// but in the future this will also return path proximity tokens and other
+// types of tokens such as symbol type (if applicable).
+// Returns the tokens which are given symbols's characteristics. For example,
+// trigrams and scopes.
+// FIXME(kbobyrev): Support more token types:
+// * Path proximity
+// * Types
+std::vector generateSearchTokens(const Symbol &Sym) {
+  std::vector Result = generateIdentifierTrigrams(Sym.Name);
+  Result.push_back(Token(Token::Kind::Scope, Sym.Scope));
+  return Result;
+}
+
+} // namespace
+
+void DexIndex::build(std::shared_ptr> Syms) {
+  llvm::DenseMap TempLookupTable;
+  llvm::DenseMap TempSymbolQuality;

[clang-tools-extra] r340262 - [clangd] Allow using experimental Dex index

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Tue Aug 21 03:32:27 2018
New Revision: 340262

URL: http://llvm.org/viewvc/llvm-project?rev=340262&view=rev
Log:
[clangd] Allow using experimental Dex index

This patch adds hidden Clangd flag ("use-dex-index") which replaces
(currently) default `MemIndex` with `DexIndex` for the static index.

Reviewed by: ioeric

Differential Revision: https://reviews.llvm.org/D50897

Modified:
clang-tools-extra/trunk/clangd/index/MemIndex.cpp
clang-tools-extra/trunk/clangd/index/MemIndex.h
clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp
clang-tools-extra/trunk/clangd/index/dex/DexIndex.h
clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp

Modified: clang-tools-extra/trunk/clangd/index/MemIndex.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/MemIndex.cpp?rev=340262&r1=340261&r2=340262&view=diff
==
--- clang-tools-extra/trunk/clangd/index/MemIndex.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/MemIndex.cpp Tue Aug 21 03:32:27 2018
@@ -28,6 +28,12 @@ void MemIndex::build(std::shared_ptr MemIndex::build(SymbolSlab Slab) {
+  auto Idx = llvm::make_unique();
+  Idx->build(getSymbolsFromSlab(std::move(Slab)));
+  return std::move(Idx);
+}
+
 bool MemIndex::fuzzyFind(
 const FuzzyFindRequest &Req,
 llvm::function_ref Callback) const {
@@ -72,7 +78,14 @@ void MemIndex::lookup(const LookupReques
   }
 }
 
-std::unique_ptr MemIndex::build(SymbolSlab Slab) {
+void MemIndex::findOccurrences(
+const OccurrencesRequest &Req,
+llvm::function_ref Callback) const {
+  log("findOccurrences is not implemented.");
+}
+
+std::shared_ptr>
+getSymbolsFromSlab(SymbolSlab Slab) {
   struct Snapshot {
 SymbolSlab Slab;
 std::vector Pointers;
@@ -81,17 +94,8 @@ std::unique_ptr MemIndex::b
   Snap->Slab = std::move(Slab);
   for (auto &Sym : Snap->Slab)
 Snap->Pointers.push_back(&Sym);
-  auto S = std::shared_ptr>(std::move(Snap),
-&Snap->Pointers);
-  auto MemIdx = llvm::make_unique();
-  MemIdx->build(std::move(S));
-  return std::move(MemIdx);
-}
-
-void MemIndex::findOccurrences(
-const OccurrencesRequest &Req,
-llvm::function_ref Callback) const {
-  log("findOccurrences is not implemented.");
+  return std::shared_ptr>(std::move(Snap),
+  &Snap->Pointers);
 }
 
 } // namespace clangd

Modified: clang-tools-extra/trunk/clangd/index/MemIndex.h
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/MemIndex.h?rev=340262&r1=340261&r2=340262&view=diff
==
--- clang-tools-extra/trunk/clangd/index/MemIndex.h (original)
+++ clang-tools-extra/trunk/clangd/index/MemIndex.h Tue Aug 21 03:32:27 2018
@@ -47,6 +47,11 @@ private:
   mutable std::mutex Mutex;
 };
 
+// Returns pointers to the symbols in given slab and bundles slab lifetime with
+// returned symbol pointers so that the pointers are never invalid.
+std::shared_ptr>
+getSymbolsFromSlab(SymbolSlab Slab);
+
 } // namespace clangd
 } // namespace clang
 

Modified: clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp?rev=340262&r1=340261&r2=340262&view=diff
==
--- clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp Tue Aug 21 03:32:27 
2018
@@ -69,6 +69,12 @@ void DexIndex::build(std::shared_ptr DexIndex::build(SymbolSlab Slab) {
+  auto Idx = llvm::make_unique();
+  Idx->build(getSymbolsFromSlab(std::move(Slab)));
+  return std::move(Idx);
+}
+
 /// Constructs iterators over tokens extracted from the query and exhausts it
 /// while applying Callback to each symbol in the order of decreasing quality
 /// of the matched symbols.

Modified: clang-tools-extra/trunk/clangd/index/dex/DexIndex.h
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/DexIndex.h?rev=340262&r1=340261&r2=340262&view=diff
==
--- clang-tools-extra/trunk/clangd/index/dex/DexIndex.h (original)
+++ clang-tools-extra/trunk/clangd/index/dex/DexIndex.h Tue Aug 21 03:32:27 2018
@@ -43,6 +43,9 @@ public:
   /// accessible as long as `Symbols` is kept alive.
   void build(std::shared_ptr> Symbols);
 
+  /// \brief Build index from a symbol slab.
+  static std::unique_ptr build(SymbolSlab Slab);
+
   bool
   fuzzyFind(const FuzzyFindRequest &Req,
 llvm::function_ref Callback) const override;

Modified: clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp?rev=340262&r1=340261&r2=340262&view

[clang-tools-extra] r340263 - [clangd] NFC: Fix broken build

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Tue Aug 21 03:40:19 2018
New Revision: 340263

URL: http://llvm.org/viewvc/llvm-project?rev=340263&view=rev
Log:
[clangd] NFC: Fix broken build

Modified:
clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp

Modified: clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp?rev=340263&r1=340262&r2=340263&view=diff
==
--- clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp (original)
+++ clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp Tue Aug 21 03:40:19 2018
@@ -29,6 +29,11 @@
 using namespace clang;
 using namespace clang::clangd;
 
+static llvm::cl::opt
+UseDex("use-dex-index",
+   llvm::cl::desc("Use experimental Dex static index."),
+   llvm::cl::init(false), llvm::cl::Hidden);
+
 namespace {
 
 enum class PCHStorageFlag { Disk, Memory };
@@ -47,7 +52,7 @@ std::unique_ptr buildStatic
   for (auto Sym : Slab)
 SymsBuilder.insert(Sym);
 
-  return UseDex ? DexIndex::build(std::move(SymsBuilder).build())
+  return UseDex ? dex::DexIndex::build(std::move(SymsBuilder).build())
 : MemIndex::build(std::move(SymsBuilder).build());
 }
 
@@ -189,11 +194,6 @@ static llvm::cl::opt Co
 "'compile_commands.json' files")),
 llvm::cl::init(FilesystemCompileArgs), llvm::cl::Hidden);
 
-static llvm::cl::opt
-UseDex("use-dex-index",
-   llvm::cl::desc("Use experimental Dex static index."),
-   llvm::cl::init(false), llvm::cl::Hidden);
-
 int main(int argc, char *argv[]) {
   llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
   llvm::cl::SetVersionPrinter([](llvm::raw_ostream &OS) {


___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] r340388 - [clangd] Cleanup after D50897

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Wed Aug 22 00:17:59 2018
New Revision: 340388

URL: http://llvm.org/viewvc/llvm-project?rev=340388&view=rev
Log:
[clangd] Cleanup after D50897

The wrong diff that was uploaded to Phabricator was building the wrong
index.

Modified:
clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp

Modified: clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp?rev=340388&r1=340387&r2=340388&view=diff
==
--- clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp Wed Aug 22 00:17:59 
2018
@@ -70,7 +70,7 @@ void DexIndex::build(std::shared_ptr DexIndex::build(SymbolSlab Slab) {
-  auto Idx = llvm::make_unique();
+  auto Idx = llvm::make_unique();
   Idx->build(getSymbolsFromSlab(std::move(Slab)));
   return std::move(Idx);
 }


___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] r340409 - [clangd] Implement BOOST iterator

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Wed Aug 22 06:44:15 2018
New Revision: 340409

URL: http://llvm.org/viewvc/llvm-project?rev=340409&view=rev
Log:
[clangd] Implement BOOST iterator

This patch introduces BOOST iterator - a substantial block for efficient
and high-quality symbol retrieval. The concept of boosting allows
performing computationally inexpensive scoring on the query side so that
the final (expensive) scoring can only be applied on the items with the
highest preliminary score while eliminating the need to score too many
items.

Reviewed by: ilya-biryukov

Differential Revision: https://reviews.llvm.org/D50970

Modified:
clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp
clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp
clang-tools-extra/trunk/clangd/index/dex/Iterator.h
clang-tools-extra/trunk/unittests/clangd/DexIndexTests.cpp

Modified: clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp?rev=340409&r1=340408&r2=340409&view=diff
==
--- clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp Wed Aug 22 06:44:15 
2018
@@ -125,11 +125,15 @@ bool DexIndex::fuzzyFind(
 // using 100x of the requested number might not be good in practice, e.g.
 // when the requested number of items is small.
 const unsigned ItemsToRetrieve = 100 * Req.MaxCandidateCount;
-std::vector SymbolDocIDs = consume(*QueryIterator, ItemsToRetrieve);
+// FIXME(kbobyrev): Add boosting to the query and utilize retrieved
+// boosting scores.
+std::vector> SymbolDocIDs =
+consume(*QueryIterator, ItemsToRetrieve);
 
 // Retrieve top Req.MaxCandidateCount items.
 std::priority_queue> Top;
-for (const auto &SymbolDocID : SymbolDocIDs) {
+for (const auto &P : SymbolDocIDs) {
+  const DocID SymbolDocID = P.first;
   const auto *Sym = (*Symbols)[SymbolDocID];
   const llvm::Optional Score = Filter.match(Sym->Name);
   if (!Score)
@@ -161,7 +165,6 @@ void DexIndex::lookup(const LookupReques
   }
 }
 
-
 void DexIndex::findOccurrences(
 const OccurrencesRequest &Req,
 llvm::function_ref Callback) const {

Modified: clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp?rev=340409&r1=340408&r2=340409&view=diff
==
--- clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp Wed Aug 22 06:44:15 
2018
@@ -46,6 +46,8 @@ public:
 return *Index;
   }
 
+  float consume(DocID ID) override { return DEFAULT_BOOST_SCORE; }
+
 private:
   llvm::raw_ostream &dump(llvm::raw_ostream &OS) const override {
 OS << '[';
@@ -103,6 +105,19 @@ public:
 
   DocID peek() const override { return Children.front()->peek(); }
 
+  // If not exhausted and points to the given item, consume() returns the
+  // product of Children->consume(ID). Otherwise, DEFAULT_BOOST_SCORE is
+  // returned.
+  float consume(DocID ID) override {
+if (reachedEnd() || peek() != ID)
+  return DEFAULT_BOOST_SCORE;
+return std::accumulate(
+begin(Children), end(Children), DEFAULT_BOOST_SCORE,
+[&](float Current, const std::unique_ptr &Child) {
+  return Current * Child->consume(ID);
+});
+  }
+
 private:
   llvm::raw_ostream &dump(llvm::raw_ostream &OS) const override {
 OS << "(& ";
@@ -209,6 +224,20 @@ public:
 return Result;
   }
 
+  // Returns the maximum boosting score among all Children when iterator is not
+  // exhausted and points to the given ID, DEFAULT_BOOST_SCORE otherwise.
+  float consume(DocID ID) override {
+if (reachedEnd() || peek() != ID)
+  return DEFAULT_BOOST_SCORE;
+return std::accumulate(
+begin(Children), end(Children), DEFAULT_BOOST_SCORE,
+[&](float Current, const std::unique_ptr &Child) {
+  return (!Child->reachedEnd() && Child->peek() == ID)
+ ? std::max(Current, Child->consume(ID))
+ : Current;
+});
+  }
+
 private:
   llvm::raw_ostream &dump(llvm::raw_ostream &OS) const override {
 OS << "(| ";
@@ -249,6 +278,8 @@ public:
 return Index;
   }
 
+  float consume(DocID) override { return DEFAULT_BOOST_SCORE; }
+
 private:
   llvm::raw_ostream &dump(llvm::raw_ostream &OS) const override {
 OS << "(TRUE {" << Index << "} out of " << Size << ")";
@@ -260,13 +291,42 @@ private:
   DocID Size;
 };
 
+/// Boost iterator is a wrapper around its child which multiplies scores of
+/// each retrieved item by a given factor.
+class BoostIterator : public Iterator {
+public:
+  BoostIterator(std::unique_ptr Child, float Factor)
+  : Child(move(Child)), Factor(Fact

[clang-tools-extra] r333411 - [clangd] Minor cleanup

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Tue May 29 04:50:51 2018
New Revision: 333411

URL: http://llvm.org/viewvc/llvm-project?rev=333411&view=rev
Log:
[clangd] Minor cleanup

This patch silences few clang-tidy warnings, removes unwanted trailing
whitespace and enforces coding guidelines.

The functionality is not affected since the cleanup is rather straightforward,
all clangd tests are still green.

Reviewers: ioeric, ilya-biryukov

Reviewed By: ioeric

Subscribers: MaskRay, jkorous, cfe-commits

Differential Revision: https://reviews.llvm.org/D47471

Modified:
clang-tools-extra/trunk/clangd/CodeComplete.cpp
clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp

Modified: clang-tools-extra/trunk/clangd/CodeComplete.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/CodeComplete.cpp?rev=333411&r1=333410&r2=333411&view=diff
==
--- clang-tools-extra/trunk/clangd/CodeComplete.cpp (original)
+++ clang-tools-extra/trunk/clangd/CodeComplete.cpp Tue May 29 04:50:51 2018
@@ -359,13 +359,13 @@ struct SpecifiedScope {
 
 // Get all scopes that will be queried in indexes.
 std::vector getQueryScopes(CodeCompletionContext &CCContext,
-const SourceManager& SM) {
-  auto GetAllAccessibleScopes = [](CodeCompletionContext& CCContext) {
+const SourceManager &SM) {
+  auto GetAllAccessibleScopes = [](CodeCompletionContext &CCContext) {
 SpecifiedScope Info;
-for (auto* Context : CCContext.getVisitedContexts()) {
+for (auto *Context : CCContext.getVisitedContexts()) {
   if (isa(Context))
 Info.AccessibleScopes.push_back(""); // global namespace
-  else if (const auto*NS = dyn_cast(Context))
+  else if (const auto *NS = dyn_cast(Context))
 Info.AccessibleScopes.push_back(NS->getQualifiedNameAsString() + "::");
 }
 return Info;
@@ -397,8 +397,9 @@ std::vector getQueryScopes(
   Info.AccessibleScopes.push_back(""); // global namespace
 
   Info.UnresolvedQualifier =
-  Lexer::getSourceText(CharSourceRange::getCharRange((*SS)->getRange()),
-   SM, clang::LangOptions()).ltrim("::");
+  Lexer::getSourceText(CharSourceRange::getCharRange((*SS)->getRange()), 
SM,
+   clang::LangOptions())
+  .ltrim("::");
   // Sema excludes the trailing "::".
   if (!Info.UnresolvedQualifier->empty())
 *Info.UnresolvedQualifier += "::";
@@ -590,7 +591,7 @@ public:
   SigHelp.signatures.push_back(ProcessOverloadCandidate(
   Candidate, *CCS,
   getParameterDocComment(S.getASTContext(), Candidate, CurrentArg,
- /*CommentsFromHeader=*/false)));
+ /*CommentsFromHeaders=*/false)));
 }
   }
 
@@ -696,7 +697,7 @@ bool semaCodeComplete(std::unique_ptrgetFrontendOpts();
@@ -1013,7 +1014,7 @@ private:
 LLVM_DEBUG(llvm::dbgs()
<< "CodeComplete: " << C.Name << (IndexResult ? " (index)" : "")
<< (SemaResult ? " (sema)" : "") << " = " << Scores.finalScore
-   << "\n" 
+   << "\n"
<< Quality << Relevance << "\n");
 
 NSema += bool(SemaResult);
@@ -1035,7 +1036,8 @@ private:
/*CommentsFromHeader=*/false);
   }
 }
-return Candidate.build(FileName, Scores, Opts, SemaCCS, Includes.get(), 
DocComment);
+return Candidate.build(FileName, Scores, Opts, SemaCCS, Includes.get(),
+   DocComment);
   }
 };
 

Modified: clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp?rev=333411&r1=333410&r2=333411&view=diff
==
--- clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp (original)
+++ clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp Tue May 29 04:50:51 2018
@@ -33,7 +33,7 @@ enum class PCHStorageFlag { Disk, Memory
 // Build an in-memory static index for global symbols from a YAML-format file.
 // The size of global symbols should be relatively small, so that all symbols
 // can be managed in memory.
-std::unique_ptr BuildStaticIndex(llvm::StringRef YamlSymbolFile) {
+std::unique_ptr buildStaticIndex(llvm::StringRef YamlSymbolFile) {
   auto Buffer = llvm::MemoryBuffer::getFile(YamlSymbolFile);
   if (!Buffer) {
 llvm::errs() << "Can't open " << YamlSymbolFile << "\n";
@@ -223,7 +223,7 @@ int main(int argc, char *argv[]) {
   Opts.BuildDynamicSymbolIndex = EnableIndex;
   std::unique_ptr StaticIdx;
   if (EnableIndex && !YamlSymbolFile.empty()) {
-StaticIdx = BuildStaticIndex(YamlSymbolFile);
+StaticIdx = buildStaticIndex(YamlSymbolFile);
 Opts.StaticIndex = StaticIdx.get();
   }
   Opts.AsyncThreadsCount = WorkerThreadsCount;


_

[clang-tools-extra] r340601 - [clangd] Log memory usage of DexIndex and MemIndex

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Fri Aug 24 02:12:54 2018
New Revision: 340601

URL: http://llvm.org/viewvc/llvm-project?rev=340601&view=rev
Log:
[clangd] Log memory usage of DexIndex and MemIndex

This patch prints information about built index size estimation to
verbose logs. This is useful for optimizing memory usage of DexIndex and
comparisons with MemIndex.

Reviewed by: sammccall

Differential Revision: https://reviews.llvm.org/D51154

Modified:
clang-tools-extra/trunk/clangd/index/FileIndex.cpp
clang-tools-extra/trunk/clangd/index/FileIndex.h
clang-tools-extra/trunk/clangd/index/Index.h
clang-tools-extra/trunk/clangd/index/MemIndex.cpp
clang-tools-extra/trunk/clangd/index/MemIndex.h
clang-tools-extra/trunk/clangd/index/Merge.cpp
clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp
clang-tools-extra/trunk/clangd/index/dex/DexIndex.h
clang-tools-extra/trunk/unittests/clangd/CodeCompleteTests.cpp

Modified: clang-tools-extra/trunk/clangd/index/FileIndex.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/FileIndex.cpp?rev=340601&r1=340600&r2=340601&view=diff
==
--- clang-tools-extra/trunk/clangd/index/FileIndex.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/FileIndex.cpp Fri Aug 24 02:12:54 2018
@@ -118,5 +118,9 @@ void FileIndex::findOccurrences(
   log("findOccurrences is not implemented.");
 }
 
+size_t FileIndex::estimateMemoryUsage() const {
+  return Index.estimateMemoryUsage();
+}
+
 } // namespace clangd
 } // namespace clang

Modified: clang-tools-extra/trunk/clangd/index/FileIndex.h
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/FileIndex.h?rev=340601&r1=340600&r2=340601&view=diff
==
--- clang-tools-extra/trunk/clangd/index/FileIndex.h (original)
+++ clang-tools-extra/trunk/clangd/index/FileIndex.h Fri Aug 24 02:12:54 2018
@@ -81,6 +81,9 @@ public:
   void findOccurrences(const OccurrencesRequest &Req,
llvm::function_ref
Callback) const override;
+
+  size_t estimateMemoryUsage() const override;
+
 private:
   FileSymbols FSymbols;
   MemIndex Index;

Modified: clang-tools-extra/trunk/clangd/index/Index.h
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Index.h?rev=340601&r1=340600&r2=340601&view=diff
==
--- clang-tools-extra/trunk/clangd/index/Index.h (original)
+++ clang-tools-extra/trunk/clangd/index/Index.h Fri Aug 24 02:12:54 2018
@@ -385,6 +385,12 @@ public:
   virtual void findOccurrences(
   const OccurrencesRequest &Req,
   llvm::function_ref Callback) const = 0;
+
+  /// Returns estimated size of index (in bytes).
+  // FIXME(kbobyrev): Currently, this only returns the size of index itself
+  // excluding the size of actual symbol slab index refers to. We should 
include
+  // both.
+  virtual size_t estimateMemoryUsage() const = 0;
 };
 
 } // namespace clangd

Modified: clang-tools-extra/trunk/clangd/index/MemIndex.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/MemIndex.cpp?rev=340601&r1=340600&r2=340601&view=diff
==
--- clang-tools-extra/trunk/clangd/index/MemIndex.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/MemIndex.cpp Fri Aug 24 02:12:54 2018
@@ -26,6 +26,9 @@ void MemIndex::build(std::shared_ptr MemIndex::build(SymbolSlab Slab) {
@@ -98,5 +101,10 @@ getSymbolsFromSlab(SymbolSlab Slab) {
   &Snap->Pointers);
 }
 
+size_t MemIndex::estimateMemoryUsage() const {
+  std::lock_guard Lock(Mutex);
+  return Index.getMemorySize();
+}
+
 } // namespace clangd
 } // namespace clang

Modified: clang-tools-extra/trunk/clangd/index/MemIndex.h
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/MemIndex.h?rev=340601&r1=340600&r2=340601&view=diff
==
--- clang-tools-extra/trunk/clangd/index/MemIndex.h (original)
+++ clang-tools-extra/trunk/clangd/index/MemIndex.h Fri Aug 24 02:12:54 2018
@@ -39,7 +39,10 @@ public:
llvm::function_ref
Callback) const override;
 
+  size_t estimateMemoryUsage() const override;
+
 private:
+
   std::shared_ptr> Symbols;
   // Index is a set of symbols that are deduplicated by symbol IDs.
   // FIXME: build smarter index structure.

Modified: clang-tools-extra/trunk/clangd/index/Merge.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Merge.cpp?rev=340601&r1=340600&r2=340601&view=diff
==
--- clang-tools-extra/trunk/clangd/in

[clang-tools-extra] r340605 - [clangd] Implement LIMIT iterator

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Fri Aug 24 04:25:43 2018
New Revision: 340605

URL: http://llvm.org/viewvc/llvm-project?rev=340605&view=rev
Log:
[clangd] Implement LIMIT iterator

This patch introduces LIMIT iterator, which is very important for
improving the quality of search query. LIMIT iterators can be applied on
top of BOOST iterators to prevent populating query request with a huge
number of low-quality symbols.

Reviewed by: sammccall

Differential Revision: https://reviews.llvm.org/D51029

Modified:
clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp
clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp
clang-tools-extra/trunk/clangd/index/dex/Iterator.h
clang-tools-extra/trunk/unittests/clangd/DexIndexTests.cpp

Modified: clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp?rev=340605&r1=340604&r2=340605&view=diff
==
--- clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp Fri Aug 24 04:25:43 
2018
@@ -128,10 +128,10 @@ bool DexIndex::fuzzyFind(
 // using 100x of the requested number might not be good in practice, e.g.
 // when the requested number of items is small.
 const unsigned ItemsToRetrieve = 100 * Req.MaxCandidateCount;
+auto Root = createLimit(move(QueryIterator), ItemsToRetrieve);
 // FIXME(kbobyrev): Add boosting to the query and utilize retrieved
 // boosting scores.
-std::vector> SymbolDocIDs =
-consume(*QueryIterator, ItemsToRetrieve);
+std::vector> SymbolDocIDs = consume(*Root);
 
 // Retrieve top Req.MaxCandidateCount items.
 std::priority_queue> Top;

Modified: clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp?rev=340605&r1=340604&r2=340605&view=diff
==
--- clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp Fri Aug 24 04:25:43 
2018
@@ -46,7 +46,7 @@ public:
 return *Index;
   }
 
-  float consume(DocID ID) override { return DEFAULT_BOOST_SCORE; }
+  float consume() override { return DEFAULT_BOOST_SCORE; }
 
 private:
   llvm::raw_ostream &dump(llvm::raw_ostream &OS) const override {
@@ -105,16 +105,12 @@ public:
 
   DocID peek() const override { return Children.front()->peek(); }
 
-  // If not exhausted and points to the given item, consume() returns the
-  // product of Children->consume(ID). Otherwise, DEFAULT_BOOST_SCORE is
-  // returned.
-  float consume(DocID ID) override {
-if (reachedEnd() || peek() != ID)
-  return DEFAULT_BOOST_SCORE;
+  float consume() override {
+assert(!reachedEnd() && "AndIterator can't consume() at the end.");
 return std::accumulate(
 begin(Children), end(Children), DEFAULT_BOOST_SCORE,
 [&](float Current, const std::unique_ptr &Child) {
-  return Current * Child->consume(ID);
+  return Current * Child->consume();
 });
   }
 
@@ -226,15 +222,16 @@ public:
 
   // Returns the maximum boosting score among all Children when iterator is not
   // exhausted and points to the given ID, DEFAULT_BOOST_SCORE otherwise.
-  float consume(DocID ID) override {
-if (reachedEnd() || peek() != ID)
-  return DEFAULT_BOOST_SCORE;
+  float consume() override {
+assert(!reachedEnd() &&
+   "OrIterator can't consume() after it reached the end.");
+const DocID ID = peek();
 return std::accumulate(
 begin(Children), end(Children), DEFAULT_BOOST_SCORE,
-[&](float Current, const std::unique_ptr &Child) {
+[&](float Boost, const std::unique_ptr &Child) {
   return (!Child->reachedEnd() && Child->peek() == ID)
- ? std::max(Current, Child->consume(ID))
- : Current;
+ ? std::max(Boost, Child->consume())
+ : Boost;
 });
   }
 
@@ -278,7 +275,7 @@ public:
 return Index;
   }
 
-  float consume(DocID) override { return DEFAULT_BOOST_SCORE; }
+  float consume() override { return DEFAULT_BOOST_SCORE; }
 
 private:
   llvm::raw_ostream &dump(llvm::raw_ostream &OS) const override {
@@ -306,7 +303,7 @@ public:
 
   DocID peek() const override { return Child->peek(); }
 
-  float consume(DocID ID) override { return Child->consume(ID) * Factor; }
+  float consume() override { return Child->consume() * Factor; }
 
 private:
   llvm::raw_ostream &dump(llvm::raw_ostream &OS) const override {
@@ -318,15 +315,50 @@ private:
   float Factor;
 };
 
+/// This iterator limits the number of items retrieved from the child iterator
+/// on top of the query tree. To ensure that query tree with LIMIT iterators
+/// inside works correctly, users have to 

[clang-tools-extra] r340729 - [clangd] Use TRUE iterator instead of complete posting list

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Mon Aug 27 02:47:50 2018
New Revision: 340729

URL: http://llvm.org/viewvc/llvm-project?rev=340729&view=rev
Log:
[clangd] Use TRUE iterator instead of complete posting list

Stop using `$$$` (empty) trigram and generating a posting list with all
items. Since TRUE iterator is already implemented and correctly inserted
when there are no real trigram posting lists, this is a valid
transformation.

Benchmarks show that this simple change allows ~30% speedup on dataset
of real completion queries.

Before

```
---
BenchmarkTime   CPU Iterations
---
DexAdHocQueries5640321 ns5640265 ns120
DexRealQ 939835603 ns  939830296 ns  1
```

After

```
---
BenchmarkTime   CPU Iterations
---
DexAdHocQueries3452014 ns3451987 ns203
DexRealQ 667455912 ns  667455750 ns  1
```

Reviewed by: ilya-biryukov

Differential Revision: https://reviews.llvm.org/D51287

Modified:
clang-tools-extra/trunk/clangd/index/dex/Trigram.cpp

Modified: clang-tools-extra/trunk/clangd/index/dex/Trigram.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/Trigram.cpp?rev=340729&r1=340728&r2=340729&view=diff
==
--- clang-tools-extra/trunk/clangd/index/dex/Trigram.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/dex/Trigram.cpp Mon Aug 27 02:47:50 
2018
@@ -67,10 +67,6 @@ std::vector generateIdentifierTri
 UniqueTrigrams.insert(Token(Token::Kind::Trigram, Chars));
   };
 
-  // FIXME(kbobyrev): Instead of producing empty trigram for each identifier,
-  // just use True Iterator on the query side when the query string is empty.
-  add({{END_MARKER, END_MARKER, END_MARKER}});
-
   if (TwoHeads.size() == 2)
 add({{TwoHeads.front(), TwoHeads.back(), END_MARKER}});
 


___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] r340749 - [docs] Mention clangd-dev in clangd documentation

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Mon Aug 27 08:38:49 2018
New Revision: 340749

URL: http://llvm.org/viewvc/llvm-project?rev=340749&view=rev
Log:
[docs] Mention clangd-dev in clangd documentation

Since the clangd-dev is intended to be the place for clangd-related
discussions, we should point new users to this mailing list while
probably mentioning cfe-dev, too.

Reviewed by: ioeric

Differential Revision: https://reviews.llvm.org/D51293

Modified:
clang-tools-extra/trunk/docs/clangd.rst

Modified: clang-tools-extra/trunk/docs/clangd.rst
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/docs/clangd.rst?rev=340749&r1=340748&r2=340749&view=diff
==
--- clang-tools-extra/trunk/docs/clangd.rst (original)
+++ clang-tools-extra/trunk/docs/clangd.rst Mon Aug 27 08:38:49 2018
@@ -111,7 +111,10 @@ extension to the protocol.
 Getting Involved
 ==
 
-A good place for interested contributors is the `Clang developer mailing list
+A good place for interested contributors is the `Clangd developer mailing list
+`_. For discussions with the
+broader community on topics not only related to Clangd, use
+`Clang developer mailing list
 `_.
 If you're also interested in contributing patches to :program:`Clangd`, take a
 look at the `LLVM Developer Policy


___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] r340759 - Cleanup after rL340729

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Mon Aug 27 10:26:43 2018
New Revision: 340759

URL: http://llvm.org/viewvc/llvm-project?rev=340759&view=rev
Log:
Cleanup after rL340729

Modified:
clang-tools-extra/trunk/unittests/clangd/DexIndexTests.cpp

Modified: clang-tools-extra/trunk/unittests/clangd/DexIndexTests.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/unittests/clangd/DexIndexTests.cpp?rev=340759&r1=340758&r2=340759&view=diff
==
--- clang-tools-extra/trunk/unittests/clangd/DexIndexTests.cpp (original)
+++ clang-tools-extra/trunk/unittests/clangd/DexIndexTests.cpp Mon Aug 27 
10:26:43 2018
@@ -335,35 +335,34 @@ trigramsAre(std::initializer_listhttp://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] r340828 - [clangd] Switch to Dex by default for the static index

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Tue Aug 28 07:55:05 2018
New Revision: 340828

URL: http://llvm.org/viewvc/llvm-project?rev=340828&view=rev
Log:
[clangd] Switch to Dex by default for the static index

Dex is now mature enough to be used as the default static index. This
patch performs the switch but introduces a hidden flag to allow users
fallback to Mem in case something happens.

Reviewed by: ioeric

Differential Revision: https://reviews.llvm.org/D51352

Modified:
clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp

Modified: clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp?rev=340828&r1=340827&r2=340828&view=diff
==
--- clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp (original)
+++ clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp Tue Aug 28 07:55:05 2018
@@ -29,10 +29,11 @@
 using namespace clang;
 using namespace clang::clangd;
 
+// FIXME: remove this option when Dex is stable enough.
 static llvm::cl::opt
 UseDex("use-dex-index",
llvm::cl::desc("Use experimental Dex static index."),
-   llvm::cl::init(false), llvm::cl::Hidden);
+   llvm::cl::init(true), llvm::cl::Hidden);
 
 namespace {
 


___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] r341057 - [clangd] Implement iterator cost

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Thu Aug 30 04:23:58 2018
New Revision: 341057

URL: http://llvm.org/viewvc/llvm-project?rev=341057&view=rev
Log:
[clangd] Implement iterator cost

This patch introduces iterator cost concept to improve the performance
of Dex query iterators (mainly, AND iterator). Benchmarks show that the
queries become ~10% faster.

Before

```
---
BenchmarkTime   CPU Iteration
---
DexAdHocQueries5883074 ns5883018 ns117
DexRealQ 959904457 ns  959898507 ns  1
```

After

```
---
BenchmarkTime   CPU Iteration
---
DexAdHocQueries5238403 ns5238361 ns130
DexRealQ 873275207 ns  873269453 ns  1
```

Reviewed by: sammccall

Differential Revision: https://reviews.llvm.org/D51310

Modified:
clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp
clang-tools-extra/trunk/clangd/index/dex/Iterator.h
clang-tools-extra/trunk/unittests/clangd/DexIndexTests.cpp

Modified: clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp?rev=341057&r1=341056&r2=341057&view=diff
==
--- clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp Thu Aug 30 04:23:58 
2018
@@ -48,6 +48,8 @@ public:
 
   float consume() override { return DEFAULT_BOOST_SCORE; }
 
+  size_t estimateSize() const override { return Documents.size(); }
+
 private:
   llvm::raw_ostream &dump(llvm::raw_ostream &OS) const override {
 OS << '[';
@@ -85,6 +87,18 @@ public:
 assert(!Children.empty() && "AndIterator should have at least one child.");
 // Establish invariants.
 sync();
+// When children are sorted by the estimateSize(), sync() calls are more
+// effective. Each sync() starts with the first child and makes sure all
+// children point to the same element. If any child is "above" the previous
+// ones, the algorithm resets and and advances the children to the next
+// highest element starting from the front. When child iterators in the
+// beginning have smaller estimated size, the sync() will have less 
restarts
+// and become more effective.
+std::sort(begin(Children), end(Children),
+  [](const std::unique_ptr &LHS,
+ const std::unique_ptr &RHS) {
+return LHS->estimateSize() < RHS->estimateSize();
+  });
   }
 
   bool reachedEnd() const override { return ReachedEnd; }
@@ -114,6 +128,10 @@ public:
 });
   }
 
+  size_t estimateSize() const override {
+return Children.front()->estimateSize();
+  }
+
 private:
   llvm::raw_ostream &dump(llvm::raw_ostream &OS) const override {
 OS << "(& ";
@@ -146,9 +164,6 @@ private:
   return;
 // If any child goes beyond given ID (i.e. ID is not the common item),
 // all children should be advanced to the next common item.
-// FIXME(kbobyrev): This is not a very optimized version; after costs
-// are introduced, cycle should break whenever ID exceeds current one
-// and cheapest children should be advanced over again.
 if (Child->peek() > SyncID) {
   SyncID = Child->peek();
   NeedsAdvance = true;
@@ -178,6 +193,7 @@ public:
   OrIterator(std::vector> AllChildren)
   : Children(std::move(AllChildren)) {
 assert(Children.size() > 0 && "Or Iterator must have at least one child.");
+std::sort(begin(Children), end(Children));
   }
 
   /// Returns true if all children are exhausted.
@@ -235,6 +251,14 @@ public:
 });
   }
 
+  size_t estimateSize() const override {
+return std::accumulate(
+begin(Children), end(Children), Children.front()->estimateSize(),
+[&](size_t Current, const std::unique_ptr &Child) {
+  return std::max(Current, Child->estimateSize());
+});
+  }
+
 private:
   llvm::raw_ostream &dump(llvm::raw_ostream &OS) const override {
 OS << "(| ";
@@ -277,6 +301,8 @@ public:
 
   float consume() override { return DEFAULT_BOOST_SCORE; }
 
+  size_t estimateSize() const override { return Size; }
+
 private:
   llvm::raw_ostream &dump(llvm::raw_ostream &OS) const override {
 OS << "(TRUE {" << Index << "} out of " << Size << ")";
@@ -305,6 +331,8 @@ public:
 
   float consume() override { return Child->consume() * Factor; }
 
+  size_t estimateSize() const override { return Child->estimateSize(); }
+
 private:
   llvm::raw_ostream &dump(llvm::raw_ostream &OS) const override {
 OS << "(BOOST " << Factor << ' ' << *Child << ')';
@@ -342,6 +370,10 @@ public:
 return Child->consume();
   }
 
+  size_

[clang-tools-extra] r341060 - [clangd] Fix tests after rL341057

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Thu Aug 30 05:29:36 2018
New Revision: 341060

URL: http://llvm.org/viewvc/llvm-project?rev=341060&view=rev
Log:
[clangd] Fix tests after rL341057

Since OR iterator children are not longer sorted by the estimated size,
string representation should be different.

Modified:
clang-tools-extra/trunk/unittests/clangd/DexIndexTests.cpp

Modified: clang-tools-extra/trunk/unittests/clangd/DexIndexTests.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/unittests/clangd/DexIndexTests.cpp?rev=341060&r1=341059&r2=341060&view=diff
==
--- clang-tools-extra/trunk/unittests/clangd/DexIndexTests.cpp (original)
+++ clang-tools-extra/trunk/unittests/clangd/DexIndexTests.cpp Thu Aug 30 
05:29:36 2018
@@ -259,7 +259,7 @@ TEST(DexIndexIterators, StringRepresenta
   createOr(create(L3), create(L4), create(L5)));
 
   EXPECT_EQ(llvm::to_string(*Nested),
-"(& (| [{END}] [0, {5}, END] [0, {1}, 5, END]) (& [{1}, 5, 7, 9, "
+"(& (| [0, {5}, END] [0, {1}, 5, END] [{END}]) (& [{1}, 5, 7, 9, "
 "END] [{1}, 3, 5, 8, 9, END]))");
 }
 


___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] r341061 - [clang-tidy] Use simple string matching instead of Regex

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Thu Aug 30 05:42:19 2018
New Revision: 341061

URL: http://llvm.org/viewvc/llvm-project?rev=341061&view=rev
Log:
[clang-tidy] Use simple string matching instead of Regex

Instead of parsing and compiling the `llvm::Regex` each time, it's
faster to use basic string matching for filename prefix check.

Reviewed by: hokein

Differential Revision: https://reviews.llvm.org/D51360

Modified:
clang-tools-extra/trunk/clang-tidy/abseil/AbseilMatcher.h

Modified: clang-tools-extra/trunk/clang-tidy/abseil/AbseilMatcher.h
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clang-tidy/abseil/AbseilMatcher.h?rev=341061&r1=341060&r2=341061&view=diff
==
--- clang-tools-extra/trunk/clang-tidy/abseil/AbseilMatcher.h (original)
+++ clang-tools-extra/trunk/clang-tidy/abseil/AbseilMatcher.h Thu Aug 30 
05:42:19 2018
@@ -6,9 +6,10 @@
 // License. See LICENSE.TXT for details.
 //
 
//===--===//
-//
+
 #include "clang/AST/ASTContext.h"
 #include "clang/ASTMatchers/ASTMatchFinder.h"
+#include 
 
 namespace clang {
 namespace ast_matchers {
@@ -28,10 +29,9 @@ namespace ast_matchers {
 ///
 /// Usable as: Matcher, Matcher, Matcher,
 /// Matcher
-
-AST_POLYMORPHIC_MATCHER(isInAbseilFile,
-AST_POLYMORPHIC_SUPPORTED_TYPES(
-Decl, Stmt, TypeLoc, NestedNameSpecifierLoc)) {
+AST_POLYMORPHIC_MATCHER(
+isInAbseilFile, AST_POLYMORPHIC_SUPPORTED_TYPES(Decl, Stmt, TypeLoc,
+NestedNameSpecifierLoc)) {
   auto &SourceManager = Finder->getASTContext().getSourceManager();
   SourceLocation Loc = Node.getBeginLoc();
   if (Loc.isInvalid())
@@ -40,11 +40,21 @@ AST_POLYMORPHIC_MATCHER(isInAbseilFile,
   SourceManager.getFileEntryForID(SourceManager.getFileID(Loc));
   if (!FileEntry)
 return false;
-  StringRef Filename = FileEntry->getName();
-  llvm::Regex RE(
-  "absl/(algorithm|base|container|debugging|memory|meta|numeric|strings|"
-  "synchronization|time|types|utility)");
-  return RE.match(Filename);
+  // Determine whether filepath contains "absl/[absl-library]" substring, where
+  // [absl-library] is AbseilLibraries list entry.
+  StringRef Path = FileEntry->getName();
+  const static llvm::SmallString<5> AbslPrefix("absl/");
+  size_t PrefixPosition = Path.find(AbslPrefix);
+  if (PrefixPosition == StringRef::npos)
+return false;
+  Path = Path.drop_front(PrefixPosition + AbslPrefix.size());
+  static const char *AbseilLibraries[] = {
+  "algorithm",   "base", "container", "debugging",
+  "memory",  "meta", "numeric",   "strings",
+  "synchronization", "time", "types", "utility"};
+  return std::any_of(
+  std::begin(AbseilLibraries), std::end(AbseilLibraries),
+  [&](const char *Library) { return Path.startswith(Library); });
 }
 
 } // namespace ast_matchers


___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] r341066 - [clangd] Remove UB introduced in rL341057

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Thu Aug 30 06:30:34 2018
New Revision: 341066

URL: http://llvm.org/viewvc/llvm-project?rev=341066&view=rev
Log:
[clangd] Remove UB introduced in rL341057

Modified:
clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp

Modified: clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp?rev=341066&r1=341065&r2=341066&view=diff
==
--- clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp Thu Aug 30 06:30:34 
2018
@@ -193,7 +193,6 @@ public:
   OrIterator(std::vector> AllChildren)
   : Children(std::move(AllChildren)) {
 assert(Children.size() > 0 && "Or Iterator must have at least one child.");
-std::sort(begin(Children), end(Children));
   }
 
   /// Returns true if all children are exhausted.


___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] r341182 - [NFC] Use LLVM naming conventions within FileDistance

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Fri Aug 31 01:19:50 2018
New Revision: 341182

URL: http://llvm.org/viewvc/llvm-project?rev=341182&view=rev
Log:
[NFC] Use LLVM naming conventions within FileDistance

Reviewed by: sammccall

Differential Revision: https://reviews.llvm.org/D51527

Modified:
clang-tools-extra/trunk/clangd/FileDistance.cpp
clang-tools-extra/trunk/clangd/FileDistance.h

Modified: clang-tools-extra/trunk/clangd/FileDistance.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/FileDistance.cpp?rev=341182&r1=341181&r2=341182&view=diff
==
--- clang-tools-extra/trunk/clangd/FileDistance.cpp (original)
+++ clang-tools-extra/trunk/clangd/FileDistance.cpp Fri Aug 31 01:19:50 2018
@@ -102,7 +102,7 @@ FileDistance::FileDistance(StringMapgetSecond();
+  Cache.try_emplace(Child, Unreachable).first->getSecond();
   if (ParentCost + Opts.DownCost < ChildCost)
 ChildCost = ParentCost + Opts.DownCost;
   Next.push(Child);
@@ -113,7 +113,7 @@ FileDistance::FileDistance(StringMap Ancestors;
   // Walk up ancestors until we find a path we know the distance for.
   for (StringRef Rest = Canonical; !Rest.empty();
@@ -129,7 +129,7 @@ unsigned FileDistance::distance(StringRe
   // Now we know the costs for (known node, queried node].
   // Fill these in, walking down the directory tree.
   for (hash_code Hash : reverse(Ancestors)) {
-if (Cost != kUnreachable)
+if (Cost != Unreachable)
   Cost += Opts.DownCost;
 Cache.try_emplace(Hash, Cost);
   }
@@ -138,7 +138,7 @@ unsigned FileDistance::distance(StringRe
 }
 
 unsigned URIDistance::distance(llvm::StringRef URI) {
-  auto R = Cache.try_emplace(llvm::hash_value(URI), 
FileDistance::kUnreachable);
+  auto R = Cache.try_emplace(llvm::hash_value(URI), FileDistance::Unreachable);
   if (!R.second)
 return R.first->getSecond();
   if (auto U = clangd::URI::parse(URI)) {

Modified: clang-tools-extra/trunk/clangd/FileDistance.h
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/FileDistance.h?rev=341182&r1=341181&r2=341182&view=diff
==
--- clang-tools-extra/trunk/clangd/FileDistance.h (original)
+++ clang-tools-extra/trunk/clangd/FileDistance.h Fri Aug 31 01:19:50 2018
@@ -66,7 +66,7 @@ struct SourceParams {
 // This object should be reused, it memoizes intermediate computations.
 class FileDistance {
 public:
-  static constexpr unsigned kUnreachable = 
std::numeric_limits::max();
+  static constexpr unsigned Unreachable = std::numeric_limits::max();
 
   FileDistance(llvm::StringMap Sources,
const FileDistanceOptions &Opts = {});


___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] r341184 - NFC: Fix build failure after rL341182

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Fri Aug 31 01:29:48 2018
New Revision: 341184

URL: http://llvm.org/viewvc/llvm-project?rev=341184&view=rev
Log:
NFC: Fix build failure after rL341182

Didn't rename another variable reference.

Modified:
clang-tools-extra/trunk/clangd/FileDistance.cpp

Modified: clang-tools-extra/trunk/clangd/FileDistance.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/FileDistance.cpp?rev=341184&r1=341183&r2=341184&view=diff
==
--- clang-tools-extra/trunk/clangd/FileDistance.cpp (original)
+++ clang-tools-extra/trunk/clangd/FileDistance.cpp Fri Aug 31 01:29:48 2018
@@ -53,7 +53,7 @@ static SmallString<128> canonicalize(Str
   return Result;
 }
 
-constexpr const unsigned FileDistance::kUnreachable;
+constexpr const unsigned FileDistance::Unreachable;
 
 FileDistance::FileDistance(StringMap Sources,
const FileDistanceOptions &Opts)


___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] r341190 - [NFC] Cleanup Dex

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Fri Aug 31 02:17:02 2018
New Revision: 341190

URL: http://llvm.org/viewvc/llvm-project?rev=341190&view=rev
Log:
[NFC] Cleanup Dex

* Use consistent assertion messages in iterators implementations
* Silence a bunch of clang-tidy warnings: use `emplace_back` instead of
  `push_back` where possible, make sure arguments have the same name in
  header and implementation file, use for loop over ranges where possible

Reviewed by: ioeric

Differential Revision: https://reviews.llvm.org/D51528

Modified:
clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp
clang-tools-extra/trunk/clangd/index/dex/DexIndex.h
clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp
clang-tools-extra/trunk/clangd/index/dex/Trigram.cpp

Modified: clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp?rev=341190&r1=341189&r2=341190&view=diff
==
--- clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp Fri Aug 31 02:17:02 
2018
@@ -30,7 +30,7 @@ namespace {
 // * Types
 std::vector generateSearchTokens(const Symbol &Sym) {
   std::vector Result = generateIdentifierTrigrams(Sym.Name);
-  Result.push_back(Token(Token::Kind::Scope, Sym.Scope));
+  Result.emplace_back(Token::Kind::Scope, Sym.Scope);
   return Result;
 }
 

Modified: clang-tools-extra/trunk/clangd/index/dex/DexIndex.h
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/DexIndex.h?rev=341190&r1=341189&r2=341190&view=diff
==
--- clang-tools-extra/trunk/clangd/index/dex/DexIndex.h (original)
+++ clang-tools-extra/trunk/clangd/index/dex/DexIndex.h Fri Aug 31 02:17:02 2018
@@ -41,7 +41,7 @@ class DexIndex : public SymbolIndex {
 public:
   /// \brief (Re-)Build index for `Symbols`. All symbol pointers must remain
   /// accessible as long as `Symbols` is kept alive.
-  void build(std::shared_ptr> Symbols);
+  void build(std::shared_ptr> Syms);
 
   /// \brief Build index from a symbol slab.
   static std::unique_ptr build(SymbolSlab Slab);

Modified: clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp?rev=341190&r1=341189&r2=341190&view=diff
==
--- clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp Fri Aug 31 02:17:02 
2018
@@ -30,23 +30,26 @@ public:
 
   /// Advances cursor to the next item.
   void advance() override {
-assert(!reachedEnd() && "DocumentIterator can't advance at the end.");
+assert(!reachedEnd() && "DOCUMENT iterator can't advance() at the end.");
 ++Index;
   }
 
   /// Applies binary search to advance cursor to the next item with DocID equal
   /// or higher than the given one.
   void advanceTo(DocID ID) override {
-assert(!reachedEnd() && "DocumentIterator can't advance at the end.");
+assert(!reachedEnd() && "DOCUMENT iterator can't advance() at the end.");
 Index = std::lower_bound(Index, std::end(Documents), ID);
   }
 
   DocID peek() const override {
-assert(!reachedEnd() && "DocumentIterator can't call peek() at the end.");
+assert(!reachedEnd() && "DOCUMENT iterator can't peek() at the end.");
 return *Index;
   }
 
-  float consume() override { return DEFAULT_BOOST_SCORE; }
+  float consume() override {
+assert(!reachedEnd() && "DOCUMENT iterator can't consume() at the end.");
+return DEFAULT_BOOST_SCORE;
+  }
 
   size_t estimateSize() const override { return Documents.size(); }
 
@@ -84,7 +87,7 @@ class AndIterator : public Iterator {
 public:
   AndIterator(std::vector> AllChildren)
   : Children(std::move(AllChildren)) {
-assert(!Children.empty() && "AndIterator should have at least one child.");
+assert(!Children.empty() && "AND iterator should have at least one 
child.");
 // Establish invariants.
 sync();
 // When children are sorted by the estimateSize(), sync() calls are more
@@ -105,14 +108,14 @@ public:
 
   /// Advances all children to the next common item.
   void advance() override {
-assert(!reachedEnd() && "AndIterator can't call advance() at the end.");
+assert(!reachedEnd() && "AND iterator can't advance() at the end.");
 Children.front()->advance();
 sync();
   }
 
   /// Advances all children to the next common item with DocumentID >= ID.
   void advanceTo(DocID ID) override {
-assert(!reachedEnd() && "AndIterator can't call advanceTo() at the end.");
+assert(!reachedEnd() && "AND iterator can't advanceTo() at the end.");
 Children.front()->advanceTo(ID);
 sync();
   }
@@ -120,7 +123,7 @@ public:
   DocID peek() c

[clang-tools-extra] r341369 - [clangd] Move buildStaticIndex() to SymbolYAML

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Tue Sep  4 08:10:40 2018
New Revision: 341369

URL: http://llvm.org/viewvc/llvm-project?rev=341369&view=rev
Log:
[clangd] Move buildStaticIndex() to SymbolYAML

`buildStaticIndex()` is used by two other tools that I'm building, now
it's useful outside of `tool/ClangdMain.cpp`.

Also, slightly refactor the code while moving it to the different source
file.

Reviewed By: sammccall

Differential Revision: https://reviews.llvm.org/D51626

Modified:
clang-tools-extra/trunk/clangd/index/SymbolYAML.cpp
clang-tools-extra/trunk/clangd/index/SymbolYAML.h
clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp

Modified: clang-tools-extra/trunk/clangd/index/SymbolYAML.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/SymbolYAML.cpp?rev=341369&r1=341368&r2=341369&view=diff
==
--- clang-tools-extra/trunk/clangd/index/SymbolYAML.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/SymbolYAML.cpp Tue Sep  4 08:10:40 2018
@@ -9,6 +9,7 @@
 
 #include "SymbolYAML.h"
 #include "Index.h"
+#include "dex/DexIndex.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/Errc.h"
@@ -25,18 +26,18 @@ using clang::clangd::Symbol;
 using clang::clangd::SymbolID;
 using clang::clangd::SymbolLocation;
 using clang::index::SymbolInfo;
-using clang::index::SymbolLanguage;
 using clang::index::SymbolKind;
+using clang::index::SymbolLanguage;
 
 // Helper to (de)serialize the SymbolID. We serialize it as a hex string.
 struct NormalizedSymbolID {
   NormalizedSymbolID(IO &) {}
-  NormalizedSymbolID(IO &, const SymbolID& ID) {
+  NormalizedSymbolID(IO &, const SymbolID &ID) {
 llvm::raw_string_ostream OS(HexString);
 OS << ID;
   }
 
-  SymbolID denormalize(IO&) {
+  SymbolID denormalize(IO &) {
 SymbolID ID;
 HexString >> ID;
 return ID;
@@ -167,7 +168,7 @@ Symbol SymbolFromYAML(llvm::yaml::Input
   return S;
 }
 
-void SymbolsToYAML(const SymbolSlab& Symbols, llvm::raw_ostream &OS) {
+void SymbolsToYAML(const SymbolSlab &Symbols, llvm::raw_ostream &OS) {
   llvm::yaml::Output Yout(OS);
   for (Symbol S : Symbols) // copy: Yout<< requires mutability.
 Yout << S;
@@ -181,5 +182,18 @@ std::string SymbolToYAML(Symbol Sym) {
   return OS.str();
 }
 
+std::unique_ptr loadIndex(llvm::StringRef SymbolFile,
+   bool UseDex) {
+  auto Buffer = llvm::MemoryBuffer::getFile(SymbolFile);
+  if (!Buffer) {
+llvm::errs() << "Can't open " << SymbolFile << "\n";
+return nullptr;
+  }
+  auto Slab = symbolsFromYAML(Buffer.get()->getBuffer());
+
+  return UseDex ? dex::DexIndex::build(std::move(Slab))
+: MemIndex::build(std::move(Slab), RefSlab());
+}
+
 } // namespace clangd
 } // namespace clang

Modified: clang-tools-extra/trunk/clangd/index/SymbolYAML.h
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/SymbolYAML.h?rev=341369&r1=341368&r2=341369&view=diff
==
--- clang-tools-extra/trunk/clangd/index/SymbolYAML.h (original)
+++ clang-tools-extra/trunk/clangd/index/SymbolYAML.h Tue Sep  4 08:10:40 2018
@@ -41,6 +41,12 @@ std::string SymbolToYAML(Symbol Sym);
 // The YAML result is safe to concatenate if you have multiple symbol slabs.
 void SymbolsToYAML(const SymbolSlab &Symbols, llvm::raw_ostream &OS);
 
+// Build an in-memory static index for global symbols from a symbol file.
+// The size of global symbols should be relatively small, so that all symbols
+// can be managed in memory.
+std::unique_ptr loadIndex(llvm::StringRef SymbolFile,
+   bool UseDex = true);
+
 } // namespace clangd
 } // namespace clang
 

Modified: clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp?rev=341369&r1=341368&r2=341369&view=diff
==
--- clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp (original)
+++ clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp Tue Sep  4 08:10:40 2018
@@ -39,24 +39,6 @@ namespace {
 
 enum class PCHStorageFlag { Disk, Memory };
 
-// Build an in-memory static index for global symbols from a YAML-format file.
-// The size of global symbols should be relatively small, so that all symbols
-// can be managed in memory.
-std::unique_ptr buildStaticIndex(llvm::StringRef YamlSymbolFile) {
-  auto Buffer = llvm::MemoryBuffer::getFile(YamlSymbolFile);
-  if (!Buffer) {
-llvm::errs() << "Can't open " << YamlSymbolFile << "\n";
-return nullptr;
-  }
-  auto Slab = symbolsFromYAML(Buffer.get()->getBuffer());
-  SymbolSlab::Builder SymsBuilder;
-  for (auto Sym : Slab)
-SymsBuilder.insert(Sym);
-
-  return UseDex ? dex::DexIndex::build(std::move(SymsBuilder).build())
-: Me

[clang-tools-extra] r341374 - [clangd] NFC: Change quality type to float

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Tue Sep  4 08:45:56 2018
New Revision: 341374

URL: http://llvm.org/viewvc/llvm-project?rev=341374&view=rev
Log:
[clangd] NFC: Change quality type to float

Reviewed by: sammccall

Differential Revision: https://reviews.llvm.org/D51636

Modified:
clang-tools-extra/trunk/clangd/index/Index.cpp
clang-tools-extra/trunk/clangd/index/Index.h

Modified: clang-tools-extra/trunk/clangd/index/Index.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Index.cpp?rev=341374&r1=341373&r2=341374&view=diff
==
--- clang-tools-extra/trunk/clangd/index/Index.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/Index.cpp Tue Sep  4 08:45:56 2018
@@ -59,7 +59,7 @@ raw_ostream &operator<<(raw_ostream &OS,
   return OS << S.Scope << S.Name;
 }
 
-double quality(const Symbol &S) {
+float quality(const Symbol &S) {
   // This avoids a sharp gradient for tail symbols, and also neatly avoids the
   // question of whether 0 references means a bad symbol or missing data.
   if (S.References < 3)

Modified: clang-tools-extra/trunk/clangd/index/Index.h
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Index.h?rev=341374&r1=341373&r2=341374&view=diff
==
--- clang-tools-extra/trunk/clangd/index/Index.h (original)
+++ clang-tools-extra/trunk/clangd/index/Index.h Tue Sep  4 08:45:56 2018
@@ -250,7 +250,7 @@ llvm::raw_ostream &operator<<(llvm::raw_
 // This currently falls in the range [1, ln(#indexed documents)].
 // FIXME: this should probably be split into symbol -> signals
 //and signals -> score, so it can be reused for Sema completions.
-double quality(const Symbol &S);
+float quality(const Symbol &S);
 
 // An immutable symbol container that stores a set of symbols.
 // The container will maintain the lifetime of the symbols.


___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] r341542 - [clangd] Implement proximity path boosting for Dex

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Thu Sep  6 05:54:43 2018
New Revision: 341542

URL: http://llvm.org/viewvc/llvm-project?rev=341542&view=rev
Log:
[clangd] Implement proximity path boosting for Dex

This patch introduces `PathURI` Search Token kind and utilizes it to
uprank symbols which are defined in files with small distance to the
directory where the fuzzy find request is coming from (e.g. files user
is editing).

Reviewed By: ioeric

Reviewers: ioeric, sammccall

Differential Revision: https://reviews.llvm.org/D51481

Modified:
clang-tools-extra/trunk/clangd/FileDistance.h
clang-tools-extra/trunk/clangd/URI.cpp
clang-tools-extra/trunk/clangd/URI.h
clang-tools-extra/trunk/clangd/index/SymbolYAML.cpp
clang-tools-extra/trunk/clangd/index/SymbolYAML.h
clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp
clang-tools-extra/trunk/clangd/index/dex/DexIndex.h
clang-tools-extra/trunk/clangd/index/dex/Token.h
clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp
clang-tools-extra/trunk/unittests/clangd/DexIndexTests.cpp

Modified: clang-tools-extra/trunk/clangd/FileDistance.h
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/FileDistance.h?rev=341542&r1=341541&r2=341542&view=diff
==
--- clang-tools-extra/trunk/clangd/FileDistance.h (original)
+++ clang-tools-extra/trunk/clangd/FileDistance.h Thu Sep  6 05:54:43 2018
@@ -37,6 +37,9 @@
 //
 
//===--===//
 
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_FILEDISTANCE_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_FILEDISTANCE_H
+
 #include "URI.h"
 #include "llvm/ADT/DenseMap.h"
 #include "llvm/ADT/DenseMapInfo.h"
@@ -107,3 +110,5 @@ private:
 
 } // namespace clangd
 } // namespace clang
+
+#endif // LLVM_CLANG_TOOLS_EXTRA_CLANGD_FILEDISTANCE_H

Modified: clang-tools-extra/trunk/clangd/URI.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/URI.cpp?rev=341542&r1=341541&r2=341542&view=diff
==
--- clang-tools-extra/trunk/clangd/URI.cpp (original)
+++ clang-tools-extra/trunk/clangd/URI.cpp Thu Sep  6 05:54:43 2018
@@ -181,6 +181,26 @@ llvm::Expected URI::create(llvm::St
   return S->get()->uriFromAbsolutePath(AbsolutePath);
 }
 
+llvm::Expected URI::create(llvm::StringRef AbsolutePath,
+const std::vector &Schemes) {
+  if (!llvm::sys::path::is_absolute(AbsolutePath))
+return make_string_error("Not a valid absolute path: " + AbsolutePath);
+  for (const auto &Scheme : Schemes) {
+auto URI = URI::create(AbsolutePath, Scheme);
+// For some paths, conversion to different URI schemes is impossible. These
+// should be just skipped.
+if (!URI) {
+  // Ignore the error.
+  llvm::consumeError(URI.takeError());
+  continue;
+}
+return URI;
+  }
+  return make_string_error(
+  "Couldn't convert " + AbsolutePath +
+  " to any given scheme: " + llvm::join(Schemes, ", "));
+}
+
 URI URI::createFile(llvm::StringRef AbsolutePath) {
   auto U = create(AbsolutePath, "file");
   if (!U)

Modified: clang-tools-extra/trunk/clangd/URI.h
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/URI.h?rev=341542&r1=341541&r2=341542&view=diff
==
--- clang-tools-extra/trunk/clangd/URI.h (original)
+++ clang-tools-extra/trunk/clangd/URI.h Thu Sep  6 05:54:43 2018
@@ -45,6 +45,11 @@ public:
   static llvm::Expected create(llvm::StringRef AbsolutePath,
 llvm::StringRef Scheme);
 
+  // Similar to above except this uses the first scheme in \p Schemes that
+  // works.
+  static llvm::Expected create(llvm::StringRef AbsolutePath,
+const std::vector &Schemes);
+
   /// This creates a file:// URI for \p AbsolutePath. The path must be 
absolute.
   static URI createFile(llvm::StringRef AbsolutePath);
 

Modified: clang-tools-extra/trunk/clangd/index/SymbolYAML.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/SymbolYAML.cpp?rev=341542&r1=341541&r2=341542&view=diff
==
--- clang-tools-extra/trunk/clangd/index/SymbolYAML.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/SymbolYAML.cpp Thu Sep  6 05:54:43 2018
@@ -185,6 +185,7 @@ std::string SymbolToYAML(Symbol Sym) {
 }
 
 std::unique_ptr loadIndex(llvm::StringRef SymbolFilename,
+   llvm::ArrayRef URISchemes,
bool UseDex) {
   trace::Span OverallTracer("LoadIndex");
   auto Buffer = llvm::MemoryBuffer::getFile(SymbolFilename);
@@ -209,7 +210,7 @@ std::unique_ptr loadIndex(l
   if (!Slab)
 return nullptr;
   trace::Span Tracer("BuildIndex");
-  retu

[clang-tools-extra] r341543 - [clangd] NFC: mark single-parameter constructors explicit

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Thu Sep  6 06:06:04 2018
New Revision: 341543

URL: http://llvm.org/viewvc/llvm-project?rev=341543&view=rev
Log:
[clangd] NFC: mark single-parameter constructors explicit

Code health: prevent implicit conversions to user-defined types.

Reviewed By: sammccall

Differential Revision: https://reviews.llvm.org/D51690

Modified:
clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp

Modified: clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp?rev=341543&r1=341542&r2=341543&view=diff
==
--- clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp Thu Sep  6 06:06:04 
2018
@@ -23,7 +23,7 @@ namespace {
 /// tree) and is simply a wrapper around PostingList::const_iterator.
 class DocumentIterator : public Iterator {
 public:
-  DocumentIterator(PostingListRef Documents)
+  explicit DocumentIterator(PostingListRef Documents)
   : Documents(Documents), Index(std::begin(Documents)) {}
 
   bool reachedEnd() const override { return Index == std::end(Documents); }
@@ -85,7 +85,7 @@ private:
 /// iterator restores the invariant: all children must point to the same item.
 class AndIterator : public Iterator {
 public:
-  AndIterator(std::vector> AllChildren)
+  explicit AndIterator(std::vector> AllChildren)
   : Children(std::move(AllChildren)) {
 assert(!Children.empty() && "AND iterator should have at least one 
child.");
 // Establish invariants.
@@ -193,7 +193,7 @@ private:
 /// soon as all of its children are exhausted.
 class OrIterator : public Iterator {
 public:
-  OrIterator(std::vector> AllChildren)
+  explicit OrIterator(std::vector> AllChildren)
   : Children(std::move(AllChildren)) {
 assert(Children.size() > 0 && "OR iterator must have at least one child.");
   }
@@ -279,7 +279,7 @@ private:
 /// in O(1).
 class TrueIterator : public Iterator {
 public:
-  TrueIterator(DocID Size) : Size(Size) {}
+  explicit TrueIterator(DocID Size) : Size(Size) {}
 
   bool reachedEnd() const override { return Index >= Size; }
 


___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] r341544 - [clangd] NFC: Use TopN instead of std::priority_queue

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Thu Sep  6 06:15:03 2018
New Revision: 341544

URL: http://llvm.org/viewvc/llvm-project?rev=341544&view=rev
Log:
[clangd] NFC: Use TopN instead of std::priority_queue

Quality.cpp defines a structure for convenient storage of Top N items,
it should be used instead of the `std::priority_queue` with slightly
obscure semantics.

This patch does not affect functionality.

Reviewed By: sammccall

Differential Revision: https://reviews.llvm.org/D51676

Modified:
clang-tools-extra/trunk/clangd/index/MemIndex.cpp

Modified: clang-tools-extra/trunk/clangd/index/MemIndex.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/MemIndex.cpp?rev=341544&r1=341543&r2=341544&view=diff
==
--- clang-tools-extra/trunk/clangd/index/MemIndex.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/MemIndex.cpp Thu Sep  6 06:15:03 2018
@@ -10,7 +10,7 @@
 #include "MemIndex.h"
 #include "../FuzzyMatch.h"
 #include "../Logger.h"
-#include 
+#include "../Quality.h"
 
 namespace clang {
 namespace clangd {
@@ -26,7 +26,7 @@ bool MemIndex::fuzzyFind(
   assert(!StringRef(Req.Query).contains("::") &&
  "There must be no :: in query.");
 
-  std::priority_queue> Top;
+  TopN> Top(Req.MaxCandidateCount);
   FuzzyMatcher Filter(Req.Query);
   bool More = false;
   for (const auto Pair : Index) {
@@ -38,16 +38,12 @@ bool MemIndex::fuzzyFind(
 if (Req.RestrictForCodeCompletion && !Sym->IsIndexedForCodeCompletion)
   continue;
 
-if (auto Score = Filter.match(Sym->Name)) {
-  Top.emplace(-*Score * quality(*Sym), Sym);
-  if (Top.size() > Req.MaxCandidateCount) {
-More = true;
-Top.pop();
-  }
-}
+if (auto Score = Filter.match(Sym->Name))
+  if (Top.push({*Score * quality(*Sym), Sym}))
+More = true; // An element with smallest score was discarded.
   }
-  for (; !Top.empty(); Top.pop())
-Callback(*Top.top().second);
+  for (const auto &Item : std::move(Top).items())
+Callback(*Item.second);
   return More;
 }
 


___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] r341552 - [clangd] Fix Dex initialization

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Thu Sep  6 08:10:10 2018
New Revision: 341552

URL: http://llvm.org/viewvc/llvm-project?rev=341552&view=rev
Log:
[clangd] Fix Dex initialization

This patch sets URI schemes of Dex to SymbolCollector's default schemes
in case callers tried to pass empty list of schemes. This was the case
for initialization in Clangd main and was a reason of incorrect
behavior.

Also, it fixes a bug with missed `continue;` after spotting invalid URI
scheme conversion.

Modified:
clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp
clang-tools-extra/trunk/clangd/index/dex/DexIndex.h

Modified: clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp?rev=341552&r1=341551&r2=341552&view=diff
==
--- clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp Thu Sep  6 08:10:10 
2018
@@ -59,6 +59,7 @@ std::vector> c
"scheme. fuzzyFind request will ignore it.",
Path);
   llvm::consumeError(PathURI.takeError());
+  continue;
 }
 const auto PathProximityURIs = generateProximityURIs(PathURI->toString());
 for (const auto &ProximityURI : PathProximityURIs)

Modified: clang-tools-extra/trunk/clangd/index/dex/DexIndex.h
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/DexIndex.h?rev=341552&r1=341551&r2=341552&view=diff
==
--- clang-tools-extra/trunk/clangd/index/dex/DexIndex.h (original)
+++ clang-tools-extra/trunk/clangd/index/dex/DexIndex.h Thu Sep  6 08:10:10 2018
@@ -22,6 +22,7 @@
 
 #include "../Index.h"
 #include "../MemIndex.h"
+#include "../SymbolCollector.h"
 #include "Iterator.h"
 #include "Token.h"
 #include "Trigram.h"
@@ -40,8 +41,14 @@ class DexIndex : public SymbolIndex {
 public:
   // All symbols must outlive this index.
   template 
-  DexIndex(Range &&Symbols, llvm::ArrayRef URISchemes)
-  : URISchemes(URISchemes) {
+  DexIndex(Range &&Symbols, llvm::ArrayRef Schemes)
+  : URISchemes(Schemes) {
+// If Schemes don't contain any items, fall back to SymbolCollector's
+// default URI schemes.
+if (URISchemes.empty()) {
+  SymbolCollector::Options Opts;
+  URISchemes = Opts.URISchemes;
+}
 for (auto &&Sym : Symbols)
   this->Symbols.push_back(&Sym);
 buildIndex();
@@ -90,7 +97,7 @@ private:
   llvm::DenseMap InvertedIndex;
   std::shared_ptr KeepAlive; // poor man's move-only std::any
 
-  const std::vector URISchemes;
+  std::vector URISchemes;
 };
 
 /// Returns Search Token for a number of parent directories of given Path.


___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] r341639 - [clangd] NFC: Document URIDistance

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Fri Sep  7 02:18:58 2018
New Revision: 341639

URL: http://llvm.org/viewvc/llvm-project?rev=341639&view=rev
Log:
[clangd] NFC: Document URIDistance

`URIDistance` constructor should mention that `Sources` must contain
*absolute paths*, not URIs. This is not very clear when looking at the
interface, especially given that `distance(...)` accepts `URI`, not an
absolute path which can give the wrong impression.

Reviewed By: sammccall

Differential Revision: https://reviews.llvm.org/D51691

Modified:
clang-tools-extra/trunk/clangd/FileDistance.h

Modified: clang-tools-extra/trunk/clangd/FileDistance.h
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/FileDistance.h?rev=341639&r1=341638&r2=341639&view=diff
==
--- clang-tools-extra/trunk/clangd/FileDistance.h (original)
+++ clang-tools-extra/trunk/clangd/FileDistance.h Fri Sep  7 02:18:58 2018
@@ -89,6 +89,7 @@ private:
 // comparison on the bodies.
 class URIDistance {
 public:
+  // \p Sources must contain absolute paths, not URIs.
   URIDistance(llvm::StringMap Sources,
   const FileDistanceOptions &Opts = {})
   : Sources(Sources), Opts(Opts) {}


___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] r341781 - [clangd] Make advanceTo() faster on Posting Lists

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Mon Sep 10 00:57:28 2018
New Revision: 341781

URL: http://llvm.org/viewvc/llvm-project?rev=341781&view=rev
Log:
[clangd] Make advanceTo() faster on Posting Lists

If the current element is already beyond advanceTo()'s DocID, just
return instead of doing binary search. This simple optimization saves up
to 6-7% performance,

Reviewed By: ilya-biryukov

Differential Revision: https://reviews.llvm.org/D51802

Modified:
clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp

Modified: clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp?rev=341781&r1=341780&r2=341781&view=diff
==
--- clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp Mon Sep 10 00:57:28 
2018
@@ -38,7 +38,10 @@ public:
   /// or higher than the given one.
   void advanceTo(DocID ID) override {
 assert(!reachedEnd() && "DOCUMENT iterator can't advance() at the end.");
-Index = std::lower_bound(Index, std::end(Documents), ID);
+// If current ID is beyond requested one, iterator is already in the right
+// state.
+if (peek() < ID)
+  Index = std::lower_bound(Index, std::end(Documents), ID);
   }
 
   DocID peek() const override {


___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] r341784 - [clangd] NFC: Rename DexIndex to Dex

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Mon Sep 10 01:23:53 2018
New Revision: 341784

URL: http://llvm.org/viewvc/llvm-project?rev=341784&view=rev
Log:
[clangd] NFC: Rename DexIndex to Dex

Also, cleanup some redundant includes.

Reviewed By: sammccall

Differential Revision: https://reviews.llvm.org/D51774

Added:
clang-tools-extra/trunk/clangd/index/dex/Dex.cpp
clang-tools-extra/trunk/clangd/index/dex/Dex.h
clang-tools-extra/trunk/unittests/clangd/DexTests.cpp
Removed:
clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp
clang-tools-extra/trunk/clangd/index/dex/DexIndex.h
clang-tools-extra/trunk/unittests/clangd/DexIndexTests.cpp
Modified:
clang-tools-extra/trunk/clangd/CMakeLists.txt
clang-tools-extra/trunk/clangd/index/SymbolYAML.cpp
clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp
clang-tools-extra/trunk/unittests/clangd/CMakeLists.txt
clang-tools-extra/trunk/unittests/clangd/TestIndex.h

Modified: clang-tools-extra/trunk/clangd/CMakeLists.txt
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/CMakeLists.txt?rev=341784&r1=341783&r2=341784&view=diff
==
--- clang-tools-extra/trunk/clangd/CMakeLists.txt (original)
+++ clang-tools-extra/trunk/clangd/CMakeLists.txt Mon Sep 10 01:23:53 2018
@@ -46,7 +46,7 @@ add_clang_library(clangDaemon
   index/SymbolCollector.cpp
   index/SymbolYAML.cpp
 
-  index/dex/DexIndex.cpp
+  index/dex/Dex.cpp
   index/dex/Iterator.cpp
   index/dex/Trigram.cpp
 

Modified: clang-tools-extra/trunk/clangd/index/SymbolYAML.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/SymbolYAML.cpp?rev=341784&r1=341783&r2=341784&view=diff
==
--- clang-tools-extra/trunk/clangd/index/SymbolYAML.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/SymbolYAML.cpp Mon Sep 10 01:23:53 2018
@@ -11,7 +11,7 @@
 #include "Index.h"
 #include "Serialization.h"
 #include "Trace.h"
-#include "dex/DexIndex.h"
+#include "dex/Dex.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/Support/Errc.h"
@@ -225,7 +225,7 @@ std::unique_ptr loadIndex(l
   if (!Slab)
 return nullptr;
   trace::Span Tracer("BuildIndex");
-  return UseDex ? dex::DexIndex::build(std::move(*Slab), URISchemes)
+  return UseDex ? dex::Dex::build(std::move(*Slab), URISchemes)
 : MemIndex::build(std::move(*Slab), RefSlab());
 }
 

Added: clang-tools-extra/trunk/clangd/index/dex/Dex.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/Dex.cpp?rev=341784&view=auto
==
--- clang-tools-extra/trunk/clangd/index/dex/Dex.cpp (added)
+++ clang-tools-extra/trunk/clangd/index/dex/Dex.cpp Mon Sep 10 01:23:53 2018
@@ -0,0 +1,270 @@
+//===--- Dex.cpp - Dex Symbol Index Implementation --*- C++ 
-*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===--===//
+
+#include "Dex.h"
+#include "FileDistance.h"
+#include "FuzzyMatch.h"
+#include "Logger.h"
+#include "Quality.h"
+#include "llvm/ADT/StringSet.h"
+#include 
+#include 
+
+namespace clang {
+namespace clangd {
+namespace dex {
+
+namespace {
+
+// Returns the tokens which are given symbol's characteristics. Currently, the
+// generated tokens only contain fuzzy matching trigrams and symbol's scope,
+// but in the future this will also return path proximity tokens and other
+// types of tokens such as symbol type (if applicable).
+// Returns the tokens which are given symbols's characteristics. For example,
+// trigrams and scopes.
+// FIXME(kbobyrev): Support more token types:
+// * Types
+// * Namespace proximity
+std::vector generateSearchTokens(const Symbol &Sym) {
+  std::vector Result = generateIdentifierTrigrams(Sym.Name);
+  Result.emplace_back(Token::Kind::Scope, Sym.Scope);
+  // Skip token generation for symbols with unknown declaration location.
+  if (!Sym.CanonicalDeclaration.FileURI.empty())
+for (const auto &ProximityURI :
+ generateProximityURIs(Sym.CanonicalDeclaration.FileURI))
+  Result.emplace_back(Token::Kind::ProximityURI, ProximityURI);
+  return Result;
+}
+
+// Constructs BOOST iterators for Path Proximities.
+std::vector> createFileProximityIterators(
+llvm::ArrayRef ProximityPaths,
+llvm::ArrayRef URISchemes,
+const llvm::DenseMap &InvertedIndex) {
+  std::vector> BoostingIterators;
+  // Deduplicate parent URIs extracted from the ProximityPaths.
+  llvm::StringSet<> ParentURIs;
+  llvm::StringMap Sources;
+  for (const auto &Path : ProximityPaths) {
+Sources[Path] = SourceParams();
+auto PathURI = URI::create(Path, URISchemes);

[clang-tools-extra] r341800 - [clangd] Add symbol slab size to index memory consumption estimates

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Mon Sep 10 04:46:07 2018
New Revision: 341800

URL: http://llvm.org/viewvc/llvm-project?rev=341800&view=rev
Log:
[clangd] Add symbol slab size to index memory consumption estimates

Currently, `SymbolIndex::estimateMemoryUsage()` returns the "overhead"
estimate, i.e. the estimate of the Index data structure excluding
backing data (such as Symbol Slab and Reference Slab). This patch
propagates information about paired data size where necessary.

Reviewed By: ioeric, sammccall

Differential Revision: https://reviews.llvm.org/D51539

Modified:
clang-tools-extra/trunk/clangd/index/FileIndex.cpp
clang-tools-extra/trunk/clangd/index/MemIndex.cpp
clang-tools-extra/trunk/clangd/index/MemIndex.h
clang-tools-extra/trunk/clangd/index/dex/Dex.cpp
clang-tools-extra/trunk/clangd/index/dex/Dex.h
clang-tools-extra/trunk/unittests/clangd/IndexTests.cpp

Modified: clang-tools-extra/trunk/clangd/index/FileIndex.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/FileIndex.cpp?rev=341800&r1=341799&r2=341800&view=diff
==
--- clang-tools-extra/trunk/clangd/index/FileIndex.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/FileIndex.cpp Mon Sep 10 04:46:07 2018
@@ -127,11 +127,18 @@ std::unique_ptr FileSymbols
 }
   }
 
+  size_t StorageSize = RefsStorage.size() * sizeof(Ref);
+  for (const auto &Slab : SymbolSlabs)
+StorageSize += Slab->bytes();
+  for (const auto &RefSlab : RefSlabs)
+StorageSize += RefSlab->bytes();
+
   // Index must keep the slabs and contiguous ranges alive.
   return llvm::make_unique(
   llvm::make_pointee_range(AllSymbols), std::move(AllRefs),
   std::make_tuple(std::move(SymbolSlabs), std::move(RefSlabs),
-  std::move(RefsStorage)));
+  std::move(RefsStorage)),
+  StorageSize);
 }
 
 void FileIndex::update(PathRef Path, ASTContext *AST,

Modified: clang-tools-extra/trunk/clangd/index/MemIndex.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/MemIndex.cpp?rev=341800&r1=341799&r2=341800&view=diff
==
--- clang-tools-extra/trunk/clangd/index/MemIndex.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/MemIndex.cpp Mon Sep 10 04:46:07 2018
@@ -16,8 +16,11 @@ namespace clang {
 namespace clangd {
 
 std::unique_ptr MemIndex::build(SymbolSlab Slab, RefSlab Refs) {
+  // Store Slab size before it is moved.
+  const auto BackingDataSize = Slab.bytes() + Refs.bytes();
   auto Data = std::make_pair(std::move(Slab), std::move(Refs));
-  return llvm::make_unique(Data.first, Data.second, std::move(Data));
+  return llvm::make_unique(Data.first, Data.second, std::move(Data),
+ BackingDataSize);
 }
 
 bool MemIndex::fuzzyFind(
@@ -70,7 +73,7 @@ void MemIndex::refs(const RefsRequest &R
 }
 
 size_t MemIndex::estimateMemoryUsage() const {
-  return Index.getMemorySize();
+  return Index.getMemorySize() + Refs.getMemorySize() + BackingDataSize;
 }
 
 } // namespace clangd

Modified: clang-tools-extra/trunk/clangd/index/MemIndex.h
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/MemIndex.h?rev=341800&r1=341799&r2=341800&view=diff
==
--- clang-tools-extra/trunk/clangd/index/MemIndex.h (original)
+++ clang-tools-extra/trunk/clangd/index/MemIndex.h Mon Sep 10 04:46:07 2018
@@ -30,11 +30,13 @@ public:
   }
   // Symbols are owned by BackingData, Index takes ownership.
   template 
-  MemIndex(SymbolRange &&Symbols, RefRange &&Refs, Payload &&BackingData)
+  MemIndex(SymbolRange &&Symbols, RefRange &&Refs, Payload &&BackingData,
+   size_t BackingDataSize)
   : MemIndex(std::forward(Symbols),
  std::forward(Refs)) {
 KeepAlive = std::shared_ptr(
 std::make_shared(std::move(BackingData)), nullptr);
+this->BackingDataSize = BackingDataSize;
   }
 
   /// Builds an index from slabs. The index takes ownership of the data.
@@ -58,6 +60,8 @@ private:
   // A map from symbol ID to symbol refs, support query by IDs.
   llvm::DenseMap> Refs;
   std::shared_ptr KeepAlive; // poor man's move-only std::any
+  // Size of memory retained by KeepAlive.
+  size_t BackingDataSize = 0;
 };
 
 } // namespace clangd

Modified: clang-tools-extra/trunk/clangd/index/dex/Dex.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/Dex.cpp?rev=341800&r1=341799&r2=341800&view=diff
==
--- clang-tools-extra/trunk/clangd/index/dex/Dex.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/dex/Dex.cpp Mon Sep 10 04:46:07 2018
@@ -227,15 +227,13 @@ void Dex::refs(const RefsRequest &Req,
 }
 
 size_t Dex::estimateMemoryUsage() const {
-  size

[clang-tools-extra] r341802 - [clangd] Implement FuzzyFindRequest JSON (de)serialization

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Mon Sep 10 04:51:05 2018
New Revision: 341802

URL: http://llvm.org/viewvc/llvm-project?rev=341802&view=rev
Log:
[clangd] Implement FuzzyFindRequest JSON (de)serialization

JSON (de)serialization of `FuzzyFindRequest` might be useful for both
D51090 and D51628. Also, this allows precise logging of the fuzzy find
requests.

Reviewed By: sammccall

Differential Revision: https://reviews.llvm.org/D51852

Modified:
clang-tools-extra/trunk/clangd/CodeComplete.cpp
clang-tools-extra/trunk/clangd/index/Index.cpp
clang-tools-extra/trunk/clangd/index/Index.h

Modified: clang-tools-extra/trunk/clangd/CodeComplete.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/CodeComplete.cpp?rev=341802&r1=341801&r2=341802&view=diff
==
--- clang-tools-extra/trunk/clangd/CodeComplete.cpp (original)
+++ clang-tools-extra/trunk/clangd/CodeComplete.cpp Mon Sep 10 04:51:05 2018
@@ -1381,8 +1381,7 @@ private:
 Req.Scopes = QueryScopes;
 // FIXME: we should send multiple weighted paths here.
 Req.ProximityPaths.push_back(FileName);
-vlog("Code complete: fuzzyFind(\"{0}\", scopes=[{1}])", Req.Query,
- llvm::join(Req.Scopes.begin(), Req.Scopes.end(), ","));
+vlog("Code complete: fuzzyFind({0:2})", toJSON(Req));
 
 if (SpecFuzzyFind)
   SpecFuzzyFind->NewReq = Req;

Modified: clang-tools-extra/trunk/clangd/index/Index.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Index.cpp?rev=341802&r1=341801&r2=341802&view=diff
==
--- clang-tools-extra/trunk/clangd/index/Index.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/Index.cpp Mon Sep 10 04:51:05 2018
@@ -175,6 +175,33 @@ std::shared_ptr SwapIndex::
   return Index;
 }
 
+bool fromJSON(const llvm::json::Value &Parameters, FuzzyFindRequest &Request) {
+  json::ObjectMapper O(Parameters);
+  llvm::Optional MaxCandidateCount;
+  bool OK =
+  O && O.map("Query", Request.Query) && O.map("Scopes", Request.Scopes) &&
+  O.map("RestrictForCodeCompletion", Request.RestrictForCodeCompletion) &&
+  O.map("ProximityPaths", Request.ProximityPaths) &&
+  O.map("MaxCandidateCount", MaxCandidateCount);
+  if (MaxCandidateCount)
+Request.MaxCandidateCount = MaxCandidateCount.getValue();
+  return OK;
+}
+
+llvm::json::Value toJSON(const FuzzyFindRequest &Request) {
+  auto Result = json::Object{
+  {"Query", Request.Query},
+  {"Scopes", json::Array{Request.Scopes}},
+  {"RestrictForCodeCompletion", Request.RestrictForCodeCompletion},
+  {"ProximityPaths", json::Array{Request.ProximityPaths}},
+  };
+  // A huge limit means no limit, leave it out.
+  if (Request.MaxCandidateCount <= std::numeric_limits::max())
+Result["MaxCandidateCount"] =
+static_cast(Request.MaxCandidateCount);
+  return Result;
+}
+
 bool SwapIndex::fuzzyFind(const FuzzyFindRequest &R,
   llvm::function_ref CB) const {
   return snapshot()->fuzzyFind(R, CB);

Modified: clang-tools-extra/trunk/clangd/index/Index.h
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Index.h?rev=341802&r1=341801&r2=341802&view=diff
==
--- clang-tools-extra/trunk/clangd/index/Index.h (original)
+++ clang-tools-extra/trunk/clangd/index/Index.h Mon Sep 10 04:51:05 2018
@@ -19,8 +19,10 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringExtras.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/Support/JSON.h"
 #include "llvm/Support/StringSaver.h"
 #include 
+#include 
 #include 
 #include 
 #include 
@@ -435,7 +437,7 @@ struct FuzzyFindRequest {
   std::vector Scopes;
   /// \brief The number of top candidates to return. The index may choose to
   /// return more than this, e.g. if it doesn't know which candidates are best.
-  size_t MaxCandidateCount = UINT_MAX;
+  size_t MaxCandidateCount = std::numeric_limits::max();
   /// If set to true, only symbols for completion support will be considered.
   bool RestrictForCodeCompletion = false;
   /// Contextually relevant files (e.g. the file we're code-completing in).
@@ -450,6 +452,8 @@ struct FuzzyFindRequest {
   }
   bool operator!=(const FuzzyFindRequest &Req) const { return !(*this == Req); 
}
 };
+bool fromJSON(const llvm::json::Value &Value, FuzzyFindRequest &Request);
+llvm::json::Value toJSON(const FuzzyFindRequest &Request);
 
 struct LookupRequest {
   llvm::DenseSet IDs;


___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] r341832 - [clangd] Unbreak buildbots after r341802

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Mon Sep 10 07:31:38 2018
New Revision: 341832

URL: http://llvm.org/viewvc/llvm-project?rev=341832&view=rev
Log:
[clangd] Unbreak buildbots after r341802

Solution: use std::move when returning result from toJSON(...).

Modified:
clang-tools-extra/trunk/clangd/index/Index.cpp

Modified: clang-tools-extra/trunk/clangd/index/Index.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Index.cpp?rev=341832&r1=341831&r2=341832&view=diff
==
--- clang-tools-extra/trunk/clangd/index/Index.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/Index.cpp Mon Sep 10 07:31:38 2018
@@ -199,7 +199,7 @@ llvm::json::Value toJSON(const FuzzyFind
   if (Request.MaxCandidateCount <= std::numeric_limits::max())
 Result["MaxCandidateCount"] =
 static_cast(Request.MaxCandidateCount);
-  return Result;
+  return std::move(Result);
 }
 
 bool SwapIndex::fuzzyFind(const FuzzyFindRequest &R,


___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] r341921 - [clangd] NFC: Use uint32_t for FuzzyFindRequest limits

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Tue Sep 11 03:31:38 2018
New Revision: 341921

URL: http://llvm.org/viewvc/llvm-project?rev=341921&view=rev
Log:
[clangd] NFC: Use uint32_t for FuzzyFindRequest limits

Reviewed By: ioeric

Differential Revision: https://reviews.llvm.org/D51860

Modified:
clang-tools-extra/trunk/clangd/index/Index.cpp
clang-tools-extra/trunk/clangd/index/Index.h

Modified: clang-tools-extra/trunk/clangd/index/Index.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Index.cpp?rev=341921&r1=341920&r2=341921&view=diff
==
--- clang-tools-extra/trunk/clangd/index/Index.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/Index.cpp Tue Sep 11 03:31:38 2018
@@ -177,29 +177,25 @@ std::shared_ptr SwapIndex::
 
 bool fromJSON(const llvm::json::Value &Parameters, FuzzyFindRequest &Request) {
   json::ObjectMapper O(Parameters);
-  llvm::Optional MaxCandidateCount;
+  int64_t MaxCandidateCount;
   bool OK =
   O && O.map("Query", Request.Query) && O.map("Scopes", Request.Scopes) &&
+  O.map("MaxCandidateCount", MaxCandidateCount) &&
   O.map("RestrictForCodeCompletion", Request.RestrictForCodeCompletion) &&
-  O.map("ProximityPaths", Request.ProximityPaths) &&
-  O.map("MaxCandidateCount", MaxCandidateCount);
-  if (MaxCandidateCount)
-Request.MaxCandidateCount = MaxCandidateCount.getValue();
+  O.map("ProximityPaths", Request.ProximityPaths);
+  if (OK && MaxCandidateCount <= std::numeric_limits::max())
+Request.MaxCandidateCount = MaxCandidateCount;
   return OK;
 }
 
 llvm::json::Value toJSON(const FuzzyFindRequest &Request) {
-  auto Result = json::Object{
+  return json::Object{
   {"Query", Request.Query},
   {"Scopes", json::Array{Request.Scopes}},
+  {"MaxCandidateCount", Request.MaxCandidateCount},
   {"RestrictForCodeCompletion", Request.RestrictForCodeCompletion},
   {"ProximityPaths", json::Array{Request.ProximityPaths}},
   };
-  // A huge limit means no limit, leave it out.
-  if (Request.MaxCandidateCount <= std::numeric_limits::max())
-Result["MaxCandidateCount"] =
-static_cast(Request.MaxCandidateCount);
-  return std::move(Result);
 }
 
 bool SwapIndex::fuzzyFind(const FuzzyFindRequest &R,

Modified: clang-tools-extra/trunk/clangd/index/Index.h
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Index.h?rev=341921&r1=341920&r2=341921&view=diff
==
--- clang-tools-extra/trunk/clangd/index/Index.h (original)
+++ clang-tools-extra/trunk/clangd/index/Index.h Tue Sep 11 03:31:38 2018
@@ -437,7 +437,9 @@ struct FuzzyFindRequest {
   std::vector Scopes;
   /// \brief The number of top candidates to return. The index may choose to
   /// return more than this, e.g. if it doesn't know which candidates are best.
-  size_t MaxCandidateCount = std::numeric_limits::max();
+  // FIXME: Use llvm::Optional; semantically, the absence of MaxCandidateCount
+  // is equivalent to setting this field to default value as below.
+  uint32_t MaxCandidateCount = std::numeric_limits::max();
   /// If set to true, only symbols for completion support will be considered.
   bool RestrictForCodeCompletion = false;
   /// Contextually relevant files (e.g. the file we're code-completing in).


___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] r342025 - [clangd] Implement a Proof-of-Concept tool for symbol index exploration

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Wed Sep 12 00:32:54 2018
New Revision: 342025

URL: http://llvm.org/viewvc/llvm-project?rev=342025&view=rev
Log:
[clangd] Implement a Proof-of-Concept tool for symbol index exploration

Reviewed By: sammccall, ilya-biryukov

Differential Revision: https://reviews.llvm.org/D51628

Added:
clang-tools-extra/trunk/clangd/index/dex/dexp/
clang-tools-extra/trunk/clangd/index/dex/dexp/CMakeLists.txt
clang-tools-extra/trunk/clangd/index/dex/dexp/Dexp.cpp
Modified:
clang-tools-extra/trunk/clangd/CMakeLists.txt

Modified: clang-tools-extra/trunk/clangd/CMakeLists.txt
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/CMakeLists.txt?rev=342025&r1=342024&r2=342025&view=diff
==
--- clang-tools-extra/trunk/clangd/CMakeLists.txt (original)
+++ clang-tools-extra/trunk/clangd/CMakeLists.txt Wed Sep 12 00:32:54 2018
@@ -74,3 +74,4 @@ if( LLVM_LIB_FUZZING_ENGINE OR LLVM_USE_
 endif()
 add_subdirectory(tool)
 add_subdirectory(global-symbol-builder)
+add_subdirectory(index/dex/dexp)

Added: clang-tools-extra/trunk/clangd/index/dex/dexp/CMakeLists.txt
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/dexp/CMakeLists.txt?rev=342025&view=auto
==
--- clang-tools-extra/trunk/clangd/index/dex/dexp/CMakeLists.txt (added)
+++ clang-tools-extra/trunk/clangd/index/dex/dexp/CMakeLists.txt Wed Sep 12 
00:32:54 2018
@@ -0,0 +1,15 @@
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../../../)
+
+set(LLVM_LINK_COMPONENTS
+  LineEditor
+  Support
+  )
+
+add_clang_executable(dexp
+  Dexp.cpp
+  )
+
+target_link_libraries(dexp
+  PRIVATE
+  clangDaemon
+  )

Added: clang-tools-extra/trunk/clangd/index/dex/dexp/Dexp.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/dexp/Dexp.cpp?rev=342025&view=auto
==
--- clang-tools-extra/trunk/clangd/index/dex/dexp/Dexp.cpp (added)
+++ clang-tools-extra/trunk/clangd/index/dex/dexp/Dexp.cpp Wed Sep 12 00:32:54 
2018
@@ -0,0 +1,161 @@
+//===--- Dexp.cpp - Dex EXPloration tool *- C++ 
-*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===--===//
+//
+// This file implements a simple interactive tool which can be used to manually
+// evaluate symbol search quality of Clangd index.
+//
+//===--===//
+
+#include "../../../index/SymbolYAML.h"
+#include "../Dex.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/StringSwitch.h"
+#include "llvm/LineEditor/LineEditor.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Signals.h"
+
+using clang::clangd::FuzzyFindRequest;
+using clang::clangd::loadIndex;
+using clang::clangd::Symbol;
+using clang::clangd::SymbolIndex;
+using llvm::StringRef;
+
+namespace {
+
+llvm::cl::opt
+SymbolCollection("symbol-collection-file",
+ llvm::cl::desc("Path to the file with symbol collection"),
+ llvm::cl::Positional, llvm::cl::Required);
+
+static const std::string Overview = R"(
+This is an **experimental** interactive tool to process user-provided search
+queries over given symbol collection obtained via global-symbol-builder. The
+tool can be used to evaluate search quality of existing index implementations
+and manually construct non-trivial test cases.
+
+Type use "help" request to get information about the details.
+)";
+
+void reportTime(StringRef Name, llvm::function_ref F) {
+  const auto TimerStart = std::chrono::high_resolution_clock::now();
+  F();
+  const auto TimerStop = std::chrono::high_resolution_clock::now();
+  const auto Duration = std::chrono::duration_cast(
+  TimerStop - TimerStart);
+  llvm::outs() << llvm::formatv("{0} took {1:ms+n}.\n", Name, Duration);
+}
+
+void fuzzyFind(llvm::StringRef UnqualifiedName, const SymbolIndex &Index) {
+  FuzzyFindRequest Request;
+  Request.MaxCandidateCount = 10;
+  Request.Query = UnqualifiedName;
+  // FIXME(kbobyrev): Print symbol final scores to see the distribution.
+  static const auto OutputFormat = "{0,-4} | {1,-40} | {2,-25}\n";
+  llvm::outs() << llvm::formatv(OutputFormat, "Rank", "Symbol ID",
+"Symbol Name");
+  size_t Rank = 0;
+  Index.fuzzyFind(Request, [&](const Symbol &Sym) {
+llvm::outs() << llvm::formatv(OutputFormat, Rank++, Sym.ID.str(), 
Sym.Name);
+  });
+}
+
+static const std::string HelpMessage = R"(dexp commands:
+
+> find Name
+
+Constructs fuzzy find request given unqualified symbol name and returns top 10
+symbols

[clang-tools-extra] r342026 - [clangd] Add index benchmarks

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Wed Sep 12 00:49:44 2018
New Revision: 342026

URL: http://llvm.org/viewvc/llvm-project?rev=342026&view=rev
Log:
[clangd] Add index benchmarks

This patch introduces index benchmarks on top of the proposed LLVM
benchmark pull.

Reviewed By: sammccall, lebedev.ri

Differential Revision: https://reviews.llvm.org/D51090

Added:
clang-tools-extra/trunk/clangd/benchmarks/
clang-tools-extra/trunk/clangd/benchmarks/CMakeLists.txt
clang-tools-extra/trunk/clangd/benchmarks/IndexBenchmark.cpp
clang-tools-extra/trunk/test/clangd/Inputs/BenchmarkHeader.h
clang-tools-extra/trunk/test/clangd/Inputs/BenchmarkSource.cpp
clang-tools-extra/trunk/test/clangd/Inputs/requests.log
clang-tools-extra/trunk/test/clangd/index-tools.test
Modified:
clang-tools-extra/trunk/clangd/CMakeLists.txt
clang-tools-extra/trunk/test/lit.cfg

Modified: clang-tools-extra/trunk/clangd/CMakeLists.txt
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/CMakeLists.txt?rev=342026&r1=342025&r2=342026&view=diff
==
--- clang-tools-extra/trunk/clangd/CMakeLists.txt (original)
+++ clang-tools-extra/trunk/clangd/CMakeLists.txt Wed Sep 12 00:49:44 2018
@@ -75,3 +75,7 @@ endif()
 add_subdirectory(tool)
 add_subdirectory(global-symbol-builder)
 add_subdirectory(index/dex/dexp)
+
+if (LLVM_INCLUDE_BENCHMARKS)
+  add_subdirectory(benchmarks)
+endif()

Added: clang-tools-extra/trunk/clangd/benchmarks/CMakeLists.txt
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/benchmarks/CMakeLists.txt?rev=342026&view=auto
==
--- clang-tools-extra/trunk/clangd/benchmarks/CMakeLists.txt (added)
+++ clang-tools-extra/trunk/clangd/benchmarks/CMakeLists.txt Wed Sep 12 
00:49:44 2018
@@ -0,0 +1,9 @@
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../)
+
+add_benchmark(IndexBenchmark IndexBenchmark.cpp)
+
+target_link_libraries(IndexBenchmark
+  PRIVATE
+  clangDaemon
+  LLVMSupport
+  )

Added: clang-tools-extra/trunk/clangd/benchmarks/IndexBenchmark.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/benchmarks/IndexBenchmark.cpp?rev=342026&view=auto
==
--- clang-tools-extra/trunk/clangd/benchmarks/IndexBenchmark.cpp (added)
+++ clang-tools-extra/trunk/clangd/benchmarks/IndexBenchmark.cpp Wed Sep 12 
00:49:44 2018
@@ -0,0 +1,114 @@
+//===--- IndexBenchmark.cpp - Clangd index benchmarks ---*- C++ 
-*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===--===//
+
+#include "../index/SymbolYAML.h"
+#include "../index/dex/Dex.h"
+#include "benchmark/benchmark.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Regex.h"
+#include 
+#include 
+#include 
+
+const char *IndexFilename;
+const char *LogFilename;
+
+namespace clang {
+namespace clangd {
+namespace {
+
+std::unique_ptr buildMem() {
+  return clang::clangd::loadIndex(IndexFilename, {}, false);
+}
+
+std::unique_ptr buildDex() {
+  return clang::clangd::loadIndex(IndexFilename, {}, true);
+}
+
+// This function processes user-provided Log file with fuzzy find requests in
+// the following format:
+//
+// fuzzyFind("UnqualifiedName", scopes=["clang::", "clang::clangd::"])
+//
+// It constructs vector of FuzzyFindRequests which is later used for the
+// benchmarks.
+std::vector extractQueriesFromLogs() {
+  llvm::Regex RequestMatcher("fuzzyFind\\(\"([a-zA-Z]*)\", scopes=\\[(.*)\\]");
+  llvm::SmallVector Matches;
+  std::ifstream InputStream(LogFilename);
+  std::string Log((std::istreambuf_iterator(InputStream)),
+  std::istreambuf_iterator());
+  llvm::StringRef Temporary(Log);
+  llvm::SmallVector Strings;
+  Temporary.split(Strings, '\n');
+
+  clang::clangd::FuzzyFindRequest R;
+  R.MaxCandidateCount = 100;
+
+  llvm::SmallVector CommaSeparatedValues;
+
+  std::vector RealRequests;
+  for (auto Line : Strings) {
+if (RequestMatcher.match(Line, &Matches)) {
+  R.Query = Matches[1];
+  CommaSeparatedValues.clear();
+  Line.split(CommaSeparatedValues, ',');
+  R.Scopes.clear();
+  for (auto C : CommaSeparatedValues) {
+R.Scopes.push_back(C);
+  }
+  RealRequests.push_back(R);
+}
+  }
+  return RealRequests;
+}
+
+static void MemQueries(benchmark::State &State) {
+  const auto Mem = buildMem();
+  const auto Requests = extractQueriesFromLogs();
+  for (auto _ : State)
+for (const auto &Request : Requests)
+  Mem->fuzzyFind(Request, [](const Symbol &S) {});
+}
+BENCHMARK(MemQueries);
+
+static void DexQueries(benchmark::Sta

[clang-tools-extra] r342036 - Fix buildbots after r342027

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Wed Sep 12 02:27:55 2018
New Revision: 342036

URL: http://llvm.org/viewvc/llvm-project?rev=342036&view=rev
Log:
Fix buildbots after r342027

Modified:
clang-tools-extra/trunk/test/clangd/index-tools.test

Modified: clang-tools-extra/trunk/test/clangd/index-tools.test
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/test/clangd/index-tools.test?rev=342036&r1=342035&r2=342036&view=diff
==
--- clang-tools-extra/trunk/test/clangd/index-tools.test (original)
+++ clang-tools-extra/trunk/test/clangd/index-tools.test Wed Sep 12 02:27:55 
2018
@@ -1,2 +1,3 @@
 # RUN: global-symbol-builder %p/Inputs/BenchmarkSource.cpp -- -I%p/Inputs > 
%t.index
-# RUN: %clangd-benchmark-dir/IndexBenchmark %t.index %p/Inputs/requests.log 
--benchmark_min_time=0.01
+# FIXME: By default, benchmarks are excluded from the list of default targets 
hence not built. Find a way to depend on benchmarks to run the next command.
+# RUN: if [ -f %clangd-benchmark-dir/IndexBenchmark ]; then 
%clangd-benchmark-dir/IndexBenchmark %t.index %p/Inputs/requests.log 
--benchmark_min_time=0.01 ; fi


___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] r342124 - [clangd] Don't create child AND and OR iterators with one posting list

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Thu Sep 13 03:02:48 2018
New Revision: 342124

URL: http://llvm.org/viewvc/llvm-project?rev=342124&view=rev
Log:
[clangd] Don't create child AND and OR iterators with one posting list

`AND( AND( Child ) ... )` -> `AND( Child ... )`
`AND( OR( Child ) ... )` -> `AND( Child ... )`

This simple optimization results in 5-6% performance improvement in the
benchmark with 2000 serialized `FuzzyFindRequest`s.

Reviewed By: ilya-biryukov

Differential Revision: https://reviews.llvm.org/D52016

Modified:
clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp

Modified: clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp?rev=342124&r1=342123&r2=342124&view=diff
==
--- clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp Thu Sep 13 03:02:48 
2018
@@ -198,7 +198,7 @@ class OrIterator : public Iterator {
 public:
   explicit OrIterator(std::vector> AllChildren)
   : Children(std::move(AllChildren)) {
-assert(Children.size() > 0 && "OR iterator must have at least one child.");
+assert(!Children.empty() && "OR iterator should have at least one child.");
   }
 
   /// Returns true if all children are exhausted.
@@ -405,12 +405,16 @@ std::unique_ptr create(Posting
 
 std::unique_ptr
 createAnd(std::vector> Children) {
-  return llvm::make_unique(move(Children));
+  // If there is exactly one child, pull it one level up: AND(Child) -> Child.
+  return Children.size() == 1 ? std::move(Children.front())
+  : llvm::make_unique(move(Children));
 }
 
 std::unique_ptr
 createOr(std::vector> Children) {
-  return llvm::make_unique(move(Children));
+  // If there is exactly one child, pull it one level up: OR(Child) -> Child.
+  return Children.size() == 1 ? std::move(Children.front())
+  : llvm::make_unique(move(Children));
 }
 
 std::unique_ptr createTrue(DocID Size) {


___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] r342129 - [docs] Provide pointers to known editor plugins and extensions

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Thu Sep 13 04:40:12 2018
New Revision: 342129

URL: http://llvm.org/viewvc/llvm-project?rev=342129&view=rev
Log:
[docs] Provide pointers to known editor plugins and extensions

Many editors provide extensions and plugins with LSP Client
functionality. Many of these are known to work with Clangd, this patch
points users to the relevant resources for better experience.

Reviewed By: ioeric, ilya-biryukov

Differential Revision

Modified:
clang-tools-extra/trunk/docs/clangd.rst

Modified: clang-tools-extra/trunk/docs/clangd.rst
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/docs/clangd.rst?rev=342129&r1=342128&r2=342129&view=diff
==
--- clang-tools-extra/trunk/docs/clangd.rst (original)
+++ clang-tools-extra/trunk/docs/clangd.rst Thu Sep 13 04:40:12 2018
@@ -108,6 +108,41 @@ extension to the protocol.
 | Gen. Getters/Setters| No |   No |
 +-++--+
 
+Editor Integration
+==
+
+Any full-featured Language Server Protocol Client implementation should work
+with :program:`Clangd`. This `list
+` contains information about
+extensions and plugins that are known to work for different editors.
+
+Vim Integration
+---
+
+LanguageClient-neovim
+~
+
+One of the options of using :program:`Clangd` in :program:`vim` (or
+:program:`nvim`) is to utilize `LanguageClient-neovim
+`_ plugin. Please see the
+`Clangd Wiki page
+`_ for
+instructions.
+
+VSCode Integration
+--
+
+:program:`VSCode` provides `vscode-clangd
+`
+which is published in Visual Studio Marketplace and can be installed direcetly
+from :program:`VSCode`.
+
+Emacs Integration
+-
+
+:program:`Emacs` provides `lsp-mode ` and
+`Eglot ` plugins for LSP integration.
+
 Getting Involved
 ==
 


___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] r342137 - [clangd] Use JSON format in benchmark requests reader

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Thu Sep 13 07:21:50 2018
New Revision: 342137

URL: http://llvm.org/viewvc/llvm-project?rev=342137&view=rev
Log:
[clangd] Use JSON format in benchmark requests reader

After `FuzzyFindRequest` JSON (de)serialization was introduced, it
should replace ad-hoc fuzzy-find request parsing implemented in the
IndexBenchmark driver.

Reviewed By: ilya-biryukov

Differential Revision: https://reviews.llvm.org/D51971

Added:
clang-tools-extra/trunk/test/clangd/Inputs/requests.json
Removed:
clang-tools-extra/trunk/test/clangd/Inputs/requests.log
Modified:
clang-tools-extra/trunk/clangd/benchmarks/IndexBenchmark.cpp
clang-tools-extra/trunk/test/clangd/index-tools.test

Modified: clang-tools-extra/trunk/clangd/benchmarks/IndexBenchmark.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/benchmarks/IndexBenchmark.cpp?rev=342137&r1=342136&r2=342137&view=diff
==
--- clang-tools-extra/trunk/clangd/benchmarks/IndexBenchmark.cpp (original)
+++ clang-tools-extra/trunk/clangd/benchmarks/IndexBenchmark.cpp Thu Sep 13 
07:21:50 2018
@@ -19,56 +19,51 @@
 #include 
 
 const char *IndexFilename;
-const char *LogFilename;
+const char *RequestsFilename;
 
 namespace clang {
 namespace clangd {
 namespace {
 
-std::unique_ptr buildMem() {
-  return clang::clangd::loadIndex(IndexFilename, {}, false);
+std::unique_ptr buildMem() {
+  return loadIndex(IndexFilename, {}, false);
 }
 
-std::unique_ptr buildDex() {
-  return clang::clangd::loadIndex(IndexFilename, {}, true);
+std::unique_ptr buildDex() {
+  return loadIndex(IndexFilename, {}, true);
 }
 
-// This function processes user-provided Log file with fuzzy find requests in
-// the following format:
-//
-// fuzzyFind("UnqualifiedName", scopes=["clang::", "clang::clangd::"])
-//
-// It constructs vector of FuzzyFindRequests which is later used for the
-// benchmarks.
-std::vector extractQueriesFromLogs() {
-  llvm::Regex RequestMatcher("fuzzyFind\\(\"([a-zA-Z]*)\", scopes=\\[(.*)\\]");
-  llvm::SmallVector Matches;
-  std::ifstream InputStream(LogFilename);
+// Reads JSON array of serialized FuzzyFindRequest's from user-provided file.
+std::vector extractQueriesFromLogs() {
+  std::ifstream InputStream(RequestsFilename);
   std::string Log((std::istreambuf_iterator(InputStream)),
   std::istreambuf_iterator());
-  llvm::StringRef Temporary(Log);
-  llvm::SmallVector Strings;
-  Temporary.split(Strings, '\n');
-
-  clang::clangd::FuzzyFindRequest R;
-  R.MaxCandidateCount = 100;
-
-  llvm::SmallVector CommaSeparatedValues;
-
-  std::vector RealRequests;
-  for (auto Line : Strings) {
-if (RequestMatcher.match(Line, &Matches)) {
-  R.Query = Matches[1];
-  CommaSeparatedValues.clear();
-  Line.split(CommaSeparatedValues, ',');
-  R.Scopes.clear();
-  for (auto C : CommaSeparatedValues) {
-R.Scopes.push_back(C);
-  }
-  RealRequests.push_back(R);
+
+  std::vector Requests;
+  auto JSONArray = llvm::json::parse(Log);
+
+  // Panic if the provided file couldn't be parsed.
+  if (!JSONArray) {
+llvm::errs() << "Error when parsing JSON requests file: "
+ << llvm::toString(JSONArray.takeError());
+exit(1);
+  }
+  if (!JSONArray->getAsArray()) {
+llvm::errs() << "Error: top-level value is not a JSON array: " << Log
+ << '\n';
+exit(1);
+  }
+
+  for (const auto &Item : *JSONArray->getAsArray()) {
+FuzzyFindRequest Request;
+// Panic if the provided file couldn't be parsed.
+if (!fromJSON(Item, Request)) {
+  llvm::errs() << "Error when deserializing request: " << Item << '\n';
+  exit(1);
 }
+Requests.push_back(Request);
   }
-  return RealRequests;
+  return Requests;
 }
 
 static void MemQueries(benchmark::State &State) {
@@ -100,12 +95,12 @@ BENCHMARK(DexQueries);
 int main(int argc, char *argv[]) {
   if (argc < 3) {
 llvm::errs() << "Usage: " << argv[0]
- << " global-symbol-index.yaml fuzzy-find-requests.log "
+ << " global-symbol-index.yaml requests.json "
 "BENCHMARK_OPTIONS...\n";
 return -1;
   }
   IndexFilename = argv[1];
-  LogFilename = argv[2];
+  RequestsFilename = argv[2];
   // Trim first two arguments of the benchmark invocation.
   argv += 3;
   argc -= 3;

Added: clang-tools-extra/trunk/test/clangd/Inputs/requests.json
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/test/clangd/Inputs/requests.json?rev=342137&view=auto
==
--- clang-tools-extra/trunk/test/clangd/Inputs/requests.json (added)
+++ clang-tools-extra/trunk/test/clangd/Inputs/requests.json Thu Sep 13 
07:21:50 2018
@@ -0,0 +1,7 @@
+[{"MaxCandidateCount":100,"ProximityPaths":["/usr/home/user/clang-tools-extra/clangd/benchmarks/IndexBenchmark.cpp"],"Query":"OMP","RestrictForCodeCompletion":true,"Sc

[clang-tools-extra] r342138 - [clangd] Cleanup FuzzyFindRequest filtering limit semantics

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Thu Sep 13 07:27:03 2018
New Revision: 342138

URL: http://llvm.org/viewvc/llvm-project?rev=342138&view=rev
Log:
[clangd] Cleanup FuzzyFindRequest filtering limit semantics

As discussed during D51860 review, it is better to use `llvm::Optional`
here as it has clear semantics which reflect intended behavior.

Reviewed By: sammccall

Differential Revision: https://reviews.llvm.org/D52028

Modified:
clang-tools-extra/trunk/clangd/CodeComplete.cpp
clang-tools-extra/trunk/clangd/FindSymbols.cpp
clang-tools-extra/trunk/clangd/index/Index.cpp
clang-tools-extra/trunk/clangd/index/Index.h
clang-tools-extra/trunk/clangd/index/MemIndex.cpp
clang-tools-extra/trunk/clangd/index/dex/Dex.cpp
clang-tools-extra/trunk/test/clangd/Inputs/requests.json
clang-tools-extra/trunk/unittests/clangd/DexTests.cpp
clang-tools-extra/trunk/unittests/clangd/IndexTests.cpp

Modified: clang-tools-extra/trunk/clangd/CodeComplete.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/CodeComplete.cpp?rev=342138&r1=342137&r2=342138&view=diff
==
--- clang-tools-extra/trunk/clangd/CodeComplete.cpp (original)
+++ clang-tools-extra/trunk/clangd/CodeComplete.cpp Thu Sep 13 07:27:03 2018
@@ -1375,7 +1375,7 @@ private:
 // Build the query.
 FuzzyFindRequest Req;
 if (Opts.Limit)
-  Req.MaxCandidateCount = Opts.Limit;
+  Req.Limit = Opts.Limit;
 Req.Query = Filter->pattern();
 Req.RestrictForCodeCompletion = true;
 Req.Scopes = QueryScopes;

Modified: clang-tools-extra/trunk/clangd/FindSymbols.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/FindSymbols.cpp?rev=342138&r1=342137&r2=342138&view=diff
==
--- clang-tools-extra/trunk/clangd/FindSymbols.cpp (original)
+++ clang-tools-extra/trunk/clangd/FindSymbols.cpp Thu Sep 13 07:27:03 2018
@@ -117,8 +117,9 @@ getWorkspaceSymbols(StringRef Query, int
   if (IsGlobalQuery || !Names.first.empty())
 Req.Scopes = {Names.first};
   if (Limit)
-Req.MaxCandidateCount = Limit;
-  TopN Top(Req.MaxCandidateCount);
+Req.Limit = Limit;
+  TopN Top(
+  Req.Limit ? *Req.Limit : std::numeric_limits::max());
   FuzzyMatcher Filter(Req.Query);
   Index->fuzzyFind(Req, [HintPath, &Top, &Filter](const Symbol &Sym) {
 // Prefer the definition over e.g. a function declaration in a header

Modified: clang-tools-extra/trunk/clangd/index/Index.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Index.cpp?rev=342138&r1=342137&r2=342138&view=diff
==
--- clang-tools-extra/trunk/clangd/index/Index.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/Index.cpp Thu Sep 13 07:27:03 2018
@@ -177,14 +177,14 @@ std::shared_ptr SwapIndex::
 
 bool fromJSON(const llvm::json::Value &Parameters, FuzzyFindRequest &Request) {
   json::ObjectMapper O(Parameters);
-  int64_t MaxCandidateCount;
+  int64_t Limit;
   bool OK =
   O && O.map("Query", Request.Query) && O.map("Scopes", Request.Scopes) &&
-  O.map("MaxCandidateCount", MaxCandidateCount) &&
+  O.map("Limit", Limit) &&
   O.map("RestrictForCodeCompletion", Request.RestrictForCodeCompletion) &&
   O.map("ProximityPaths", Request.ProximityPaths);
-  if (OK && MaxCandidateCount <= std::numeric_limits::max())
-Request.MaxCandidateCount = MaxCandidateCount;
+  if (OK && Limit <= std::numeric_limits::max())
+Request.Limit = Limit;
   return OK;
 }
 
@@ -192,7 +192,7 @@ llvm::json::Value toJSON(const FuzzyFind
   return json::Object{
   {"Query", Request.Query},
   {"Scopes", json::Array{Request.Scopes}},
-  {"MaxCandidateCount", Request.MaxCandidateCount},
+  {"Limit", Request.Limit},
   {"RestrictForCodeCompletion", Request.RestrictForCodeCompletion},
   {"ProximityPaths", json::Array{Request.ProximityPaths}},
   };

Modified: clang-tools-extra/trunk/clangd/index/Index.h
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Index.h?rev=342138&r1=342137&r2=342138&view=diff
==
--- clang-tools-extra/trunk/clangd/index/Index.h (original)
+++ clang-tools-extra/trunk/clangd/index/Index.h Thu Sep 13 07:27:03 2018
@@ -437,9 +437,7 @@ struct FuzzyFindRequest {
   std::vector Scopes;
   /// \brief The number of top candidates to return. The index may choose to
   /// return more than this, e.g. if it doesn't know which candidates are best.
-  // FIXME: Use llvm::Optional; semantically, the absence of MaxCandidateCount
-  // is equivalent to setting this field to default value as below.
-  uint32_t MaxCandidateCount = std::numeric_limits::max();
+  llvm::Optional Limit;
   /// If set to true, only symbols for completion support will be con

[clang-tools-extra] r342143 - [clangd] Fix Dexp build

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Thu Sep 13 08:35:55 2018
New Revision: 342143

URL: http://llvm.org/viewvc/llvm-project?rev=342143&view=rev
Log:
[clangd] Fix Dexp build

%s/MaxCandidateCount/Limit/g after rL342138.

Modified:
clang-tools-extra/trunk/clangd/index/dex/dexp/Dexp.cpp

Modified: clang-tools-extra/trunk/clangd/index/dex/dexp/Dexp.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/dexp/Dexp.cpp?rev=342143&r1=342142&r2=342143&view=diff
==
--- clang-tools-extra/trunk/clangd/index/dex/dexp/Dexp.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/dex/dexp/Dexp.cpp Thu Sep 13 08:35:55 
2018
@@ -54,7 +54,7 @@ void reportTime(StringRef Name, llvm::fu
 
 void fuzzyFind(llvm::StringRef UnqualifiedName, const SymbolIndex &Index) {
   FuzzyFindRequest Request;
-  Request.MaxCandidateCount = 10;
+  Request.Limit = 10;
   Request.Query = UnqualifiedName;
   // FIXME(kbobyrev): Print symbol final scores to see the distribution.
   static const auto OutputFormat = "{0,-4} | {1,-40} | {2,-25}\n";


___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] r342155 - [clangd] Introduce PostingList interface

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Thu Sep 13 10:11:03 2018
New Revision: 342155

URL: http://llvm.org/viewvc/llvm-project?rev=342155&view=rev
Log:
[clangd] Introduce PostingList interface

This patch abstracts `PostingList` interface and reuses existing
implementation. It will be used later to test different `PostingList`
representations.

No functionality change is introduced, this patch is mostly refactoring
so that the following patches could focus on functionality while not
being too hard to review.

Reviewed By: sammccall, ioeric

Differential Revision: https://reviews.llvm.org/D51982

Added:
clang-tools-extra/trunk/clangd/index/dex/PostingList.cpp
clang-tools-extra/trunk/clangd/index/dex/PostingList.h
Modified:
clang-tools-extra/trunk/clangd/CMakeLists.txt
clang-tools-extra/trunk/clangd/index/dex/Dex.cpp
clang-tools-extra/trunk/clangd/index/dex/Dex.h
clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp
clang-tools-extra/trunk/clangd/index/dex/Iterator.h
clang-tools-extra/trunk/unittests/clangd/DexTests.cpp

Modified: clang-tools-extra/trunk/clangd/CMakeLists.txt
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/CMakeLists.txt?rev=342155&r1=342154&r2=342155&view=diff
==
--- clang-tools-extra/trunk/clangd/CMakeLists.txt (original)
+++ clang-tools-extra/trunk/clangd/CMakeLists.txt Thu Sep 13 10:11:03 2018
@@ -48,6 +48,7 @@ add_clang_library(clangDaemon
 
   index/dex/Dex.cpp
   index/dex/Iterator.cpp
+  index/dex/PostingList.cpp
   index/dex/Trigram.cpp
 
   LINK_LIBS

Modified: clang-tools-extra/trunk/clangd/index/dex/Dex.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/Dex.cpp?rev=342155&r1=342154&r2=342155&view=diff
==
--- clang-tools-extra/trunk/clangd/index/dex/Dex.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/dex/Dex.cpp Thu Sep 13 10:11:03 2018
@@ -82,7 +82,7 @@ std::vector> c
   // FIXME(kbobyrev): Append LIMIT on top of every BOOST iterator.
   PathProximitySignals.SymbolURI = ParentURI;
   BoostingIterators.push_back(
-  createBoost(create(It->second), PathProximitySignals.evaluate()));
+  createBoost(It->second.iterator(), PathProximitySignals.evaluate()));
 }
   }
   return BoostingIterators;
@@ -112,13 +112,19 @@ void Dex::buildIndex() {
 Symbols[I] = ScoredSymbols[I].second;
   }
 
-  // Populate TempInvertedIndex with posting lists for index symbols.
+  // Populate TempInvertedIndex with lists for index symbols.
+  llvm::DenseMap> TempInvertedIndex;
   for (DocID SymbolRank = 0; SymbolRank < Symbols.size(); ++SymbolRank) {
 const auto *Sym = Symbols[SymbolRank];
 for (const auto &Token : generateSearchTokens(*Sym))
-  InvertedIndex[Token].push_back(SymbolRank);
+  TempInvertedIndex[Token].push_back(SymbolRank);
   }
 
+  // Convert lists of items to posting lists.
+  for (const auto &TokenToPostingList : TempInvertedIndex)
+InvertedIndex.insert({TokenToPostingList.first,
+  PostingList(move(TokenToPostingList.second))});
+
   vlog("Built Dex with estimated memory usage {0} bytes.",
estimateMemoryUsage());
 }
@@ -142,7 +148,7 @@ bool Dex::fuzzyFind(const FuzzyFindReque
   for (const auto &Trigram : TrigramTokens) {
 const auto It = InvertedIndex.find(Trigram);
 if (It != InvertedIndex.end())
-  TrigramIterators.push_back(create(It->second));
+  TrigramIterators.push_back(It->second.iterator());
   }
   if (!TrigramIterators.empty())
 TopLevelChildren.push_back(createAnd(move(TrigramIterators)));
@@ -152,7 +158,7 @@ bool Dex::fuzzyFind(const FuzzyFindReque
   for (const auto &Scope : Req.Scopes) {
 const auto It = InvertedIndex.find(Token(Token::Kind::Scope, Scope));
 if (It != InvertedIndex.end())
-  ScopeIterators.push_back(create(It->second));
+  ScopeIterators.push_back(It->second.iterator());
   }
   // Add OR iterator for scopes if there are any Scope Iterators.
   if (!ScopeIterators.empty())
@@ -233,7 +239,7 @@ size_t Dex::estimateMemoryUsage() const
   Bytes += LookupTable.getMemorySize();
   Bytes += InvertedIndex.getMemorySize();
   for (const auto &P : InvertedIndex)
-Bytes += P.second.size() * sizeof(DocID);
+Bytes += P.second.bytes();
   return Bytes + BackingDataSize;
 }
 

Modified: clang-tools-extra/trunk/clangd/index/dex/Dex.h
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/Dex.h?rev=342155&r1=342154&r2=342155&view=diff
==
--- clang-tools-extra/trunk/clangd/index/dex/Dex.h (original)
+++ clang-tools-extra/trunk/clangd/index/dex/Dex.h Thu Sep 13 10:11:03 2018
@@ -21,6 +21,7 @@
 #define LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_DEX_DEX_H
 
 #include "Iterator.h"
+#include "PostingList.h"
 #include "Toke

[clang-tools-extra] r342227 - [clangd] NFC: Fix IndexBenchmark CLI arguments handling

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Fri Sep 14 05:21:09 2018
New Revision: 342227

URL: http://llvm.org/viewvc/llvm-project?rev=342227&view=rev
Log:
[clangd] NFC: Fix IndexBenchmark CLI arguments handling

Modified:
clang-tools-extra/trunk/clangd/benchmarks/IndexBenchmark.cpp

Modified: clang-tools-extra/trunk/clangd/benchmarks/IndexBenchmark.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/benchmarks/IndexBenchmark.cpp?rev=342227&r1=342226&r2=342227&view=diff
==
--- clang-tools-extra/trunk/clangd/benchmarks/IndexBenchmark.cpp (original)
+++ clang-tools-extra/trunk/clangd/benchmarks/IndexBenchmark.cpp Fri Sep 14 
05:21:09 2018
@@ -101,9 +101,11 @@ int main(int argc, char *argv[]) {
   }
   IndexFilename = argv[1];
   RequestsFilename = argv[2];
-  // Trim first two arguments of the benchmark invocation.
-  argv += 3;
-  argc -= 3;
+  // Trim first two arguments of the benchmark invocation and pretend no
+  // arguments were passed in the first place.
+  argv[2] = argv[0];
+  argv += 2;
+  argc -= 2;
   ::benchmark::Initialize(&argc, argv);
   ::benchmark::RunSpecifiedBenchmarks();
 }


___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

Re: [clang-tools-extra] r342036 - Fix buildbots after r342027

[Kirill Bobyrev via cfe-commits]

Oh, I see. Thank you very much for fixing it!

I had concerns about platform dependence, but I’ve seen seen some tests in 
libcxx which use `if […]` so I thought it should be alright (because there was 
no REQUIRES there). I can see now, that it is a Darwin test, my bad.

Again, thank you very much for helping and apologies for inconvenience.

Kind regards,
Kirill

> On 14 Sep 2018, at 22:53, Reid Kleckner  wrote:
> 
> That construct does not work on Windows, where we don't use bash. We use the 
> lit internal shell, which doesn't support 'if [ ...]'. I marked it REQUIRES: 
> shell in r342282.
> 
> On Wed, Sep 12, 2018 at 2:29 AM Kirill Bobyrev via cfe-commits 
> mailto:cfe-commits@lists.llvm.org>> wrote:
> Author: omtcyfz
> Date: Wed Sep 12 02:27:55 2018
> New Revision: 342036
> 
> URL: http://llvm.org/viewvc/llvm-project?rev=342036&view=rev 
> <http://llvm.org/viewvc/llvm-project?rev=342036&view=rev>
> Log:
> Fix buildbots after r342027
> 
> Modified:
> clang-tools-extra/trunk/test/clangd/index-tools.test
> 
> Modified: clang-tools-extra/trunk/test/clangd/index-tools.test
> URL: 
> http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/test/clangd/index-tools.test?rev=342036&r1=342035&r2=342036&view=diff
>  
> <http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/test/clangd/index-tools.test?rev=342036&r1=342035&r2=342036&view=diff>
> ==
> --- clang-tools-extra/trunk/test/clangd/index-tools.test (original)
> +++ clang-tools-extra/trunk/test/clangd/index-tools.test Wed Sep 12 02:27:55 
> 2018
> @@ -1,2 +1,3 @@
>  # RUN: global-symbol-builder %p/Inputs/BenchmarkSource.cpp -- -I%p/Inputs > 
> %t.index
> -# RUN: %clangd-benchmark-dir/IndexBenchmark %t.index %p/Inputs/requests.log 
> --benchmark_min_time=0.01
> +# FIXME: By default, benchmarks are excluded from the list of default 
> targets hence not built. Find a way to depend on benchmarks to run the next 
> command.
> +# RUN: if [ -f %clangd-benchmark-dir/IndexBenchmark ]; then 
> %clangd-benchmark-dir/IndexBenchmark %t.index %p/Inputs/requests.log 
> --benchmark_min_time=0.01 ; fi
> 
> 
> ___
> cfe-commits mailing list
> cfe-commits@lists.llvm.org <mailto:cfe-commits@lists.llvm.org>
> http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits 
> <http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits>

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] r337901 - [clangd] Introduce Dex symbol index search tokens

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Wed Jul 25 03:34:57 2018
New Revision: 337901

URL: http://llvm.org/viewvc/llvm-project?rev=337901&view=rev
Log:
[clangd] Introduce Dex symbol index search tokens

This patch introduces the core building block of the next-generation
Clangd symbol index - Dex. Search tokens are the keys in the inverted
index and represent a characteristic of a specific symbol: examples of
search token types (Token Namespaces) are

* Trigrams -  these are essential for unqualified symbol name fuzzy
search * Scopes for filtering the symbols by the namespace * Paths, e.g.
these can be used to uprank symbols defined close to the edited file

This patch outlines the generic for such token namespaces, but only
implements trigram generation.

The intuition behind trigram generation algorithm is that each extracted
trigram is a valid sequence for Fuzzy Matcher jumps, proposed
implementation utilize existing FuzzyMatcher API for segmentation and
trigram extraction.

However, trigrams generation algorithm for the query string is different
from the previous one: it simply yields sequences of 3 consecutive
lowercased valid characters (letters, digits).

Dex RFC in the mailing list:
http://lists.llvm.org/pipermail/clangd-dev/2018-July/22.html

The trigram generation techniques are described in detail in the
proposal:
https://docs.google.com/document/d/1C-A6PGT6TynyaX4PXyExNMiGmJ2jL1UwV91Kyx11gOI/edit#heading=h.903u1zon9nkj

Reviewers: sammccall, ioeric, ilya-biryukovA

Subscribers: cfe-commits, klimek, mgorny, MaskRay, jkorous, arphaman

Differential Revision: https://reviews.llvm.org/D49591

Added:
clang-tools-extra/trunk/clangd/index/dex/
clang-tools-extra/trunk/clangd/index/dex/Token.h
clang-tools-extra/trunk/clangd/index/dex/Trigram.cpp
clang-tools-extra/trunk/clangd/index/dex/Trigram.h
clang-tools-extra/trunk/unittests/clangd/DexIndexTests.cpp
Modified:
clang-tools-extra/trunk/clangd/CMakeLists.txt
clang-tools-extra/trunk/unittests/clangd/CMakeLists.txt

Modified: clang-tools-extra/trunk/clangd/CMakeLists.txt
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/CMakeLists.txt?rev=337901&r1=337900&r2=337901&view=diff
==
--- clang-tools-extra/trunk/clangd/CMakeLists.txt (original)
+++ clang-tools-extra/trunk/clangd/CMakeLists.txt Wed Jul 25 03:34:57 2018
@@ -34,6 +34,7 @@ add_clang_library(clangDaemon
   TUScheduler.cpp
   URI.cpp
   XRefs.cpp
+
   index/CanonicalIncludes.cpp
   index/FileIndex.cpp
   index/Index.cpp
@@ -42,6 +43,8 @@ add_clang_library(clangDaemon
   index/SymbolCollector.cpp
   index/SymbolYAML.cpp
 
+  index/dex/Trigram.cpp
+
   LINK_LIBS
   clangAST
   clangASTMatchers

Added: clang-tools-extra/trunk/clangd/index/dex/Token.h
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/Token.h?rev=337901&view=auto
==
--- clang-tools-extra/trunk/clangd/index/dex/Token.h (added)
+++ clang-tools-extra/trunk/clangd/index/dex/Token.h Wed Jul 25 03:34:57 2018
@@ -0,0 +1,112 @@
+//===--- Token.h - Symbol Search primitive --*- C++ 
-*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===--===//
+//
+// Token objects represent a characteristic of a symbol, which can be used to
+// perform efficient search. Tokens are keys for inverted index which are 
mapped
+// to the corresponding posting lists.
+//
+// The symbol std::cout might have the tokens:
+// * Scope "std::"
+// * Trigram "cou"
+// * Trigram "out"
+// * Type "std::ostream"
+//
+//===--===//
+
+#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_DEX_TOKEN_H
+#define LLVM_CLANG_TOOLS_EXTRA_CLANGD_DEX_TOKEN_H
+
+#include "llvm/ADT/DenseMap.h"
+#include "llvm/Support/raw_ostream.h"
+
+#include 
+#include 
+
+namespace clang {
+namespace clangd {
+namespace dex {
+
+/// A Token represents an attribute of a symbol, such as a particular trigram
+/// present in the name (used for fuzzy search).
+///
+/// Tokens can be used to perform more sophisticated search queries by
+/// constructing complex iterator trees.
+struct Token {
+  /// Kind specifies Token type which defines semantics for the internal
+  /// representation. Each Kind has different representation stored in Data
+  /// field.
+  enum class Kind {
+/// Represents trigram used for fuzzy search of unqualified symbol names.
+///
+/// Data contains 3 bytes with trigram contents.
+Trigram,
+/// Scope primitives, e.g. "symbol belongs to namespace foo::bar".
+///
+/// Data stroes full scope name , e.g. "foo::bar::baz::" or "" (for global
+/// scope).
+Scope,

[clang-tools-extra] r338015 - [clangd] Give an example for symbol-builder usage

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Thu Jul 26 02:41:24 2018
New Revision: 338015

URL: http://llvm.org/viewvc/llvm-project?rev=338015&view=rev
Log:
[clangd] Give an example for symbol-builder usage

`global-symbol-builder` help message mentions `-executor=`
option, but doesn't give any example of what the value could be

Assuming the most popular use case to be building the whole project
index, help message should probably give an example of such usage.

Reviewers: ioeric

Subscribers: cfe-commits

Differential Revision: https://reviews.llvm.org/D49785

Modified:

clang-tools-extra/trunk/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp

Modified: 
clang-tools-extra/trunk/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp?rev=338015&r1=338014&r2=338015&view=diff
==
--- 
clang-tools-extra/trunk/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp
 (original)
+++ 
clang-tools-extra/trunk/clangd/global-symbol-builder/GlobalSymbolBuilderMain.cpp
 Thu Jul 26 02:41:24 2018
@@ -150,10 +150,23 @@ SymbolSlab mergeSymbols(tooling::ToolRes
 int main(int argc, const char **argv) {
   llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
 
-  const char* Overview =
-  "This is an **experimental** tool to generate YAML-format "
-  "project-wide symbols for clangd (global code completion). It would be "
-  "changed and deprecated eventually. Don't use it in production code!";
+  const char *Overview = R"(
+  This is an **experimental** tool to generate YAML-format project-wide symbols
+  for clangd (global code completion). It would be changed and deprecated
+  eventually. Don't use it in production code!
+
+  Example usage for building index for the whole project using CMake compile
+  commands:
+
+  $ global-symbol-builder --executor=all-TUs compile_commands.json > index.yaml
+
+  Example usage for file sequence index without flags:
+
+  $ global-symbol-builder File1.cpp File2.cpp ... FileN.cpp > index.yaml
+
+  Note: only symbols from header files will be collected.
+  )";
+
   auto Executor = clang::tooling::createExecutorFromCommandLineArgs(
   argc, argv, cl::GeneralCategory, Overview);
 


___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] r338017 - [clangd] Proof-of-concept query iterators for Dex symbol index

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Thu Jul 26 03:42:31 2018
New Revision: 338017

URL: http://llvm.org/viewvc/llvm-project?rev=338017&view=rev
Log:
[clangd] Proof-of-concept query iterators for Dex symbol index

This patch introduces three essential types of query iterators:
`DocumentIterator`, `AndIterator`, `OrIterator`. It provides a
convenient API for query tree generation and serves as a building block
for the next generation symbol index - Dex. Currently, many
optimizations are missed to improve code readability and to serve as the
reference implementation. Potential improvements are briefly mentioned
in `FIXME`s and will be addressed in the following patches.

Dex RFC in the mailing list:
http://lists.llvm.org/pipermail/clangd-dev/2018-July/22.html

Iterators, their applications and potential extensions are explained in
detail in the design proposal:
https://docs.google.com/document/d/1C-A6PGT6TynyaX4PXyExNMiGmJ2jL1UwV91Kyx11gOI/edit#heading=h.903u1zon9nkj

Reviewers: ioeric, sammccall, ilya-biryukov

Subscribers: cfe-commits, klimek, jfb, mgrang, mgorny, MaskRay, jkorous,
arphaman

Differential Revision: https://reviews.llvm.org/D49546

Added:
clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp
clang-tools-extra/trunk/clangd/index/dex/Iterator.h
Modified:
clang-tools-extra/trunk/clangd/CMakeLists.txt
clang-tools-extra/trunk/unittests/clangd/DexIndexTests.cpp

Modified: clang-tools-extra/trunk/clangd/CMakeLists.txt
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/CMakeLists.txt?rev=338017&r1=338016&r2=338017&view=diff
==
--- clang-tools-extra/trunk/clangd/CMakeLists.txt (original)
+++ clang-tools-extra/trunk/clangd/CMakeLists.txt Thu Jul 26 03:42:31 2018
@@ -43,6 +43,7 @@ add_clang_library(clangDaemon
   index/SymbolCollector.cpp
   index/SymbolYAML.cpp
 
+  index/dex/Iterator.cpp
   index/dex/Trigram.cpp
 
   LINK_LIBS

Added: clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp?rev=338017&view=auto
==
--- clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp (added)
+++ clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp Thu Jul 26 03:42:31 
2018
@@ -0,0 +1,244 @@
+//===--- Iterator.cpp - Query Symbol Retrieval --*- C++ 
-*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===--===//
+
+#include "Iterator.h"
+#include 
+#include 
+#include 
+
+namespace clang {
+namespace clangd {
+namespace dex {
+
+namespace {
+
+/// Implements Iterator over a PostingList. DocumentIterator is the most basic
+/// iterator: it doesn't have any children (hence it is the leaf of iterator
+/// tree) and is simply a wrapper around PostingList::const_iterator.
+class DocumentIterator : public Iterator {
+public:
+  DocumentIterator(PostingListRef Documents)
+  : Documents(Documents), Index(std::begin(Documents)) {}
+
+  bool reachedEnd() const override { return Index == std::end(Documents); }
+
+  /// Advances cursor to the next item.
+  void advance() override {
+assert(!reachedEnd() && "DocumentIterator can't advance at the end.");
+++Index;
+  }
+
+  /// Applies binary search to advance cursor to the next item with DocID equal
+  /// or higher than the given one.
+  void advanceTo(DocID ID) override {
+assert(!reachedEnd() && "DocumentIterator can't advance at the end.");
+Index = std::lower_bound(Index, std::end(Documents), ID);
+  }
+
+  DocID peek() const override {
+assert(!reachedEnd() && "DocumentIterator can't call peek() at the end.");
+return *Index;
+  }
+
+  llvm::raw_ostream &dump(llvm::raw_ostream &OS) const override {
+OS << '[';
+auto Separator = "";
+for (const auto &ID : Documents) {
+  OS << Separator << ID;
+  Separator = ", ";
+}
+OS << ']';
+return OS;
+  }
+
+private:
+  PostingListRef Documents;
+  PostingListRef::const_iterator Index;
+};
+
+/// Implements Iterator over the intersection of other iterators.
+///
+/// AndIterator iterates through common items among all children. It becomes
+/// exhausted as soon as any child becomes exhausted. After each mutation, the
+/// iterator restores the invariant: all children must point to the same item.
+class AndIterator : public Iterator {
+public:
+  AndIterator(std::vector> AllChildren)
+  : Children(std::move(AllChildren)) {
+assert(!Children.empty() && "AndIterator should have at least one child.");
+// Establish invariants.
+sync();
+  }
+
+  bool reachedEnd() const override { return ReachedEnd; }
+
+  /// Advances all children to the next common item.
+  void advance

[clang-tools-extra] r338028 - [clangd] Fix unit tests for Dex

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Thu Jul 26 07:00:00 2018
New Revision: 338028

URL: http://llvm.org/viewvc/llvm-project?rev=338028&view=rev
Log:
[clangd] Fix unit tests for Dex

Iterators took temporary objects in constructors, objects were
invalidated when built with recent Clang which resulted in crashes.

Modified:
clang-tools-extra/trunk/unittests/clangd/DexIndexTests.cpp

Modified: clang-tools-extra/trunk/unittests/clangd/DexIndexTests.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/unittests/clangd/DexIndexTests.cpp?rev=338028&r1=338027&r2=338028&view=diff
==
--- clang-tools-extra/trunk/unittests/clangd/DexIndexTests.cpp (original)
+++ clang-tools-extra/trunk/unittests/clangd/DexIndexTests.cpp Thu Jul 26 
07:00:00 2018
@@ -24,7 +24,8 @@ namespace dex {
 using ::testing::ElementsAre;
 
 TEST(DexIndexIterators, DocumentIterator) {
-  auto DocIterator = create({4, 7, 8, 20, 42, 100});
+  const PostingList L = {4, 7, 8, 20, 42, 100};
+  auto DocIterator = create(L);
 
   EXPECT_EQ(DocIterator->peek(), 4U);
   EXPECT_EQ(DocIterator->reachedEnd(), false);
@@ -194,12 +195,18 @@ TEST(DexIndexIterators, QueryTree) {
   //  |1, 3, 5, 8, 9| |1, 5, 7, 9|   |Empty||0, 5||0, 1, 5|
   //  +-+ +--+   +-++++---+
 
+  const PostingList L0 = {1, 3, 5, 8, 9};
+  const PostingList L1 = {1, 5, 7, 9};
+  const PostingList L2 = {0, 5};
+  const PostingList L3 = {0, 1, 5};
+  const PostingList L4;
+
   // Root of the query tree: [1, 5]
   auto Root = createAnd({
   // Lower And Iterator: [1, 5, 9]
-  createAnd({create({1, 3, 5, 8, 9}), create({1, 5, 7, 9})}),
+  createAnd({create(L0), create(L1)}),
   // Lower Or Iterator: [0, 1, 5]
-  createOr({create({0, 5}), create({0, 1, 5}), create({})}),
+  createOr({create(L2), create(L3), create(L4)}),
   });
 
   EXPECT_EQ(Root->reachedEnd(), false);
@@ -218,12 +225,18 @@ TEST(DexIndexIterators, QueryTree) {
 }
 
 TEST(DexIndexIterators, StringRepresentation) {
-  EXPECT_EQ(llvm::to_string(*(create({4, 7, 8, 20, 42, 100}))),
-"[4, 7, 8, 20, 42, 100]");
+  const PostingList L0 = {4, 7, 8, 20, 42, 100};
+  const PostingList L1 = {1, 3, 5, 8, 9};
+  const PostingList L2 = {1, 5, 7, 9};
+  const PostingList L3 = {0, 5};
+  const PostingList L4 = {0, 1, 5};
+  const PostingList L5;
+
+  EXPECT_EQ(llvm::to_string(*(create(L0))), "[4, 7, 8, 20, 42, 100]");
 
   auto Nested = createAnd({
-  createAnd({create({1, 3, 5, 8, 9}), create({1, 5, 7, 9})}),
-  createOr({create({0, 5}), create({0, 1, 5}), create({})}),
+  createAnd({create(L1), create(L2)}),
+  createOr({create(L3), create(L4), create(L5)}),
   });
 
   EXPECT_EQ(llvm::to_string(*Nested),


___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] r338054 - Revert Clangd Dex Iterators patch

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Thu Jul 26 11:25:48 2018
New Revision: 338054

URL: http://llvm.org/viewvc/llvm-project?rev=338054&view=rev
Log:
Revert Clangd Dex Iterators patch

This reverts two revisions:

* https://reviews.llvm.org/rL338017
* https://reviews.llvm.org/rL338028

They caused crash for Clang 3.6 & Clang 3.7 buildbots, it was
reported by Jeremy Morse.

Removed:
clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp
clang-tools-extra/trunk/clangd/index/dex/Iterator.h
Modified:
clang-tools-extra/trunk/clangd/CMakeLists.txt
clang-tools-extra/trunk/unittests/clangd/DexIndexTests.cpp

Modified: clang-tools-extra/trunk/clangd/CMakeLists.txt
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/CMakeLists.txt?rev=338054&r1=338053&r2=338054&view=diff
==
--- clang-tools-extra/trunk/clangd/CMakeLists.txt (original)
+++ clang-tools-extra/trunk/clangd/CMakeLists.txt Thu Jul 26 11:25:48 2018
@@ -43,7 +43,6 @@ add_clang_library(clangDaemon
   index/SymbolCollector.cpp
   index/SymbolYAML.cpp
 
-  index/dex/Iterator.cpp
   index/dex/Trigram.cpp
 
   LINK_LIBS

Removed: clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp?rev=338053&view=auto
==
--- clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp (removed)
@@ -1,244 +0,0 @@
-//===--- Iterator.cpp - Query Symbol Retrieval --*- C++ 
-*-===//
-//
-// The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===--===//
-
-#include "Iterator.h"
-#include 
-#include 
-#include 
-
-namespace clang {
-namespace clangd {
-namespace dex {
-
-namespace {
-
-/// Implements Iterator over a PostingList. DocumentIterator is the most basic
-/// iterator: it doesn't have any children (hence it is the leaf of iterator
-/// tree) and is simply a wrapper around PostingList::const_iterator.
-class DocumentIterator : public Iterator {
-public:
-  DocumentIterator(PostingListRef Documents)
-  : Documents(Documents), Index(std::begin(Documents)) {}
-
-  bool reachedEnd() const override { return Index == std::end(Documents); }
-
-  /// Advances cursor to the next item.
-  void advance() override {
-assert(!reachedEnd() && "DocumentIterator can't advance at the end.");
-++Index;
-  }
-
-  /// Applies binary search to advance cursor to the next item with DocID equal
-  /// or higher than the given one.
-  void advanceTo(DocID ID) override {
-assert(!reachedEnd() && "DocumentIterator can't advance at the end.");
-Index = std::lower_bound(Index, std::end(Documents), ID);
-  }
-
-  DocID peek() const override {
-assert(!reachedEnd() && "DocumentIterator can't call peek() at the end.");
-return *Index;
-  }
-
-  llvm::raw_ostream &dump(llvm::raw_ostream &OS) const override {
-OS << '[';
-auto Separator = "";
-for (const auto &ID : Documents) {
-  OS << Separator << ID;
-  Separator = ", ";
-}
-OS << ']';
-return OS;
-  }
-
-private:
-  PostingListRef Documents;
-  PostingListRef::const_iterator Index;
-};
-
-/// Implements Iterator over the intersection of other iterators.
-///
-/// AndIterator iterates through common items among all children. It becomes
-/// exhausted as soon as any child becomes exhausted. After each mutation, the
-/// iterator restores the invariant: all children must point to the same item.
-class AndIterator : public Iterator {
-public:
-  AndIterator(std::vector> AllChildren)
-  : Children(std::move(AllChildren)) {
-assert(!Children.empty() && "AndIterator should have at least one child.");
-// Establish invariants.
-sync();
-  }
-
-  bool reachedEnd() const override { return ReachedEnd; }
-
-  /// Advances all children to the next common item.
-  void advance() override {
-assert(!reachedEnd() && "AndIterator can't call advance() at the end.");
-Children.front()->advance();
-sync();
-  }
-
-  /// Advances all children to the next common item with DocumentID >= ID.
-  void advanceTo(DocID ID) override {
-assert(!reachedEnd() && "AndIterator can't call advanceTo() at the end.");
-Children.front()->advanceTo(ID);
-sync();
-  }
-
-  DocID peek() const override { return Children.front()->peek(); }
-
-  llvm::raw_ostream &dump(llvm::raw_ostream &OS) const override {
-OS << "(& ";
-auto Separator = "";
-for (const auto &Child : Children) {
-  OS << Separator << *Child;
-  Separator = " ";
-}
-OS << ')';
-return OS;
-  }
-
-private:
-  /// Restores class invariants: each child will point to the sam

Re: [clang-tools-extra] r338017 - [clangd] Proof-of-concept query iterators for Dex symbol index

[Kirill Bobyrev via cfe-commits]

Hi Jeremy,

Thank you for reporting the issue and apologies for inconvenience. There
was another bug which caused buildbots crashes, I was looking into that
before so I missed this one.

I built Clang 3.6 and Clang 3.7 locally, simply wrapping everything into
std::move didn't resolve the issue, I reverted the changes (
https://reviews.llvm.org/rL338054) and will investigate.

Thank you for reaching out,
Kirill Bobyrev

On Thu, Jul 26, 2018 at 7:16 PM Jeremy Morse 
wrote:

> [Resending with cfe-commits@, whoops]
>
> Hi Kirill,
>
> We believe this patch might be breaking one of the buildbots:
>
>
> http://lab.llvm.org:8011/builders/clang-x86_64-linux-abi-test/builds/29719
>
> Which is complaining about the std::unique_ptr copy constructor being
> called in DexIndexTests.cpp. It seems likely that returning a unique_ptr:
>
> > std::unique_ptr
> > createAnd(std::vector> Children) {
> >   return llvm::make_unique(move(Children));
> > }
>
> And sites such as:
>
> >  auto AndEmpty = createAnd({create(L0)});
>
> Are triggering a bug in clang 3.7 and 3.8 where the move constructor isn't
> correctly inferred:
>
> https://godbolt.org/g/Aquug4
>
> Wrapping everything in std::move will likely fix that, those two versions
> of clang are still officially supported by LLVM/Clang.
>
> --
> Thanks,
> Jeremy
>
>
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] r338116 - [clangd] Return Dex Iterators

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Fri Jul 27 02:54:27 2018
New Revision: 338116

URL: http://llvm.org/viewvc/llvm-project?rev=338116&view=rev
Log:
[clangd] Return Dex Iterators

The original Dex Iterators patch (https://reviews.llvm.org/rL338017)
caused problems for Clang 3.6 and Clang 3.7 due to the compiler bug
which prevented inferring template parameter (`Size`) in create(And|Or)?
functions. It was reverted in https://reviews.llvm.org/rL338054.

In this revision the mentioned helper functions were replaced with
variadic templated versions.

Proposed changes were tested on multiple compiler versions, including
Clang 3.6 which originally caused the failure.

Added:
clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp
clang-tools-extra/trunk/clangd/index/dex/Iterator.h
Modified:
clang-tools-extra/trunk/clangd/CMakeLists.txt
clang-tools-extra/trunk/unittests/clangd/DexIndexTests.cpp

Modified: clang-tools-extra/trunk/clangd/CMakeLists.txt
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/CMakeLists.txt?rev=338116&r1=338115&r2=338116&view=diff
==
--- clang-tools-extra/trunk/clangd/CMakeLists.txt (original)
+++ clang-tools-extra/trunk/clangd/CMakeLists.txt Fri Jul 27 02:54:27 2018
@@ -43,6 +43,7 @@ add_clang_library(clangDaemon
   index/SymbolCollector.cpp
   index/SymbolYAML.cpp
 
+  index/dex/Iterator.cpp
   index/dex/Trigram.cpp
 
   LINK_LIBS

Added: clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp?rev=338116&view=auto
==
--- clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp (added)
+++ clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp Fri Jul 27 02:54:27 
2018
@@ -0,0 +1,244 @@
+//===--- Iterator.cpp - Query Symbol Retrieval --*- C++ 
-*-===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===--===//
+
+#include "Iterator.h"
+#include 
+#include 
+#include 
+
+namespace clang {
+namespace clangd {
+namespace dex {
+
+namespace {
+
+/// Implements Iterator over a PostingList. DocumentIterator is the most basic
+/// iterator: it doesn't have any children (hence it is the leaf of iterator
+/// tree) and is simply a wrapper around PostingList::const_iterator.
+class DocumentIterator : public Iterator {
+public:
+  DocumentIterator(PostingListRef Documents)
+  : Documents(Documents), Index(std::begin(Documents)) {}
+
+  bool reachedEnd() const override { return Index == std::end(Documents); }
+
+  /// Advances cursor to the next item.
+  void advance() override {
+assert(!reachedEnd() && "DocumentIterator can't advance at the end.");
+++Index;
+  }
+
+  /// Applies binary search to advance cursor to the next item with DocID equal
+  /// or higher than the given one.
+  void advanceTo(DocID ID) override {
+assert(!reachedEnd() && "DocumentIterator can't advance at the end.");
+Index = std::lower_bound(Index, std::end(Documents), ID);
+  }
+
+  DocID peek() const override {
+assert(!reachedEnd() && "DocumentIterator can't call peek() at the end.");
+return *Index;
+  }
+
+  llvm::raw_ostream &dump(llvm::raw_ostream &OS) const override {
+OS << '[';
+auto Separator = "";
+for (const auto &ID : Documents) {
+  OS << Separator << ID;
+  Separator = ", ";
+}
+OS << ']';
+return OS;
+  }
+
+private:
+  PostingListRef Documents;
+  PostingListRef::const_iterator Index;
+};
+
+/// Implements Iterator over the intersection of other iterators.
+///
+/// AndIterator iterates through common items among all children. It becomes
+/// exhausted as soon as any child becomes exhausted. After each mutation, the
+/// iterator restores the invariant: all children must point to the same item.
+class AndIterator : public Iterator {
+public:
+  AndIterator(std::vector> AllChildren)
+  : Children(std::move(AllChildren)) {
+assert(!Children.empty() && "AndIterator should have at least one child.");
+// Establish invariants.
+sync();
+  }
+
+  bool reachedEnd() const override { return ReachedEnd; }
+
+  /// Advances all children to the next common item.
+  void advance() override {
+assert(!reachedEnd() && "AndIterator can't call advance() at the end.");
+Children.front()->advance();
+sync();
+  }
+
+  /// Advances all children to the next common item with DocumentID >= ID.
+  void advanceTo(DocID ID) override {
+assert(!reachedEnd() && "AndIterator can't call advanceTo() at the end.");
+Children.front()->advanceTo(ID);
+sync();
+  }
+
+  DocID peek() const override { return Children.front()->peek(); }
+
+  llvm::raw_ostream &dump(l

[clang-tools-extra] r326051 - [clangd] Address FIXME and fix comment

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Sat Feb 24 23:21:16 2018
New Revision: 326051

URL: http://llvm.org/viewvc/llvm-project?rev=326051&view=rev
Log:
[clangd] Address FIXME and fix comment

* Address a FIXME by warning the user that both -run-synchronously and -j X are
  passed.
* Fix a comment to suppress clang-tidy warning by passing the correct argument
  name.

Reviewers: ioeric

Subscribers: ilya-biryukov, jkorous-apple, cfe-commits

Differential Revision: https://reviews.llvm.org/D43671

Modified:
clang-tools-extra/trunk/clangd/Context.cpp
clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp

Modified: clang-tools-extra/trunk/clangd/Context.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/Context.cpp?rev=326051&r1=326050&r2=326051&view=diff
==
--- clang-tools-extra/trunk/clangd/Context.cpp (original)
+++ clang-tools-extra/trunk/clangd/Context.cpp Sat Feb 24 23:21:16 2018
@@ -13,7 +13,7 @@
 namespace clang {
 namespace clangd {
 
-Context Context::empty() { return Context(/*Data=*/nullptr); }
+Context Context::empty() { return Context(/*DataPtr=*/nullptr); }
 
 Context::Context(std::shared_ptr DataPtr)
 : DataPtr(std::move(DataPtr)) {}

Modified: clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp?rev=326051&r1=326050&r2=326051&view=diff
==
--- clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp (original)
+++ clang-tools-extra/trunk/clangd/tool/ClangdMain.cpp Sat Feb 24 23:21:16 2018
@@ -149,10 +149,11 @@ int main(int argc, char *argv[]) {
 return 1;
   }
 
-  // Ignore -j option if -run-synchonously is used.
-  // FIXME: a warning should be shown here.
-  if (RunSynchronously)
+  if (RunSynchronously) {
+if (WorkerThreadsCount.getNumOccurrences())
+  llvm::errs() << "Ignoring -j because -run-synchronously is set.\n";
 WorkerThreadsCount = 0;
+  }
 
   // Validate command line arguments.
   llvm::Optional InputMirrorStream;


___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

Re: [clang-tools-extra] r342227 - [clangd] NFC: Fix IndexBenchmark CLI arguments handling

[Kirill Bobyrev via cfe-commits]

Hi Roman,

Is there any benefit of doing so? Also, I’m not sure whether I understood you 
correctly. Consuming benchmark options *before* trimming would probably not be 
the desired behaviour since the first two arguments arguments are passed 
directly to the tool driver.

I might have misunderstood you, could you please elaborate on the proposed idea?

Kind regards,
Kirill

> On 14 Sep 2018, at 14:46, Roman Lebedev  wrote:
> 
> On Fri, Sep 14, 2018 at 3:21 PM, Kirill Bobyrev via cfe-commits
>  wrote:
>> Author: omtcyfz
>> Date: Fri Sep 14 05:21:09 2018
>> New Revision: 342227
>> 
>> URL: http://llvm.org/viewvc/llvm-project?rev=342227&view=rev
>> Log:
>> [clangd] NFC: Fix IndexBenchmark CLI arguments handling
>> 
>> Modified:
>>clang-tools-extra/trunk/clangd/benchmarks/IndexBenchmark.cpp
>> 
>> Modified: clang-tools-extra/trunk/clangd/benchmarks/IndexBenchmark.cpp
>> URL: 
>> http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/benchmarks/IndexBenchmark.cpp?rev=342227&r1=342226&r2=342227&view=diff
>> ==
>> --- clang-tools-extra/trunk/clangd/benchmarks/IndexBenchmark.cpp (original)
>> +++ clang-tools-extra/trunk/clangd/benchmarks/IndexBenchmark.cpp Fri Sep 14 
>> 05:21:09 2018
>> @@ -101,9 +101,11 @@ int main(int argc, char *argv[]) {
>>   }
> 
>>   IndexFilename = argv[1];
>>   RequestsFilename = argv[2];
>> -  // Trim first two arguments of the benchmark invocation.
>> -  argv += 3;
>> -  argc -= 3;
>> +  // Trim first two arguments of the benchmark invocation and pretend no
>> +  // arguments were passed in the first place.
>> +  argv[2] = argv[0];
>> +  argv += 2;
>> +  argc -= 2;
>>   ::benchmark::Initialize(&argc, argv);
> Passing-by thought: why is this being done in *this* order?
> Why not first let the ::benchmark::Initialize() consume it's flags first?
> 
>>   ::benchmark::RunSpecifiedBenchmarks();
>> }
>> 
> Roman.
> 
>> ___
>> cfe-commits mailing list
>> cfe-commits@lists.llvm.org
>> http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

Re: [clang-tools-extra] r342227 - [clangd] NFC: Fix IndexBenchmark CLI arguments handling

[Kirill Bobyrev via cfe-commits]

Thanks for the explanation! I didn’t know that benchmark’s Initialize does 
that, that was probably the source of my confusion. The suggestion looks 
reasonable, I should try this approach, it looks to be cleaner.

-Kirill

> On 18 Sep 2018, at 21:16, Roman Lebedev  wrote:
> 
> On Tue, Sep 18, 2018 at 10:09 PM, Kirill Bobyrev
>  wrote:
>> Hi Roman,
>> 
>> Is there any benefit of doing so? Also, I’m not sure whether I understood 
>> you correctly. Consuming benchmark options *before* trimming would probably 
>> not be the desired behaviour since the first two arguments arguments are 
>> passed directly to the tool driver.
> Currently, you have to call it like
> $ IndexBenchmark IndexFilename RequestsFilename --benchmark_something
> 
> If you do
> $ IndexBenchmark --benchmark_something IndexFilename RequestsFilename
> your code will still consume argv[1] (i.e. "--benchmark_something") as
> IndexFilename
> 
> But if you do call  ::benchmark::Initialize(&argc, argv);  first it
> will work as you'd expect,
> i.e. you would still consume argv[1], but it then would have been
> adjusted to IndexFilename
> 
>> I might have misunderstood you, could you please elaborate on the proposed 
>> idea?
>> 
>> Kind regards,
>> Kirill
> Roman.
> 
>>> On 14 Sep 2018, at 14:46, Roman Lebedev  wrote:
>>> 
>>> On Fri, Sep 14, 2018 at 3:21 PM, Kirill Bobyrev via cfe-commits
>>>  wrote:
>>>> Author: omtcyfz
>>>> Date: Fri Sep 14 05:21:09 2018
>>>> New Revision: 342227
>>>> 
>>>> URL: http://llvm.org/viewvc/llvm-project?rev=342227&view=rev
>>>> Log:
>>>> [clangd] NFC: Fix IndexBenchmark CLI arguments handling
>>>> 
>>>> Modified:
>>>>   clang-tools-extra/trunk/clangd/benchmarks/IndexBenchmark.cpp
>>>> 
>>>> Modified: clang-tools-extra/trunk/clangd/benchmarks/IndexBenchmark.cpp
>>>> URL: 
>>>> http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/benchmarks/IndexBenchmark.cpp?rev=342227&r1=342226&r2=342227&view=diff
>>>> ==
>>>> --- clang-tools-extra/trunk/clangd/benchmarks/IndexBenchmark.cpp (original)
>>>> +++ clang-tools-extra/trunk/clangd/benchmarks/IndexBenchmark.cpp Fri Sep 
>>>> 14 05:21:09 2018
>>>> @@ -101,9 +101,11 @@ int main(int argc, char *argv[]) {
>>>>  }
>>> 
>>>>  IndexFilename = argv[1];
>>>>  RequestsFilename = argv[2];
>>>> -  // Trim first two arguments of the benchmark invocation.
>>>> -  argv += 3;
>>>> -  argc -= 3;
>>>> +  // Trim first two arguments of the benchmark invocation and pretend no
>>>> +  // arguments were passed in the first place.
>>>> +  argv[2] = argv[0];
>>>> +  argv += 2;
>>>> +  argc -= 2;
>>>>  ::benchmark::Initialize(&argc, argv);
>>> Passing-by thought: why is this being done in *this* order?
>>> Why not first let the ::benchmark::Initialize() consume it's flags first?
>>> 
>>>>  ::benchmark::RunSpecifiedBenchmarks();
>>>> }
>>>> 
>>> Roman.
>>> 
>>>> ___
>>>> cfe-commits mailing list
>>>> cfe-commits@lists.llvm.org
>>>> http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
>> 

___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] r342866 - [clangd] Force Dex to respect symbol collector flags

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Mon Sep 24 01:45:18 2018
New Revision: 342866

URL: http://llvm.org/viewvc/llvm-project?rev=342866&view=rev
Log:
[clangd] Force Dex to respect symbol collector flags

`Dex` should utilize `FuzzyFindRequest.RestrictForCodeCompletion` flags
and omit symbols not meant for code completion when asked for it.

The measurements below were conducted with setting
`FuzzyFindRequest.RestrictForCodeCompletion` to `true` (so that it's
more realistic). Sadly, the average latency goes down, I suspect that is
mostly because of the empty queries where the number of posting lists is
critical.

| Metrics  | Before | After | Relative difference
| -  | -  | -   | -
| Cumulative query latency (7000 `FuzzyFindRequest`s over LLVM static index)  | 
6182735043 ns| 7202442053 ns | +16%
| Whole Index size | 81.24 MB| 81.79 MB | +0.6%

Out of 292252 symbols collected from LLVM codebase 136926 appear to be
restricted for code completion.

Reviewers: ioeric

Differential Revision: https://reviews.llvm.org/D52357

Modified:
clang-tools-extra/trunk/clangd/index/dex/Dex.cpp
clang-tools-extra/trunk/unittests/clangd/DexTests.cpp

Modified: clang-tools-extra/trunk/clangd/index/dex/Dex.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/Dex.cpp?rev=342866&r1=342865&r2=342866&view=diff
==
--- clang-tools-extra/trunk/clangd/index/dex/Dex.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/dex/Dex.cpp Mon Sep 24 01:45:18 2018
@@ -22,6 +22,10 @@ namespace dex {
 
 namespace {
 
+// Mark symbols which are can be used for code completion.
+static const Token RestrictedForCodeCompletion =
+Token(Token::Kind::Sentinel, "Restricted For Code Completion");
+
 // Returns the tokens which are given symbol's characteristics. Currently, the
 // generated tokens only contain fuzzy matching trigrams and symbol's scope,
 // but in the future this will also return path proximity tokens and other
@@ -39,6 +43,8 @@ std::vector generateSearchTokens(
 for (const auto &ProximityURI :
  generateProximityURIs(Sym.CanonicalDeclaration.FileURI))
   Result.emplace_back(Token::Kind::ProximityURI, ProximityURI);
+  if (Sym.Flags & Symbol::IndexedForCodeCompletion)
+Result.emplace_back(RestrictedForCodeCompletion);
   return Result;
 }
 
@@ -175,6 +181,10 @@ bool Dex::fuzzyFind(const FuzzyFindReque
 TopLevelChildren.push_back(createOr(move(BoostingIterators)));
   }
 
+  if (Req.RestrictForCodeCompletion)
+TopLevelChildren.push_back(
+InvertedIndex.find(RestrictedForCodeCompletion)->second.iterator());
+
   // Use TRUE iterator if both trigrams and scopes from the query are not
   // present in the symbol index.
   auto QueryIterator = TopLevelChildren.empty()

Modified: clang-tools-extra/trunk/unittests/clangd/DexTests.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/unittests/clangd/DexTests.cpp?rev=342866&r1=342865&r2=342866&view=diff
==
--- clang-tools-extra/trunk/unittests/clangd/DexTests.cpp (original)
+++ clang-tools-extra/trunk/unittests/clangd/DexTests.cpp Mon Sep 24 01:45:18 
2018
@@ -583,6 +583,20 @@ TEST(DexTest, Lookup) {
   EXPECT_THAT(lookup(*I, SymbolID("ns::nonono")), UnorderedElementsAre());
 }
 
+TEST(DexTest, SymbolIndexOptionsFilter) {
+  auto CodeCompletionSymbol = symbol("Completion");
+  auto NonCodeCompletionSymbol = symbol("NoCompletion");
+  CodeCompletionSymbol.Flags = Symbol::SymbolFlag::IndexedForCodeCompletion;
+  NonCodeCompletionSymbol.Flags = Symbol::SymbolFlag::None;
+  std::vector Symbols{CodeCompletionSymbol, NonCodeCompletionSymbol};
+  Dex I(Symbols, URISchemes);
+  FuzzyFindRequest Req;
+  Req.RestrictForCodeCompletion = false;
+  EXPECT_THAT(match(I, Req), ElementsAre("Completion", "NoCompletion"));
+  Req.RestrictForCodeCompletion = true;
+  EXPECT_THAT(match(I, Req), ElementsAre("Completion"));
+}
+
 TEST(DexTest, ProximityPathsBoosting) {
   auto RootSymbol = symbol("root::abc");
   RootSymbol.CanonicalDeclaration.FileURI = "unittest:///file.h";


___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] r342957 - [clangd] NFC: Remove test duplicate

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Tue Sep 25 02:47:01 2018
New Revision: 342957

URL: http://llvm.org/viewvc/llvm-project?rev=342957&view=rev
Log:
[clangd] NFC: Remove test duplicate

`FuzzyMatchQ` test was a duplicate of `FuzzyMatch` pulled from MemIndex
tests.

Modified:
clang-tools-extra/trunk/unittests/clangd/DexTests.cpp

Modified: clang-tools-extra/trunk/unittests/clangd/DexTests.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/unittests/clangd/DexTests.cpp?rev=342957&r1=342956&r2=342957&view=diff
==
--- clang-tools-extra/trunk/unittests/clangd/DexTests.cpp (original)
+++ clang-tools-extra/trunk/unittests/clangd/DexTests.cpp Tue Sep 25 02:47:01 
2018
@@ -475,17 +475,6 @@ TEST(Dex, FuzzyFind) {
"other::A"));
 }
 
-TEST(DexTest, FuzzyMatchQ) {
-  auto I = Dex::build(
-  generateSymbols({"LaughingOutLoud", "LionPopulation", "LittleOldLady"}),
-  URISchemes);
-  FuzzyFindRequest Req;
-  Req.Query = "lol";
-  Req.Limit = 2;
-  EXPECT_THAT(match(*I, Req),
-  UnorderedElementsAre("LaughingOutLoud", "LittleOldLady"));
-}
-
 // FIXME(kbobyrev): This test is different for Dex and MemIndex: while
 // MemIndex manages response deduplication, Dex simply returns all matched
 // symbols which means there might be equivalent symbols in the response.


___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] r342965 - [clangd] Implement VByte PostingList compression

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Tue Sep 25 04:54:51 2018
New Revision: 342965

URL: http://llvm.org/viewvc/llvm-project?rev=342965&view=rev
Log:
[clangd] Implement VByte PostingList compression

This patch implements Variable-length Byte compression of `PostingList`s
to sacrifice some performance for lower memory consumption.

`PostingList` compression and decompression was extensively tested using
fuzzer for multiple hours and runnning significant number of realistic
`FuzzyFindRequests`. AddressSanitizer and UndefinedBehaviorSanitizer
were used to ensure the correct behaviour.

Performance evaluation was conducted with recent LLVM symbol index (292k
symbols) and the collection of user-recorded queries (7751
`FuzzyFindRequest` JSON dumps):

| Metrics | Before| After | Change (%)
| -  | -  | -   | -
| Memory consumption (posting lists only), MB  |  54.4 | 23.5 | -60%
| Time to process queries, sec | 7.70 | 9.4 | +25%

Reviewers: sammccall, ioeric

Reviewed By: sammccall

Differential Revision: https://reviews.llvm.org/D52300

Modified:
clang-tools-extra/trunk/clangd/index/dex/Dex.cpp
clang-tools-extra/trunk/clangd/index/dex/PostingList.cpp
clang-tools-extra/trunk/clangd/index/dex/PostingList.h
clang-tools-extra/trunk/unittests/clangd/DexTests.cpp

Modified: clang-tools-extra/trunk/clangd/index/dex/Dex.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/Dex.cpp?rev=342965&r1=342964&r2=342965&view=diff
==
--- clang-tools-extra/trunk/clangd/index/dex/Dex.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/dex/Dex.cpp Tue Sep 25 04:54:51 2018
@@ -128,8 +128,8 @@ void Dex::buildIndex() {
 
   // Convert lists of items to posting lists.
   for (const auto &TokenToPostingList : TempInvertedIndex)
-InvertedIndex.insert({TokenToPostingList.first,
-  PostingList(move(TokenToPostingList.second))});
+InvertedIndex.insert(
+{TokenToPostingList.first, PostingList(TokenToPostingList.second)});
 
   vlog("Built Dex with estimated memory usage {0} bytes.",
estimateMemoryUsage());

Modified: clang-tools-extra/trunk/clangd/index/dex/PostingList.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/PostingList.cpp?rev=342965&r1=342964&r2=342965&view=diff
==
--- clang-tools-extra/trunk/clangd/index/dex/PostingList.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/dex/PostingList.cpp Tue Sep 25 
04:54:51 2018
@@ -9,6 +9,8 @@
 
 #include "PostingList.h"
 #include "Iterator.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/MathExtras.h"
 
 namespace clang {
 namespace clangd {
@@ -16,21 +18,27 @@ namespace dex {
 
 namespace {
 
-/// Implements Iterator over std::vector. This is the most basic
-/// iterator and is simply a wrapper around
-/// std::vector::const_iterator.
-class PlainIterator : public Iterator {
+/// Implements iterator of PostingList chunks. This requires iterating over two
+/// levels: the first level iterator iterates over the chunks and decompresses
+/// them on-the-fly when the contents of chunk are to be seen.
+class ChunkIterator : public Iterator {
 public:
-  explicit PlainIterator(llvm::ArrayRef Documents)
-  : Documents(Documents), Index(std::begin(Documents)) {}
+  explicit ChunkIterator(llvm::ArrayRef Chunks)
+  : Chunks(Chunks), CurrentChunk(Chunks.begin()) {
+if (!Chunks.empty()) {
+  DecompressedChunk = CurrentChunk->decompress();
+  CurrentID = DecompressedChunk.begin();
+}
+  }
 
-  bool reachedEnd() const override { return Index == std::end(Documents); }
+  bool reachedEnd() const override { return CurrentChunk == Chunks.end(); }
 
   /// Advances cursor to the next item.
   void advance() override {
 assert(!reachedEnd() &&
"Posting List iterator can't advance() at the end.");
-++Index;
+++CurrentID;
+normalizeCursor();
   }
 
   /// Applies binary search to advance cursor to the next item with DocID
@@ -38,16 +46,17 @@ public:
   void advanceTo(DocID ID) override {
 assert(!reachedEnd() &&
"Posting List iterator can't advance() at the end.");
-// If current ID is beyond requested one, iterator is already in the right
-// state.
-if (peek() < ID)
-  Index = std::lower_bound(Index, std::end(Documents), ID);
+if (ID <= peek())
+  return;
+advanceToChunk(ID);
+// Try to find ID within current chunk.
+CurrentID = std::lower_bound(CurrentID, std::end(DecompressedChunk), ID);
+normalizeCursor();
   }
 
   DocID peek() const override {
-assert(!reachedEnd() &&
-   "Posting List iterator can't peek() at the end.");
-return *Index;
+assert(!reachedEnd() && "Posting List iterator can't peek() at the end.");
+return *CurrentID;
   }
 
   float consume() override {
@@ -56,2

[clang-tools-extra] r342970 - [clangd] Fix some buildbots after r342965

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Tue Sep 25 06:14:11 2018
New Revision: 342970

URL: http://llvm.org/viewvc/llvm-project?rev=342970&view=rev
Log:
[clangd] Fix some buildbots after r342965

Some compilers fail to parse struct default member initializer.

Modified:
clang-tools-extra/trunk/clangd/index/dex/PostingList.h

Modified: clang-tools-extra/trunk/clangd/index/dex/PostingList.h
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/PostingList.h?rev=342970&r1=342969&r2=342970&view=diff
==
--- clang-tools-extra/trunk/clangd/index/dex/PostingList.h (original)
+++ clang-tools-extra/trunk/clangd/index/dex/PostingList.h Tue Sep 25 06:14:11 
2018
@@ -50,7 +50,7 @@ struct Chunk {
   /// The first element of decompressed Chunk.
   DocID Head;
   /// VByte-encoded deltas.
-  std::array Payload = std::array();
+  std::array Payload;
 };
 static_assert(sizeof(Chunk) == 32, "Chunk should take 32 bytes of memory.");
 


___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] r342974 - [clangd] NFC: Simplify code, enforce LLVM Coding Standards

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Tue Sep 25 06:58:48 2018
New Revision: 342974

URL: http://llvm.org/viewvc/llvm-project?rev=342974&view=rev
Log:
[clangd] NFC: Simplify code, enforce LLVM Coding Standards

For consistency, functional-style code pieces are replaced with their
simple counterparts to improve readability.

Also, file headers are fixed to comply with LLVM Coding Standards.

`static` member of anonymous namespace is not marked `static` anymore,
because it is redundant.

Reviewed By: sammccall

Differential Revision: https://reviews.llvm.org/D52466

Modified:
clang-tools-extra/trunk/clangd/index/dex/Dex.cpp
clang-tools-extra/trunk/clangd/index/dex/Dex.h
clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp
clang-tools-extra/trunk/clangd/index/dex/Iterator.h
clang-tools-extra/trunk/clangd/index/dex/Token.h
clang-tools-extra/trunk/clangd/index/dex/Trigram.h

Modified: clang-tools-extra/trunk/clangd/index/dex/Dex.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/Dex.cpp?rev=342974&r1=342973&r2=342974&view=diff
==
--- clang-tools-extra/trunk/clangd/index/dex/Dex.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/dex/Dex.cpp Tue Sep 25 06:58:48 2018
@@ -23,7 +23,7 @@ namespace dex {
 namespace {
 
 // Mark symbols which are can be used for code completion.
-static const Token RestrictedForCodeCompletion =
+const Token RestrictedForCodeCompletion =
 Token(Token::Kind::Sentinel, "Restricted For Code Completion");
 
 // Returns the tokens which are given symbol's characteristics. Currently, the

Modified: clang-tools-extra/trunk/clangd/index/dex/Dex.h
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/Dex.h?rev=342974&r1=342973&r2=342974&view=diff
==
--- clang-tools-extra/trunk/clangd/index/dex/Dex.h (original)
+++ clang-tools-extra/trunk/clangd/index/dex/Dex.h Tue Sep 25 06:58:48 2018
@@ -6,15 +6,16 @@
 // License. See LICENSE.TXT for details.
 //
 
//===--===//
-//
-// This defines Dex - a symbol index implementation based on query iterators
-// over symbol tokens, such as fuzzy matching trigrams, scopes, types, etc.
-// While consuming more memory and having longer build stage due to
-// preprocessing, Dex will have substantially lower latency. It will also allow
-// efficient symbol searching which is crucial for operations like code
-// completion, and can be very important for a number of different code
-// transformations which will be eventually supported by Clangd.
-//
+///
+/// \file
+/// This defines Dex - a symbol index implementation based on query iterators
+/// over symbol tokens, such as fuzzy matching trigrams, scopes, types, etc.
+/// While consuming more memory and having longer build stage due to
+/// preprocessing, Dex will have substantially lower latency. It will also 
allow
+/// efficient symbol searching which is crucial for operations like code
+/// completion, and can be very important for a number of different code
+/// transformations which will be eventually supported by Clangd.
+///
 
//===--===//
 
 #ifndef LLVM_CLANG_TOOLS_EXTRA_CLANGD_INDEX_DEX_DEX_H

Modified: clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp?rev=342974&r1=342973&r2=342974&view=diff
==
--- clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp Tue Sep 25 06:58:48 
2018
@@ -64,11 +64,10 @@ public:
 
   float consume() override {
 assert(!reachedEnd() && "AND iterator can't consume() at the end.");
-return std::accumulate(
-begin(Children), end(Children), DEFAULT_BOOST_SCORE,
-[&](float Current, const std::unique_ptr &Child) {
-  return Current * Child->consume();
-});
+float Boost = DEFAULT_BOOST_SCORE;
+for (const auto &Child : Children)
+  Boost *= Child->consume();
+return Boost;
   }
 
   size_t estimateSize() const override {
@@ -140,10 +139,10 @@ public:
 
   /// Returns true if all children are exhausted.
   bool reachedEnd() const override {
-return std::all_of(begin(Children), end(Children),
-   [](const std::unique_ptr &Child) {
- return Child->reachedEnd();
-   });
+for (const auto &Child : Children)
+  if (!Child->reachedEnd())
+return false;
+return true;
   }
 
   /// Moves each child pointing to the smallest DocID to the next item.
@@ -181,21 +180,18 @@ public:
   float consume() override {
 assert(!reachedEnd() && "OR iterator can't

[clang-tools-extra] r343116 - [docs] Update PostingList string representation format

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Wed Sep 26 07:59:49 2018
New Revision: 343116

URL: http://llvm.org/viewvc/llvm-project?rev=343116&view=rev
Log:
[docs] Update PostingList string representation format

Because `PostingList` objects are compressed, it is now impossible to
see elements other than the current one and the documentation doesn't
match implementation anymore.

Reviewed By: ioeric

Differential Revision: https://reviews.llvm.org/D52545

Modified:
clang-tools-extra/trunk/clangd/index/dex/Iterator.h

Modified: clang-tools-extra/trunk/clangd/index/dex/Iterator.h
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/Iterator.h?rev=343116&r1=343115&r2=343116&view=diff
==
--- clang-tools-extra/trunk/clangd/index/dex/Iterator.h (original)
+++ clang-tools-extra/trunk/clangd/index/dex/Iterator.h Wed Sep 26 07:59:49 2018
@@ -94,9 +94,8 @@ public:
   ///
   /// Where Type is the iterator type representation: "&" for And, "|" for Or,
   /// ChildN is N-th iterator child. Raw iterators over PostingList are
-  /// represented as "[ID1, ID2, ..., {IDN}, ... END]" where IDN is N-th
-  /// PostingList entry and the element which is pointed to by the PostingList
-  /// iterator is enclosed in {} braces.
+  /// represented as "[... CurID ...]" where CurID is the current PostingList
+  /// entry being inspected.
   friend llvm::raw_ostream &operator<<(llvm::raw_ostream &OS,
const Iterator &Iterator) {
 return Iterator.dump(OS);


___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] r343117 - [clangd] Fix bugs with incorrect memory estimate report

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Wed Sep 26 08:06:23 2018
New Revision: 343117

URL: http://llvm.org/viewvc/llvm-project?rev=343117&view=rev
Log:
[clangd] Fix bugs with incorrect memory estimate report

* With the current implementation, `sizeof(std::vector)` is added
twice to the `Dex` memory estimate which is incorrect
* `Dex` logs memory usage estimation before `BackingDataSize` is set and
hence the log report excludes size of the external `SymbolSlab` which is
coupled with `Dex` instance

Reviewed By: ioeric

Differential Revision: https://reviews.llvm.org/D52503

Modified:
clang-tools-extra/trunk/clangd/index/Serialization.cpp
clang-tools-extra/trunk/clangd/index/dex/Dex.cpp
clang-tools-extra/trunk/clangd/index/dex/PostingList.h

Modified: clang-tools-extra/trunk/clangd/index/Serialization.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Serialization.cpp?rev=343117&r1=343116&r2=343117&view=diff
==
--- clang-tools-extra/trunk/clangd/index/Serialization.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/Serialization.cpp Wed Sep 26 08:06:23 
2018
@@ -6,8 +6,10 @@
 // License. See LICENSE.TXT for details.
 //
 
//===--===//
+
 #include "Serialization.h"
 #include "Index.h"
+#include "Logger.h"
 #include "RIFF.h"
 #include "Trace.h"
 #include "dex/Dex.h"
@@ -433,8 +435,12 @@ std::unique_ptr loadIndex(l
   }
 
   trace::Span Tracer("BuildIndex");
-  return UseDex ? dex::Dex::build(std::move(Symbols), URISchemes)
-: MemIndex::build(std::move(Symbols), std::move(Refs));
+  auto Index = UseDex ? dex::Dex::build(std::move(Symbols), URISchemes)
+  : MemIndex::build(std::move(Symbols), std::move(Refs));
+  vlog("Loaded {0} from {1} with estimated memory usage {2}",
+   UseDex ? "Dex" : "MemIndex", SymbolFilename,
+   Index->estimateMemoryUsage());
+  return Index;
 }
 
 } // namespace clangd

Modified: clang-tools-extra/trunk/clangd/index/dex/Dex.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/Dex.cpp?rev=343117&r1=343116&r2=343117&view=diff
==
--- clang-tools-extra/trunk/clangd/index/dex/Dex.cpp (original)
+++ clang-tools-extra/trunk/clangd/index/dex/Dex.cpp Wed Sep 26 08:06:23 2018
@@ -130,9 +130,6 @@ void Dex::buildIndex() {
   for (const auto &TokenToPostingList : TempInvertedIndex)
 InvertedIndex.insert(
 {TokenToPostingList.first, PostingList(TokenToPostingList.second)});
-
-  vlog("Built Dex with estimated memory usage {0} bytes.",
-   estimateMemoryUsage());
 }
 
 /// Constructs iterators over tokens extracted from the query and exhausts it
@@ -248,8 +245,8 @@ size_t Dex::estimateMemoryUsage() const
   Bytes += SymbolQuality.size() * sizeof(float);
   Bytes += LookupTable.getMemorySize();
   Bytes += InvertedIndex.getMemorySize();
-  for (const auto &P : InvertedIndex)
-Bytes += P.second.bytes();
+  for (const auto &TokenToPostingList : InvertedIndex)
+Bytes += TokenToPostingList.second.bytes();
   return Bytes + BackingDataSize;
 }
 

Modified: clang-tools-extra/trunk/clangd/index/dex/PostingList.h
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/PostingList.h?rev=343117&r1=343116&r2=343117&view=diff
==
--- clang-tools-extra/trunk/clangd/index/dex/PostingList.h (original)
+++ clang-tools-extra/trunk/clangd/index/dex/PostingList.h Wed Sep 26 08:06:23 
2018
@@ -66,10 +66,8 @@ public:
   /// go through the chunks and decompress them on-the-fly when necessary.
   std::unique_ptr iterator() const;
 
-  /// Returns in-memory size.
-  size_t bytes() const {
-return sizeof(Chunk) + Chunks.capacity() * sizeof(Chunk);
-  }
+  /// Returns in-memory size of external storage.
+  size_t bytes() const { return Chunks.capacity() * sizeof(Chunk); }
 
 private:
   const std::vector Chunks;


___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] r343306 - [docs] Fix links in Clangd documentation

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Fri Sep 28 02:32:47 2018
New Revision: 343306

URL: http://llvm.org/viewvc/llvm-project?rev=343306&view=rev
Log:
[docs] Fix links in Clangd documentation

Add missing `_` after each `external link `_, as
required by the reStructuredText specification.

Modified:
clang-tools-extra/trunk/docs/clangd.rst

Modified: clang-tools-extra/trunk/docs/clangd.rst
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/docs/clangd.rst?rev=343306&r1=343305&r2=343306&view=diff
==
--- clang-tools-extra/trunk/docs/clangd.rst (original)
+++ clang-tools-extra/trunk/docs/clangd.rst Fri Sep 28 02:32:47 2018
@@ -133,15 +133,15 @@ VSCode Integration
 --
 
 :program:`VSCode` provides `vscode-clangd
-`
+`_
 which is published in Visual Studio Marketplace and can be installed direcetly
 from :program:`VSCode`.
 
 Emacs Integration
 -
 
-:program:`Emacs` provides `lsp-mode ` and
-`Eglot ` plugins for LSP integration.
+:program:`Emacs` provides `lsp-mode `_ and
+`Eglot `_ plugins for LSP integration.
 
 Getting Involved
 ==


___
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] r343937 - [clangd] NFC: Migrate to LLVM STLExtras API where possible

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Sun Oct  7 07:49:41 2018
New Revision: 343937

URL: http://llvm.org/viewvc/llvm-project?rev=343937&view=rev
Log:
[clangd] NFC: Migrate to LLVM STLExtras API where possible

This patch improves readability by migrating `std::function(ForwardIt
start, ForwardIt end, ...)` to LLVM's STLExtras range-based equivalent
`llvm::function(RangeT &&Range, ...)`.

Similar change in Clang: D52576.

Reviewed By: sammccall

Differential Revision: https://reviews.llvm.org/D52650

Modified:
clang-tools-extra/trunk/clangd/ClangdLSPServer.cpp
clang-tools-extra/trunk/clangd/ClangdServer.cpp
clang-tools-extra/trunk/clangd/CodeComplete.cpp
clang-tools-extra/trunk/clangd/TUScheduler.cpp
clang-tools-extra/trunk/clangd/XRefs.cpp
clang-tools-extra/trunk/clangd/index/CanonicalIncludes.cpp
clang-tools-extra/trunk/clangd/index/FileIndex.cpp
clang-tools-extra/trunk/clangd/index/Index.cpp
clang-tools-extra/trunk/clangd/index/Serialization.cpp
clang-tools-extra/trunk/clangd/index/SymbolCollector.cpp
clang-tools-extra/trunk/clangd/index/dex/Dex.cpp
clang-tools-extra/trunk/clangd/index/dex/Iterator.cpp

Modified: clang-tools-extra/trunk/clangd/ClangdLSPServer.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/ClangdLSPServer.cpp?rev=343937&r1=343936&r2=343937&view=diff
==
--- clang-tools-extra/trunk/clangd/ClangdLSPServer.cpp (original)
+++ clang-tools-extra/trunk/clangd/ClangdLSPServer.cpp Sun Oct  7 07:49:41 2018
@@ -549,8 +549,7 @@ void ClangdLSPServer::onDiagnosticsReady
   DiagnosticsJSON.push_back(std::move(LSPDiag));
 
   auto &FixItsForDiagnostic = LocalFixIts[Diag];
-  std::copy(Fixes.begin(), Fixes.end(),
-std::back_inserter(FixItsForDiagnostic));
+  llvm::copy(Fixes, std::back_inserter(FixItsForDiagnostic));
 });
   }
 

Modified: clang-tools-extra/trunk/clangd/ClangdServer.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/ClangdServer.cpp?rev=343937&r1=343936&r2=343937&view=diff
==
--- clang-tools-extra/trunk/clangd/ClangdServer.cpp (original)
+++ clang-tools-extra/trunk/clangd/ClangdServer.cpp Sun Oct  7 07:49:41 2018
@@ -361,17 +361,15 @@ llvm::Optional ClangdServer::switc
 
   // Lookup in a list of known extensions.
   auto SourceIter =
-  std::find_if(std::begin(SourceExtensions), std::end(SourceExtensions),
-   [&PathExt](PathRef SourceExt) {
- return SourceExt.equals_lower(PathExt);
-   });
+  llvm::find_if(SourceExtensions, [&PathExt](PathRef SourceExt) {
+return SourceExt.equals_lower(PathExt);
+  });
   bool IsSource = SourceIter != std::end(SourceExtensions);
 
   auto HeaderIter =
-  std::find_if(std::begin(HeaderExtensions), std::end(HeaderExtensions),
-   [&PathExt](PathRef HeaderExt) {
- return HeaderExt.equals_lower(PathExt);
-   });
+  llvm::find_if(HeaderExtensions, [&PathExt](PathRef HeaderExt) {
+return HeaderExt.equals_lower(PathExt);
+  });
 
   bool IsHeader = HeaderIter != std::end(HeaderExtensions);
 

Modified: clang-tools-extra/trunk/clangd/CodeComplete.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/CodeComplete.cpp?rev=343937&r1=343936&r2=343937&view=diff
==
--- clang-tools-extra/trunk/clangd/CodeComplete.cpp (original)
+++ clang-tools-extra/trunk/clangd/CodeComplete.cpp Sun Oct  7 07:49:41 2018
@@ -306,11 +306,10 @@ struct CodeCompletionBuilder {
 Completion.FixIts.push_back(
 toTextEdit(FixIt, ASTCtx.getSourceManager(), 
ASTCtx.getLangOpts()));
   }
-  std::sort(Completion.FixIts.begin(), Completion.FixIts.end(),
-[](const TextEdit &X, const TextEdit &Y) {
-  return std::tie(X.range.start.line, X.range.start.character) 
<
- std::tie(Y.range.start.line, Y.range.start.character);
-});
+  llvm::sort(Completion.FixIts, [](const TextEdit &X, const TextEdit &Y) {
+return std::tie(X.range.start.line, X.range.start.character) <
+   std::tie(Y.range.start.line, Y.range.start.character);
+  });
   Completion.Deprecated |=
   (C.SemaResult->Availability == CXAvailability_Deprecated);
 }
@@ -861,8 +860,8 @@ public:
   IndexRequest.IDs.size(), FetchedDocs.size());
 }
 
-std::sort(
-ScoredSignatures.begin(), ScoredSignatures.end(),
+llvm::sort(
+ScoredSignatures,
 [](const ScoredSignature &L, const ScoredSignature &R) {
   // Ordering follows:
   // - Less number of parameters is better.
@@ -1164,13 +1163,12 @@ llvm::SmallVector
 getRankedIncludes(const Symbol &S

[clang-tools-extra] r336257 - [clang-tools-extra] Cleanup documentation routine

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Wed Jul  4 03:18:03 2018
New Revision: 336257

URL: http://llvm.org/viewvc/llvm-project?rev=336257&view=rev
Log:
[clang-tools-extra] Cleanup documentation routine

The following issues are resolved:

* Doxygen didn't generate documentation for a bunch of existing tools
due to the absence of their directories in the doxygen configuration
file. This patch adds all relevant directories to the appropriate list.

* clang-tools-extra/docs/Doxyfile seems to be unused and irrelevant,
doxygen.cfg.in is passed to the CMake's Doxygen invocation, hence
Doxyfile is removed.

The validity of proposed changes was manually checked by building
doxygen-clang-tools and making sure that clangd and other tools are
present in Doxygen-generated docs of clang-tools-extra.

Reviewers: ioeric

Subscribers: cfe-commits

Differential Revision: https://reviews.llvm.org/D47537

Removed:
clang-tools-extra/trunk/docs/Doxyfile
Modified:
clang-tools-extra/trunk/docs/doxygen.cfg.in

Removed: clang-tools-extra/trunk/docs/Doxyfile
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/docs/Doxyfile?rev=336256&view=auto
==
--- clang-tools-extra/trunk/docs/Doxyfile (original)
+++ clang-tools-extra/trunk/docs/Doxyfile (removed)
@@ -1,1808 +0,0 @@
-# Doxyfile 1.7.6.1
-
-# This file describes the settings to be used by the documentation system
-# doxygen (www.doxygen.org) for a project
-#
-# All text after a hash (#) is considered a comment and will be ignored
-# The format is:
-#   TAG = value [value, ...]
-# For lists items can also be appended using:
-#   TAG += value [value, ...]
-# Values that contain spaces should be placed between quotes (" ")
-
-#---
-# Project related configuration options
-#---
-
-# This tag specifies the encoding used for all characters in the config file
-# that follow. The default is UTF-8 which is also the encoding used for all
-# text before the first occurrence of this tag. Doxygen uses libiconv (or the
-# iconv built into libc) for the transcoding. See
-# http://www.gnu.org/software/libiconv for the list of possible encodings.
-
-DOXYFILE_ENCODING  = UTF-8
-
-# The PROJECT_NAME tag is a single word (or sequence of words) that should
-# identify the project. Note that if you do not use Doxywizard you need
-# to put quotes around the project name if it contains spaces.
-
-PROJECT_NAME   = clang-tools-extra
-
-# The PROJECT_NUMBER tag can be used to enter a project or revision number.
-# This could be handy for archiving the generated documentation or
-# if some version control system is used.
-
-PROJECT_NUMBER =
-
-# Using the PROJECT_BRIEF tag one can provide an optional one line description
-# for a project that appears at the top of each page and should give viewer
-# a quick idea about the purpose of the project. Keep the description short.
-
-PROJECT_BRIEF  =
-
-# With the PROJECT_LOGO tag one can specify an logo or icon that is
-# included in the documentation. The maximum height of the logo should not
-# exceed 55 pixels and the maximum width should not exceed 200 pixels.
-# Doxygen will copy the logo to the output directory.
-
-PROJECT_LOGO   =
-
-# The OUTPUT_DIRECTORY tag is used to specify the (relative or absolute)
-# base path where the generated documentation will be put.
-# If a relative path is entered, it will be relative to the location
-# where doxygen was started. If left blank the current directory will be used.
-
-# Same directory that Sphinx uses.
-OUTPUT_DIRECTORY   = ./_build/
-
-# If the CREATE_SUBDIRS tag is set to YES, then doxygen will create
-# 4096 sub-directories (in 2 levels) under the output directory of each output
-# format and will distribute the generated files over these directories.
-# Enabling this option can be useful when feeding doxygen a huge amount of
-# source files, where putting all generated files in the same directory would
-# otherwise cause performance problems for the file system.
-
-CREATE_SUBDIRS = NO
-
-# The OUTPUT_LANGUAGE tag is used to specify the language in which all
-# documentation generated by doxygen is written. Doxygen will use this
-# information to generate all constant output in the proper language.
-# The default language is English, other supported languages are:
-# Afrikaans, Arabic, Brazilian, Catalan, Chinese, Chinese-Traditional,
-# Croatian, Czech, Danish, Dutch, Esperanto, Farsi, Finnish, French, German,
-# Greek, Hungarian, Italian, Japanese, Japanese-en (Japanese with English
-# messages), Korean, Korean-en, Lithuanian, Norwegian, Macedonian, Persian,
-# Polish, Portuguese, Romanian, Russian, Serbian, Serbian-Cyrillic, Slovak,
-# Slovene, Spanish, Swedish, Ukrainian, and Vietnamese.
-
-OUTPUT_LANGUAGE= English
-
-# If

[clang-tools-extra] r336330 - [NFS] Wipe trailing whitespaces

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Thu Jul  5 02:37:26 2018
New Revision: 336330

URL: http://llvm.org/viewvc/llvm-project?rev=336330&view=rev
Log:
[NFS] Wipe trailing whitespaces

This patch is a preparation for another one containing meaningful
changes. This patch simply removes trailing whitespaces in few files
affected by the upcoming patch and reformats

Modified:
clang-tools-extra/trunk/clangd/Quality.cpp

Modified: clang-tools-extra/trunk/clangd/Quality.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/Quality.cpp?rev=336330&r1=336329&r2=336330&view=diff
==
--- clang-tools-extra/trunk/clangd/Quality.cpp (original)
+++ clang-tools-extra/trunk/clangd/Quality.cpp Thu Jul  5 02:37:26 2018
@@ -64,7 +64,8 @@ static SymbolQualitySignals::SymbolCateg
   return Switch().Visit(&ND);
 }
 
-static SymbolQualitySignals::SymbolCategory categorize(const 
CodeCompletionResult &R) {
+static SymbolQualitySignals::SymbolCategory
+categorize(const CodeCompletionResult &R) {
   if (R.Declaration)
 return categorize(*R.Declaration);
   if (R.Kind == CodeCompletionResult::RK_Macro)
@@ -72,57 +73,57 @@ static SymbolQualitySignals::SymbolCateg
   // Everything else is a keyword or a pattern. Patterns are mostly keywords
   // too, except a few which we recognize by cursor kind.
   switch (R.CursorKind) {
-case CXCursor_CXXMethod:
-  return SymbolQualitySignals::Function;
-case CXCursor_ModuleImportDecl:
-  return SymbolQualitySignals::Namespace;
-case CXCursor_MacroDefinition:
-  return SymbolQualitySignals::Macro;
-case CXCursor_TypeRef:
-  return SymbolQualitySignals::Type;
-case CXCursor_MemberRef:
-  return SymbolQualitySignals::Variable;
-default:
-  return SymbolQualitySignals::Keyword;
+  case CXCursor_CXXMethod:
+return SymbolQualitySignals::Function;
+  case CXCursor_ModuleImportDecl:
+return SymbolQualitySignals::Namespace;
+  case CXCursor_MacroDefinition:
+return SymbolQualitySignals::Macro;
+  case CXCursor_TypeRef:
+return SymbolQualitySignals::Type;
+  case CXCursor_MemberRef:
+return SymbolQualitySignals::Variable;
+  default:
+return SymbolQualitySignals::Keyword;
   }
 }
 
 static SymbolQualitySignals::SymbolCategory
 categorize(const index::SymbolInfo &D) {
   switch (D.Kind) {
-case index::SymbolKind::Namespace:
-case index::SymbolKind::NamespaceAlias:
-  return SymbolQualitySignals::Namespace;
-case index::SymbolKind::Macro:
-  return SymbolQualitySignals::Macro;
-case index::SymbolKind::Enum:
-case index::SymbolKind::Struct:
-case index::SymbolKind::Class:
-case index::SymbolKind::Protocol:
-case index::SymbolKind::Extension:
-case index::SymbolKind::Union:
-case index::SymbolKind::TypeAlias:
-  return SymbolQualitySignals::Type;
-case index::SymbolKind::Function:
-case index::SymbolKind::ClassMethod:
-case index::SymbolKind::InstanceMethod:
-case index::SymbolKind::StaticMethod:
-case index::SymbolKind::InstanceProperty:
-case index::SymbolKind::ClassProperty:
-case index::SymbolKind::StaticProperty:
-case index::SymbolKind::Constructor:
-case index::SymbolKind::Destructor:
-case index::SymbolKind::ConversionFunction:
-  return SymbolQualitySignals::Function;
-case index::SymbolKind::Variable:
-case index::SymbolKind::Field:
-case index::SymbolKind::EnumConstant:
-case index::SymbolKind::Parameter:
-  return SymbolQualitySignals::Variable;
-case index::SymbolKind::Using:
-case index::SymbolKind::Module:
-case index::SymbolKind::Unknown:
-  return SymbolQualitySignals::Unknown;
+  case index::SymbolKind::Namespace:
+  case index::SymbolKind::NamespaceAlias:
+return SymbolQualitySignals::Namespace;
+  case index::SymbolKind::Macro:
+return SymbolQualitySignals::Macro;
+  case index::SymbolKind::Enum:
+  case index::SymbolKind::Struct:
+  case index::SymbolKind::Class:
+  case index::SymbolKind::Protocol:
+  case index::SymbolKind::Extension:
+  case index::SymbolKind::Union:
+  case index::SymbolKind::TypeAlias:
+return SymbolQualitySignals::Type;
+  case index::SymbolKind::Function:
+  case index::SymbolKind::ClassMethod:
+  case index::SymbolKind::InstanceMethod:
+  case index::SymbolKind::StaticMethod:
+  case index::SymbolKind::InstanceProperty:
+  case index::SymbolKind::ClassProperty:
+  case index::SymbolKind::StaticProperty:
+  case index::SymbolKind::Constructor:
+  case index::SymbolKind::Destructor:
+  case index::SymbolKind::ConversionFunction:
+return SymbolQualitySignals::Function;
+  case index::SymbolKind::Variable:
+  case index::SymbolKind::Field:
+  case index::SymbolKind::EnumConstant:
+  case index::SymbolKind::Parameter:
+return SymbolQualitySignals::Variable;
+  case index::SymbolKind::Using:
+  case index::SymbolKind::Module:
+  case index::SymbolKind::Unknown:
+ 

r336810 - [clangd] Uprank delcarations when "using q::name" is present in the main file

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Wed Jul 11 07:49:49 2018
New Revision: 336810

URL: http://llvm.org/viewvc/llvm-project?rev=336810&view=rev
Log:
[clangd] Uprank delcarations when "using q::name" is present in the main file

Having `using qualified::name;` for some symbol is an important signal
for clangd code completion as the user is more likely to use such
symbol.  This patch helps to uprank the relevant symbols by saving
UsingShadowDecl in the new field of CodeCompletionResult and checking
whether the corresponding UsingShadowDecl is located in the main file
later in ClangD code completion routine. While the relative importance
of such signal is a subject to change in the future, this patch simply
bumps DeclProximity score to the value of 1.0 which should be enough for
now.

The patch was tested using

`$ ninja check-clang check-clang-tools`

No unexpected failures were noticed after running the relevant testsets.

Reviewers: sammccall, ioeric

Subscribers: MaskRay, jkorous, cfe-commits

Differential Revision: https://reviews.llvm.org/D49012

Modified:
cfe/trunk/include/clang/Sema/CodeCompleteConsumer.h
cfe/trunk/lib/Sema/SemaCodeComplete.cpp

Modified: cfe/trunk/include/clang/Sema/CodeCompleteConsumer.h
URL: 
http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Sema/CodeCompleteConsumer.h?rev=336810&r1=336809&r2=336810&view=diff
==
--- cfe/trunk/include/clang/Sema/CodeCompleteConsumer.h (original)
+++ cfe/trunk/include/clang/Sema/CodeCompleteConsumer.h Wed Jul 11 07:49:49 2018
@@ -45,8 +45,9 @@ class LangOptions;
 class NamedDecl;
 class NestedNameSpecifier;
 class Preprocessor;
-class Sema;
 class RawComment;
+class Sema;
+class UsingShadowDecl;
 
 /// Default priority values for code-completion results based
 /// on their kind.
@@ -836,6 +837,12 @@ public:
   /// informative rather than required.
   NestedNameSpecifier *Qualifier = nullptr;
 
+  /// If this Decl was unshadowed by using declaration, this can store a
+  /// pointer to the UsingShadowDecl which was used in the unshadowing process.
+  /// This information can be used to uprank CodeCompletionResults / which have
+  /// corresponding `using decl::qualified::name;` nearby.
+  const UsingShadowDecl *ShadowDecl = nullptr;
+
   /// Build a result that refers to a declaration.
   CodeCompletionResult(const NamedDecl *Declaration, unsigned Priority,
NestedNameSpecifier *Qualifier = nullptr,
@@ -847,7 +854,7 @@ public:
 QualifierIsInformative(QualifierIsInformative),
 StartsNestedNameSpecifier(false), AllParametersAreInformative(false),
 DeclaringEntity(false), Qualifier(Qualifier) {
-//FIXME: Add assert to check FixIts range requirements.
+// FIXME: Add assert to check FixIts range requirements.
 computeCursorKindAndAvailability(Accessible);
   }
 

Modified: cfe/trunk/lib/Sema/SemaCodeComplete.cpp
URL: 
http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Sema/SemaCodeComplete.cpp?rev=336810&r1=336809&r2=336810&view=diff
==
--- cfe/trunk/lib/Sema/SemaCodeComplete.cpp (original)
+++ cfe/trunk/lib/Sema/SemaCodeComplete.cpp Wed Jul 11 07:49:49 2018
@@ -859,12 +859,12 @@ void ResultBuilder::MaybeAddResult(Resul
   }
 
   // Look through using declarations.
-  if (const UsingShadowDecl *Using =
-  dyn_cast(R.Declaration)) {
-MaybeAddResult(Result(Using->getTargetDecl(),
-  getBasePriority(Using->getTargetDecl()),
-  R.Qualifier),
-   CurContext);
+  if (const UsingShadowDecl *Using = dyn_cast(R.Declaration)) 
{
+CodeCompletionResult Result(Using->getTargetDecl(),
+getBasePriority(Using->getTargetDecl()),
+R.Qualifier);
+Result.ShadowDecl = Using;
+MaybeAddResult(Result, CurContext);
 return;
   }
 
@@ -977,10 +977,11 @@ void ResultBuilder::AddResult(Result R,
 
   // Look through using declarations.
   if (const UsingShadowDecl *Using = dyn_cast(R.Declaration)) 
{
-AddResult(Result(Using->getTargetDecl(),
- getBasePriority(Using->getTargetDecl()),
- R.Qualifier),
-  CurContext, Hiding);
+CodeCompletionResult Result(Using->getTargetDecl(),
+getBasePriority(Using->getTargetDecl()),
+R.Qualifier);
+Result.ShadowDecl = Using;
+AddResult(Result, CurContext, Hiding);
 return;
   }
 
@@ -1004,10 +1005,10 @@ void ResultBuilder::AddResult(Result R,
   if (AsNestedNameSpecifier) {
 R.StartsNestedNameSpecifier = true;
 R.Priority = CCP_NestedNameSpecifier;
-  }
-  else if (Filter == &ResultBuilder::IsMember && !R.Qualifier && InBaseClass &&
-   isa(R.Declaration->getDeclContext()
-  ->getRedeclContext

[clang-tools-extra] r336810 - [clangd] Uprank delcarations when "using q::name" is present in the main file

[Kirill Bobyrev via cfe-commits]

Author: omtcyfz
Date: Wed Jul 11 07:49:49 2018
New Revision: 336810

URL: http://llvm.org/viewvc/llvm-project?rev=336810&view=rev
Log:
[clangd] Uprank delcarations when "using q::name" is present in the main file

Having `using qualified::name;` for some symbol is an important signal
for clangd code completion as the user is more likely to use such
symbol.  This patch helps to uprank the relevant symbols by saving
UsingShadowDecl in the new field of CodeCompletionResult and checking
whether the corresponding UsingShadowDecl is located in the main file
later in ClangD code completion routine. While the relative importance
of such signal is a subject to change in the future, this patch simply
bumps DeclProximity score to the value of 1.0 which should be enough for
now.

The patch was tested using

`$ ninja check-clang check-clang-tools`

No unexpected failures were noticed after running the relevant testsets.

Reviewers: sammccall, ioeric

Subscribers: MaskRay, jkorous, cfe-commits

Differential Revision: https://reviews.llvm.org/D49012

Modified:
clang-tools-extra/trunk/clangd/Quality.cpp
clang-tools-extra/trunk/unittests/clangd/QualityTests.cpp

Modified: clang-tools-extra/trunk/clangd/Quality.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/Quality.cpp?rev=336810&r1=336809&r2=336810&view=diff
==
--- clang-tools-extra/trunk/clangd/Quality.cpp (original)
+++ clang-tools-extra/trunk/clangd/Quality.cpp Wed Jul 11 07:49:49 2018
@@ -41,6 +41,17 @@ static bool hasDeclInMainFile(const Decl
   return false;
 }
 
+static bool hasUsingDeclInMainFile(const CodeCompletionResult &R) {
+  const auto &Context = R.Declaration->getASTContext();
+  const auto &SourceMgr = Context.getSourceManager();
+  if (R.ShadowDecl) {
+const auto Loc = SourceMgr.getExpansionLoc(R.ShadowDecl->getLocation());
+if (SourceMgr.isWrittenInMainFile(Loc))
+  return true;
+  }
+  return false;
+}
+
 static SymbolQualitySignals::SymbolCategory categorize(const NamedDecl &ND) {
   class Switch
   : public ConstDeclVisitor {
@@ -231,8 +242,10 @@ void SymbolRelevanceSignals::merge(const
 // We boost things that have decls in the main file. We give a fixed score
 // for all other declarations in sema as they are already included in the
 // translation unit.
-float DeclProximity =
-hasDeclInMainFile(*SemaCCResult.Declaration) ? 1.0 : 0.6;
+float DeclProximity = (hasDeclInMainFile(*SemaCCResult.Declaration) ||
+   hasUsingDeclInMainFile(SemaCCResult))
+  ? 1.0
+  : 0.6;
 SemaProximityScore = std::max(DeclProximity, SemaProximityScore);
   }
 

Modified: clang-tools-extra/trunk/unittests/clangd/QualityTests.cpp
URL: 
http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/unittests/clangd/QualityTests.cpp?rev=336810&r1=336809&r2=336810&view=diff
==
--- clang-tools-extra/trunk/unittests/clangd/QualityTests.cpp (original)
+++ clang-tools-extra/trunk/unittests/clangd/QualityTests.cpp Wed Jul 11 
07:49:49 2018
@@ -77,18 +77,31 @@ TEST(QualityTests, SymbolQualitySignalEx
 TEST(QualityTests, SymbolRelevanceSignalExtraction) {
   TestTU Test;
   Test.HeaderCode = R"cpp(
-int header();
-int header_main();
-)cpp";
+  int header();
+  int header_main();
+
+  namespace hdr { class Bar {}; } // namespace hdr
+
+  #define DEFINE_FLAG(X) \
+  namespace flags { \
+  int FLAGS_##X; \
+  } \
+
+  DEFINE_FLAG(FOO)
+  )cpp";
   Test.Code = R"cpp(
-int ::header_main() {}
-int main();
+  using hdr::Bar;
+
+  using flags::FLAGS_FOO;
+
+  int ::header_main() {}
+  int main();
 
-[[deprecated]]
-int deprecated() { return 0; }
+  [[deprecated]]
+  int deprecated() { return 0; }
 
-namespace { struct X { void y() { int z; } }; }
-struct S{}
+  namespace { struct X { void y() { int z; } }; }
+  struct S{}
   )cpp";
   auto AST = Test.build();
 
@@ -111,6 +124,32 @@ TEST(QualityTests, SymbolRelevanceSignal
   EXPECT_FLOAT_EQ(Relevance.SemaProximityScore, 1.0f)
   << "Current file and header";
 
+  auto constructShadowDeclCompletionResult = [&](const std::string DeclName) {
+auto *Shadow =
+*dyn_cast(
+ &findAnyDecl(AST,
+  [&](const NamedDecl &ND) {
+if (const UsingDecl *Using =
+dyn_cast(&ND))
+  if (Using->shadow_size() &&
+  Using->getQualifiedNameAsString() == 
DeclName)
+return true;
+return false;
+  }))
+ ->shadow_begin();
+CodeCompletionResult Result(Shadow->getTargetDecl(), 42);
+Result.ShadowDecl = Shadow;
+return Result;
+  };
+
+  Relevance 

[Differential] D83826: [clangd] Don't send invalid messages from remote index

[Kirill Bobyrev via cfe-commits]

This revision was not accepted when it landed; it landed in state "Needs 
Review".
This revision was landed with ongoing or failed builds.
This revision was automatically updated to reflect the committed changes.
Closed by commit rGeef162c330b0: [clangd] Don't send invalid messages from 
remote index (authored by kbobyrev).

Changed prior to commit:
  https://reviews.llvm.org/D83826?vs=278683&id=279041#toc

Repository:
  rG LLVM Github Monorepo

CHANGES SINCE LAST ACTION
  https://reviews.llvm.org/D83826/new/


  https://reviews.llvm.org/D83826

Files:
  clang-tools-extra/clangd/index/remote/Client.cpp
  clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
  clang-tools-extra/clangd/index/remote/marshalling/Marshalling.h
  clang-tools-extra/clangd/index/remote/server/Server.cpp
  clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp

Index: clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp
===
--- clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp
+++ clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp
@@ -13,6 +13,7 @@
 #include "index/Serialization.h"
 #include "index/Symbol.h"
 #include "index/SymbolID.h"
+#include "index/SymbolLocation.h"
 #include "index/remote/marshalling/Marshalling.h"
 #include "clang/Index/IndexSymbol.h"
 #include "llvm/ADT/SmallString.h"
@@ -39,29 +40,35 @@
 TEST(RemoteMarshallingTest, URITranslation) {
   llvm::BumpPtrAllocator Arena;
   llvm::UniqueStringSaver Strings(Arena);
+  Marshaller ProtobufMarshaller(
+  testPath("remote/machine/projects/llvm-project/"),
+  testPath("home/my-projects/llvm-project/"));
   clangd::Ref Original;
   Original.Location.FileURI =
   testPathURI("remote/machine/projects/llvm-project/clang-tools-extra/"
   "clangd/unittests/remote/MarshallingTests.cpp",
   Strings);
-  auto Serialized =
-  toProtobuf(Original, testPath("remote/machine/projects/llvm-project/"));
-  EXPECT_EQ(Serialized.location().file_path(),
+  auto Serialized = ProtobufMarshaller.toProtobuf(Original);
+  EXPECT_TRUE(Serialized);
+  EXPECT_EQ(Serialized->location().file_path(),
 "clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp");
-  const std::string LocalIndexPrefix = testPath("local/machine/project/");
-  auto Deserialized = fromProtobuf(Serialized, &Strings,
-   testPath("home/my-projects/llvm-project/"));
+  auto Deserialized = ProtobufMarshaller.fromProtobuf(*Serialized);
   EXPECT_TRUE(Deserialized);
-  EXPECT_EQ(Deserialized->Location.FileURI,
-testPathURI("home/my-projects/llvm-project/clang-tools-extra/"
-"clangd/unittests/remote/MarshallingTests.cpp",
-Strings));
+  EXPECT_STREQ(Deserialized->Location.FileURI,
+   testPathURI("home/my-projects/llvm-project/clang-tools-extra/"
+   "clangd/unittests/remote/MarshallingTests.cpp",
+   Strings));
+
+  // Can't have empty paths.
+  *Serialized->mutable_location()->mutable_file_path() = std::string();
+  Deserialized = ProtobufMarshaller.fromProtobuf(*Serialized);
+  EXPECT_FALSE(Deserialized);
 
   clangd::Ref WithInvalidURI;
-  // Invalid URI results in empty path.
+  // Invalid URI results in serialization failure.
   WithInvalidURI.Location.FileURI = "This is not a URI";
-  Serialized = toProtobuf(WithInvalidURI, testPath("home/"));
-  EXPECT_EQ(Serialized.location().file_path(), "");
+  Serialized = ProtobufMarshaller.toProtobuf(WithInvalidURI);
+  EXPECT_FALSE(Serialized);
 
   // Can not use URIs with scheme different from "file".
   auto UnittestURI =
@@ -69,15 +76,15 @@
   EXPECT_TRUE(bool(UnittestURI));
   WithInvalidURI.Location.FileURI =
   Strings.save(UnittestURI->toString()).begin();
-  Serialized = toProtobuf(WithInvalidURI, testPath("project/lib/"));
-  EXPECT_EQ(Serialized.location().file_path(), "");
+  Serialized = ProtobufMarshaller.toProtobuf(WithInvalidURI);
+  EXPECT_FALSE(Serialized);
 
+  // Paths transmitted over the wire can not be absolute, they have to be
+  // relative.
   Ref WithAbsolutePath;
   *WithAbsolutePath.mutable_location()->mutable_file_path() =
   "/usr/local/user/home/HelloWorld.cpp";
-  Deserialized = fromProtobuf(WithAbsolutePath, &Strings, LocalIndexPrefix);
-  // Paths transmitted over the wire can not be absolute, they have to be
-  // relative.
+  Deserialized = ProtobufMarshaller.fromProtobuf(WithAbsolutePath);
   EXPECT_FALSE(Deserialized);
 }
 
@@ -128,48 +135,63 @@
 
   Sym.Flags = clangd::Symbol::SymbolFlag::IndexedForCodeCompletion;
 
+  Marshaller ProtobufMarshaller(testPath("home/"), testPath("home/"));
+
   // Check that symbols are exactly the same if the path to indexed project is
   // the same on indexing machine and the client.
-  auto Serialized = toProtobuf(Sym, testPath("home/"));
-  auto Deserialized = fromPr

[clang-tools-extra] eef162c - [clangd] Don't send invalid messages from remote index

[Kirill Bobyrev via cfe-commits]

Author: Kirill Bobyrev
Date: 2020-07-21T11:05:03+02:00
New Revision: eef162c330b02fdc53ec33e5549635be5c5911fa

URL: 
https://github.com/llvm/llvm-project/commit/eef162c330b02fdc53ec33e5549635be5c5911fa
DIFF: 
https://github.com/llvm/llvm-project/commit/eef162c330b02fdc53ec33e5549635be5c5911fa.diff

LOG: [clangd] Don't send invalid messages from remote index

Summary:
Remote server should not send messages that are invalid and will cause problems
on the client side. The client should not be affected by server's failures
whenever possible.

Also add more error messages and logs.

Reviewers: sammccall

Reviewed By: sammccall

Subscribers: ilya-biryukov, MaskRay, jkorous, arphaman, kadircet, usaxena95, 
cfe-commits

Tags: #clang

Differential Revision: https://reviews.llvm.org/D83826

Added: 


Modified: 
clang-tools-extra/clangd/index/remote/Client.cpp
clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
clang-tools-extra/clangd/index/remote/marshalling/Marshalling.h
clang-tools-extra/clangd/index/remote/server/Server.cpp
clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp

Removed: 




diff  --git a/clang-tools-extra/clangd/index/remote/Client.cpp 
b/clang-tools-extra/clangd/index/remote/Client.cpp
index b46883193076..35ce84068f40 100644
--- a/clang-tools-extra/clangd/index/remote/Client.cpp
+++ b/clang-tools-extra/clangd/index/remote/Client.cpp
@@ -29,16 +29,6 @@ class IndexClient : public clangd::SymbolIndex {
   using StreamingCall = std::unique_ptr> (
   remote::SymbolIndex::Stub::*)(grpc::ClientContext *, const RequestT &);
 
-  template 
-  RequestT serializeRequest(ClangdRequestT Request) const {
-return toProtobuf(Request);
-  }
-
-  template <>
-  FuzzyFindRequest serializeRequest(clangd::FuzzyFindRequest Request) const {
-return toProtobuf(Request, ProjectRoot);
-  }
-
   template 
   bool streamRPC(ClangdRequestT Request,
@@ -46,24 +36,23 @@ class IndexClient : public clangd::SymbolIndex {
  CallbackT Callback) const {
 bool FinalResult = false;
 trace::Span Tracer(RequestT::descriptor()->name());
-const auto RPCRequest = serializeRequest(Request);
+const auto RPCRequest = ProtobufMarshaller->toProtobuf(Request);
 grpc::ClientContext Context;
 std::chrono::system_clock::time_point Deadline =
 std::chrono::system_clock::now() + DeadlineWaitingTime;
 Context.set_deadline(Deadline);
 auto Reader = (Stub.get()->*RPCCall)(&Context, RPCRequest);
-llvm::BumpPtrAllocator Arena;
-llvm::UniqueStringSaver Strings(Arena);
 ReplyT Reply;
 while (Reader->Read(&Reply)) {
   if (!Reply.has_stream_result()) {
 FinalResult = Reply.final_result();
 continue;
   }
-  auto Response =
-  fromProtobuf(Reply.stream_result(), &Strings, ProjectRoot);
-  if (!Response)
+  auto Response = ProtobufMarshaller->fromProtobuf(Reply.stream_result());
+  if (!Response) {
 elog("Received invalid {0}", ReplyT::descriptor()->name());
+continue;
+  }
   Callback(*Response);
 }
 SPAN_ATTACH(Tracer, "status", Reader->Finish().ok());
@@ -74,7 +63,9 @@ class IndexClient : public clangd::SymbolIndex {
   IndexClient(
   std::shared_ptr Channel, llvm::StringRef ProjectRoot,
   std::chrono::milliseconds DeadlineTime = std::chrono::milliseconds(1000))
-  : Stub(remote::SymbolIndex::NewStub(Channel)), ProjectRoot(ProjectRoot),
+  : Stub(remote::SymbolIndex::NewStub(Channel)),
+ProtobufMarshaller(new Marshaller(/*RemoteIndexRoot=*/"",
+  /*LocalIndexRoot=*/ProjectRoot)),
 DeadlineWaitingTime(DeadlineTime) {}
 
   void lookup(const clangd::LookupRequest &Request,
@@ -105,7 +96,7 @@ class IndexClient : public clangd::SymbolIndex {
 
 private:
   std::unique_ptr Stub;
-  std::string ProjectRoot;
+  std::unique_ptr ProtobufMarshaller;
   // Each request will be terminated if it takes too long.
   std::chrono::milliseconds DeadlineWaitingTime;
 };

diff  --git a/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp 
b/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
index db895bf039ba..059c42ee0eaa 100644
--- a/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
+++ b/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
@@ -30,101 +30,28 @@ namespace clang {
 namespace clangd {
 namespace remote {
 
-namespace {
-
-clangd::SymbolLocation::Position fromProtobuf(const Position &Message) {
-  clangd::SymbolLocation::Position Result;
-  Result.setColumn(static_cast(Message.column()));
-  Result.setLine(static_cast(Message.line()));
-  return Result;
-}
-
-Position toProtobuf(const clangd::SymbolLocation::Position &Position) {
-  remote::Position Result;
-  Result.set_column(Position.column());
-  Result.set_line(Position.line());
-  ret

[clang-tools-extra] 4470b8c - [clangd] Fix assertions for D83826

[Kirill Bobyrev via cfe-commits]

Author: Kirill Bobyrev
Date: 2020-07-21T11:20:31+02:00
New Revision: 4470b8c6a6b16de3b4f1f3c4cf81137a9fe4c8a1

URL: 
https://github.com/llvm/llvm-project/commit/4470b8c6a6b16de3b4f1f3c4cf81137a9fe4c8a1
DIFF: 
https://github.com/llvm/llvm-project/commit/4470b8c6a6b16de3b4f1f3c4cf81137a9fe4c8a1.diff

LOG: [clangd] Fix assertions for D83826

FuzzyFindRequest's toProtobuf is called on the client side (hence
LocalIndexRoot must be present) and fromProtobuf - on the server.

Added: 


Modified: 
clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp

Removed: 




diff  --git a/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp 
b/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
index 059c42ee0eaa..e8393d17b01e 100644
--- a/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
+++ b/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
@@ -51,7 +51,7 @@ Marshaller::Marshaller(llvm::StringRef RemoteIndexRoot,
 
 clangd::FuzzyFindRequest
 Marshaller::fromProtobuf(const FuzzyFindRequest *Request) {
-  assert(LocalIndexRoot);
+  assert(RemoteIndexRoot);
   clangd::FuzzyFindRequest Result;
   Result.Query = Request->query();
   for (const auto &Scope : Request->scopes())
@@ -146,7 +146,7 @@ LookupRequest Marshaller::toProtobuf(const 
clangd::LookupRequest &From) {
 }
 
 FuzzyFindRequest Marshaller::toProtobuf(const clangd::FuzzyFindRequest &From) {
-  assert(RemoteIndexRoot);
+  assert(LocalIndexRoot);
   FuzzyFindRequest RPCRequest;
   RPCRequest.set_query(From.Query);
   for (const auto &Scope : From.Scopes)



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] 7d591e1 - [clangd] Complete the fix for (Local|Remote)IndexRoot confusion

[Kirill Bobyrev via cfe-commits]

Author: Kirill Bobyrev
Date: 2020-07-21T11:53:17+02:00
New Revision: 7d591e123e0eb2d3415840de9c2b45c3742f0eaa

URL: 
https://github.com/llvm/llvm-project/commit/7d591e123e0eb2d3415840de9c2b45c3742f0eaa
DIFF: 
https://github.com/llvm/llvm-project/commit/7d591e123e0eb2d3415840de9c2b45c3742f0eaa.diff

LOG: [clangd] Complete the fix for (Local|Remote)IndexRoot confusion

Related revision: https://reviews.llvm.org/D83826

Added: 


Modified: 
clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp

Removed: 




diff  --git a/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp 
b/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
index e8393d17b01e..b6c83c974072 100644
--- a/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
+++ b/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
@@ -61,7 +61,7 @@ Marshaller::fromProtobuf(const FuzzyFindRequest *Request) {
 Result.Limit = Request->limit();
   Result.RestrictForCodeCompletion = Request->restricted_for_code_completion();
   for (const auto &Path : Request->proximity_paths()) {
-llvm::SmallString<256> LocalPath = llvm::StringRef(*LocalIndexRoot);
+llvm::SmallString<256> LocalPath = llvm::StringRef(*RemoteIndexRoot);
 llvm::sys::path::append(LocalPath, Path);
 Result.ProximityPaths.push_back(std::string(LocalPath));
   }
@@ -157,7 +157,7 @@ FuzzyFindRequest Marshaller::toProtobuf(const 
clangd::FuzzyFindRequest &From) {
   
RPCRequest.set_restricted_for_code_completion(From.RestrictForCodeCompletion);
   for (const auto &Path : From.ProximityPaths) {
 llvm::SmallString<256> RelativePath = llvm::StringRef(Path);
-if (llvm::sys::path::replace_path_prefix(RelativePath, *RemoteIndexRoot,
+if (llvm::sys::path::replace_path_prefix(RelativePath, *LocalIndexRoot,
  ""))
   RPCRequest.add_proximity_paths(llvm::sys::path::convert_to_slash(
   RelativePath, llvm::sys::path::Style::posix));

diff  --git a/clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp 
b/clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp
index 6cc08144bef8..7db7c03d61c9 100644
--- a/clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp
+++ b/clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp
@@ -282,16 +282,16 @@ TEST(RemoteMarshallingTest, IncludeHeaderURIs) {
 
 TEST(RemoteMarshallingTest, FuzzyFindRequestSerialization) {
   clangd::FuzzyFindRequest Request;
-  Request.ProximityPaths = {testPath("remote/Header.h"),
-testPath("remote/subdir/OtherHeader.h"),
-testPath("notremote/File.h"), "Not a Path."};
-  Marshaller ProtobufMarshaller(testPath("remote/"), testPath("home/"));
+  Request.ProximityPaths = {testPath("local/Header.h"),
+testPath("local/subdir/OtherHeader.h"),
+testPath("remote/File.h"), "Not a Path."};
+  Marshaller ProtobufMarshaller(testPath("remote/"), testPath("local/"));
   auto Serialized = ProtobufMarshaller.toProtobuf(Request);
   EXPECT_EQ(Serialized.proximity_paths_size(), 2);
   auto Deserialized = ProtobufMarshaller.fromProtobuf(&Serialized);
   EXPECT_THAT(Deserialized.ProximityPaths,
-  testing::ElementsAre(testPath("home/Header.h"),
-   testPath("home/subdir/OtherHeader.h")));
+  testing::ElementsAre(testPath("remote/Header.h"),
+   testPath("remote/subdir/OtherHeader.h")));
 }
 
 TEST(RemoteMarshallingTest, RelativePathToURITranslation) {



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] 3975c3b - [clangd] Fix conversion from Windows UNC paths to file URI format.

[Kirill Bobyrev via cfe-commits]

Author: Ilya Golovenko
Date: 2020-07-22T12:13:09+02:00
New Revision: 3975c3be80412bb6b1376bcef53ebce53984ddd7

URL: 
https://github.com/llvm/llvm-project/commit/3975c3be80412bb6b1376bcef53ebce53984ddd7
DIFF: 
https://github.com/llvm/llvm-project/commit/3975c3be80412bb6b1376bcef53ebce53984ddd7.diff

LOG: [clangd] Fix conversion from Windows UNC paths to file URI format.

Summary:
The fix improves handling of Windows UNC paths to align with Appendix E. 
Nonstandard Syntax Variations of RFC 8089.

Before this fix it was difficult to use Windows UNC paths in 
compile_commands.json database as such paths were converted to file URIs using 
'file:auth/share/file.cpp' notation instead of recommended 
'file://auth/share/file.cpp'.

As an example, VS.Code cannot understand file URIs with 4 starting slashes, 
thus such features as go-to-definition, jump-to-file, hover tooltip, etc. stop 
working. This also applicable to files which reside on Windows network-mapped 
drives because clangd internally resolves file paths to real paths in some 
cases and such paths get resolved to UNC paths.

Reviewers: sammccall, kadircet

Reviewed By: sammccall

Subscribers: ormris, ilya-biryukov, MaskRay, jkorous, arphaman, kadircet, 
usaxena95, kbobyrev, cfe-commits

Tags: #clang

Differential Revision: https://reviews.llvm.org/D84172

Added: 


Modified: 
clang-tools-extra/clangd/URI.cpp
clang-tools-extra/clangd/unittests/URITests.cpp

Removed: 




diff  --git a/clang-tools-extra/clangd/URI.cpp 
b/clang-tools-extra/clangd/URI.cpp
index 2061a5601c58..fad93143a30d 100644
--- a/clang-tools-extra/clangd/URI.cpp
+++ b/clang-tools-extra/clangd/URI.cpp
@@ -26,6 +26,15 @@ inline llvm::Error make_string_error(const llvm::Twine 
&Message) {
  llvm::inconvertibleErrorCode());
 }
 
+bool isWindowsPath(llvm::StringRef Path) {
+  return Path.size() > 1 && llvm::isAlpha(Path[0]) && Path[1] == ':';
+}
+
+bool isNetworkPath(llvm::StringRef Path) {
+  return Path.size() > 2 && Path[0] == Path[1] &&
+ llvm::sys::path::is_separator(Path[0]);
+}
+
 /// This manages file paths in the file system. All paths in the scheme
 /// are absolute (with leading '/').
 /// Note that this scheme is hardcoded into the library and not registered in
@@ -33,28 +42,40 @@ inline llvm::Error make_string_error(const llvm::Twine 
&Message) {
 class FileSystemScheme : public URIScheme {
 public:
   llvm::Expected
-  getAbsolutePath(llvm::StringRef /*Authority*/, llvm::StringRef Body,
+  getAbsolutePath(llvm::StringRef Authority, llvm::StringRef Body,
   llvm::StringRef /*HintPath*/) const override {
 if (!Body.startswith("/"))
   return make_string_error("File scheme: expect body to be an absolute "
"path starting with '/': " +
Body);
-// For Windows paths e.g. /X:
-if (Body.size() > 2 && Body[0] == '/' && Body[2] == ':')
+llvm::SmallString<128> Path;
+if (!Authority.empty()) {
+  // Windows UNC paths e.g. file://server/share => \\server\share
+  ("//" + Authority).toVector(Path);
+} else if (isWindowsPath(Body.substr(1))) {
+  // Windows paths e.g. file:///X:/path => X:\path
   Body.consume_front("/");
-llvm::SmallVector Path(Body.begin(), Body.end());
+}
+Path.append(Body);
 llvm::sys::path::native(Path);
-return std::string(Path.begin(), Path.end());
+return std::string(Path);
   }
 
   llvm::Expected
   uriFromAbsolutePath(llvm::StringRef AbsolutePath) const override {
 std::string Body;
-// For Windows paths e.g. X:
-if (AbsolutePath.size() > 1 && AbsolutePath[1] == ':')
+llvm::StringRef Authority;
+llvm::StringRef Root = llvm::sys::path::root_name(AbsolutePath);
+if (isNetworkPath(Root)) {
+  // Windows UNC paths e.g. \\server\share => file://server/share
+  Authority = Root.drop_front(2);
+  AbsolutePath.consume_front(Root);
+} else if (isWindowsPath(Root)) {
+  // Windows paths e.g. X:\path => file:///X:/path
   Body = "/";
+}
 Body += llvm::sys::path::convert_to_slash(AbsolutePath);
-return URI("file", /*Authority=*/"", Body);
+return URI("file", Authority, Body);
   }
 };
 
@@ -96,13 +117,13 @@ bool shouldEscape(unsigned char C) {
 void percentEncode(llvm::StringRef Content, std::string &Out) {
   std::string Result;
   for (unsigned char C : Content)
-if (shouldEscape(C))
-{
+if (shouldEscape(C)) {
   Out.push_back('%');
   Out.push_back(llvm::hexdigit(C / 16));
   Out.push_back(llvm::hexdigit(C % 16));
-} else
-{ Out.push_back(C); }
+} else {
+  Out.push_back(C);
+}
 }
 
 /// Decodes a string according to percent-encoding.

diff  --git a/clang-tools-extra/clangd/unittests/URITests.cpp 
b/clang-tools-extra/clangd/unittests/URITests.cpp
index 52ca7b4447cd..cd734cfe

[clang-tools-extra] 974ffee - [clangd] Switch from EXPECT_TRUE to ASSERT_TRUE in remote marshalling tests

[Kirill Bobyrev via cfe-commits]

Author: Kirill Bobyrev
Date: 2020-07-27T10:44:23+02:00
New Revision: 974ffee9ccd70703c6edb880ac4934a5dc12e56d

URL: 
https://github.com/llvm/llvm-project/commit/974ffee9ccd70703c6edb880ac4934a5dc12e56d
DIFF: 
https://github.com/llvm/llvm-project/commit/974ffee9ccd70703c6edb880ac4934a5dc12e56d.diff

LOG: [clangd] Switch from EXPECT_TRUE to ASSERT_TRUE in remote marshalling tests

Summary:
When dereferencing Optional's it makes sense to use ASSERT_TRUE for better
test failures readability. Switch from EXPECT_TRUE to ASSERT_TRUE where
it is appropriate.

Reviewers: kadircet

Reviewed By: kadircet

Subscribers: ilya-biryukov, MaskRay, jkorous, arphaman, usaxena95, cfe-commits

Tags: #clang

Differential Revision: https://reviews.llvm.org/D84535

Signed-off-by: Kirill Bobyrev 

Added: 


Modified: 
clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp

Removed: 




diff  --git a/clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp 
b/clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp
index 7db7c03d61c9..147601b665c4 100644
--- a/clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp
+++ b/clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp
@@ -49,11 +49,11 @@ TEST(RemoteMarshallingTest, URITranslation) {
   "clangd/unittests/remote/MarshallingTests.cpp",
   Strings);
   auto Serialized = ProtobufMarshaller.toProtobuf(Original);
-  EXPECT_TRUE(Serialized);
+  ASSERT_TRUE(Serialized);
   EXPECT_EQ(Serialized->location().file_path(),
 "clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp");
   auto Deserialized = ProtobufMarshaller.fromProtobuf(*Serialized);
-  EXPECT_TRUE(Deserialized);
+  ASSERT_TRUE(Deserialized);
   EXPECT_STREQ(Deserialized->Location.FileURI,
testPathURI("home/my-projects/llvm-project/clang-tools-extra/"
"clangd/unittests/remote/MarshallingTests.cpp",
@@ -61,38 +61,34 @@ TEST(RemoteMarshallingTest, URITranslation) {
 
   // Can't have empty paths.
   *Serialized->mutable_location()->mutable_file_path() = std::string();
-  Deserialized = ProtobufMarshaller.fromProtobuf(*Serialized);
-  EXPECT_FALSE(Deserialized);
+  EXPECT_FALSE(ProtobufMarshaller.fromProtobuf(*Serialized));
 
   clangd::Ref WithInvalidURI;
   // Invalid URI results in serialization failure.
   WithInvalidURI.Location.FileURI = "This is not a URI";
-  Serialized = ProtobufMarshaller.toProtobuf(WithInvalidURI);
-  EXPECT_FALSE(Serialized);
+  EXPECT_FALSE(ProtobufMarshaller.toProtobuf(WithInvalidURI));
 
   // Can not use URIs with scheme 
diff erent from "file".
   auto UnittestURI =
   URI::create(testPath("project/lib/HelloWorld.cpp"), "unittest");
-  EXPECT_TRUE(bool(UnittestURI));
+  ASSERT_TRUE(bool(UnittestURI));
   WithInvalidURI.Location.FileURI =
   Strings.save(UnittestURI->toString()).begin();
-  Serialized = ProtobufMarshaller.toProtobuf(WithInvalidURI);
-  EXPECT_FALSE(Serialized);
+  EXPECT_FALSE(ProtobufMarshaller.toProtobuf(WithInvalidURI));
 
   // Paths transmitted over the wire can not be absolute, they have to be
   // relative.
   Ref WithAbsolutePath;
   *WithAbsolutePath.mutable_location()->mutable_file_path() =
   "/usr/local/user/home/HelloWorld.cpp";
-  Deserialized = ProtobufMarshaller.fromProtobuf(WithAbsolutePath);
-  EXPECT_FALSE(Deserialized);
+  EXPECT_FALSE(ProtobufMarshaller.fromProtobuf(WithAbsolutePath));
 }
 
 TEST(RemoteMarshallingTest, SymbolSerialization) {
   clangd::Symbol Sym;
 
   auto ID = SymbolID::fromStr("057557CEBF6E6B2D");
-  EXPECT_TRUE(bool(ID));
+  ASSERT_TRUE(bool(ID));
   Sym.ID = *ID;
 
   index::SymbolInfo Info;
@@ -140,9 +136,9 @@ TEST(RemoteMarshallingTest, SymbolSerialization) {
   // Check that symbols are exactly the same if the path to indexed project is
   // the same on indexing machine and the client.
   auto Serialized = ProtobufMarshaller.toProtobuf(Sym);
-  EXPECT_TRUE(Serialized);
+  ASSERT_TRUE(Serialized);
   auto Deserialized = ProtobufMarshaller.fromProtobuf(*Serialized);
-  EXPECT_TRUE(Deserialized);
+  ASSERT_TRUE(Deserialized);
   EXPECT_EQ(toYAML(Sym), toYAML(*Deserialized));
   // Serialized paths are relative and have UNIX slashes.
   EXPECT_EQ(convert_to_slash(Serialized->definition().file_path(),
@@ -154,44 +150,39 @@ TEST(RemoteMarshallingTest, SymbolSerialization) {
   // Missing definition is OK.
   Sym.Definition = clangd::SymbolLocation();
   Serialized = ProtobufMarshaller.toProtobuf(Sym);
-  EXPECT_TRUE(Serialized);
-  Deserialized = ProtobufMarshaller.fromProtobuf(*Serialized);
-  EXPECT_TRUE(Deserialized);
+  ASSERT_TRUE(Serialized);
+  EXPECT_TRUE(ProtobufMarshaller.fromProtobuf(*Serialized));
 
   // Relative path is absolute.
   *Serialized->mutable_canonical_declaration()->mutable_file_path() =
   convert_to_slash("/path/to/Declaration.h");
-  Deserialized = ProtobufMarshaller.fro

[clang-tools-extra] 37ac559 - [clangd] Add option to use remote index as static index

[Kirill Bobyrev via cfe-commits]

Author: Kirill Bobyrev
Date: 2020-07-27T11:24:15+02:00
New Revision: 37ac559fccd46dcec246ceb3907c8d3910728c69

URL: 
https://github.com/llvm/llvm-project/commit/37ac559fccd46dcec246ceb3907c8d3910728c69
DIFF: 
https://github.com/llvm/llvm-project/commit/37ac559fccd46dcec246ceb3907c8d3910728c69.diff

LOG: [clangd] Add option to use remote index as static index

Reviewers: hokein

Reviewed By: hokein

Subscribers: usaxena95, mgorny, ilya-biryukov, MaskRay, jkorous, arphaman, 
kadircet, cfe-commits

Tags: #clang

Differential Revision: https://reviews.llvm.org/D83817

Added: 


Modified: 
clang-tools-extra/clangd/Features.inc.in
clang-tools-extra/clangd/tool/CMakeLists.txt
clang-tools-extra/clangd/tool/ClangdMain.cpp

Removed: 




diff  --git a/clang-tools-extra/clangd/Features.inc.in 
b/clang-tools-extra/clangd/Features.inc.in
index da75aa67a65b..8584b87c6205 100644
--- a/clang-tools-extra/clangd/Features.inc.in
+++ b/clang-tools-extra/clangd/Features.inc.in
@@ -1 +1,2 @@
 #define CLANGD_BUILD_XPC @CLANGD_BUILD_XPC@
+#define CLANGD_ENABLE_REMOTE @CLANGD_ENABLE_REMTE@

diff  --git a/clang-tools-extra/clangd/tool/CMakeLists.txt 
b/clang-tools-extra/clangd/tool/CMakeLists.txt
index 3368013f5079..670e5a17013a 100644
--- a/clang-tools-extra/clangd/tool/CMakeLists.txt
+++ b/clang-tools-extra/clangd/tool/CMakeLists.txt
@@ -27,6 +27,7 @@ clang_target_link_libraries(clangd
   clangToolingCore
   clangToolingRefactoring
   clangToolingSyntax
+  clangdRemoteIndex
   )
 target_link_libraries(clangd
   PRIVATE

diff  --git a/clang-tools-extra/clangd/tool/ClangdMain.cpp 
b/clang-tools-extra/clangd/tool/ClangdMain.cpp
index 7bce1c062e81..8d1bf5c42260 100644
--- a/clang-tools-extra/clangd/tool/ClangdMain.cpp
+++ b/clang-tools-extra/clangd/tool/ClangdMain.cpp
@@ -14,6 +14,7 @@
 #include "Transport.h"
 #include "index/Background.h"
 #include "index/Serialization.h"
+#include "index/remote/Client.h"
 #include "refactor/Rename.h"
 #include "support/Path.h"
 #include "support/Shutdown.h"
@@ -449,6 +450,21 @@ opt EnableConfig{
 init(true),
 };
 
+#ifdef CLANGD_ENABLE_REMOTE
+opt RemoteIndexAddress{
+"remote-index-address",
+cat(Features),
+desc("Address of the remote index server"),
+};
+
+// FIXME(kirillbobyrev): Should this be the location of compile_commands.json?
+opt ProjectRoot{
+"project-root",
+cat(Features),
+desc("Path to the project root. Requires remote-index-address to be set."),
+};
+#endif
+
 /// Supports a test URI scheme with relaxed constraints for lit tests.
 /// The path in a test URI will be combined with a platform-specific fake
 /// directory to form an absolute path. For example, test:///a.cpp is resolved
@@ -680,6 +696,23 @@ clangd accepts flags on the commandline, and in the 
CLANGD_FLAGS environment var
 if (Sync)
   AsyncIndexLoad.wait();
   }
+#ifdef CLANGD_ENABLE_REMOTE
+  if (RemoteIndexAddress.empty() != ProjectRoot.empty()) {
+llvm::errs() << "remote-index-address and project-path have to be "
+"specified at the same time.";
+return 1;
+  }
+  if (!RemoteIndexAddress.empty()) {
+if (IndexFile.empty()) {
+  log("Connecting to remote index at {0}", RemoteIndexAddress);
+  StaticIdx = remote::getClient(RemoteIndexAddress, ProjectRoot);
+  EnableBackgroundIndex = false;
+} else {
+  elog("When enabling remote index, IndexFile should not be specified. "
+   "Only one can be used at time. Remote index will ignored.");
+}
+  }
+#endif
   Opts.StaticIndex = StaticIdx.get();
   Opts.AsyncThreadsCount = WorkerThreadsCount;
   Opts.BuildRecoveryAST = RecoveryAST;



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] f49a7ad - [clangd] Add marshalling code for all request types

[Kirill Bobyrev via cfe-commits]

Author: Kirill Bobyrev
Date: 2020-07-27T14:16:03+02:00
New Revision: f49a7ad8c0854a01b945c27de2fd313b9013ae0d

URL: 
https://github.com/llvm/llvm-project/commit/f49a7ad8c0854a01b945c27de2fd313b9013ae0d
DIFF: 
https://github.com/llvm/llvm-project/commit/f49a7ad8c0854a01b945c27de2fd313b9013ae0d.diff

LOG: [clangd] Add marshalling code for all request types

Summary:
Only FuzzyFindRequest is implemented via Marshaller even though other requests
also follow a similar pattern. Unify them under the marshalling umbrella and
make the server requests even more uniform to complement D84499.

Reviewers: kadircet

Reviewed By: kadircet

Subscribers: ilya-biryukov, MaskRay, jkorous, arphaman, usaxena95, cfe-commits, 
sammccall

Tags: #clang

Differential Revision: https://reviews.llvm.org/D84525

Added: 


Modified: 
clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
clang-tools-extra/clangd/index/remote/marshalling/Marshalling.h
clang-tools-extra/clangd/index/remote/server/Server.cpp
clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp

Removed: 




diff  --git a/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp 
b/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
index b6c83c974072..b2085bc21f48 100644
--- a/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
+++ b/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
@@ -17,6 +17,7 @@
 #include "index/SymbolOrigin.h"
 #include "support/Logger.h"
 #include "clang/Index/IndexSymbol.h"
+#include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/None.h"
 #include "llvm/ADT/Optional.h"
 #include "llvm/ADT/SmallString.h"
@@ -30,6 +31,22 @@ namespace clang {
 namespace clangd {
 namespace remote {
 
+namespace {
+
+template 
+llvm::Expected> getIDs(MessageT *Message) {
+  llvm::DenseSet Result;
+  for (const auto &ID : Message->ids()) {
+auto SID = SymbolID::fromStr(StringRef(ID));
+if (!SID)
+  return SID.takeError();
+Result.insert(*SID);
+  }
+  return Result;
+}
+
+} // namespace
+
 Marshaller::Marshaller(llvm::StringRef RemoteIndexRoot,
llvm::StringRef LocalIndexRoot)
 : Strings(Arena) {
@@ -49,27 +66,50 @@ Marshaller::Marshaller(llvm::StringRef RemoteIndexRoot,
   assert(!RemoteIndexRoot.empty() || !LocalIndexRoot.empty());
 }
 
-clangd::FuzzyFindRequest
-Marshaller::fromProtobuf(const FuzzyFindRequest *Request) {
+llvm::Expected
+Marshaller::fromProtobuf(const LookupRequest *Message) {
+  clangd::LookupRequest Req;
+  auto IDs = getIDs(Message);
+  if (!IDs)
+return IDs.takeError();
+  Req.IDs = std::move(*IDs);
+  return Req;
+}
+
+llvm::Expected
+Marshaller::fromProtobuf(const FuzzyFindRequest *Message) {
   assert(RemoteIndexRoot);
   clangd::FuzzyFindRequest Result;
-  Result.Query = Request->query();
-  for (const auto &Scope : Request->scopes())
+  Result.Query = Message->query();
+  for (const auto &Scope : Message->scopes())
 Result.Scopes.push_back(Scope);
-  Result.AnyScope = Request->any_scope();
-  if (Request->limit())
-Result.Limit = Request->limit();
-  Result.RestrictForCodeCompletion = Request->restricted_for_code_completion();
-  for (const auto &Path : Request->proximity_paths()) {
+  Result.AnyScope = Message->any_scope();
+  if (Message->limit())
+Result.Limit = Message->limit();
+  Result.RestrictForCodeCompletion = Message->restricted_for_code_completion();
+  for (const auto &Path : Message->proximity_paths()) {
 llvm::SmallString<256> LocalPath = llvm::StringRef(*RemoteIndexRoot);
 llvm::sys::path::append(LocalPath, Path);
 Result.ProximityPaths.push_back(std::string(LocalPath));
   }
-  for (const auto &Type : Request->preferred_types())
+  for (const auto &Type : Message->preferred_types())
 Result.ProximityPaths.push_back(Type);
   return Result;
 }
 
+llvm::Expected
+Marshaller::fromProtobuf(const RefsRequest *Message) {
+  clangd::RefsRequest Req;
+  auto IDs = getIDs(Message);
+  if (!IDs)
+return IDs.takeError();
+  Req.IDs = std::move(*IDs);
+  Req.Filter = static_cast(Message->filter());
+  if (Message->limit())
+Req.Limit = Message->limit();
+  return Req;
+}
+
 llvm::Optional Marshaller::fromProtobuf(const Symbol &Message) 
{
   if (!Message.has_info() || !Message.has_canonical_declaration()) {
 elog("Cannot convert Symbol from protobuf (missing info, definition or "
@@ -157,8 +197,7 @@ FuzzyFindRequest Marshaller::toProtobuf(const 
clangd::FuzzyFindRequest &From) {
   
RPCRequest.set_restricted_for_code_completion(From.RestrictForCodeCompletion);
   for (const auto &Path : From.ProximityPaths) {
 llvm::SmallString<256> RelativePath = llvm::StringRef(Path);
-if (llvm::sys::path::replace_path_prefix(RelativePath, *LocalIndexRoot,
- ""))
+if (llvm::sys::path::replace_path_prefix(RelativePath, *LocalIndexRoot, 
"")

Re: [PATCH] D83817: [clangd] Add option to use remote index as static index

[Kirill Bobyrev via cfe-commits]

Ah, that's why... Thank you again and apologies for inconovinience!

On Mon, Jul 27, 2020 at 3:06 PM Hans Wennborg via Phabricator <
revi...@reviews.llvm.org> wrote:

> hans added inline comments.
>
>
> 
> Comment at: clang-tools-extra/clangd/tool/ClangdMain.cpp:702
>}
> +  if (RemoteIndexAddress.empty() != ProjectPath.empty()) {
> +llvm::errs() << "remote-index-address and project-path have to be "
> 
> hokein wrote:
> > the new code section here should be guarded under `#ifdef
> CLANGD_ENABLE_REMOTE`
> I think it should be an #if since the macro is always defined (0 or 1).
> Doing that in 40d11a878044711708fb6738e4b78a4c9ac3de7b
>
>
> Repository:
>   rG LLVM Github Monorepo
>
> CHANGES SINCE LAST ACTION
>   https://reviews.llvm.org/D83817/new/
>
> https://reviews.llvm.org/D83817
>
>
>
>
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

Re: [PATCH] D83817: [clangd] Add option to use remote index as static index

[Kirill Bobyrev via cfe-commits]

Uh, you're right, thank you for the fix! I wonder why that was building for
me...

Again, thank you for noticing!

Kirill

On Mon, Jul 27, 2020 at 2:41 PM Hans Wennborg via Phabricator <
revi...@reviews.llvm.org> wrote:

> hans added inline comments.
>
>
> 
> Comment at: clang-tools-extra/clangd/Features.inc.in:2
>  #define CLANGD_BUILD_XPC @CLANGD_BUILD_XPC@
> +#define CLANGD_ENABLE_REMOTE @CLANGD_ENABLE_REMTE@
> 
> I'm guessing this should be @CLANGD_ENABLE_REMOTE@
> The gn build was upset: http://45.33.8.238/linux/23942/step_4.txt
> Speculatively fixed in 47a0254229ca425aa4e169c2db14e92b8db86784. Please
> shout if this was wrong.
>
>
> Repository:
>   rG LLVM Github Monorepo
>
> CHANGES SINCE LAST ACTION
>   https://reviews.llvm.org/D83817/new/
>
> https://reviews.llvm.org/D83817
>
>
>
>
___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] 731043c - [clangd] Add more logs and attach tracers to remote index server routines

[Kirill Bobyrev via cfe-commits]

Author: Kirill Bobyrev
Date: 2020-07-27T20:45:18+02:00
New Revision: 731043c0c494672efe1eeea9ee0f0c7788813dea

URL: 
https://github.com/llvm/llvm-project/commit/731043c0c494672efe1eeea9ee0f0c7788813dea
DIFF: 
https://github.com/llvm/llvm-project/commit/731043c0c494672efe1eeea9ee0f0c7788813dea.diff

LOG: [clangd] Add more logs and attach tracers to remote index server routines

Reviewers: kadircet

Reviewed By: kadircet

Subscribers: sammccall, ilya-biryukov, MaskRay, jkorous, arphaman, usaxena95, 
cfe-commits

Tags: #clang

Differential Revision: https://reviews.llvm.org/D84499

Added: 


Modified: 
clang-tools-extra/clangd/index/remote/Client.cpp
clang-tools-extra/clangd/index/remote/server/Server.cpp

Removed: 




diff  --git a/clang-tools-extra/clangd/index/remote/Client.cpp 
b/clang-tools-extra/clangd/index/remote/Client.cpp
index 35ce84068f40..5a33fd2eaf14 100644
--- a/clang-tools-extra/clangd/index/remote/Client.cpp
+++ b/clang-tools-extra/clangd/index/remote/Client.cpp
@@ -37,12 +37,15 @@ class IndexClient : public clangd::SymbolIndex {
 bool FinalResult = false;
 trace::Span Tracer(RequestT::descriptor()->name());
 const auto RPCRequest = ProtobufMarshaller->toProtobuf(Request);
+SPAN_ATTACH(Tracer, "Request", RPCRequest.DebugString());
 grpc::ClientContext Context;
 std::chrono::system_clock::time_point Deadline =
 std::chrono::system_clock::now() + DeadlineWaitingTime;
 Context.set_deadline(Deadline);
 auto Reader = (Stub.get()->*RPCCall)(&Context, RPCRequest);
 ReplyT Reply;
+unsigned Successful = 0;
+unsigned FailedToParse = 0;
 while (Reader->Read(&Reply)) {
   if (!Reply.has_stream_result()) {
 FinalResult = Reply.final_result();
@@ -51,11 +54,15 @@ class IndexClient : public clangd::SymbolIndex {
   auto Response = ProtobufMarshaller->fromProtobuf(Reply.stream_result());
   if (!Response) {
 elog("Received invalid {0}", ReplyT::descriptor()->name());
+++FailedToParse;
 continue;
   }
   Callback(*Response);
+  ++Successful;
 }
-SPAN_ATTACH(Tracer, "status", Reader->Finish().ok());
+SPAN_ATTACH(Tracer, "Status", Reader->Finish().ok());
+SPAN_ATTACH(Tracer, "Successful", Successful);
+SPAN_ATTACH(Tracer, "Failed to parse", FailedToParse);
 return FinalResult;
   }
 

diff  --git a/clang-tools-extra/clangd/index/remote/server/Server.cpp 
b/clang-tools-extra/clangd/index/remote/server/Server.cpp
index 7bf47a288e79..364e4db6503c 100644
--- a/clang-tools-extra/clangd/index/remote/server/Server.cpp
+++ b/clang-tools-extra/clangd/index/remote/server/Server.cpp
@@ -6,10 +6,13 @@
 //
 
//===--===//
 
+#include "Index.pb.h"
 #include "index/Index.h"
 #include "index/Serialization.h"
+#include "index/Symbol.h"
 #include "index/remote/marshalling/Marshalling.h"
 #include "support/Logger.h"
+#include "support/Trace.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Path.h"
@@ -36,6 +39,16 @@ llvm::cl::opt IndexPath(llvm::cl::desc(""),
 llvm::cl::opt IndexRoot(llvm::cl::desc(""),
  llvm::cl::Positional, llvm::cl::Required);
 
+llvm::cl::opt TraceFile(
+"trace-file",
+llvm::cl::desc("Path to the file where tracer logs will be stored"));
+
+llvm::cl::opt PrettyPrint{
+"pretty",
+llvm::cl::desc("Pretty-print JSON output in the trace"),
+llvm::cl::init(false),
+};
+
 llvm::cl::opt ServerAddress(
 "server-address", llvm::cl::init("0.0.0.0:50051"),
 llvm::cl::desc("Address of the invoked server. Defaults to 
0.0.0.0:50051"));
@@ -60,66 +73,90 @@ class RemoteIndexServer final : public SymbolIndex::Service 
{
   grpc::Status Lookup(grpc::ServerContext *Context,
   const LookupRequest *Request,
   grpc::ServerWriter *Reply) override {
+trace::Span Tracer(LookupRequest::descriptor()->name());
 auto Req = ProtobufMarshaller->fromProtobuf(Request);
 if (!Req) {
   elog("Can not parse LookupRequest from protobuf: {0}", Req.takeError());
   return grpc::Status::CANCELLED;
 }
-Index->lookup(*Req, [&](const clangd::Symbol &Sym) {
-  auto SerializedSymbol = ProtobufMarshaller->toProtobuf(Sym);
-  if (!SerializedSymbol)
+unsigned Sent = 0;
+unsigned FailedToSend = 0;
+Index->lookup(*Req, [&](const auto &Item) {
+  auto SerializedItem = ProtobufMarshaller->toProtobuf(Item);
+  if (!SerializedItem) {
+++FailedToSend;
 return;
+  }
   LookupReply NextMessage;
-  *NextMessage.mutable_stream_result() = *SerializedSymbol;
+  *NextMessage.mutable_stream_result() = *SerializedItem;
   Reply->Write(NextMessage);
+  ++Sent;
 });
 LookupReply LastMessage;
 LastMessage.set_final_resul

[clang-tools-extra] fb22678 - [clangd] Use elog instead of llvm::errs, log instead of llvm::outs

[Kirill Bobyrev via cfe-commits]

Author: Kirill Bobyrev
Date: 2020-07-29T01:48:24+02:00
New Revision: fb22678cd67801e41af4917acceb4014ab5d007e

URL: 
https://github.com/llvm/llvm-project/commit/fb22678cd67801e41af4917acceb4014ab5d007e
DIFF: 
https://github.com/llvm/llvm-project/commit/fb22678cd67801e41af4917acceb4014ab5d007e.diff

LOG: [clangd] Use elog instead of llvm::errs, log instead of llvm::outs

Reviewed By: sammccall

Differential Revision: https://reviews.llvm.org/D84697

Added: 


Modified: 
clang-tools-extra/clangd/index/remote/server/Server.cpp
clang-tools-extra/clangd/indexer/IndexerMain.cpp
clang-tools-extra/clangd/tool/ClangdMain.cpp

Removed: 




diff  --git a/clang-tools-extra/clangd/index/remote/server/Server.cpp 
b/clang-tools-extra/clangd/index/remote/server/Server.cpp
index 364e4db6503c..c3f9efeb7ec0 100644
--- a/clang-tools-extra/clangd/index/remote/server/Server.cpp
+++ b/clang-tools-extra/clangd/index/remote/server/Server.cpp
@@ -39,6 +39,15 @@ llvm::cl::opt IndexPath(llvm::cl::desc(""),
 llvm::cl::opt IndexRoot(llvm::cl::desc(""),
  llvm::cl::Positional, llvm::cl::Required);
 
+llvm::cl::opt LogLevel{
+"log",
+llvm::cl::desc("Verbosity of log messages written to stderr"),
+values(clEnumValN(Logger::Error, "error", "Error messages only"),
+   clEnumValN(Logger::Info, "info", "High level execution tracing"),
+   clEnumValN(Logger::Debug, "verbose", "Low level details")),
+llvm::cl::init(Logger::Info),
+};
+
 llvm::cl::opt TraceFile(
 "trace-file",
 llvm::cl::desc("Path to the file where tracer logs will be stored"));
@@ -173,7 +182,7 @@ void runServer(std::unique_ptr Index,
   Builder.AddListeningPort(ServerAddress, grpc::InsecureServerCredentials());
   Builder.RegisterService(&Service);
   std::unique_ptr Server(Builder.BuildAndStart());
-  llvm::outs() << "Server listening on " << ServerAddress << '\n';
+  log("Server listening on {0}", ServerAddress);
 
   Server->Wait();
 }
@@ -191,10 +200,16 @@ int main(int argc, char *argv[]) {
   llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
 
   if (!llvm::sys::path::is_absolute(IndexRoot)) {
-elog("Index root should be an absolute path.");
+llvm::errs() << "Index root should be an absolute path.\n";
 return -1;
   }
 
+  llvm::errs().SetBuffered();
+  // Don't flush stdout when logging for thread safety.
+  llvm::errs().tie(nullptr);
+  clang::clangd::StreamLogger Logger(llvm::errs(), LogLevel);
+  clang::clangd::LoggingSession LoggingSession(Logger);
+
   llvm::Optional TracerStream;
   std::unique_ptr Tracer;
   if (!TraceFile.empty()) {
@@ -220,7 +235,7 @@ int main(int argc, char *argv[]) {
   std::unique_ptr Index = openIndex(IndexPath);
 
   if (!Index) {
-elog("Failed to open the index.");
+llvm::errs() << "Failed to open the index.\n";
 return -1;
   }
 

diff  --git a/clang-tools-extra/clangd/indexer/IndexerMain.cpp 
b/clang-tools-extra/clangd/indexer/IndexerMain.cpp
index dac038308d9e..46224238c3fc 100644
--- a/clang-tools-extra/clangd/indexer/IndexerMain.cpp
+++ b/clang-tools-extra/clangd/indexer/IndexerMain.cpp
@@ -16,6 +16,7 @@
 #include "index/Serialization.h"
 #include "index/Symbol.h"
 #include "index/SymbolCollector.h"
+#include "support/Logger.h"
 #include "clang/Tooling/ArgumentsAdjusters.h"
 #include "clang/Tooling/CommonOptionsParser.h"
 #include "clang/Tooling/Execution.h"
@@ -122,7 +123,7 @@ int main(int argc, const char **argv) {
   std::make_unique(Data),
   clang::tooling::getStripPluginsAdjuster());
   if (Err) {
-llvm::errs() << llvm::toString(std::move(Err)) << "\n";
+clang::clangd::elog("{0}", std::move(Err));
   }
 
   // Emit collected data.

diff  --git a/clang-tools-extra/clangd/tool/ClangdMain.cpp 
b/clang-tools-extra/clangd/tool/ClangdMain.cpp
index 0d4267774c92..daf87d11c384 100644
--- a/clang-tools-extra/clangd/tool/ClangdMain.cpp
+++ b/clang-tools-extra/clangd/tool/ClangdMain.cpp
@@ -657,16 +657,16 @@ clangd accepts flags on the commandline, and in the 
CLANGD_FLAGS environment var
   // continuing.
   llvm::SmallString<128> Path(CompileCommandsDir);
   if (std::error_code EC = llvm::sys::fs::make_absolute(Path)) {
-llvm::errs() << "Error while converting the relative path specified by 
"
-"--compile-commands-dir to an absolute path: "
- << EC.message() << ". The argument will be ignored.\n";
+elog("Error while converting the relative path specified by "
+ "--compile-commands-dir to an absolute path: {0}. The argument "
+ "will be ignored.",
+ EC.message());
   } else {
 CompileCommandsDirPath = std::string(Path.str());
   }
 } else {
-  llvm::errs()
-  << "Path specified by --compile-commands-dir does not exist. The "
- "argument will be ignored.\n";
+  elog("Path

[clang-tools-extra] 1603470 - [clangd] Fix clangd-indexeer builds after D84697

[Kirill Bobyrev via cfe-commits]

Author: Kirill Bobyrev
Date: 2020-07-29T10:27:11+02:00
New Revision: 1603470e59a99a39ebdc4bf62a3a16c8c4ebea36

URL: 
https://github.com/llvm/llvm-project/commit/1603470e59a99a39ebdc4bf62a3a16c8c4ebea36
DIFF: 
https://github.com/llvm/llvm-project/commit/1603470e59a99a39ebdc4bf62a3a16c8c4ebea36.diff

LOG: [clangd] Fix clangd-indexeer builds after D84697

Some buildbots require explicit clangdSupport dependency:

http://lab.llvm.org:8011/builders/llvm-avr-linux/builds/3996/steps/build%20stage%201/logs/stdio

Added: 


Modified: 
clang-tools-extra/clangd/indexer/CMakeLists.txt

Removed: 




diff  --git a/clang-tools-extra/clangd/indexer/CMakeLists.txt 
b/clang-tools-extra/clangd/indexer/CMakeLists.txt
index edbced1410bb..ff110693c706 100644
--- a/clang-tools-extra/clangd/indexer/CMakeLists.txt
+++ b/clang-tools-extra/clangd/indexer/CMakeLists.txt
@@ -20,4 +20,5 @@ clang_target_link_libraries(clangd-indexer
 target_link_libraries(clangd-indexer
   PRIVATE
   clangDaemon
+  clangdSupport
 )



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] a262f0f - [clangd] Implement Relations request for remote index

[Kirill Bobyrev via cfe-commits]

Author: Kirill Bobyrev
Date: 2020-07-30T12:57:33+02:00
New Revision: a262f0fea46ce08008f3462c336c3d7107e98b27

URL: 
https://github.com/llvm/llvm-project/commit/a262f0fea46ce08008f3462c336c3d7107e98b27
DIFF: 
https://github.com/llvm/llvm-project/commit/a262f0fea46ce08008f3462c336c3d7107e98b27.diff

LOG: [clangd] Implement Relations request for remote index

This is the last missing bit in the core remote index implementation. The only
remaining bits are some API refactorings (replacing Optional with Expected and
being better at reporting errors).

Reviewed By: kadircet

Differential Revision: https://reviews.llvm.org/D84894

Added: 


Modified: 
clang-tools-extra/clangd/index/remote/Client.cpp
clang-tools-extra/clangd/index/remote/Index.proto
clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
clang-tools-extra/clangd/index/remote/marshalling/Marshalling.h
clang-tools-extra/clangd/index/remote/server/Server.cpp
clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp

Removed: 




diff  --git a/clang-tools-extra/clangd/index/remote/Client.cpp 
b/clang-tools-extra/clangd/index/remote/Client.cpp
index 5a33fd2eaf14..af58645d1795 100644
--- a/clang-tools-extra/clangd/index/remote/Client.cpp
+++ b/clang-tools-extra/clangd/index/remote/Client.cpp
@@ -91,11 +91,16 @@ class IndexClient : public clangd::SymbolIndex {
 return streamRPC(Request, &remote::SymbolIndex::Stub::Refs, Callback);
   }
 
-  // FIXME(kirillbobyrev): Implement this.
   void
-  relations(const clangd::RelationsRequest &,
-llvm::function_ref)
-  const {}
+  relations(const clangd::RelationsRequest &Request,
+llvm::function_ref
+Callback) const {
+streamRPC(Request, &remote::SymbolIndex::Stub::Relations,
+  // Unpack protobuf Relation.
+  [&](std::pair SubjectAndObject) {
+Callback(SubjectAndObject.first, SubjectAndObject.second);
+  });
+  }
 
   // IndexClient does not take any space since the data is stored on the
   // server.

diff  --git a/clang-tools-extra/clangd/index/remote/Index.proto 
b/clang-tools-extra/clangd/index/remote/Index.proto
index 99c4e3329d67..305164ffef77 100644
--- a/clang-tools-extra/clangd/index/remote/Index.proto
+++ b/clang-tools-extra/clangd/index/remote/Index.proto
@@ -18,6 +18,8 @@ service SymbolIndex {
   rpc FuzzyFind(FuzzyFindRequest) returns (stream FuzzyFindReply) {}
 
   rpc Refs(RefsRequest) returns (stream RefsReply) {}
+
+  rpc Relations(RelationsRequest) returns (stream RelationsReply) {}
 }
 
 message LookupRequest { repeated string ids = 1; }
@@ -114,3 +116,25 @@ message HeaderWithReferences {
   string header = 1;
   uint32 references = 2;
 }
+
+message RelationsRequest {
+  repeated string subjects = 1;
+  uint32 predicate = 2;
+  uint32 limit = 3;
+}
+
+// The response is a stream of reference messages, and one terminating has_more
+// message.
+message RelationsReply {
+  oneof kind {
+Relation stream_result = 1;
+bool final_result = 2; // HasMore
+  }
+}
+
+// This struct does not mirror clangd::Relation but rather the arguments of
+// SymbolIndex::relations callback.
+message Relation {
+  string subject_id = 1;
+  Symbol object = 2;
+}

diff  --git a/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp 
b/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
index b2085bc21f48..270e15347ee0 100644
--- a/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
+++ b/clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
@@ -10,6 +10,7 @@
 #include "Headers.h"
 #include "Index.pb.h"
 #include "Protocol.h"
+#include "index/Index.h"
 #include "index/Serialization.h"
 #include "index/Symbol.h"
 #include "index/SymbolID.h"
@@ -33,10 +34,10 @@ namespace remote {
 
 namespace {
 
-template 
-llvm::Expected> getIDs(MessageT *Message) {
+template 
+llvm::Expected> getIDs(IDRange IDs) {
   llvm::DenseSet Result;
-  for (const auto &ID : Message->ids()) {
+  for (const auto &ID : IDs) {
 auto SID = SymbolID::fromStr(StringRef(ID));
 if (!SID)
   return SID.takeError();
@@ -69,7 +70,7 @@ Marshaller::Marshaller(llvm::StringRef RemoteIndexRoot,
 llvm::Expected
 Marshaller::fromProtobuf(const LookupRequest *Message) {
   clangd::LookupRequest Req;
-  auto IDs = getIDs(Message);
+  auto IDs = getIDs(Message->ids());
   if (!IDs)
 return IDs.takeError();
   Req.IDs = std::move(*IDs);
@@ -100,7 +101,7 @@ Marshaller::fromProtobuf(const FuzzyFindRequest *Message) {
 llvm::Expected
 Marshaller::fromProtobuf(const RefsRequest *Message) {
   clangd::RefsRequest Req;
-  auto IDs = getIDs(Message);
+  auto IDs = getIDs(Message->ids());
   if (!IDs)
 return IDs.takeError();
   Req.IDs = std::move(*IDs);
@@ -110,6 +111,19 @@ Marshaller::fromProtobuf(const RefsRequest *Message) {
   return Req;
 }
 
+llvm::Expected
+Mar

[clang-tools-extra] c4b7bfd - [clangd] NFC: Spell out types in index callback arguments

[Kirill Bobyrev via cfe-commits]

Author: Kirill Bobyrev
Date: 2020-07-30T17:12:29+02:00
New Revision: c4b7bfdff65e5883bfe64248ac244b9fadf531ee

URL: 
https://github.com/llvm/llvm-project/commit/c4b7bfdff65e5883bfe64248ac244b9fadf531ee
DIFF: 
https://github.com/llvm/llvm-project/commit/c4b7bfdff65e5883bfe64248ac244b9fadf531ee.diff

LOG: [clangd] NFC: Spell out types in index callback arguments

Added: 


Modified: 
clang-tools-extra/clangd/index/remote/server/Server.cpp

Removed: 




diff  --git a/clang-tools-extra/clangd/index/remote/server/Server.cpp 
b/clang-tools-extra/clangd/index/remote/server/Server.cpp
index 077d58ea7db0..27e89cbbb48e 100644
--- a/clang-tools-extra/clangd/index/remote/server/Server.cpp
+++ b/clang-tools-extra/clangd/index/remote/server/Server.cpp
@@ -90,7 +90,7 @@ class RemoteIndexServer final : public SymbolIndex::Service {
 }
 unsigned Sent = 0;
 unsigned FailedToSend = 0;
-Index->lookup(*Req, [&](const auto &Item) {
+Index->lookup(*Req, [&](const clangd::Symbol &Item) {
   auto SerializedItem = ProtobufMarshaller->toProtobuf(Item);
   if (!SerializedItem) {
 ++FailedToSend;
@@ -121,7 +121,7 @@ class RemoteIndexServer final : public SymbolIndex::Service 
{
 }
 unsigned Sent = 0;
 unsigned FailedToSend = 0;
-bool HasMore = Index->fuzzyFind(*Req, [&](const auto &Item) {
+bool HasMore = Index->fuzzyFind(*Req, [&](const clangd::Symbol &Item) {
   auto SerializedItem = ProtobufMarshaller->toProtobuf(Item);
   if (!SerializedItem) {
 ++FailedToSend;
@@ -150,7 +150,7 @@ class RemoteIndexServer final : public SymbolIndex::Service 
{
 }
 unsigned Sent = 0;
 unsigned FailedToSend = 0;
-bool HasMore = Index->refs(*Req, [&](const auto &Item) {
+bool HasMore = Index->refs(*Req, [&](const clangd::Ref &Item) {
   auto SerializedItem = ProtobufMarshaller->toProtobuf(Item);
   if (!SerializedItem) {
 ++FailedToSend;



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] f058673 - [dexp] NFC: Change positional argument format

[Kirill Bobyrev via cfe-commits]

Author: Kirill Bobyrev
Date: 2020-04-15T13:28:07+02:00
New Revision: f05867339737bb203b64a95e096fee509313122d

URL: 
https://github.com/llvm/llvm-project/commit/f05867339737bb203b64a95e096fee509313122d
DIFF: 
https://github.com/llvm/llvm-project/commit/f05867339737bb203b64a95e096fee509313122d.diff

LOG: [dexp] NFC: Change positional argument format

Summary:
Before:

  USAGE: dexp [options] --index-path Path to the index

After:

  USAGE: dexp [options] 

Reviewers: sammccall

Reviewed By: sammccall

Subscribers: jkorous, arphaman, kadircet, usaxena95, cfe-commits

Tags: #clang

Differential Revision: https://reviews.llvm.org/D78089

Added: 


Modified: 
clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp

Removed: 




diff  --git a/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp 
b/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
index 9bdc88d7d886..ae49f9437211 100644
--- a/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
+++ b/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
@@ -26,8 +26,7 @@ namespace clang {
 namespace clangd {
 namespace {
 
-llvm::cl::opt IndexPath("index-path",
- llvm::cl::desc("Path to the index"),
+llvm::cl::opt IndexPath(llvm::cl::desc(""),
  llvm::cl::Positional, llvm::cl::Required);
 
 llvm::cl::opt



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] 348364b - [clangd] Don't produce snippets when completion location is followed by parenthesis

[Kirill Bobyrev via cfe-commits]

Author: Kirill Bobyrev
Date: 2020-06-09T13:59:29+02:00
New Revision: 348364bffd379e291501dc49b192cdd2adf83811

URL: 
https://github.com/llvm/llvm-project/commit/348364bffd379e291501dc49b192cdd2adf83811
DIFF: 
https://github.com/llvm/llvm-project/commit/348364bffd379e291501dc49b192cdd2adf83811.diff

LOG: [clangd] Don't produce snippets when completion location is followed by 
parenthesis

Summary:
Prevent a second pair of parenthesis from being added when there already is one
right after cursor.

Related issue and more context: https://github.com/clangd/clangd/issues/387

Reviewers: sammccall

Reviewed By: sammccall

Subscribers: ilya-biryukov, MaskRay, jkorous, arphaman, kadircet, usaxena95, 
cfe-commits

Tags: #clang

Differential Revision: https://reviews.llvm.org/D81380

Added: 


Modified: 
clang-tools-extra/clangd/CodeComplete.cpp
clang-tools-extra/clangd/unittests/CodeCompleteTests.cpp

Removed: 




diff  --git a/clang-tools-extra/clangd/CodeComplete.cpp 
b/clang-tools-extra/clangd/CodeComplete.cpp
index 3cfdac1c3bc8..cf79673d821e 100644
--- a/clang-tools-extra/clangd/CodeComplete.cpp
+++ b/clang-tools-extra/clangd/CodeComplete.cpp
@@ -45,10 +45,12 @@
 #include "clang/Basic/CharInfo.h"
 #include "clang/Basic/LangOptions.h"
 #include "clang/Basic/SourceLocation.h"
+#include "clang/Basic/TokenKinds.h"
 #include "clang/Format/Format.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Frontend/FrontendActions.h"
 #include "clang/Lex/ExternalPreprocessorSource.h"
+#include "clang/Lex/Lexer.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Lex/PreprocessorOptions.h"
 #include "clang/Sema/CodeCompleteConsumer.h"
@@ -255,10 +257,11 @@ struct CodeCompletionBuilder {
 const IncludeInserter &Includes,
 llvm::StringRef FileName,
 CodeCompletionContext::Kind ContextKind,
-const CodeCompleteOptions &Opts, bool GenerateSnippets)
+const CodeCompleteOptions &Opts,
+bool IsUsingDeclaration, tok::TokenKind NextTokenKind)
   : ASTCtx(ASTCtx), ExtractDocumentation(Opts.IncludeComments),
 EnableFunctionArgSnippets(Opts.EnableFunctionArgSnippets),
-GenerateSnippets(GenerateSnippets) {
+IsUsingDeclaration(IsUsingDeclaration), NextTokenKind(NextTokenKind) {
 add(C, SemaCCS);
 if (C.SemaResult) {
   assert(ASTCtx);
@@ -429,7 +432,13 @@ struct CodeCompletionBuilder {
   }
 
   std::string summarizeSnippet() const {
-if (!GenerateSnippets)
+if (IsUsingDeclaration)
+  return "";
+// Suppress function argument snippets if args are already present.
+if ((Completion.Kind == CompletionItemKind::Function ||
+ Completion.Kind == CompletionItemKind::Method ||
+ Completion.Kind == CompletionItemKind::Constructor) &&
+NextTokenKind == tok::l_paren)
   return "";
 auto *Snippet = onlyValue<&BundledEntry::SnippetSuffix>();
 if (!Snippet)
@@ -488,8 +497,10 @@ struct CodeCompletionBuilder {
   llvm::SmallVector Bundled;
   bool ExtractDocumentation;
   bool EnableFunctionArgSnippets;
-  /// When false, no snippets are generated argument lists.
-  bool GenerateSnippets;
+  // No snippets will be generated for using declarations and when the function
+  // arguments are already present.
+  bool IsUsingDeclaration;
+  tok::TokenKind NextTokenKind;
 };
 
 // Determine the symbol ID for a Sema code completion result, if possible.
@@ -1223,6 +1234,10 @@ class CodeCompleteFlow {
   CompletionRecorder *Recorder = nullptr;
   CodeCompletionContext::Kind CCContextKind = CodeCompletionContext::CCC_Other;
   bool IsUsingDeclaration = false;
+  // The snippets will not be generated if the token following completion
+  // location is an opening parenthesis (tok::l_paren) because this would add
+  // extra parenthesis.
+  tok::TokenKind NextTokenKind = tok::eof;
   // Counters for logging.
   int NSema = 0, NIndex = 0, NSemaAndIndex = 0, NIdent = 0;
   bool Incomplete = false; // Would more be available with a higher limit?
@@ -1277,6 +1292,11 @@ class CodeCompleteFlow {
   auto Style = getFormatStyleForFile(
   SemaCCInput.FileName, SemaCCInput.ParseInput.Contents,
   SemaCCInput.ParseInput.FSProvider->getFileSystem().get());
+  const auto NextToken = Lexer::findNextToken(
+  Recorder->CCSema->getPreprocessor().getCodeCompletionLoc(),
+  Recorder->CCSema->getSourceManager(), Recorder->CCSema->LangOpts);
+  if (NextToken)
+NextTokenKind = NextToken->getKind();
   // If preprocessor was run, inclusions from preprocessor callback should
   // already be added to Includes.
   Inserter.emplace(
@@ -1694,8 +1714,7 @@ class CodeCompleteFlow {
   if (!Builder)
 Builder.emplace(Recorder ? &Recorder->CCSema->getASTContext() : 
nullptr,
  

[clang] 550c456 - Revert "Prevent IR-gen from emitting consteval declarations"

[Kirill Bobyrev via cfe-commits]

Author: Kirill Bobyrev
Date: 2020-06-15T12:14:15+02:00
New Revision: 550c4562d18a6ab6ce60a062dceb163313da0ddd

URL: 
https://github.com/llvm/llvm-project/commit/550c4562d18a6ab6ce60a062dceb163313da0ddd
DIFF: 
https://github.com/llvm/llvm-project/commit/550c4562d18a6ab6ce60a062dceb163313da0ddd.diff

LOG: Revert "Prevent IR-gen from emitting consteval declarations"

This reverts commit 3bab88b7baa20b276faaee0aa7ca87f636c91877.

This patch causes test failures:
http://lab.llvm.org:8011/builders/clang-cmake-armv7-quick/builds/17260

Added: 


Modified: 
clang/lib/AST/Expr.cpp
clang/lib/AST/ExprConstant.cpp
clang/lib/CodeGen/CGCall.cpp
clang/lib/CodeGen/CGDecl.cpp
clang/lib/CodeGen/CGExpr.cpp
clang/lib/CodeGen/CGExprAgg.cpp
clang/lib/CodeGen/CGExprComplex.cpp
clang/lib/CodeGen/CGExprConstant.cpp
clang/lib/CodeGen/CGExprScalar.cpp
clang/lib/CodeGen/CGStmt.cpp
clang/lib/CodeGen/CodeGenFunction.h
clang/lib/CodeGen/CodeGenModule.cpp
clang/lib/CodeGen/ConstantEmitter.h
clang/lib/Sema/SemaExpr.cpp
clang/test/SemaCXX/cxx2a-consteval.cpp

Removed: 
clang/test/CodeGenCXX/cxx2a-consteval.cpp



diff  --git a/clang/lib/AST/Expr.cpp b/clang/lib/AST/Expr.cpp
index 27a12b880a7b..89eb8e9c0220 100644
--- a/clang/lib/AST/Expr.cpp
+++ b/clang/lib/AST/Expr.cpp
@@ -272,7 +272,6 @@ void ConstantExpr::DefaultInit(ResultStorageKind 
StorageKind) {
   ConstantExprBits.ResultKind = StorageKind;
   ConstantExprBits.APValueKind = APValue::None;
   ConstantExprBits.HasCleanup = false;
-  ConstantExprBits.IsImmediateInvocation = false;
   if (StorageKind == ConstantExpr::RSK_APValue)
 ::new (getTrailingObjects()) APValue();
 }

diff  --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp
index 870bf7696093..c8f1a92d0725 100644
--- a/clang/lib/AST/ExprConstant.cpp
+++ b/clang/lib/AST/ExprConstant.cpp
@@ -9986,6 +9986,8 @@ class IntExprEvaluator
   //Visitor Methods
   
//======//
 
+  bool VisitConstantExpr(const ConstantExpr *E);
+
   bool VisitIntegerLiteral(const IntegerLiteral *E) {
 return Success(E->getValue(), E);
   }
@@ -10767,6 +10769,13 @@ static bool tryEvaluateBuiltinObjectSize(const Expr 
*E, unsigned Type,
   return true;
 }
 
+bool IntExprEvaluator::VisitConstantExpr(const ConstantExpr *E) {
+  llvm::SaveAndRestore InConstantContext(Info.InConstantContext, true);
+  if (E->getResultAPValueKind() != APValue::None)
+return Success(E->getAPValueResult(), E);
+  return ExprEvaluatorBaseTy::VisitConstantExpr(E);
+}
+
 bool IntExprEvaluator::VisitCallExpr(const CallExpr *E) {
   if (unsigned BuiltinOp = E->getBuiltinCallee())
 return VisitBuiltinCallExpr(E, BuiltinOp);

diff  --git a/clang/lib/CodeGen/CGCall.cpp b/clang/lib/CodeGen/CGCall.cpp
index 78a0ecef4f3f..17282e2bafe6 100644
--- a/clang/lib/CodeGen/CGCall.cpp
+++ b/clang/lib/CodeGen/CGCall.cpp
@@ -1272,17 +1272,18 @@ static llvm::Value *CreateCoercedLoad(Address Src, 
llvm::Type *Ty,
 // store the elements rather than the aggregate to be more friendly to
 // fast-isel.
 // FIXME: Do we need to recurse here?
-void CodeGenFunction::EmitAggregateStore(llvm::Value *Val, Address Dest,
- bool DestIsVolatile) {
+static void BuildAggStore(CodeGenFunction &CGF, llvm::Value *Val,
+  Address Dest, bool DestIsVolatile) {
   // Prefer scalar stores to first-class aggregate stores.
-  if (llvm::StructType *STy = dyn_cast(Val->getType())) {
+  if (llvm::StructType *STy =
+dyn_cast(Val->getType())) {
 for (unsigned i = 0, e = STy->getNumElements(); i != e; ++i) {
-  Address EltPtr = Builder.CreateStructGEP(Dest, i);
-  llvm::Value *Elt = Builder.CreateExtractValue(Val, i);
-  Builder.CreateStore(Elt, EltPtr, DestIsVolatile);
+  Address EltPtr = CGF.Builder.CreateStructGEP(Dest, i);
+  llvm::Value *Elt = CGF.Builder.CreateExtractValue(Val, i);
+  CGF.Builder.CreateStore(Elt, EltPtr, DestIsVolatile);
 }
   } else {
-Builder.CreateStore(Val, Dest, DestIsVolatile);
+CGF.Builder.CreateStore(Val, Dest, DestIsVolatile);
   }
 }
 
@@ -1333,7 +1334,7 @@ static void CreateCoercedStore(llvm::Value *Src,
   // If store is legal, just bitcast the src pointer.
   if (SrcSize <= DstSize) {
 Dst = CGF.Builder.CreateElementBitCast(Dst, SrcTy);
-CGF.EmitAggregateStore(Src, Dst, DstIsVolatile);
+BuildAggStore(CGF, Src, Dst, DstIsVolatile);
   } else {
 // Otherwise do coercion through memory. This is stupid, but
 // simple.
@@ -5069,7 +5070,7 @@ RValue CodeGenFunction::EmitCall(const CGFunctionInfo 
&CallInfo,
 DestPtr = CreateMemTemp(RetTy, "agg.tmp");
 DestIsVolatile = false;
   }
-  EmitAggregateStore(CI, DestPtr, DestIsVolatile);
+  BuildAggStore(*this, C

[clang-tools-extra] 22a3e40 - [clangd] Set gRPC deadlines to all remote index requests

[Kirill Bobyrev via cfe-commits]

Author: Kirill Bobyrev
Date: 2020-07-01T12:46:29+02:00
New Revision: 22a3e4055f4382e8ebbf67705501e6969c6b398b

URL: 
https://github.com/llvm/llvm-project/commit/22a3e4055f4382e8ebbf67705501e6969c6b398b
DIFF: 
https://github.com/llvm/llvm-project/commit/22a3e4055f4382e8ebbf67705501e6969c6b398b.diff

LOG: [clangd] Set gRPC deadlines to all remote index requests

Summary: "TL;DR: Always set a deadline.", https://grpc.io/blog/deadlines/

Reviewers: sammccall

Reviewed By: sammccall

Subscribers: ilya-biryukov, MaskRay, jkorous, arphaman, kadircet, usaxena95, 
cfe-commits

Tags: #clang

Differential Revision: https://reviews.llvm.org/D82844

Added: 


Modified: 
clang-tools-extra/clangd/index/remote/Client.cpp

Removed: 




diff  --git a/clang-tools-extra/clangd/index/remote/Client.cpp 
b/clang-tools-extra/clangd/index/remote/Client.cpp
index 366a37a135a4..c43afd2573a9 100644
--- a/clang-tools-extra/clangd/index/remote/Client.cpp
+++ b/clang-tools-extra/clangd/index/remote/Client.cpp
@@ -15,6 +15,8 @@
 #include "support/Logger.h"
 #include "support/Trace.h"
 
+#include 
+
 namespace clang {
 namespace clangd {
 namespace remote {
@@ -25,7 +27,6 @@ class IndexClient : public clangd::SymbolIndex {
   using StreamingCall = std::unique_ptr> (
   remote::SymbolIndex::Stub::*)(grpc::ClientContext *, const RequestT &);
 
-  // FIXME(kirillbobyrev): Set deadlines for requests.
   template 
   bool streamRPC(ClangdRequestT Request,
@@ -35,6 +36,9 @@ class IndexClient : public clangd::SymbolIndex {
 trace::Span Tracer(RequestT::descriptor()->name());
 const auto RPCRequest = toProtobuf(Request);
 grpc::ClientContext Context;
+std::chrono::system_clock::time_point Deadline =
+std::chrono::system_clock::now() + DeadlineWaitingTime;
+Context.set_deadline(Deadline);
 auto Reader = (Stub.get()->*RPCCall)(&Context, RPCRequest);
 llvm::BumpPtrAllocator Arena;
 llvm::UniqueStringSaver Strings(Arena);
@@ -54,8 +58,11 @@ class IndexClient : public clangd::SymbolIndex {
   }
 
 public:
-  IndexClient(std::shared_ptr Channel)
-  : Stub(remote::SymbolIndex::NewStub(Channel)) {}
+  IndexClient(
+  std::shared_ptr Channel,
+  std::chrono::milliseconds DeadlineTime = std::chrono::milliseconds(1000))
+  : Stub(remote::SymbolIndex::NewStub(Channel)),
+DeadlineWaitingTime(DeadlineTime) {}
 
   void lookup(const clangd::LookupRequest &Request,
   llvm::function_ref Callback) const 
{
@@ -84,6 +91,8 @@ class IndexClient : public clangd::SymbolIndex {
 
 private:
   std::unique_ptr Stub;
+  // Each request will be terminated if it takes too long.
+  std::chrono::milliseconds DeadlineWaitingTime;
 };
 
 } // namespace



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] 93bb994 - [clangd] Implement path and URI translation for remote index

[Kirill Bobyrev via cfe-commits]

Author: Kirill Bobyrev
Date: 2020-07-09T12:52:55+02:00
New Revision: 93bb9944cb577f0529636dc5acfba16026740962

URL: 
https://github.com/llvm/llvm-project/commit/93bb9944cb577f0529636dc5acfba16026740962
DIFF: 
https://github.com/llvm/llvm-project/commit/93bb9944cb577f0529636dc5acfba16026740962.diff

LOG: [clangd] Implement path and URI translation for remote index

Reviewers: sammccall

Reviewed By: sammccall

Subscribers: ormris, ilya-biryukov, MaskRay, jkorous, arphaman, kadircet, 
usaxena95, cfe-commits

Tags: #clang

Differential Revision: https://reviews.llvm.org/D82938

Added: 


Modified: 
clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
clang-tools-extra/clangd/index/remote/Client.cpp
clang-tools-extra/clangd/index/remote/Client.h
clang-tools-extra/clangd/index/remote/Index.proto
clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
clang-tools-extra/clangd/index/remote/marshalling/Marshalling.h
clang-tools-extra/clangd/index/remote/server/Server.cpp
clang-tools-extra/clangd/index/remote/unimplemented/UnimplementedClient.cpp
clang-tools-extra/clangd/unittests/remote/MarshallingTests.cpp

Removed: 




diff  --git a/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp 
b/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
index 3e25da385d7a..6fc844c18931 100644
--- a/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
+++ b/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
@@ -32,6 +32,9 @@ llvm::cl::opt IndexLocation(
 llvm::cl::opt
 ExecCommand("c", llvm::cl::desc("Command to execute and then exit"));
 
+llvm::cl::opt ProjectRoot("project-root",
+   llvm::cl::desc("Path to the project"));
+
 static constexpr char Overview[] = R"(
 This is an **experimental** interactive tool to process user-provided search
 queries over given symbol collection obtained via clangd-indexer. The
@@ -326,7 +329,8 @@ struct {
 
 std::unique_ptr openIndex(llvm::StringRef Index) {
   return Index.startswith("remote:")
- ? remote::getClient(Index.drop_front(strlen("remote:")))
+ ? remote::getClient(Index.drop_front(strlen("remote:")),
+ ProjectRoot)
  : loadIndex(Index, /*UseDex=*/true);
 }
 

diff  --git a/clang-tools-extra/clangd/index/remote/Client.cpp 
b/clang-tools-extra/clangd/index/remote/Client.cpp
index c43afd2573a9..b46883193076 100644
--- a/clang-tools-extra/clangd/index/remote/Client.cpp
+++ b/clang-tools-extra/clangd/index/remote/Client.cpp
@@ -10,10 +10,12 @@
 
 #include "Client.h"
 #include "Index.grpc.pb.h"
+#include "index/Index.h"
 #include "index/Serialization.h"
 #include "marshalling/Marshalling.h"
 #include "support/Logger.h"
 #include "support/Trace.h"
+#include "llvm/ADT/StringRef.h"
 
 #include 
 
@@ -27,6 +29,16 @@ class IndexClient : public clangd::SymbolIndex {
   using StreamingCall = std::unique_ptr> (
   remote::SymbolIndex::Stub::*)(grpc::ClientContext *, const RequestT &);
 
+  template 
+  RequestT serializeRequest(ClangdRequestT Request) const {
+return toProtobuf(Request);
+  }
+
+  template <>
+  FuzzyFindRequest serializeRequest(clangd::FuzzyFindRequest Request) const {
+return toProtobuf(Request, ProjectRoot);
+  }
+
   template 
   bool streamRPC(ClangdRequestT Request,
@@ -34,7 +46,7 @@ class IndexClient : public clangd::SymbolIndex {
  CallbackT Callback) const {
 bool FinalResult = false;
 trace::Span Tracer(RequestT::descriptor()->name());
-const auto RPCRequest = toProtobuf(Request);
+const auto RPCRequest = serializeRequest(Request);
 grpc::ClientContext Context;
 std::chrono::system_clock::time_point Deadline =
 std::chrono::system_clock::now() + DeadlineWaitingTime;
@@ -48,10 +60,11 @@ class IndexClient : public clangd::SymbolIndex {
 FinalResult = Reply.final_result();
 continue;
   }
-  auto Sym = fromProtobuf(Reply.stream_result(), &Strings);
-  if (!Sym)
+  auto Response =
+  fromProtobuf(Reply.stream_result(), &Strings, ProjectRoot);
+  if (!Response)
 elog("Received invalid {0}", ReplyT::descriptor()->name());
-  Callback(*Sym);
+  Callback(*Response);
 }
 SPAN_ATTACH(Tracer, "status", Reader->Finish().ok());
 return FinalResult;
@@ -59,9 +72,9 @@ class IndexClient : public clangd::SymbolIndex {
 
 public:
   IndexClient(
-  std::shared_ptr Channel,
+  std::shared_ptr Channel, llvm::StringRef ProjectRoot,
   std::chrono::milliseconds DeadlineTime = std::chrono::milliseconds(1000))
-  : Stub(remote::SymbolIndex::NewStub(Channel)),
+  : Stub(remote::SymbolIndex::NewStub(Channel)), ProjectRoot(ProjectRoot),
 DeadlineWaitingTime(DeadlineTime) {}
 
   void lookup(const clangd::LookupRequest &Request,
@@ -86,22 +99,26 @@ class IndexClient : public clangd::SymbolIndex {
 llvm::f

[clang-tools-extra] d7d1af3 - [clangd] Fix DocumentSymbol ranges

[Kirill Bobyrev via cfe-commits]

Author: Kirill Bobyrev
Date: 2020-07-13T15:02:53+02:00
New Revision: d7d1af39168ce8afd041f3ae8db1d1fd3d4f70ac

URL: 
https://github.com/llvm/llvm-project/commit/d7d1af39168ce8afd041f3ae8db1d1fd3d4f70ac
DIFF: 
https://github.com/llvm/llvm-project/commit/d7d1af39168ce8afd041f3ae8db1d1fd3d4f70ac.diff

LOG: [clangd] Fix DocumentSymbol ranges

Summary:
DocumentSymbol ranges were not previously tested and, as a result, had invalid
end location. This patch addresses the issue.

Reviewers: sammccall

Reviewed By: sammccall

Subscribers: ilya-biryukov, MaskRay, jkorous, arphaman, kadircet, usaxena95, 
cfe-commits

Tags: #clang

Differential Revision: https://reviews.llvm.org/D83668

Added: 


Modified: 
clang-tools-extra/clangd/FindSymbols.cpp
clang-tools-extra/clangd/unittests/FindSymbolsTests.cpp

Removed: 




diff  --git a/clang-tools-extra/clangd/FindSymbols.cpp 
b/clang-tools-extra/clangd/FindSymbols.cpp
index 58e2ee1e21c7..f5d6a95aa713 100644
--- a/clang-tools-extra/clangd/FindSymbols.cpp
+++ b/clang-tools-extra/clangd/FindSymbols.cpp
@@ -136,17 +136,11 @@ llvm::Optional declToSym(ASTContext &Ctx, 
const NamedDecl &ND) {
   auto &SM = Ctx.getSourceManager();
 
   SourceLocation NameLoc = nameLocation(ND, SM);
-  // getFileLoc is a good choice for us, but we also need to make sure
-  // sourceLocToPosition won't switch files, so we call getSpellingLoc on top 
of
-  // that to make sure it does not switch files.
-  // FIXME: sourceLocToPosition should not switch files!
   SourceLocation BeginLoc = SM.getSpellingLoc(SM.getFileLoc(ND.getBeginLoc()));
   SourceLocation EndLoc = SM.getSpellingLoc(SM.getFileLoc(ND.getEndLoc()));
-  if (NameLoc.isInvalid() || BeginLoc.isInvalid() || EndLoc.isInvalid())
-return llvm::None;
-
-  if (!SM.isWrittenInMainFile(NameLoc) || !SM.isWrittenInMainFile(BeginLoc) ||
-  !SM.isWrittenInMainFile(EndLoc))
+  const auto SymbolRange =
+  toHalfOpenFileRange(SM, Ctx.getLangOpts(), {BeginLoc, EndLoc});
+  if (!SymbolRange)
 return llvm::None;
 
   Position NameBegin = sourceLocToPosition(SM, NameLoc);
@@ -162,8 +156,8 @@ llvm::Optional declToSym(ASTContext &Ctx, 
const NamedDecl &ND) {
   SI.name = printName(Ctx, ND);
   SI.kind = SK;
   SI.deprecated = ND.isDeprecated();
-  SI.range =
-  Range{sourceLocToPosition(SM, BeginLoc), sourceLocToPosition(SM, 
EndLoc)};
+  SI.range = Range{sourceLocToPosition(SM, SymbolRange->getBegin()),
+   sourceLocToPosition(SM, SymbolRange->getEnd())};
   SI.selectionRange = Range{NameBegin, NameEnd};
   if (!SI.range.contains(SI.selectionRange)) {
 // 'selectionRange' must be contained in 'range', so in cases where clang

diff  --git a/clang-tools-extra/clangd/unittests/FindSymbolsTests.cpp 
b/clang-tools-extra/clangd/unittests/FindSymbolsTests.cpp
index 31879e356ce0..07c42fcf2030 100644
--- a/clang-tools-extra/clangd/unittests/FindSymbolsTests.cpp
+++ b/clang-tools-extra/clangd/unittests/FindSymbolsTests.cpp
@@ -35,7 +35,7 @@ MATCHER_P(QName, Name, "") {
 }
 MATCHER_P(WithName, N, "") { return arg.name == N; }
 MATCHER_P(WithKind, Kind, "") { return arg.kind == Kind; }
-MATCHER_P(SymRange, Range, "") { return arg.location.range == Range; }
+MATCHER_P(SymRange, Range, "") { return arg.range == Range; }
 
 // GMock helpers for matching DocumentSymbol.
 MATCHER_P(SymNameRange, Range, "") { return arg.selectionRange == Range; }
@@ -712,6 +712,72 @@ TEST(DocumentSymbols, QualifiersWithTemplateArgs) {
WithName("Foo_type::method3")));
 }
 
+TEST(DocumentSymbolsTest, Ranges) {
+  TestTU TU;
+  Annotations Main(R"(
+  $foo[[int foo(bool Argument) {
+return 42;
+  }]]
+
+  $variable[[char GLOBAL_VARIABLE]];
+
+  $ns[[namespace ns {
+  $bar[[class Bar {
+  public:
+$ctor[[Bar() {}]]
+$dtor[[~Bar()]];
+
+  private:
+$field[[unsigned Baz]];
+
+$getbaz[[unsigned getBaz() { return Baz; }]]
+  }]];
+  }]] // namespace ns
+
+  $forwardclass[[class ForwardClassDecl]];
+
+  $struct[[struct StructDefinition {
+$structfield[[int *Pointer = nullptr]];
+  }]];
+  $forwardstruct[[struct StructDeclaration]];
+
+  $forwardfunc[[void forwardFunctionDecl(int Something)]];
+)");
+  TU.Code = Main.code().str();
+  EXPECT_THAT(
+  getSymbols(TU.build()),
+  UnorderedElementsAre(
+  AllOf(WithName("foo"), WithKind(SymbolKind::Function),
+SymRange(Main.range("foo"))),
+  AllOf(WithName("GLOBAL_VARIABLE"), WithKind(SymbolKind::Variable),
+SymRange(Main.range("variable"))),
+  AllOf(
+  WithName("ns"), WithKind(SymbolKind::Namespace),
+  SymRange(Main.range("ns")),
+  Children(AllOf(
+  WithName("Bar"), WithKind(SymbolKind::Class),
+  SymRange(Main.range("bar")),
+  Children(

[clang-tools-extra] 7a514c9 - [clangd] Implement textDocument/foldingRange

[Kirill Bobyrev via cfe-commits]

Author: Kirill Bobyrev
Date: 2020-07-14T09:28:42+02:00
New Revision: 7a514c9bf8f2513b57aee685879dd2c104381d99

URL: 
https://github.com/llvm/llvm-project/commit/7a514c9bf8f2513b57aee685879dd2c104381d99
DIFF: 
https://github.com/llvm/llvm-project/commit/7a514c9bf8f2513b57aee685879dd2c104381d99.diff

LOG: [clangd] Implement textDocument/foldingRange

Summary:
This patch introduces basic textDocument/foldingRange support. It relies on
textDocument/documentSymbols to collect all symbols and uses takes ranges
to create folds.

The next steps for textDocument/foldingRange support would be:

* Implementing FoldingRangeClientCapabilities and respecting respect client
  preferences
* Specifying folding range kind
* Migrating from DocumentSymbol implementation to custom RecursiveASTVisitor 
flow that will allow more flexibility
* Supporting more folding range types: comments, PP conditional regions, 
includes and other code regions (e.g. public/private/protected sections of 
classes, control flow statement bodies)

Tested: (Neo)Vim (coc-clangd) and VSCode.

Related issue: https://github.com/clangd/clangd/issues/310

Reviewers: sammccall

Reviewed By: sammccall

Subscribers: nridge, ilya-biryukov, MaskRay, jkorous, arphaman, kadircet, 
usaxena95, cfe-commits

Tags: #clang

Differential Revision: https://reviews.llvm.org/D82436

Added: 


Modified: 
clang-tools-extra/clangd/ClangdLSPServer.cpp
clang-tools-extra/clangd/ClangdLSPServer.h
clang-tools-extra/clangd/ClangdServer.cpp
clang-tools-extra/clangd/ClangdServer.h
clang-tools-extra/clangd/Protocol.cpp
clang-tools-extra/clangd/Protocol.h
clang-tools-extra/clangd/SemanticSelection.cpp
clang-tools-extra/clangd/SemanticSelection.h
clang-tools-extra/clangd/tool/ClangdMain.cpp
clang-tools-extra/clangd/unittests/SemanticSelectionTests.cpp

Removed: 




diff  --git a/clang-tools-extra/clangd/ClangdLSPServer.cpp 
b/clang-tools-extra/clangd/ClangdLSPServer.cpp
index b0aba886edbe..0408b0498488 100644
--- a/clang-tools-extra/clangd/ClangdLSPServer.cpp
+++ b/clang-tools-extra/clangd/ClangdLSPServer.cpp
@@ -637,6 +637,8 @@ void ClangdLSPServer::onInitialize(const InitializeParams 
&Params,
 ->insert(
 {"semanticHighlighting",
  llvm::json::Object{{"scopes", 
buildHighlightScopeLookupTable()}}});
+  if (ClangdServerOpts.FoldingRanges)
+Result.getObject("capabilities")->insert({"foldingRangeProvider", true});
   Reply(std::move(Result));
 }
 
@@ -929,7 +931,6 @@ void ClangdLSPServer::onDocumentFormatting(
 static std::vector
 flattenSymbolHierarchy(llvm::ArrayRef Symbols,
const URIForFile &FileURI) {
-
   std::vector Results;
   std::function Process =
   [&](const DocumentSymbol &S, llvm::Optional ParentName) 
{
@@ -968,6 +969,12 @@ void ClangdLSPServer::onDocumentSymbol(const 
DocumentSymbolParams &Params,
   });
 }
 
+void ClangdLSPServer::onFoldingRange(
+const FoldingRangeParams &Params,
+Callback> Reply) {
+  Server->foldingRanges(Params.textDocument.uri.file(), std::move(Reply));
+}
+
 static llvm::Optional asCommand(const CodeAction &Action) {
   Command Cmd;
   if (Action.command && Action.edit)
@@ -1395,6 +1402,8 @@ ClangdLSPServer::ClangdLSPServer(
   MsgHandler->bind("textDocument/documentLink", 
&ClangdLSPServer::onDocumentLink);
   MsgHandler->bind("textDocument/semanticTokens/full", 
&ClangdLSPServer::onSemanticTokens);
   MsgHandler->bind("textDocument/semanticTokens/full/delta", 
&ClangdLSPServer::onSemanticTokensDelta);
+  if (Opts.FoldingRanges)
+MsgHandler->bind("textDocument/foldingRange", 
&ClangdLSPServer::onFoldingRange);
   // clang-format on
 }
 

diff  --git a/clang-tools-extra/clangd/ClangdLSPServer.h 
b/clang-tools-extra/clangd/ClangdLSPServer.h
index a779e9036c4a..d0c0e814c641 100644
--- a/clang-tools-extra/clangd/ClangdLSPServer.h
+++ b/clang-tools-extra/clangd/ClangdLSPServer.h
@@ -87,6 +87,8 @@ class ClangdLSPServer : private ClangdServer::Callbacks {
   // otherwise.
   void onDocumentSymbol(const DocumentSymbolParams &,
 Callback);
+  void onFoldingRange(const FoldingRangeParams &,
+  Callback>);
   void onCodeAction(const CodeActionParams &, Callback);
   void onCompletion(const CompletionParams &, Callback);
   void onSignatureHelp(const TextDocumentPositionParams &,

diff  --git a/clang-tools-extra/clangd/ClangdServer.cpp 
b/clang-tools-extra/clangd/ClangdServer.cpp
index 5d99104dadaf..c33cdffcb0ca 100644
--- a/clang-tools-extra/clangd/ClangdServer.cpp
+++ b/clang-tools-extra/clangd/ClangdServer.cpp
@@ -674,6 +674,18 @@ void ClangdServer::documentSymbols(llvm::StringRef File,
TUScheduler::InvalidateOnUpdate);
 }
 
+void ClangdServer::foldingRanges(llvm::StringRef File,
+ Callback> CB) {
+  auto Action =
+  [CB = std::move(CB)](llvm:

[clang-tools-extra] c11c78a - [clangd] Use llvm::errs() instead of outs() for errors

[Kirill Bobyrev via cfe-commits]

Author: Kirill Bobyrev
Date: 2020-07-15T14:34:29+02:00
New Revision: c11c78a1bd0b3275bf845604aae3c94e97acceed

URL: 
https://github.com/llvm/llvm-project/commit/c11c78a1bd0b3275bf845604aae3c94e97acceed
DIFF: 
https://github.com/llvm/llvm-project/commit/c11c78a1bd0b3275bf845604aae3c94e97acceed.diff

LOG: [clangd] Use llvm::errs() instead of outs() for errors

Summary: errs() is more appropriate for error messages in dexp and 
clangd-index-server.

Reviewers: sammccall

Reviewed By: sammccall

Subscribers: ilya-biryukov, MaskRay, jkorous, arphaman, kadircet, usaxena95, 
cfe-commits

Tags: #clang

Differential Revision: https://reviews.llvm.org/D83827

Added: 


Modified: 
clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
clang-tools-extra/clangd/index/remote/server/Server.cpp

Removed: 




diff  --git a/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp 
b/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
index 6fc844c18931..80d87aa3f9f5 100644
--- a/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
+++ b/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
@@ -181,7 +181,7 @@ class Lookup : public Command {
 
   void run() override {
 if (ID.getNumOccurrences() == 0 && Name.getNumOccurrences() == 0) {
-  llvm::outs()
+  llvm::errs()
   << "Missing required argument: please provide id or -name.\n";
   return;
 }
@@ -189,7 +189,7 @@ class Lookup : public Command {
 if (ID.getNumOccurrences()) {
   auto SID = SymbolID::fromStr(ID);
   if (!SID) {
-llvm::outs() << llvm::toString(SID.takeError()) << "\n";
+llvm::errs() << llvm::toString(SID.takeError()) << "\n";
 return;
   }
   IDs.push_back(*SID);
@@ -205,7 +205,7 @@ class Lookup : public Command {
   llvm::outs() << toYAML(Sym);
 });
 if (!FoundSymbol)
-  llvm::outs() << "not found\n";
+  llvm::errs() << "not found\n";
   }
 };
 
@@ -228,7 +228,7 @@ class Refs : public Command {
 
   void run() override {
 if (ID.getNumOccurrences() == 0 && Name.getNumOccurrences() == 0) {
-  llvm::outs()
+  llvm::errs()
   << "Missing required argument: please provide id or -name.\n";
   return;
 }
@@ -236,14 +236,14 @@ class Refs : public Command {
 if (ID.getNumOccurrences()) {
   auto SID = SymbolID::fromStr(ID);
   if (!SID) {
-llvm::outs() << llvm::toString(SID.takeError()) << "\n";
+llvm::errs() << llvm::toString(SID.takeError()) << "\n";
 return;
   }
   IDs.push_back(*SID);
 } else {
   IDs = getSymbolIDsFromIndex(Name, Index);
   if (IDs.size() > 1) {
-llvm::outs() << llvm::formatv(
+llvm::errs() << llvm::formatv(
 "The name {0} is ambiguous, found {1} 
diff erent "
 "symbols. Please use id flag to disambiguate.\n",
 Name, IDs.size());
@@ -256,7 +256,7 @@ class Refs : public Command {
 Index->refs(RefRequest, [&RegexFilter](const Ref &R) {
   auto U = URI::parse(R.Location.FileURI);
   if (!U) {
-llvm::outs() << U.takeError();
+llvm::errs() << U.takeError();
 return;
   }
   if (RegexFilter.match(U->body()))
@@ -358,7 +358,7 @@ bool runCommand(std::string Request, const SymbolIndex 
&Index) {
   return Cmd.Implementation()->parseAndRun(FakeArgv, Cmd.Description,
Index);
   }
-  llvm::outs() << "Unknown command. Try 'help'.\n";
+  llvm::errs() << "Unknown command. Try 'help'.\n";
   return false;
 }
 
@@ -380,7 +380,7 @@ int main(int argc, const char *argv[]) {
  [&]() { Index = openIndex(IndexLocation); });
 
   if (!Index) {
-llvm::outs() << "Failed to open the index.\n";
+llvm::errs() << "Failed to open the index.\n";
 return -1;
   }
 

diff  --git a/clang-tools-extra/clangd/index/remote/server/Server.cpp 
b/clang-tools-extra/clangd/index/remote/server/Server.cpp
index 718d623a4845..fecd72806cbc 100644
--- a/clang-tools-extra/clangd/index/remote/server/Server.cpp
+++ b/clang-tools-extra/clangd/index/remote/server/Server.cpp
@@ -141,14 +141,14 @@ int main(int argc, char *argv[]) {
   llvm::sys::PrintStackTraceOnErrorSignal(argv[0]);
 
   if (!llvm::sys::path::is_absolute(IndexRoot)) {
-llvm::outs() << "Index root should be an absolute path.\n";
+llvm::errs() << "Index root should be an absolute path.\n";
 return -1;
   }
 
   std::unique_ptr Index = openIndex(IndexPath);
 
   if (!Index) {
-llvm::outs() << "Failed to open the index.\n";
+llvm::errs() << "Failed to open the index.\n";
 return -1;
   }
 



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits

[clang-tools-extra] cee80c0 - [clangd] Pull installed gRPC and introduce clangd-remote-(server|client)

[Kirill Bobyrev via cfe-commits]

Author: Kirill Bobyrev
Date: 2020-04-16T13:55:08+02:00
New Revision: cee80c0489e96c36269388b2aacd4da1c5714a66

URL: 
https://github.com/llvm/llvm-project/commit/cee80c0489e96c36269388b2aacd4da1c5714a66
DIFF: 
https://github.com/llvm/llvm-project/commit/cee80c0489e96c36269388b2aacd4da1c5714a66.diff

LOG: [clangd] Pull installed gRPC and introduce clangd-remote-(server|client)

Summary:
This patch allows using installed gRPC to build two simple tools which
currently provide the functionality of looking up the symbol by name.
remote-index-client is a simplified version of dexp which connects to
remote-index-server passes lookup requests.

I also significantly reduced the scope of this patch to prevent large changelist
and more bugs. The next steps would be:

* Extending Protocol for deep copies of Symbol and inherit RemoteIndex from
  Index to unify the interfaces
* Make remote-index-server more generic and merge the remote index client with
  dexp
* Modify Clangd to allow using remote index instead of the local one for all
  global index requests

Reviewers: sammccall

Reviewed By: sammccall

Subscribers: mgorny, ilya-biryukov, MaskRay, jkorous, arphaman, kadircet, 
usaxena95, cfe-commits

Tags: #clang

Differential Revision: https://reviews.llvm.org/D77794

Added: 
clang-tools-extra/clangd/index/remote/CMakeLists.txt
clang-tools-extra/clangd/index/remote/Index.proto
clang-tools-extra/clangd/index/remote/README.md
clang-tools-extra/clangd/index/remote/client/CMakeLists.txt
clang-tools-extra/clangd/index/remote/client/Client.cpp
clang-tools-extra/clangd/index/remote/server/CMakeLists.txt
clang-tools-extra/clangd/index/remote/server/Server.cpp
llvm/cmake/modules/FindGRPC.cmake

Modified: 
clang-tools-extra/clangd/CMakeLists.txt

Removed: 




diff  --git a/clang-tools-extra/clangd/CMakeLists.txt 
b/clang-tools-extra/clangd/CMakeLists.txt
index 7a9a4f7932ae..1c2cbf398b77 100644
--- a/clang-tools-extra/clangd/CMakeLists.txt
+++ b/clang-tools-extra/clangd/CMakeLists.txt
@@ -153,3 +153,12 @@ if(CLANG_INCLUDE_TESTS)
 add_subdirectory(test)
 add_subdirectory(unittests)
 endif()
+
+# FIXME(kirillbobyrev): Document this in the LLVM docs once remote index is 
stable.
+option(CLANGD_ENABLE_REMOTE "Use gRPC library to enable remote index support 
for Clangd" OFF)
+set(GRPC_INSTALL_PATH "" CACHE PATH "Path to gRPC library manual 
installation.")
+
+if (CLANGD_ENABLE_REMOTE)
+  include(FindGRPC)
+  add_subdirectory(index/remote)
+endif()

diff  --git a/clang-tools-extra/clangd/index/remote/CMakeLists.txt 
b/clang-tools-extra/clangd/index/remote/CMakeLists.txt
new file mode 100644
index ..b946958f3c5f
--- /dev/null
+++ b/clang-tools-extra/clangd/index/remote/CMakeLists.txt
@@ -0,0 +1,7 @@
+generate_grpc_protos(RemoteIndexProtos "Index.proto")
+
+include_directories("${CMAKE_CURRENT_BINARY_DIR}")
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/../../)
+
+add_subdirectory(client)
+add_subdirectory(server)

diff  --git a/clang-tools-extra/clangd/index/remote/Index.proto 
b/clang-tools-extra/clangd/index/remote/Index.proto
new file mode 100644
index ..399036ed72b7
--- /dev/null
+++ b/clang-tools-extra/clangd/index/remote/Index.proto
@@ -0,0 +1,19 @@
+//===--- Index.proto - Remote index Protocol Buffers definition 
---===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM 
Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===--===//
+
+syntax = "proto3";
+
+package clang.clangd.remote;
+
+service Index {
+  rpc Lookup(LookupRequest) returns (stream LookupReply) {}
+}
+
+message LookupRequest { string id = 1; }
+
+message LookupReply { string symbol_yaml = 1; }

diff  --git a/clang-tools-extra/clangd/index/remote/README.md 
b/clang-tools-extra/clangd/index/remote/README.md
new file mode 100644
index ..b56b2fc1011e
--- /dev/null
+++ b/clang-tools-extra/clangd/index/remote/README.md
@@ -0,0 +1,59 @@
+# Clangd remote index
+
+Clangd uses a global index for project-wide code completion, navigation and
+other features.  For large projects, building this can take many hours and
+keeping it loaded uses a lot of memory.
+
+To relieve that burden, we're building remote index — a global index
+served on a 
diff erent machine and shared between developers. This directory
+contains code that is used as Proof of Concept for the upcoming remote index
+feature.
+
+## Building
+
+This feature uses gRPC and Protobuf libraries, so you will need to install 
them.
+There are two ways of doing that.
+
+However you install dependencies, to enable this feature and build remote index
+tools you will need to set this CMake flag — `-DCLANGD_ENABLE_REMOTE=On`.
+
+### System-installed libraries
+
+On Debian-like s

[clang-tools-extra] 67b2dbd - [clangd] Extend dexp to support remote index

[Kirill Bobyrev via cfe-commits]

Author: Kirill Bobyrev
Date: 2020-04-24T13:59:21+02:00
New Revision: 67b2dbd5a33583fe148fd12f141e15301cfe99d1

URL: 
https://github.com/llvm/llvm-project/commit/67b2dbd5a33583fe148fd12f141e15301cfe99d1
DIFF: 
https://github.com/llvm/llvm-project/commit/67b2dbd5a33583fe148fd12f141e15301cfe99d1.diff

LOG: [clangd] Extend dexp to support remote index

Summary:
* Merge clangd-remote-client into dexp
* Implement `clangd::remote::IndexClient` that is derived from `SymbolIndex`
* Upgrade remote mode-related CMake infrastructure

Reviewers: sammccall

Reviewed By: sammccall

Subscribers: mgorny, ilya-biryukov, MaskRay, jkorous, arphaman, kadircet, 
usaxena95, cfe-commits

Tags: #clang

Differential Revision: https://reviews.llvm.org/D78521

Added: 
clang-tools-extra/clangd/index/remote/Client.cpp
clang-tools-extra/clangd/index/remote/Client.h
clang-tools-extra/clangd/index/remote/marshalling/CMakeLists.txt
clang-tools-extra/clangd/index/remote/marshalling/Marshalling.cpp
clang-tools-extra/clangd/index/remote/marshalling/Marshalling.h
clang-tools-extra/clangd/index/remote/unimplemented/CMakeLists.txt
clang-tools-extra/clangd/index/remote/unimplemented/UnimplementedClient.cpp

Modified: 
clang-tools-extra/clangd/CMakeLists.txt
clang-tools-extra/clangd/Features.inc.in
clang-tools-extra/clangd/index/Serialization.h
clang-tools-extra/clangd/index/YAMLSerialization.cpp
clang-tools-extra/clangd/index/dex/dexp/CMakeLists.txt
clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
clang-tools-extra/clangd/index/remote/CMakeLists.txt
clang-tools-extra/clangd/index/remote/Index.proto
clang-tools-extra/clangd/index/remote/server/CMakeLists.txt
clang-tools-extra/clangd/index/remote/server/Server.cpp

Removed: 
clang-tools-extra/clangd/index/remote/client/CMakeLists.txt
clang-tools-extra/clangd/index/remote/client/Client.cpp



diff  --git a/clang-tools-extra/clangd/CMakeLists.txt 
b/clang-tools-extra/clangd/CMakeLists.txt
index 1c2cbf398b77..124f087589d6 100644
--- a/clang-tools-extra/clangd/CMakeLists.txt
+++ b/clang-tools-extra/clangd/CMakeLists.txt
@@ -140,7 +140,6 @@ if (${CMAKE_SYSTEM_NAME} STREQUAL "Linux")
 endif()
 add_subdirectory(tool)
 add_subdirectory(indexer)
-add_subdirectory(index/dex/dexp)
 
 if (LLVM_INCLUDE_BENCHMARKS)
   add_subdirectory(benchmarks)
@@ -160,5 +159,6 @@ set(GRPC_INSTALL_PATH "" CACHE PATH "Path to gRPC library 
manual installation.")
 
 if (CLANGD_ENABLE_REMOTE)
   include(FindGRPC)
-  add_subdirectory(index/remote)
 endif()
+add_subdirectory(index/remote)
+add_subdirectory(index/dex/dexp)

diff  --git a/clang-tools-extra/clangd/Features.inc.in 
b/clang-tools-extra/clangd/Features.inc.in
index da75aa67a65b..6797232ddac7 100644
--- a/clang-tools-extra/clangd/Features.inc.in
+++ b/clang-tools-extra/clangd/Features.inc.in
@@ -1 +1,2 @@
 #define CLANGD_BUILD_XPC @CLANGD_BUILD_XPC@
+#define CLANGD_ENABLE_REMOTE @CLANGD_ENABLE_REMOTE@

diff  --git a/clang-tools-extra/clangd/index/Serialization.h 
b/clang-tools-extra/clangd/index/Serialization.h
index 47317c0401fc..99510630f1ca 100644
--- a/clang-tools-extra/clangd/index/Serialization.h
+++ b/clang-tools-extra/clangd/index/Serialization.h
@@ -77,6 +77,13 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, const 
IndexFileOut &O);
 std::string toYAML(const Symbol &);
 std::string toYAML(const std::pair> &);
 std::string toYAML(const Relation &);
+std::string toYAML(const Ref &);
+
+// Deserialize a single symbol from YAML.
+llvm::Expected symbolFromYAML(StringRef YAML,
+  llvm::UniqueStringSaver 
*Strings);
+llvm::Expected refFromYAML(StringRef YAML,
+llvm::UniqueStringSaver *Strings);
 
 // Build an in-memory static index from an index file.
 // The size should be relatively small, so data can be managed in memory.

diff  --git a/clang-tools-extra/clangd/index/YAMLSerialization.cpp 
b/clang-tools-extra/clangd/index/YAMLSerialization.cpp
index 79965ceb1634..fc515a17d100 100644
--- a/clang-tools-extra/clangd/index/YAMLSerialization.cpp
+++ b/clang-tools-extra/clangd/index/YAMLSerialization.cpp
@@ -517,5 +517,40 @@ std::string toYAML(const Relation &R) {
   return Buf;
 }
 
+std::string toYAML(const Ref &R) {
+  std::string Buf;
+  {
+llvm::raw_string_ostream OS(Buf);
+llvm::yaml::Output Yout(OS);
+Ref Reference = R; // copy: Yout<< requires mutability.
+Yout << Reference;
+  }
+  return Buf;
+}
+
+llvm::Expected
+symbolFromYAML(StringRef YAML, llvm::UniqueStringSaver *Strings) {
+  clangd::Symbol Deserialized;
+  llvm::yaml::Input YAMLInput(YAML, Strings);
+  if (YAMLInput.error())
+return llvm::make_error(
+llvm::formatv("Unable to deserialize Symbol from YAML: {0}", YAML),
+llvm::inconvertibleErrorCode());
+  YAMLInput >> Deserialized;
+  return Deserialized;
+}
+
+llvm::Expected 

[clang-tools-extra] 1ccfe47 - [clangd] Fix build when CLANGD_REMOTE is not enabled

[Kirill Bobyrev via cfe-commits]

Author: Kirill Bobyrev
Date: 2020-04-24T14:07:39+02:00
New Revision: 1ccfe475a753a93d647be78edff000b9d6667ff7

URL: 
https://github.com/llvm/llvm-project/commit/1ccfe475a753a93d647be78edff000b9d6667ff7
DIFF: 
https://github.com/llvm/llvm-project/commit/1ccfe475a753a93d647be78edff000b9d6667ff7.diff

LOG: [clangd] Fix build when CLANGD_REMOTE is not enabled

Added: 


Modified: 
clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp

Removed: 




diff  --git a/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp 
b/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
index 101367ce7c67..8738f9cd144c 100644
--- a/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
+++ b/clang-tools-extra/clangd/index/dex/dexp/Dexp.cpp
@@ -286,9 +286,9 @@ class Export : public Command {
   void run() {
 using namespace clang::clangd;
 // Read input file (as specified in global option)
-auto Buffer = llvm::MemoryBuffer::getFile(IndexPath);
+auto Buffer = llvm::MemoryBuffer::getFile(IndexLocation);
 if (!Buffer) {
-  llvm::errs() << llvm::formatv("Can't open {0}", IndexPath) << "\n";
+  llvm::errs() << llvm::formatv("Can't open {0}", IndexLocation) << "\n";
   return;
 }
 



___
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits