kbobyrev updated this revision to Diff 163786.
kbobyrev marked 3 inline comments as done.
kbobyrev added a comment.
Rebase on top of recent revisions.
https://reviews.llvm.org/D51539
Files:
clang-tools-extra/clangd/index/FileIndex.cpp
clang-tools-extra/clangd/index/FileIndex.h
clang-tools-extra/clangd/index/MemIndex.cpp
clang-tools-extra/clangd/index/MemIndex.h
clang-tools-extra/clangd/index/dex/DexIndex.cpp
clang-tools-extra/clangd/index/dex/DexIndex.h
Index: clang-tools-extra/clangd/index/dex/DexIndex.h
===================================================================
--- clang-tools-extra/clangd/index/dex/DexIndex.h
+++ clang-tools-extra/clangd/index/dex/DexIndex.h
@@ -40,25 +40,29 @@
public:
// All symbols must outlive this index.
template <typename Range>
- DexIndex(Range &&Symbols, llvm::ArrayRef<std::string> URISchemes)
- : URISchemes(URISchemes) {
+ DexIndex(Range &&Symbols, llvm::ArrayRef<std::string> URISchemes,
+ size_t BackingMemory = 0)
+ : BackingMemory(BackingMemory), URISchemes(URISchemes) {
for (auto &&Sym : Symbols)
this->Symbols.push_back(&Sym);
buildIndex();
}
// Symbols are owned by BackingData, Index takes ownership.
template <typename Range, typename Payload>
DexIndex(Range &&Symbols, Payload &&BackingData,
- llvm::ArrayRef<std::string> URISchemes)
- : DexIndex(std::forward<Range>(Symbols), URISchemes) {
+ llvm::ArrayRef<std::string> URISchemes, size_t BackingMemory = 0)
+ : DexIndex(std::forward<Range>(Symbols), URISchemes, BackingMemory) {
KeepAlive = std::shared_ptr<void>(
std::make_shared<Payload>(std::move(BackingData)), nullptr);
}
/// Builds an index from a slab. The index takes ownership of the slab.
static std::unique_ptr<SymbolIndex>
build(SymbolSlab Slab, llvm::ArrayRef<std::string> URISchemes) {
- return llvm::make_unique<DexIndex>(Slab, std::move(Slab), URISchemes);
+ // Store SymbolSlab size before it's moved.
+ size_t BackingMemory = Slab.size();
+ return llvm::make_unique<DexIndex>(Slab, std::move(Slab), URISchemes,
+ BackingMemory);
}
bool
@@ -90,6 +94,10 @@
/// during the fuzzyFind process.
llvm::DenseMap<Token, PostingList> InvertedIndex;
std::shared_ptr<void> KeepAlive; // poor man's move-only std::any
+ /// When paired with the SymbolSlab, the index owns underlying SymbolSlab and
+ /// keeps it alive. BackingMemory is the only way to know the size of that
+ /// Slab.
+ size_t BackingMemory;
const std::vector<std::string> URISchemes;
};
Index: clang-tools-extra/clangd/index/dex/DexIndex.cpp
===================================================================
--- clang-tools-extra/clangd/index/dex/DexIndex.cpp
+++ clang-tools-extra/clangd/index/dex/DexIndex.cpp
@@ -213,15 +213,14 @@
}
size_t DexIndex::estimateMemoryUsage() const {
- size_t Bytes =
- LookupTable.size() * sizeof(std::pair<SymbolID, const Symbol *>);
- Bytes += SymbolQuality.size() * sizeof(std::pair<const Symbol *, float>);
- Bytes += InvertedIndex.size() * sizeof(Token);
+ size_t Bytes = LookupTable.getMemorySize();
+ Bytes += SymbolQuality.size() * sizeof(float);
+ Bytes += InvertedIndex.getMemorySize();
for (const auto &P : InvertedIndex) {
Bytes += P.second.size() * sizeof(DocID);
}
- return Bytes;
+ return Bytes + BackingMemory;
}
} // namespace dex
Index: clang-tools-extra/clangd/index/MemIndex.h
===================================================================
--- clang-tools-extra/clangd/index/MemIndex.h
+++ clang-tools-extra/clangd/index/MemIndex.h
@@ -28,15 +28,18 @@
// All symbols and occurrences must outlive this index.
// TODO: find a better type for Occurrences here.
template <typename SymbolRange>
- MemIndex(SymbolRange &&Symbols, OccurrenceMap Occurrences)
- : Occurrences(std::move(Occurrences)) {
+ MemIndex(SymbolRange &&Symbols, OccurrenceMap Occurrences,
+ size_t BackingMemory = 0)
+ : Occurrences(std::move(Occurrences)), BackingMemory(BackingMemory) {
for (const Symbol &S : Symbols)
Index[S.ID] = &S;
}
// Symbols are owned by BackingData, Index takes ownership.
template <typename Range, typename Payload>
- MemIndex(Range &&Symbols, OccurrenceMap Occurrences, Payload &&BackingData)
- : MemIndex(std::forward<Range>(Symbols), std::move(Occurrences)) {
+ MemIndex(Range &&Symbols, OccurrenceMap Occurrences, Payload &&BackingData,
+ size_t BackingMemory = 0)
+ : MemIndex(std::forward<Range>(Symbols), std::move(Occurrences),
+ BackingMemory) {
KeepAlive = std::shared_ptr<void>(
std::make_shared<Payload>(std::move(BackingData)), nullptr);
}
@@ -64,6 +67,10 @@
// A map from symbol ID to symbol occurrences, support query by IDs.
OccurrenceMap Occurrences;
std::shared_ptr<void> KeepAlive; // poor man's move-only std::any
+ /// When paired with the SymbolSlab, the index owns underlying SymbolSlab and
+ /// keeps it alive. BackingMemory is the only way to know the size of that
+ /// Slab.
+ size_t BackingMemory;
};
} // namespace clangd
Index: clang-tools-extra/clangd/index/MemIndex.cpp
===================================================================
--- clang-tools-extra/clangd/index/MemIndex.cpp
+++ clang-tools-extra/clangd/index/MemIndex.cpp
@@ -23,8 +23,10 @@
for (const auto &Occurrence : SymbolAndOccurrences.second)
Entry.push_back(&Occurrence);
}
+ size_t BackingData = Slab.bytes();
auto Data = std::make_pair(std::move(Slab), std::move(Occurrences));
- return llvm::make_unique<MemIndex>(Data.first, std::move(M), std::move(Data));
+ return llvm::make_unique<MemIndex>(Data.first, std::move(M), std::move(Data),
+ BackingData);
}
bool MemIndex::fuzzyFind(
@@ -82,7 +84,7 @@
}
size_t MemIndex::estimateMemoryUsage() const {
- return Index.getMemorySize();
+ return Index.getMemorySize() + BackingMemory;
}
} // namespace clangd
Index: clang-tools-extra/clangd/index/FileIndex.h
===================================================================
--- clang-tools-extra/clangd/index/FileIndex.h
+++ clang-tools-extra/clangd/index/FileIndex.h
@@ -47,6 +47,8 @@
// The index keeps the symbols alive.
std::unique_ptr<SymbolIndex> buildMemIndex();
+ size_t estimateMemoryUsage() const;
+
private:
mutable std::mutex Mutex;
@@ -73,6 +75,8 @@
update(PathRef Path, ASTContext *AST, std::shared_ptr<Preprocessor> PP,
llvm::Optional<llvm::ArrayRef<Decl *>> TopLevelDecls = llvm::None);
+ size_t estimateMemoryUsage() const;
+
private:
// Only update() should swap the index.
using SwapIndex::reset;
Index: clang-tools-extra/clangd/index/FileIndex.cpp
===================================================================
--- clang-tools-extra/clangd/index/FileIndex.cpp
+++ clang-tools-extra/clangd/index/FileIndex.cpp
@@ -113,7 +113,15 @@
// Index must keep the slabs alive.
return llvm::make_unique<MemIndex>(
llvm::make_pointee_range(AllSymbols), std::move(AllOccurrences),
- std::make_pair(std::move(Slabs), std::move(OccurrenceSlabs)));
+ std::make_pair(std::move(Slabs), std::move(OccurrenceSlabs)),
+ /*BackingMemory=*/0);
+}
+
+size_t FileSymbols::estimateMemoryUsage() const {
+ size_t Result = 0;
+ for (const auto &FileAndSlab : FileToSlabs)
+ Result += FileAndSlab.second->bytes();
+ return Result;
}
void FileIndex::update(PathRef Path, ASTContext *AST,
@@ -132,5 +140,9 @@
reset(FSymbols.buildMemIndex());
}
+size_t FileIndex::estimateMemoryUsage() const {
+ return FSymbols.estimateMemoryUsage();
+}
+
} // namespace clangd
} // namespace clang
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
