kbobyrev created this revision.
kbobyrev added reviewers: ioeric, ilya-biryukov, sammccall.
kbobyrev added a project: clang-tools-extra.
Herald added subscribers: kadircet, arphaman, jkorous, MaskRay.
This patch prints information about built index size estimation to verbose
logs. This is useful for optimizing memory usage of DexIndex and comparisons
with MemIndex.
https://reviews.llvm.org/D51154
Files:
clang-tools-extra/clangd/index/MemIndex.cpp
clang-tools-extra/clangd/index/MemIndex.h
clang-tools-extra/clangd/index/dex/DexIndex.cpp
clang-tools-extra/clangd/index/dex/DexIndex.h
Index: clang-tools-extra/clangd/index/dex/DexIndex.h
===================================================================
--- clang-tools-extra/clangd/index/dex/DexIndex.h
+++ clang-tools-extra/clangd/index/dex/DexIndex.h
@@ -58,6 +58,9 @@
Callback) const override;
private:
+ /// Returns estimate size of the index in megabytes.
+ size_t estimateMemoryUsage();
+
mutable std::mutex Mutex;
std::shared_ptr<std::vector<const Symbol *>> Symbols /*GUARDED_BY(Mutex)*/;
Index: clang-tools-extra/clangd/index/dex/DexIndex.cpp
===================================================================
--- clang-tools-extra/clangd/index/dex/DexIndex.cpp
+++ clang-tools-extra/clangd/index/dex/DexIndex.cpp
@@ -67,6 +67,9 @@
InvertedIndex = std::move(TempInvertedIndex);
SymbolQuality = std::move(TempSymbolQuality);
}
+
+ vlog("Built DexIndex with estimated memory usage {0} MB.",
+ estimateMemoryUsage());
}
std::unique_ptr<SymbolIndex> DexIndex::build(SymbolSlab Slab) {
@@ -171,6 +174,20 @@
log("findOccurrences is not implemented.");
}
+size_t DexIndex::estimateMemoryUsage() {
+ size_t Bytes = LookupTable.size() * sizeof(std::pair<SymbolID, const Symbol
*>);
+ Bytes += SymbolQuality.size() * sizeof(std::pair<const Symbol *, float>);
+ Bytes += InvertedIndex.size() * sizeof(Token);
+ {
+ std::lock_guard<std::mutex> Lock(Mutex);
+
+ for (const auto &P : InvertedIndex) {
+ Bytes += P.second.size() * sizeof(DocID);
+ }
+ }
+ return Bytes / (1000 * 1000);
+}
+
} // namespace dex
} // namespace clangd
} // namespace clang
Index: clang-tools-extra/clangd/index/MemIndex.h
===================================================================
--- clang-tools-extra/clangd/index/MemIndex.h
+++ clang-tools-extra/clangd/index/MemIndex.h
@@ -40,6 +40,9 @@
Callback) const override;
private:
+ /// Returns estimate size of the index in megabytes.
+ size_t estimateMemoryUsage();
+
std::shared_ptr<std::vector<const Symbol *>> Symbols;
// Index is a set of symbols that are deduplicated by symbol IDs.
// FIXME: build smarter index structure.
Index: clang-tools-extra/clangd/index/MemIndex.cpp
===================================================================
--- clang-tools-extra/clangd/index/MemIndex.cpp
+++ clang-tools-extra/clangd/index/MemIndex.cpp
@@ -26,6 +26,9 @@
Index = std::move(TempIndex);
Symbols = std::move(Syms); // Relase old symbols.
}
+
+ vlog("Built MemIndex with estimated memory usage {0} MB.",
+ estimateMemoryUsage());
}
std::unique_ptr<SymbolIndex> MemIndex::build(SymbolSlab Slab) {
@@ -98,5 +101,10 @@
&Snap->Pointers);
}
+size_t MemIndex::estimateMemoryUsage() {
+ size_t Bytes = Index.size() * sizeof(std::pair<SymbolID, const Symbol *>);
+ return Bytes / (1000 * 1000);
+}
+
} // namespace clangd
} // namespace clang
Index: clang-tools-extra/clangd/index/dex/DexIndex.h
===================================================================
--- clang-tools-extra/clangd/index/dex/DexIndex.h
+++ clang-tools-extra/clangd/index/dex/DexIndex.h
@@ -58,6 +58,9 @@
Callback) const override;
private:
+ /// Returns estimate size of the index in megabytes.
+ size_t estimateMemoryUsage();
+
mutable std::mutex Mutex;
std::shared_ptr<std::vector<const Symbol *>> Symbols /*GUARDED_BY(Mutex)*/;
Index: clang-tools-extra/clangd/index/dex/DexIndex.cpp
===================================================================
--- clang-tools-extra/clangd/index/dex/DexIndex.cpp
+++ clang-tools-extra/clangd/index/dex/DexIndex.cpp
@@ -67,6 +67,9 @@
InvertedIndex = std::move(TempInvertedIndex);
SymbolQuality = std::move(TempSymbolQuality);
}
+
+ vlog("Built DexIndex with estimated memory usage {0} MB.",
+ estimateMemoryUsage());
}
std::unique_ptr<SymbolIndex> DexIndex::build(SymbolSlab Slab) {
@@ -171,6 +174,20 @@
log("findOccurrences is not implemented.");
}
+size_t DexIndex::estimateMemoryUsage() {
+ size_t Bytes = LookupTable.size() * sizeof(std::pair<SymbolID, const Symbol *>);
+ Bytes += SymbolQuality.size() * sizeof(std::pair<const Symbol *, float>);
+ Bytes += InvertedIndex.size() * sizeof(Token);
+ {
+ std::lock_guard<std::mutex> Lock(Mutex);
+
+ for (const auto &P : InvertedIndex) {
+ Bytes += P.second.size() * sizeof(DocID);
+ }
+ }
+ return Bytes / (1000 * 1000);
+}
+
} // namespace dex
} // namespace clangd
} // namespace clang
Index: clang-tools-extra/clangd/index/MemIndex.h
===================================================================
--- clang-tools-extra/clangd/index/MemIndex.h
+++ clang-tools-extra/clangd/index/MemIndex.h
@@ -40,6 +40,9 @@
Callback) const override;
private:
+ /// Returns estimate size of the index in megabytes.
+ size_t estimateMemoryUsage();
+
std::shared_ptr<std::vector<const Symbol *>> Symbols;
// Index is a set of symbols that are deduplicated by symbol IDs.
// FIXME: build smarter index structure.
Index: clang-tools-extra/clangd/index/MemIndex.cpp
===================================================================
--- clang-tools-extra/clangd/index/MemIndex.cpp
+++ clang-tools-extra/clangd/index/MemIndex.cpp
@@ -26,6 +26,9 @@
Index = std::move(TempIndex);
Symbols = std::move(Syms); // Relase old symbols.
}
+
+ vlog("Built MemIndex with estimated memory usage {0} MB.",
+ estimateMemoryUsage());
}
std::unique_ptr<SymbolIndex> MemIndex::build(SymbolSlab Slab) {
@@ -98,5 +101,10 @@
&Snap->Pointers);
}
+size_t MemIndex::estimateMemoryUsage() {
+ size_t Bytes = Index.size() * sizeof(std::pair<SymbolID, const Symbol *>);
+ return Bytes / (1000 * 1000);
+}
+
} // namespace clangd
} // namespace clang
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
