Author: omtcyfz Date: Fri Aug 24 02:12:54 2018 New Revision: 340601 URL: http://llvm.org/viewvc/llvm-project?rev=340601&view=rev Log: [clangd] Log memory usage of DexIndex and MemIndex
This patch prints information about built index size estimation to verbose logs. This is useful for optimizing memory usage of DexIndex and comparisons with MemIndex. Reviewed by: sammccall Differential Revision: https://reviews.llvm.org/D51154 Modified: clang-tools-extra/trunk/clangd/index/FileIndex.cpp clang-tools-extra/trunk/clangd/index/FileIndex.h clang-tools-extra/trunk/clangd/index/Index.h clang-tools-extra/trunk/clangd/index/MemIndex.cpp clang-tools-extra/trunk/clangd/index/MemIndex.h clang-tools-extra/trunk/clangd/index/Merge.cpp clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp clang-tools-extra/trunk/clangd/index/dex/DexIndex.h clang-tools-extra/trunk/unittests/clangd/CodeCompleteTests.cpp Modified: clang-tools-extra/trunk/clangd/index/FileIndex.cpp URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/FileIndex.cpp?rev=340601&r1=340600&r2=340601&view=diff ============================================================================== --- clang-tools-extra/trunk/clangd/index/FileIndex.cpp (original) +++ clang-tools-extra/trunk/clangd/index/FileIndex.cpp Fri Aug 24 02:12:54 2018 @@ -118,5 +118,9 @@ void FileIndex::findOccurrences( log("findOccurrences is not implemented."); } +size_t FileIndex::estimateMemoryUsage() const { + return Index.estimateMemoryUsage(); +} + } // namespace clangd } // namespace clang Modified: clang-tools-extra/trunk/clangd/index/FileIndex.h URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/FileIndex.h?rev=340601&r1=340600&r2=340601&view=diff ============================================================================== --- clang-tools-extra/trunk/clangd/index/FileIndex.h (original) +++ clang-tools-extra/trunk/clangd/index/FileIndex.h Fri Aug 24 02:12:54 2018 @@ -81,6 +81,9 @@ public: void findOccurrences(const OccurrencesRequest &Req, llvm::function_ref<void(const SymbolOccurrence &)> Callback) const override; + + size_t estimateMemoryUsage() const override; + private: FileSymbols FSymbols; MemIndex Index; Modified: clang-tools-extra/trunk/clangd/index/Index.h URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Index.h?rev=340601&r1=340600&r2=340601&view=diff ============================================================================== --- clang-tools-extra/trunk/clangd/index/Index.h (original) +++ clang-tools-extra/trunk/clangd/index/Index.h Fri Aug 24 02:12:54 2018 @@ -385,6 +385,12 @@ public: virtual void findOccurrences( const OccurrencesRequest &Req, llvm::function_ref<void(const SymbolOccurrence &)> Callback) const = 0; + + /// Returns estimated size of index (in bytes). + // FIXME(kbobyrev): Currently, this only returns the size of index itself + // excluding the size of actual symbol slab index refers to. We should include + // both. + virtual size_t estimateMemoryUsage() const = 0; }; } // namespace clangd Modified: clang-tools-extra/trunk/clangd/index/MemIndex.cpp URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/MemIndex.cpp?rev=340601&r1=340600&r2=340601&view=diff ============================================================================== --- clang-tools-extra/trunk/clangd/index/MemIndex.cpp (original) +++ clang-tools-extra/trunk/clangd/index/MemIndex.cpp Fri Aug 24 02:12:54 2018 @@ -26,6 +26,9 @@ void MemIndex::build(std::shared_ptr<std Index = std::move(TempIndex); Symbols = std::move(Syms); // Relase old symbols. } + + vlog("Built MemIndex with estimated memory usage {0} bytes.", + estimateMemoryUsage()); } std::unique_ptr<SymbolIndex> MemIndex::build(SymbolSlab Slab) { @@ -98,5 +101,10 @@ getSymbolsFromSlab(SymbolSlab Slab) { &Snap->Pointers); } +size_t MemIndex::estimateMemoryUsage() const { + std::lock_guard<std::mutex> Lock(Mutex); + return Index.getMemorySize(); +} + } // namespace clangd } // namespace clang Modified: clang-tools-extra/trunk/clangd/index/MemIndex.h URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/MemIndex.h?rev=340601&r1=340600&r2=340601&view=diff ============================================================================== --- clang-tools-extra/trunk/clangd/index/MemIndex.h (original) +++ clang-tools-extra/trunk/clangd/index/MemIndex.h Fri Aug 24 02:12:54 2018 @@ -39,7 +39,10 @@ public: llvm::function_ref<void(const SymbolOccurrence &)> Callback) const override; + size_t estimateMemoryUsage() const override; + private: + std::shared_ptr<std::vector<const Symbol *>> Symbols; // Index is a set of symbols that are deduplicated by symbol IDs. // FIXME: build smarter index structure. Modified: clang-tools-extra/trunk/clangd/index/Merge.cpp URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/Merge.cpp?rev=340601&r1=340600&r2=340601&view=diff ============================================================================== --- clang-tools-extra/trunk/clangd/index/Merge.cpp (original) +++ clang-tools-extra/trunk/clangd/index/Merge.cpp Fri Aug 24 02:12:54 2018 @@ -84,6 +84,10 @@ class MergedIndex : public SymbolIndex { log("findOccurrences is not implemented."); } + size_t estimateMemoryUsage() const override { + return Dynamic->estimateMemoryUsage() + Static->estimateMemoryUsage(); + } + private: const SymbolIndex *Dynamic, *Static; }; Modified: clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp?rev=340601&r1=340600&r2=340601&view=diff ============================================================================== --- clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp (original) +++ clang-tools-extra/trunk/clangd/index/dex/DexIndex.cpp Fri Aug 24 02:12:54 2018 @@ -67,6 +67,9 @@ void DexIndex::build(std::shared_ptr<std InvertedIndex = std::move(TempInvertedIndex); SymbolQuality = std::move(TempSymbolQuality); } + + vlog("Built DexIndex with estimated memory usage {0} bytes.", + estimateMemoryUsage()); } std::unique_ptr<SymbolIndex> DexIndex::build(SymbolSlab Slab) { @@ -171,6 +174,20 @@ void DexIndex::findOccurrences( log("findOccurrences is not implemented."); } +size_t DexIndex::estimateMemoryUsage() const { + std::lock_guard<std::mutex> Lock(Mutex); + + size_t Bytes = + LookupTable.size() * sizeof(std::pair<SymbolID, const Symbol *>); + Bytes += SymbolQuality.size() * sizeof(std::pair<const Symbol *, float>); + Bytes += InvertedIndex.size() * sizeof(Token); + + for (const auto &P : InvertedIndex) { + Bytes += P.second.size() * sizeof(DocID); + } + return Bytes; +} + } // namespace dex } // namespace clangd } // namespace clang Modified: clang-tools-extra/trunk/clangd/index/dex/DexIndex.h URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/DexIndex.h?rev=340601&r1=340600&r2=340601&view=diff ============================================================================== --- clang-tools-extra/trunk/clangd/index/dex/DexIndex.h (original) +++ clang-tools-extra/trunk/clangd/index/dex/DexIndex.h Fri Aug 24 02:12:54 2018 @@ -57,7 +57,10 @@ public: llvm::function_ref<void(const SymbolOccurrence &)> Callback) const override; + size_t estimateMemoryUsage() const override; + private: + mutable std::mutex Mutex; std::shared_ptr<std::vector<const Symbol *>> Symbols /*GUARDED_BY(Mutex)*/; Modified: clang-tools-extra/trunk/unittests/clangd/CodeCompleteTests.cpp URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/unittests/clangd/CodeCompleteTests.cpp?rev=340601&r1=340600&r2=340601&view=diff ============================================================================== --- clang-tools-extra/trunk/unittests/clangd/CodeCompleteTests.cpp (original) +++ clang-tools-extra/trunk/unittests/clangd/CodeCompleteTests.cpp Fri Aug 24 02:12:54 2018 @@ -923,6 +923,10 @@ public: llvm::function_ref<void(const SymbolOccurrence &)> Callback) const override {} + // This is incorrect, but IndexRequestCollector is not an actual index and it + // isn't used in production code. + size_t estimateMemoryUsage() const override { return 0; } + const std::vector<FuzzyFindRequest> allRequests() const { return Requests; } private: _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
