Author: omtcyfz Date: Mon Sep 10 04:46:07 2018 New Revision: 341800 URL: http://llvm.org/viewvc/llvm-project?rev=341800&view=rev Log: [clangd] Add symbol slab size to index memory consumption estimates
Currently, `SymbolIndex::estimateMemoryUsage()` returns the "overhead" estimate, i.e. the estimate of the Index data structure excluding backing data (such as Symbol Slab and Reference Slab). This patch propagates information about paired data size where necessary. Reviewed By: ioeric, sammccall Differential Revision: https://reviews.llvm.org/D51539 Modified: clang-tools-extra/trunk/clangd/index/FileIndex.cpp clang-tools-extra/trunk/clangd/index/MemIndex.cpp clang-tools-extra/trunk/clangd/index/MemIndex.h clang-tools-extra/trunk/clangd/index/dex/Dex.cpp clang-tools-extra/trunk/clangd/index/dex/Dex.h clang-tools-extra/trunk/unittests/clangd/IndexTests.cpp Modified: clang-tools-extra/trunk/clangd/index/FileIndex.cpp URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/FileIndex.cpp?rev=341800&r1=341799&r2=341800&view=diff ============================================================================== --- clang-tools-extra/trunk/clangd/index/FileIndex.cpp (original) +++ clang-tools-extra/trunk/clangd/index/FileIndex.cpp Mon Sep 10 04:46:07 2018 @@ -127,11 +127,18 @@ std::unique_ptr<SymbolIndex> FileSymbols } } + size_t StorageSize = RefsStorage.size() * sizeof(Ref); + for (const auto &Slab : SymbolSlabs) + StorageSize += Slab->bytes(); + for (const auto &RefSlab : RefSlabs) + StorageSize += RefSlab->bytes(); + // Index must keep the slabs and contiguous ranges alive. return llvm::make_unique<MemIndex>( llvm::make_pointee_range(AllSymbols), std::move(AllRefs), std::make_tuple(std::move(SymbolSlabs), std::move(RefSlabs), - std::move(RefsStorage))); + std::move(RefsStorage)), + StorageSize); } void FileIndex::update(PathRef Path, ASTContext *AST, Modified: clang-tools-extra/trunk/clangd/index/MemIndex.cpp URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/MemIndex.cpp?rev=341800&r1=341799&r2=341800&view=diff ============================================================================== --- clang-tools-extra/trunk/clangd/index/MemIndex.cpp (original) +++ clang-tools-extra/trunk/clangd/index/MemIndex.cpp Mon Sep 10 04:46:07 2018 @@ -16,8 +16,11 @@ namespace clang { namespace clangd { std::unique_ptr<SymbolIndex> MemIndex::build(SymbolSlab Slab, RefSlab Refs) { + // Store Slab size before it is moved. + const auto BackingDataSize = Slab.bytes() + Refs.bytes(); auto Data = std::make_pair(std::move(Slab), std::move(Refs)); - return llvm::make_unique<MemIndex>(Data.first, Data.second, std::move(Data)); + return llvm::make_unique<MemIndex>(Data.first, Data.second, std::move(Data), + BackingDataSize); } bool MemIndex::fuzzyFind( @@ -70,7 +73,7 @@ void MemIndex::refs(const RefsRequest &R } size_t MemIndex::estimateMemoryUsage() const { - return Index.getMemorySize(); + return Index.getMemorySize() + Refs.getMemorySize() + BackingDataSize; } } // namespace clangd Modified: clang-tools-extra/trunk/clangd/index/MemIndex.h URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/MemIndex.h?rev=341800&r1=341799&r2=341800&view=diff ============================================================================== --- clang-tools-extra/trunk/clangd/index/MemIndex.h (original) +++ clang-tools-extra/trunk/clangd/index/MemIndex.h Mon Sep 10 04:46:07 2018 @@ -30,11 +30,13 @@ public: } // Symbols are owned by BackingData, Index takes ownership. template <typename SymbolRange, typename RefRange, typename Payload> - MemIndex(SymbolRange &&Symbols, RefRange &&Refs, Payload &&BackingData) + MemIndex(SymbolRange &&Symbols, RefRange &&Refs, Payload &&BackingData, + size_t BackingDataSize) : MemIndex(std::forward<SymbolRange>(Symbols), std::forward<RefRange>(Refs)) { KeepAlive = std::shared_ptr<void>( std::make_shared<Payload>(std::move(BackingData)), nullptr); + this->BackingDataSize = BackingDataSize; } /// Builds an index from slabs. The index takes ownership of the data. @@ -58,6 +60,8 @@ private: // A map from symbol ID to symbol refs, support query by IDs. llvm::DenseMap<SymbolID, llvm::ArrayRef<Ref>> Refs; std::shared_ptr<void> KeepAlive; // poor man's move-only std::any + // Size of memory retained by KeepAlive. + size_t BackingDataSize = 0; }; } // namespace clangd Modified: clang-tools-extra/trunk/clangd/index/dex/Dex.cpp URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/Dex.cpp?rev=341800&r1=341799&r2=341800&view=diff ============================================================================== --- clang-tools-extra/trunk/clangd/index/dex/Dex.cpp (original) +++ clang-tools-extra/trunk/clangd/index/dex/Dex.cpp Mon Sep 10 04:46:07 2018 @@ -227,15 +227,13 @@ void Dex::refs(const RefsRequest &Req, } size_t Dex::estimateMemoryUsage() const { - size_t Bytes = - LookupTable.size() * sizeof(std::pair<SymbolID, const Symbol *>); - Bytes += SymbolQuality.size() * sizeof(std::pair<const Symbol *, float>); - Bytes += InvertedIndex.size() * sizeof(Token); - - for (const auto &P : InvertedIndex) { + size_t Bytes = Symbols.size() * sizeof(const Symbol *); + Bytes += SymbolQuality.size() * sizeof(float); + Bytes += LookupTable.getMemorySize(); + Bytes += InvertedIndex.getMemorySize(); + for (const auto &P : InvertedIndex) Bytes += P.second.size() * sizeof(DocID); - } - return Bytes; + return Bytes + BackingDataSize; } std::vector<std::string> generateProximityURIs(llvm::StringRef URIPath) { Modified: clang-tools-extra/trunk/clangd/index/dex/Dex.h URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/clangd/index/dex/Dex.h?rev=341800&r1=341799&r2=341800&view=diff ============================================================================== --- clang-tools-extra/trunk/clangd/index/dex/Dex.h (original) +++ clang-tools-extra/trunk/clangd/index/dex/Dex.h Mon Sep 10 04:46:07 2018 @@ -55,17 +55,21 @@ public: } // Symbols are owned by BackingData, Index takes ownership. template <typename Range, typename Payload> - Dex(Range &&Symbols, Payload &&BackingData, + Dex(Range &&Symbols, Payload &&BackingData, size_t BackingDataSize, llvm::ArrayRef<std::string> URISchemes) : Dex(std::forward<Range>(Symbols), URISchemes) { KeepAlive = std::shared_ptr<void>( std::make_shared<Payload>(std::move(BackingData)), nullptr); + this->BackingDataSize = BackingDataSize; } /// Builds an index from a slab. The index takes ownership of the slab. static std::unique_ptr<SymbolIndex> build(SymbolSlab Slab, llvm::ArrayRef<std::string> URISchemes) { - return llvm::make_unique<Dex>(Slab, std::move(Slab), URISchemes); + // Store Slab size before it is moved. + const auto BackingDataSize = Slab.bytes(); + return llvm::make_unique<Dex>(Slab, std::move(Slab), BackingDataSize, + URISchemes); } bool @@ -96,6 +100,8 @@ private: /// during the fuzzyFind process. llvm::DenseMap<Token, PostingList> InvertedIndex; std::shared_ptr<void> KeepAlive; // poor man's move-only std::any + // Size of memory retained by KeepAlive. + size_t BackingDataSize = 0; std::vector<std::string> URISchemes; }; Modified: clang-tools-extra/trunk/unittests/clangd/IndexTests.cpp URL: http://llvm.org/viewvc/llvm-project/clang-tools-extra/trunk/unittests/clangd/IndexTests.cpp?rev=341800&r1=341799&r2=341800&view=diff ============================================================================== --- clang-tools-extra/trunk/unittests/clangd/IndexTests.cpp (original) +++ clang-tools-extra/trunk/unittests/clangd/IndexTests.cpp Mon Sep 10 04:46:07 2018 @@ -10,11 +10,11 @@ #include "Annotations.h" #include "TestIndex.h" #include "TestTU.h" -#include "gmock/gmock.h" #include "index/FileIndex.h" #include "index/Index.h" #include "index/MemIndex.h" #include "index/Merge.h" +#include "gmock/gmock.h" #include "gtest/gtest.h" using testing::_; @@ -58,11 +58,11 @@ TEST(SwapIndexTest, OldIndexRecycled) { auto Token = std::make_shared<int>(); std::weak_ptr<int> WeakToken = Token; - SwapIndex S( - llvm::make_unique<MemIndex>(SymbolSlab(), RefSlab(), std::move(Token))); - EXPECT_FALSE(WeakToken.expired()); // Current MemIndex keeps it alive. + SwapIndex S(llvm::make_unique<MemIndex>( + SymbolSlab(), RefSlab(), std::move(Token), /*BackingDataSize=*/0)); + EXPECT_FALSE(WeakToken.expired()); // Current MemIndex keeps it alive. S.reset(llvm::make_unique<MemIndex>()); // Now the MemIndex is destroyed. - EXPECT_TRUE(WeakToken.expired()); // So the token is too. + EXPECT_TRUE(WeakToken.expired()); // So the token is too. } TEST(MemIndexTest, MemIndexDeduplicate) { _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
