Author: Kadir Cetinkaya Date: 2022-04-22T17:44:47+02:00 New Revision: 2efccf5166f4dd7d188eea2de4f4b7f50b64e477
URL: https://github.com/llvm/llvm-project/commit/2efccf5166f4dd7d188eea2de4f4b7f50b64e477 DIFF: https://github.com/llvm/llvm-project/commit/2efccf5166f4dd7d188eea2de4f4b7f50b64e477.diff LOG: [clangd][NFC] Reduce memory usage while building dex Reduce peak memory usage by tearing down the intermediate representation as we build the final one. Rather than deleting it in the end. Differential Revision: https://reviews.llvm.org/D124240 Added: Modified: clang-tools-extra/clangd/index/dex/Dex.cpp Removed: ################################################################################ diff --git a/clang-tools-extra/clangd/index/dex/Dex.cpp b/clang-tools-extra/clangd/index/dex/Dex.cpp index 5829d780c6072..18779077b2c43 100644 --- a/clang-tools-extra/clangd/index/dex/Dex.cpp +++ b/clang-tools-extra/clangd/index/dex/Dex.cpp @@ -13,14 +13,19 @@ #include "URI.h" #include "index/Index.h" #include "index/dex/Iterator.h" +#include "index/dex/Token.h" #include "index/dex/Trigram.h" #include "support/Logger.h" #include "support/Trace.h" +#include "llvm/ADT/DenseMap.h" +#include "llvm/ADT/StringMap.h" #include "llvm/ADT/StringSet.h" #include "llvm/Support/Path.h" #include "llvm/Support/ScopedPrinter.h" #include <algorithm> #include <queue> +#include <utility> +#include <vector> namespace clang { namespace clangd { @@ -76,23 +81,38 @@ class IndexBuilder { } // Assemble the final compressed posting lists for the added symbols. - llvm::DenseMap<Token, PostingList> build() { + llvm::DenseMap<Token, PostingList> build() && { llvm::DenseMap<Token, PostingList> Result(/*InitialReserve=*/ TrigramDocs.size() + RestrictedCCDocs.size() + TypeDocs.size() + ScopeDocs.size() + ProximityDocs.size()); - for (const auto &E : TrigramDocs) + // Tear down intermediate structs as we go to reduce memory usage. + // Since we're trying to get rid of underlying allocations, clearing the + // containers is not enough. + auto CreatePostingList = + [&Result](Token::Kind TK, llvm::StringMap<std::vector<DocID>> &Docs) { + for (auto &E : Docs) { + Result.try_emplace(Token(TK, E.first()), E.second); + E.second = {}; + } + Docs = {}; + }; + CreatePostingList(Token::Kind::Type, TypeDocs); + CreatePostingList(Token::Kind::Scope, ScopeDocs); + CreatePostingList(Token::Kind::ProximityURI, ProximityDocs); + + // TrigramDocs are stored in a DenseMap and RestrictedCCDocs is not even a + // map, treat them specially. + for (auto &E : TrigramDocs) { Result.try_emplace(Token(Token::Kind::Trigram, E.first.str()), E.second); - for (const auto &E : TypeDocs) - Result.try_emplace(Token(Token::Kind::Type, E.first()), E.second); - for (const auto &E : ScopeDocs) - Result.try_emplace(Token(Token::Kind::Scope, E.first()), E.second); - for (const auto &E : ProximityDocs) - Result.try_emplace(Token(Token::Kind::ProximityURI, E.first()), E.second); + E.second = {}; + } + TrigramDocs = llvm::DenseMap<Trigram, std::vector<DocID>>{}; if (!RestrictedCCDocs.empty()) - Result.try_emplace(RestrictedForCodeCompletion, RestrictedCCDocs); + Result.try_emplace(RestrictedForCodeCompletion, + std::move(RestrictedCCDocs)); return Result; } }; @@ -125,7 +145,7 @@ void Dex::buildIndex() { IndexBuilder Builder; for (DocID SymbolRank = 0; SymbolRank < Symbols.size(); ++SymbolRank) Builder.add(*Symbols[SymbolRank], SymbolRank); - InvertedIndex = Builder.build(); + InvertedIndex = std::move(Builder).build(); } std::unique_ptr<Iterator> Dex::iterator(const Token &Tok) const { _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
