Author: hokein Date: Wed Apr 11 01:13:07 2018 New Revision: 329784 URL: http://llvm.org/viewvc/llvm-project?rev=329784&view=rev Log: [Tooling] Optimize memory usage in InMemoryToolResults.
Avoid storing duplicated "std::string"s. clangd's global-symbol-builder takes 20+GB memory running across LLVM repository. With this patch, the used memory is ~10GB (running on 48 threads, most of meory are AST-related). Subscribers: klimek, cfe-commits Differential Revision: https://reviews.llvm.org/D45479 Modified: cfe/trunk/include/clang/Tooling/Execution.h cfe/trunk/lib/Tooling/AllTUsExecution.cpp cfe/trunk/lib/Tooling/Execution.cpp Modified: cfe/trunk/include/clang/Tooling/Execution.h URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/include/clang/Tooling/Execution.h?rev=329784&r1=329783&r2=329784&view=diff ============================================================================== --- cfe/trunk/include/clang/Tooling/Execution.h (original) +++ cfe/trunk/include/clang/Tooling/Execution.h Wed Apr 11 01:13:07 2018 @@ -32,6 +32,7 @@ #include "clang/Tooling/Tooling.h" #include "llvm/Support/Error.h" #include "llvm/Support/Registry.h" +#include "llvm/Support/StringSaver.h" namespace clang { namespace tooling { @@ -45,20 +46,30 @@ class ToolResults { public: virtual ~ToolResults() = default; virtual void addResult(StringRef Key, StringRef Value) = 0; - virtual std::vector<std::pair<std::string, std::string>> AllKVResults() = 0; + virtual std::vector<std::pair<llvm::StringRef, llvm::StringRef>> + AllKVResults() = 0; virtual void forEachResult( llvm::function_ref<void(StringRef Key, StringRef Value)> Callback) = 0; }; +/// \brief Stores the key-value results in memory. It maintains the lifetime of +/// the result. Clang tools using this class are expected to generate a small +/// set of different results, or a large set of duplicated results. class InMemoryToolResults : public ToolResults { public: + InMemoryToolResults() : StringsPool(Arena) {} void addResult(StringRef Key, StringRef Value) override; - std::vector<std::pair<std::string, std::string>> AllKVResults() override; + std::vector<std::pair<llvm::StringRef, llvm::StringRef>> + AllKVResults() override; void forEachResult(llvm::function_ref<void(StringRef Key, StringRef Value)> Callback) override; private: - std::vector<std::pair<std::string, std::string>> KVResults; + llvm::BumpPtrAllocator Arena; + llvm::StringSaver StringsPool; + llvm::DenseSet<llvm::StringRef> Strings; + + std::vector<std::pair<llvm::StringRef, llvm::StringRef>> KVResults; }; /// \brief The context of an execution, including the information about Modified: cfe/trunk/lib/Tooling/AllTUsExecution.cpp URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Tooling/AllTUsExecution.cpp?rev=329784&r1=329783&r2=329784&view=diff ============================================================================== --- cfe/trunk/lib/Tooling/AllTUsExecution.cpp (original) +++ cfe/trunk/lib/Tooling/AllTUsExecution.cpp Wed Apr 11 01:13:07 2018 @@ -36,7 +36,8 @@ public: Results.addResult(Key, Value); } - std::vector<std::pair<std::string, std::string>> AllKVResults() override { + std::vector<std::pair<llvm::StringRef, llvm::StringRef>> + AllKVResults() override { return Results.AllKVResults(); } Modified: cfe/trunk/lib/Tooling/Execution.cpp URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/lib/Tooling/Execution.cpp?rev=329784&r1=329783&r2=329784&view=diff ============================================================================== --- cfe/trunk/lib/Tooling/Execution.cpp (original) +++ cfe/trunk/lib/Tooling/Execution.cpp Wed Apr 11 01:13:07 2018 @@ -21,10 +21,19 @@ static llvm::cl::opt<std::string> llvm::cl::init("standalone")); void InMemoryToolResults::addResult(StringRef Key, StringRef Value) { - KVResults.push_back({Key.str(), Value.str()}); + auto Intern = [&](StringRef &V) { + auto R = Strings.insert(V); + if (R.second) { // A new entry, create a new string copy. + *R.first = StringsPool.save(V); + } + V = *R.first; + }; + Intern(Key); + Intern(Value); + KVResults.push_back({Key, Value}); } -std::vector<std::pair<std::string, std::string>> +std::vector<std::pair<llvm::StringRef, llvm::StringRef>> InMemoryToolResults::AllKVResults() { return KVResults; } _______________________________________________ cfe-commits mailing list cfe-commits@lists.llvm.org http://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
