[PATCH] D86077: [clangd] Add a way for exporting memory usage metrics

[Kadir Cetinkaya via Phabricator via cfe-commits]

kadircet created this revision.
kadircet added reviewers: sammccall, adamcz.
Herald added subscribers: cfe-commits, usaxena95, arphaman, jkorous, 
javed.absar.
Herald added a project: clang.
kadircet requested review of this revision.
Herald added subscribers: MaskRay, ilya-biryukov.

Introduces a helper to collect memory usage metrics from multiple
components.
Adds measurements for dynamic index, ast cache and preambles.


Repository:
  rG LLVM Github Monorepo

https://reviews.llvm.org/D86077

Files:
  clang-tools-extra/clangd/TUScheduler.cpp
  clang-tools-extra/clangd/index/FileIndex.cpp
  clang-tools-extra/clangd/support/Trace.cpp
  clang-tools-extra/clangd/support/Trace.h


Index: clang-tools-extra/clangd/support/Trace.h
===================================================================
--- clang-tools-extra/clangd/support/Trace.h
+++ clang-tools-extra/clangd/support/Trace.h
@@ -68,6 +68,10 @@
   const llvm::StringLiteral LabelName;
 };
 
+/// Convenient helper for collecting memory usage metrics from across multiple
+/// components. Results are recorded under a metric named "memory_usage".
+void recordMemoryUsage(double Value, llvm::StringRef ComponentName);
+
 /// A consumer of trace events and measurements. The events are produced by
 /// Spans and trace::log, the measurements are produced by Metrics::record.
 /// Implementations of this interface must be thread-safe.
Index: clang-tools-extra/clangd/support/Trace.cpp
===================================================================
--- clang-tools-extra/clangd/support/Trace.cpp
+++ clang-tools-extra/clangd/support/Trace.cpp
@@ -326,6 +326,13 @@
 Context EventTracer::beginSpan(llvm::StringRef Name, llvm::json::Object *Args) {
   return Context::current().clone();
 }
+
+void recordMemoryUsage(double Value, llvm::StringRef ComponentName) {
+  static constexpr Metric MemoryUsage("memory_usage", Metric::Value,
+                                      "component_name");
+
+  MemoryUsage.record(Value, ComponentName);
+}
 } // namespace trace
 } // namespace clangd
 } // namespace clang
Index: clang-tools-extra/clangd/index/FileIndex.cpp
===================================================================
--- clang-tools-extra/clangd/index/FileIndex.cpp
+++ clang-tools-extra/clangd/index/FileIndex.cpp
@@ -23,6 +23,7 @@
 #include "index/dex/Dex.h"
 #include "support/Logger.h"
 #include "support/Path.h"
+#include "support/Trace.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/Index/IndexingAction.h"
 #include "clang/Index/IndexingOptions.h"
@@ -399,6 +400,7 @@
   auto NewIndex =
       PreambleSymbols.buildIndex(UseDex ? IndexType::Heavy : IndexType::Light,
                                  DuplicateHandling::PickOne, &IndexVersion);
+  trace::recordMemoryUsage(NewIndex->estimateMemoryUsage(), "preamble_index");
   {
     std::lock_guard<std::mutex> Lock(UpdateIndexMu);
     if (IndexVersion <= PreambleIndexVersion) {
@@ -424,6 +426,7 @@
   size_t IndexVersion = 0;
   auto NewIndex = MainFileSymbols.buildIndex(
       IndexType::Light, DuplicateHandling::Merge, &IndexVersion);
+  trace::recordMemoryUsage(NewIndex->estimateMemoryUsage(), "main_file_index");
   {
     std::lock_guard<std::mutex> Lock(UpdateIndexMu);
     if (IndexVersion <= MainIndexVersion) {
Index: clang-tools-extra/clangd/TUScheduler.cpp
===================================================================
--- clang-tools-extra/clangd/TUScheduler.cpp
+++ clang-tools-extra/clangd/TUScheduler.cpp
@@ -77,6 +77,7 @@
 #include <algorithm>
 #include <chrono>
 #include <condition_variable>
+#include <cstddef>
 #include <functional>
 #include <memory>
 #include <mutex>
@@ -164,6 +165,14 @@
     return llvm::Optional<std::unique_ptr<ParsedAST>>(std::move(V));
   }
 
+  size_t getUsedBytes() {
+    size_t TotalBytes = 0;
+    std::lock_guard<std::mutex> Lock(Mut);
+    for (const auto &Elem : LRU)
+      TotalBytes += Elem.second->getUsedBytes();
+    return TotalBytes;
+  }
+
 private:
   using KVPair = std::pair<Key, std::unique_ptr<ParsedAST>>;
 
@@ -1286,6 +1295,20 @@
     FD->Contents = Inputs.Contents;
   }
   FD->Worker->update(std::move(Inputs), WantDiags);
+
+  // Update memory usage metrics. Note that these are just estimates.
+  size_t ASTCacheBytes = IdleASTs->getUsedBytes();
+  trace::recordMemoryUsage(ASTCacheBytes, "ast_cache");
+
+  size_t PreambleBytes = 0;
+  // Otherwise preambles are stored on disk and we only keep filename in memory.
+  if (Opts.StorePreamblesInMemory) {
+    for (const auto &Elem : fileStats())
+      PreambleBytes += Elem.second.UsedBytes;
+    // fileStats results include ast cache sizes too, subtract them.
+    PreambleBytes -= ASTCacheBytes;
+  }
+  trace::recordMemoryUsage(PreambleBytes, "preambles");
   return NewFile;
 }
 

_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits