This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 9861f81630e [branch-2.1](memory) Fix Jemalloc Cache Memory Tracker
(#37905)
9861f81630e is described below
commit 9861f81630ec8c81bd28342590413fee5d670f79
Author: Xinyi Zou <[email protected]>
AuthorDate: Tue Jul 16 19:01:31 2024 +0800
[branch-2.1](memory) Fix Jemalloc Cache Memory Tracker (#37905)
pick #37464
---
be/src/common/config.cpp | 1 -
be/src/runtime/memory/mem_tracker_limiter.cpp | 12 ++++++++++--
be/src/util/mem_info.cpp | 27 ++++++++++++++++++++-------
be/src/util/mem_info.h | 24 ++++++++++++++++++++++++
4 files changed, 54 insertions(+), 10 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index acf75dbc04a..da7c45d1a19 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1062,7 +1062,6 @@ DEFINE_mInt32(schema_cache_sweep_time_sec, "100");
DEFINE_mInt32(segment_cache_capacity, "-1");
DEFINE_mInt32(estimated_num_columns_per_segment, "200");
DEFINE_mInt32(estimated_mem_per_column_reader, "1024");
-// The value is calculate by storage_page_cache_limit *
index_page_cache_percentage
DEFINE_mInt32(segment_cache_memory_percentage, "2");
// enable feature binlog, default false
diff --git a/be/src/runtime/memory/mem_tracker_limiter.cpp
b/be/src/runtime/memory/mem_tracker_limiter.cpp
index e79ca1bfb3a..1c0ab863e6c 100644
--- a/be/src/runtime/memory/mem_tracker_limiter.cpp
+++ b/be/src/runtime/memory/mem_tracker_limiter.cpp
@@ -191,13 +191,21 @@ void
MemTrackerLimiter::make_process_snapshots(std::vector<MemTracker::Snapshot>
}
snapshot.type = "overview";
- snapshot.label = "tc/jemalloc cache";
+ snapshot.label = "tc/jemalloc_cache";
snapshot.limit = -1;
snapshot.cur_consumption = MemInfo::allocator_cache_mem();
snapshot.peak_consumption = -1;
(*snapshots).emplace_back(snapshot);
all_tracker_mem_sum += MemInfo::allocator_cache_mem();
+ snapshot.type = "overview";
+ snapshot.label = "tc/jemalloc_metadata";
+ snapshot.limit = -1;
+ snapshot.cur_consumption = MemInfo::allocator_metadata_mem();
+ snapshot.peak_consumption = -1;
+ (*snapshots).emplace_back(snapshot);
+ all_tracker_mem_sum += MemInfo::allocator_metadata_mem();
+
snapshot.type = "overview";
snapshot.label = "sum of all trackers"; // is virtual memory
snapshot.limit = -1;
@@ -217,7 +225,7 @@ void
MemTrackerLimiter::make_process_snapshots(std::vector<MemTracker::Snapshot>
(*snapshots).emplace_back(snapshot);
snapshot.type = "overview";
- snapshot.label = "reserved memory";
+ snapshot.label = "reserve_memory";
snapshot.limit = -1;
snapshot.cur_consumption =
GlobalMemoryArbitrator::process_reserved_memory();
snapshot.peak_consumption = -1;
diff --git a/be/src/util/mem_info.cpp b/be/src/util/mem_info.cpp
index b6f35df5d9c..baddefcc27f 100644
--- a/be/src/util/mem_info.cpp
+++ b/be/src/util/mem_info.cpp
@@ -52,6 +52,7 @@ std::atomic<int64_t> MemInfo::_s_mem_limit =
std::numeric_limits<int64_t>::max()
std::atomic<int64_t> MemInfo::_s_soft_mem_limit =
std::numeric_limits<int64_t>::max();
std::atomic<int64_t> MemInfo::_s_allocator_cache_mem = 0;
+std::atomic<int64_t> MemInfo::_s_allocator_metadata_mem = 0;
std::atomic<int64_t> MemInfo::_s_je_dirty_pages_mem =
std::numeric_limits<int64_t>::min();
std::atomic<int64_t> MemInfo::_s_je_dirty_pages_mem_limit =
std::numeric_limits<int64_t>::max();
std::atomic<int64_t> MemInfo::_s_virtual_memory_used = 0;
@@ -75,6 +76,10 @@ std::atomic<bool> MemInfo::je_purge_dirty_pages_notify
{false};
void MemInfo::refresh_allocator_mem() {
#if defined(ADDRESS_SANITIZER) || defined(LEAK_SANITIZER) ||
defined(THREAD_SANITIZER)
#elif defined(USE_JEMALLOC)
+ // jemalloc mallctl refer to : https://jemalloc.net/jemalloc.3.html
+ // https://www.bookstack.cn/read/aliyun-rds-core/4a0cdf677f62feb3.md
+ // Check the Doris BE web page `http://ip:webserver_port/memz` to get the
Jemalloc Profile.
+
// 'epoch' is a special mallctl -- it updates the statistics. Without it,
all
// the following calls will return stale values. It increments and returns
// the current epoch number, which might be useful to log as a sanity
check.
@@ -82,14 +87,22 @@ void MemInfo::refresh_allocator_mem() {
size_t sz = sizeof(epoch);
jemallctl("epoch", &epoch, &sz, &epoch, sz);
- // https://jemalloc.net/jemalloc.3.html
- // https://www.bookstack.cn/read/aliyun-rds-core/4a0cdf677f62feb3.md
- _s_allocator_cache_mem.store(get_je_all_arena_metrics("tcache_bytes") +
- get_je_metrics("stats.metadata") +
- get_je_all_arena_metrics("pdirty") *
get_page_size(),
+ // Number of extents of the given type in this arena in the bucket
corresponding to page size index.
+ // Large size class starts at 16384, the extents have three sizes before
16384: 4096, 8192, and 12288, so + 3
+ int64_t dirty_pages_bytes = 0;
+ for (unsigned i = 0; i < get_je_unsigned_metrics("arenas.nlextents") + 3;
i++) {
+ dirty_pages_bytes += get_je_all_arena_extents_metrics(i,
"dirty_bytes");
+ }
+ _s_je_dirty_pages_mem.store(dirty_pages_bytes, std::memory_order_relaxed);
+
+ // Doris uses Jemalloc as default Allocator, Jemalloc Cache consists of
two parts:
+ // - Thread Cache, cache a specified number of Pages in Thread Cache.
+ // - Dirty Page, memory Page that can be reused in all Arenas.
+ _s_allocator_cache_mem.store(get_je_all_arena_metrics("tcache_bytes") +
dirty_pages_bytes,
std::memory_order_relaxed);
- _s_je_dirty_pages_mem.store(get_je_all_arena_metrics("pdirty") *
get_page_size(),
- std::memory_order_relaxed);
+ // Total number of bytes dedicated to metadata, which comprise base
allocations used
+ // for bootstrap-sensitive allocator metadata structures.
+ _s_allocator_metadata_mem.store(get_je_metrics("stats.metadata"),
std::memory_order_relaxed);
_s_virtual_memory_used.store(get_je_metrics("stats.mapped"),
std::memory_order_relaxed);
#else
_s_allocator_cache_mem.store(get_tc_metrics("tcmalloc.pageheap_free_bytes") +
diff --git a/be/src/util/mem_info.h b/be/src/util/mem_info.h
index 6b66d05033f..ee83239c128 100644
--- a/be/src/util/mem_info.h
+++ b/be/src/util/mem_info.h
@@ -105,6 +105,17 @@ public:
return 0;
}
+ static inline unsigned get_je_unsigned_metrics(const std::string& name) {
+#ifdef USE_JEMALLOC
+ unsigned value = 0;
+ size_t sz = sizeof(value);
+ if (jemallctl(name.c_str(), &value, &sz, nullptr, 0) == 0) {
+ return value;
+ }
+#endif
+ return 0;
+ }
+
static inline int64_t get_je_all_arena_metrics(const std::string& name) {
#ifdef USE_JEMALLOC
return get_je_metrics(fmt::format("stats.arenas.{}.{}",
MALLCTL_ARENAS_ALL, name));
@@ -112,6 +123,15 @@ public:
return 0;
}
+ static inline int64_t get_je_all_arena_extents_metrics(int64_t
page_size_index,
+ const std::string&
extent_type) {
+#ifdef USE_JEMALLOC
+ return get_je_metrics(fmt::format("stats.arenas.{}.extents.{}.{}",
MALLCTL_ARENAS_ALL,
+ page_size_index, extent_type));
+#endif
+ return 0;
+ }
+
static inline void je_purge_all_arena_dirty_pages() {
#ifdef USE_JEMALLOC
// https://github.com/jemalloc/jemalloc/issues/2470
@@ -144,6 +164,9 @@ public:
static inline size_t allocator_cache_mem() {
return _s_allocator_cache_mem.load(std::memory_order_relaxed);
}
+ static inline size_t allocator_metadata_mem() {
+ return _s_allocator_metadata_mem.load(std::memory_order_relaxed);
+ }
static inline int64_t je_dirty_pages_mem() {
return _s_je_dirty_pages_mem.load(std::memory_order_relaxed);
}
@@ -184,6 +207,7 @@ private:
static std::atomic<int64_t> _s_soft_mem_limit;
static std::atomic<int64_t> _s_allocator_cache_mem;
+ static std::atomic<int64_t> _s_allocator_metadata_mem;
static std::atomic<int64_t> _s_je_dirty_pages_mem;
static std::atomic<int64_t> _s_je_dirty_pages_mem_limit;
static std::atomic<int64_t> _s_virtual_memory_used;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
