This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-1.1-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.1-lts by this push:
new 00c8bb6eb8 [fix](memtracker) Refactor load channel mem tracker to
improve accuracy (#12791) (#12948)
00c8bb6eb8 is described below
commit 00c8bb6eb84f26bac0a9b6b54e60aeefecec5599
Author: Xinyi Zou <zouxiny...@gmail.com>
AuthorDate: Sun Sep 25 09:41:44 2022 +0800
[fix](memtracker) Refactor load channel mem tracker to improve accuracy
(#12791) (#12948)
The mem hook record tracker cannot guarantee that the final consumption is
0, nor can it guarantee that the memory alloc and free are recorded in a
one-to-one correspondence.
In the life cycle of a memtable from insert to flush, the memory free of
hook is more than that of alloc, resulting in tracker consumption less than 0.
In order to avoid the cumulative error of the upper load channel tracker,
the memtable tracker consumption is reset to zero on destructor.
---
be/src/runtime/memory/mem_tracker_limiter.cpp | 5 +++--
be/src/runtime/memory/mem_tracker_limiter.h | 6 ++++++
be/src/runtime/memory/mem_tracker_task_pool.cpp | 12 +++++-------
be/src/runtime/runtime_state.cpp | 1 +
be/src/util/mem_info.cpp | 2 ++
be/src/util/mem_info.h | 6 ++++++
6 files changed, 23 insertions(+), 9 deletions(-)
diff --git a/be/src/runtime/memory/mem_tracker_limiter.cpp
b/be/src/runtime/memory/mem_tracker_limiter.cpp
index 28ce5389b7..aaba7c6b19 100644
--- a/be/src/runtime/memory/mem_tracker_limiter.cpp
+++ b/be/src/runtime/memory/mem_tracker_limiter.cpp
@@ -82,12 +82,13 @@ MemTrackerLimiter::~MemTrackerLimiter() {
// the first layer: process;
// the second layer: a tracker that will not be destructed globally
(query/load pool, load channel mgr, etc.);
// the third layer: a query/load/compaction task generates a tracker
(query tracker, load channel tracker, etc.).
- if (_parent->parent()->label() == "Process") {
+ if ((_parent && _parent->label() == "Process") ||
+ (_parent->parent() && _parent->parent()->label() == "Process")) {
ExecEnv::GetInstance()->orphan_mem_tracker_raw()->cache_consume_local(
_consumption->current_value());
}
#endif
-
+ if (_reset_zero) cache_consume_local(-_consumption->current_value());
if (_parent) {
std::lock_guard<std::mutex> l(_parent->_child_tracker_limiter_lock);
if (_child_tracker_it != _parent->_child_tracker_limiters.end()) {
diff --git a/be/src/runtime/memory/mem_tracker_limiter.h
b/be/src/runtime/memory/mem_tracker_limiter.h
index 7f294ef998..09b42cb7e6 100644
--- a/be/src/runtime/memory/mem_tracker_limiter.h
+++ b/be/src/runtime/memory/mem_tracker_limiter.h
@@ -130,6 +130,7 @@ public:
}
void enable_print_log_usage() { _print_log_usage = true; }
+ void enable_reset_zero() { _reset_zero = true; }
// Logs the usage of this tracker limiter and optionally its children
(recursively).
// If 'logged_consumption' is non-nullptr, sets the consumption value
logged.
@@ -251,6 +252,11 @@ private:
std::atomic_size_t _had_child_count = 0;
bool _print_log_usage = false;
+ // mem hook record tracker cannot guarantee that the final consumption is
0,
+ // nor can it guarantee that the memory alloc and free are recorded in a
one-to-one correspondence.
+ // In some cases, in order to avoid the cumulative error of the upper
global tracker,
+ // the consumption of the current tracker is reset to zero.
+ bool _reset_zero = false;
};
inline void MemTrackerLimiter::consume(int64_t bytes) {
diff --git a/be/src/runtime/memory/mem_tracker_task_pool.cpp
b/be/src/runtime/memory/mem_tracker_task_pool.cpp
index a08d876370..143e7486fa 100644
--- a/be/src/runtime/memory/mem_tracker_task_pool.cpp
+++ b/be/src/runtime/memory/mem_tracker_task_pool.cpp
@@ -89,14 +89,12 @@ void MemTrackerTaskPool::logout_task_mem_tracker() {
// between the two trackers.
// At present, it is impossible to effectively locate which memory
consume and release on different trackers,
// so query memory leaks cannot be found.
- //
- // In order to ensure that the query pool mem tracker is the sum
of all currently running query mem trackers,
- // the effect of the ended query mem tracker on the query pool mem
tracker should be cleared, that is,
- // the negative number of the current value of consume.
-
it->second->parent()->cache_consume_local(-it->second->consumption());
LOG(INFO) << fmt::format(
- "Deregister query/load memory tracker, queryId={},
Limit={}, PeakUsed={}",
- it->first, it->second->limit(),
it->second->peak_consumption());
+ "Deregister query/load memory tracker, queryId={},
Limit={}, CurrUsed={}, "
+ "PeakUsed={}",
+ it->first, PrettyPrinter::print(it->second->limit(),
TUnit::BYTES),
+ PrettyPrinter::print(it->second->consumption(),
TUnit::BYTES),
+ PrettyPrinter::print(it->second->peak_consumption(),
TUnit::BYTES));
expired_task_ids.emplace_back(it->first);
}
}
diff --git a/be/src/runtime/runtime_state.cpp b/be/src/runtime/runtime_state.cpp
index 28c00ba48b..6040718a33 100644
--- a/be/src/runtime/runtime_state.cpp
+++ b/be/src/runtime/runtime_state.cpp
@@ -230,6 +230,7 @@ Status RuntimeState::init_mem_trackers(const TUniqueId&
query_id) {
DCHECK(false);
_new_query_mem_tracker =
ExecEnv::GetInstance()->query_pool_mem_tracker();
}
+ _new_query_mem_tracker->enable_reset_zero();
_new_instance_mem_tracker = std::make_shared<MemTrackerLimiter>(
-1, "RuntimeState:instance:" + print_id(_fragment_instance_id),
diff --git a/be/src/util/mem_info.cpp b/be/src/util/mem_info.cpp
index fe4e4e8f10..66e9a97ff1 100644
--- a/be/src/util/mem_info.cpp
+++ b/be/src/util/mem_info.cpp
@@ -40,12 +40,14 @@ int64_t MemInfo::_s_mem_limit = -1;
std::string MemInfo::_s_mem_limit_str = "";
int64_t MemInfo::_s_hard_mem_limit = -1;
size_t MemInfo::_s_allocator_physical_mem = 0;
+size_t MemInfo::_s_pageheap_unmapped_bytes = 0;
size_t MemInfo::_s_tcmalloc_pageheap_free_bytes = 0;
size_t MemInfo::_s_tcmalloc_central_bytes = 0;
size_t MemInfo::_s_tcmalloc_transfer_bytes = 0;
size_t MemInfo::_s_tcmalloc_thread_bytes = 0;
size_t MemInfo::_s_allocator_cache_mem = 0;
std::string MemInfo::_s_allocator_cache_mem_str = "";
+size_t MemInfo::_s_virtual_memory_used = 0;
void MemInfo::init() {
// Read from /proc/meminfo
diff --git a/be/src/util/mem_info.h b/be/src/util/mem_info.h
index d07f608f4e..770657ace6 100644
--- a/be/src/util/mem_info.h
+++ b/be/src/util/mem_info.h
@@ -42,6 +42,7 @@ public:
}
static inline size_t current_mem() { return _s_allocator_physical_mem; }
+ static inline size_t allocator_virtual_mem() { return
_s_virtual_memory_used; }
static inline size_t allocator_cache_mem() { return
_s_allocator_cache_mem; }
static inline std::string allocator_cache_mem_str() { return
_s_allocator_cache_mem_str; }
@@ -50,6 +51,8 @@ public:
static inline void refresh_allocator_mem() {
MallocExtension::instance()->GetNumericProperty("generic.total_physical_bytes",
&_s_allocator_physical_mem);
+
MallocExtension::instance()->GetNumericProperty("tcmalloc.pageheap_unmapped_bytes",
+
&_s_pageheap_unmapped_bytes);
MallocExtension::instance()->GetNumericProperty("tcmalloc.pageheap_free_bytes",
&_s_tcmalloc_pageheap_free_bytes);
MallocExtension::instance()->GetNumericProperty("tcmalloc.central_cache_free_bytes",
@@ -61,6 +64,7 @@ public:
_s_allocator_cache_mem = _s_tcmalloc_pageheap_free_bytes +
_s_tcmalloc_central_bytes +
_s_tcmalloc_transfer_bytes +
_s_tcmalloc_thread_bytes;
_s_allocator_cache_mem_str =
PrettyPrinter::print(_s_allocator_cache_mem, TUnit::BYTES);
+ _s_virtual_memory_used = _s_allocator_physical_mem +
_s_pageheap_unmapped_bytes;
}
static inline int64_t mem_limit() {
@@ -82,12 +86,14 @@ private:
static std::string _s_mem_limit_str;
static int64_t _s_hard_mem_limit;
static size_t _s_allocator_physical_mem;
+ static size_t _s_pageheap_unmapped_bytes;
static size_t _s_tcmalloc_pageheap_free_bytes;
static size_t _s_tcmalloc_central_bytes;
static size_t _s_tcmalloc_transfer_bytes;
static size_t _s_tcmalloc_thread_bytes;
static size_t _s_allocator_cache_mem;
static std::string _s_allocator_cache_mem_str;
+ static size_t _s_virtual_memory_used;
};
} // namespace doris
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
