This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch spill_and_reserve
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/spill_and_reserve by this push:
new 5d18956013f [fix](memory) Not check process memory limit when thread
reserve memory is sufficient (#40548)
5d18956013f is described below
commit 5d18956013fef7a873749ac79aa03951da437222
Author: Xinyi Zou <[email protected]>
AuthorDate: Tue Sep 10 11:24:04 2024 +0800
[fix](memory) Not check process memory limit when thread reserve memory is
sufficient (#40548)
---
be/src/runtime/memory/global_memory_arbitrator.cpp | 8 ++++++++
be/src/runtime/memory/global_memory_arbitrator.h | 15 +++++++++++++++
2 files changed, 23 insertions(+)
diff --git a/be/src/runtime/memory/global_memory_arbitrator.cpp
b/be/src/runtime/memory/global_memory_arbitrator.cpp
index 344bcbc5984..4a152098c1d 100644
--- a/be/src/runtime/memory/global_memory_arbitrator.cpp
+++ b/be/src/runtime/memory/global_memory_arbitrator.cpp
@@ -79,4 +79,12 @@ void
GlobalMemoryArbitrator::release_process_reserved_memory(int64_t bytes) {
}
}
+int64_t GlobalMemoryArbitrator::sub_thread_reserve_memory(int64_t bytes) {
+ doris::ThreadContext* thread_context = doris::thread_context(true);
+ if (thread_context) {
+ return bytes -
doris::thread_context()->thread_mem_tracker_mgr->reserved_mem();
+ }
+ return bytes;
+}
+
} // namespace doris
diff --git a/be/src/runtime/memory/global_memory_arbitrator.h
b/be/src/runtime/memory/global_memory_arbitrator.h
index f8fda18d0e9..a2f55a8fa4f 100644
--- a/be/src/runtime/memory/global_memory_arbitrator.h
+++ b/be/src/runtime/memory/global_memory_arbitrator.h
@@ -124,12 +124,27 @@ public:
return _s_process_reserved_memory.load(std::memory_order_relaxed);
}
+ // `process_memory_usage` includes all reserved memory. if a thread has
`reserved_memory`,
+ // and the memory allocated by thread is less than the thread
`reserved_memory`,
+ // even if `process_memory_usage` is greater than `process_mem_limit`,
memory can still be allocated.
+ // At this time, `process_memory_usage` will not increase, process
physical memory will increase,
+ // and `reserved_memory` will be reduced.
+ static int64_t sub_thread_reserve_memory(int64_t bytes);
+
static bool is_exceed_soft_mem_limit(int64_t bytes = 0) {
+ bytes = sub_thread_reserve_memory(bytes);
+ if (bytes <= 0) {
+ return false;
+ }
return process_memory_usage() + bytes >= MemInfo::soft_mem_limit() ||
sys_mem_available() - bytes <
MemInfo::sys_mem_available_warning_water_mark();
}
static bool is_exceed_hard_mem_limit(int64_t bytes = 0) {
+ bytes = sub_thread_reserve_memory(bytes);
+ if (bytes <= 0) {
+ return false;
+ }
// Limit process memory usage using the actual physical memory of the
process in `/proc/self/status`.
// This is independent of the consumption value of the mem tracker,
which counts the virtual memory
// of the process malloc.
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
