This is an automated email from the ASF dual-hosted git repository.
liaoxin pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 232202b71f0 [improve](load) reduce memory reserved in memtable limiter
(#37511) (#37699)
232202b71f0 is described below
commit 232202b71f0dd4c84651840db2fe063659a25d36
Author: Kaijie Chen <[email protected]>
AuthorDate: Mon Jul 15 11:09:09 2024 +0800
[improve](load) reduce memory reserved in memtable limiter (#37511) (#37699)
cherry-pick #37511
---
be/src/common/config.cpp | 2 ++
be/src/common/config.h | 3 +++
be/src/olap/memtable_memory_limiter.cpp | 22 ++++++++++------------
be/src/olap/memtable_memory_limiter.h | 4 ++--
4 files changed, 17 insertions(+), 14 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index ae935bd39b5..17bca94a459 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -115,6 +115,8 @@ DEFINE_mInt32(double_resize_threshold, "23");
DEFINE_Int64(max_sys_mem_available_low_water_mark_bytes, "6871947673");
+DEFINE_Int64(memtable_limiter_reserved_memory_bytes, "838860800");
+
// The size of the memory that gc wants to release each time, as a percentage
of the mem limit.
DEFINE_mString(process_minor_gc_size, "10%");
DEFINE_mString(process_full_gc_size, "20%");
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 9afa08f16e6..5fa5b23e977 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -162,6 +162,9 @@ DECLARE_mInt32(double_resize_threshold);
// Turn down max. will use as much memory as possible.
DECLARE_Int64(max_sys_mem_available_low_water_mark_bytes);
+// reserve a small amount of memory so we do not trigger MinorGC
+DECLARE_Int64(memtable_limiter_reserved_memory_bytes);
+
// The size of the memory that gc wants to release each time, as a percentage
of the mem limit.
DECLARE_mString(process_minor_gc_size);
DECLARE_mString(process_full_gc_size);
diff --git a/be/src/olap/memtable_memory_limiter.cpp
b/be/src/olap/memtable_memory_limiter.cpp
index 1eaad31ec22..23b760284b8 100644
--- a/be/src/olap/memtable_memory_limiter.cpp
+++ b/be/src/olap/memtable_memory_limiter.cpp
@@ -77,20 +77,17 @@ void
MemTableMemoryLimiter::register_writer(std::weak_ptr<MemTableWriter> writer
_writers.push_back(writer);
}
-int64_t MemTableMemoryLimiter::_avail_mem_lack() {
+bool MemTableMemoryLimiter::_sys_avail_mem_less_than_warning_water_mark() {
// reserve a small amount of memory so we do not trigger MinorGC
- auto reserved_mem = doris::MemInfo::sys_mem_available_low_water_mark();
- auto avail_mem_lack =
doris::MemInfo::sys_mem_available_warning_water_mark() -
- doris::GlobalMemoryArbitrator::sys_mem_available();
- return avail_mem_lack + reserved_mem;
+ return doris::GlobalMemoryArbitrator::sys_mem_available() <
+ doris::MemInfo::sys_mem_available_warning_water_mark() +
+ config::memtable_limiter_reserved_memory_bytes;
}
-int64_t MemTableMemoryLimiter::_proc_mem_extra() {
+bool MemTableMemoryLimiter::_process_used_mem_more_than_soft_mem_limit() {
// reserve a small amount of memory so we do not trigger MinorGC
- auto reserved_mem = doris::MemInfo::sys_mem_available_low_water_mark();
- auto proc_mem_extra =
- GlobalMemoryArbitrator::process_memory_usage() -
MemInfo::soft_mem_limit();
- return proc_mem_extra + reserved_mem;
+ return GlobalMemoryArbitrator::process_memory_usage() >
+ MemInfo::soft_mem_limit() -
config::memtable_limiter_reserved_memory_bytes;
}
bool MemTableMemoryLimiter::_soft_limit_reached() {
@@ -98,8 +95,9 @@ bool MemTableMemoryLimiter::_soft_limit_reached() {
}
bool MemTableMemoryLimiter::_hard_limit_reached() {
- return _mem_tracker->consumption() >= _load_hard_mem_limit ||
_avail_mem_lack() >= 0 ||
- _proc_mem_extra() >= 0;
+ return _mem_tracker->consumption() >= _load_hard_mem_limit ||
+ _sys_avail_mem_less_than_warning_water_mark() ||
+ _process_used_mem_more_than_soft_mem_limit();
}
bool MemTableMemoryLimiter::_load_usage_low() {
diff --git a/be/src/olap/memtable_memory_limiter.h
b/be/src/olap/memtable_memory_limiter.h
index 895b0bbe2ca..9a0189e4881 100644
--- a/be/src/olap/memtable_memory_limiter.h
+++ b/be/src/olap/memtable_memory_limiter.h
@@ -51,8 +51,8 @@ public:
int64_t mem_usage() const { return _mem_usage; }
private:
- static int64_t _avail_mem_lack();
- static int64_t _proc_mem_extra();
+ static inline bool _sys_avail_mem_less_than_warning_water_mark();
+ static inline bool _process_used_mem_more_than_soft_mem_limit();
bool _soft_limit_reached();
bool _hard_limit_reached();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
