This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new cec412a853d branch-3.0: [opt](cache) Reset initial capacity of all
caches after Cgroup memory limit changes (#51698)
cec412a853d is described below
commit cec412a853d040c3973f495bbcdbb9a56c1134b3
Author: Xinyi Zou <[email protected]>
AuthorDate: Fri Jun 20 10:01:19 2025 +0800
branch-3.0: [opt](cache) Reset initial capacity of all caches after Cgroup
memory limit changes (#51698)
### What problem does this PR solve?
pick #51216
---
be/src/olap/lru_cache.cpp | 4 +++
be/src/runtime/memory/cache_manager.cpp | 7 ++++
be/src/runtime/memory/cache_manager.h | 2 ++
be/src/runtime/memory/cache_policy.h | 1 +
be/src/runtime/memory/lru_cache_policy.h | 25 ++++++++++++--
be/src/util/mem_info.cpp | 8 ++++-
be/test/olap/lru_cache_test.cpp | 57 ++++++++++++++++++++++++++++++++
7 files changed, 100 insertions(+), 4 deletions(-)
diff --git a/be/src/olap/lru_cache.cpp b/be/src/olap/lru_cache.cpp
index 8b004d294af..9bb21ef717d 100644
--- a/be/src/olap/lru_cache.cpp
+++ b/be/src/olap/lru_cache.cpp
@@ -181,6 +181,10 @@ PrunedInfo LRUCache::set_capacity(size_t capacity) {
LRUHandle* last_ref_list = nullptr;
{
std::lock_guard l(_mutex);
+ if (capacity > _capacity) {
+ _capacity = capacity;
+ return {0, 0};
+ }
_capacity = capacity;
_evict_from_lru(0, &last_ref_list);
}
diff --git a/be/src/runtime/memory/cache_manager.cpp
b/be/src/runtime/memory/cache_manager.cpp
index ec57ffba50d..f823d23df2b 100644
--- a/be/src/runtime/memory/cache_manager.cpp
+++ b/be/src/runtime/memory/cache_manager.cpp
@@ -81,4 +81,11 @@ int64_t CacheManager::for_each_cache_refresh_capacity(double
adjust_weighted,
return freed_size;
}
+void CacheManager::for_each_cache_reset_initial_capacity(double
adjust_weighted) {
+ std::lock_guard<std::mutex> l(_caches_lock);
+ for (const auto& pair : _caches) {
+ pair.second->reset_initial_capacity(adjust_weighted);
+ }
+}
+
} // namespace doris
diff --git a/be/src/runtime/memory/cache_manager.h
b/be/src/runtime/memory/cache_manager.h
index a2a089b929d..04e611c5a87 100644
--- a/be/src/runtime/memory/cache_manager.h
+++ b/be/src/runtime/memory/cache_manager.h
@@ -84,6 +84,8 @@ public:
int64_t for_each_cache_refresh_capacity(double adjust_weighted,
RuntimeProfile* profile = nullptr);
+ void for_each_cache_reset_initial_capacity(double adjust_weighted);
+
private:
std::mutex _caches_lock;
std::unordered_map<CachePolicy::CacheType, CachePolicy*> _caches;
diff --git a/be/src/runtime/memory/cache_policy.h
b/be/src/runtime/memory/cache_policy.h
index b7f7b65ee16..5c5c3bfddb5 100644
--- a/be/src/runtime/memory/cache_policy.h
+++ b/be/src/runtime/memory/cache_policy.h
@@ -156,6 +156,7 @@ public:
CacheType type() { return _type; }
size_t initial_capacity() const { return _initial_capacity; }
+ virtual int64_t reset_initial_capacity(double adjust_weighted) = 0;
bool enable_prune() const { return _enable_prune; }
RuntimeProfile* profile() { return _profile.get(); }
diff --git a/be/src/runtime/memory/lru_cache_policy.h
b/be/src/runtime/memory/lru_cache_policy.h
index 7bd2ca486ac..83c7f46585a 100644
--- a/be/src/runtime/memory/lru_cache_policy.h
+++ b/be/src/runtime/memory/lru_cache_policy.h
@@ -240,9 +240,9 @@ public:
}
}
- int64_t adjust_capacity_weighted(double adjust_weighted) override {
- std::lock_guard<std::mutex> l(_lock);
- auto capacity = static_cast<size_t>(_initial_capacity *
adjust_weighted);
+ int64_t adjust_capacity_weighted_unlocked(double adjust_weighted) {
+ auto capacity =
+ static_cast<size_t>(static_cast<double>(_initial_capacity) *
adjust_weighted);
COUNTER_SET(_freed_entrys_counter, (int64_t)0);
COUNTER_SET(_freed_memory_counter, (int64_t)0);
COUNTER_SET(_cost_timer, (int64_t)0);
@@ -271,6 +271,25 @@ public:
return _freed_entrys_counter->value();
}
+ int64_t adjust_capacity_weighted(double adjust_weighted) override {
+ std::lock_guard<std::mutex> l(_lock);
+ return adjust_capacity_weighted_unlocked(adjust_weighted);
+ }
+
+ int64_t reset_initial_capacity(double adjust_weighted) override {
+ DCHECK(adjust_weighted != 0.0); // otherwise initial_capacity will
always to be 0.
+ std::lock_guard<std::mutex> l(_lock);
+ int64_t prune_num = adjust_capacity_weighted_unlocked(adjust_weighted);
+ size_t old_capacity = _initial_capacity;
+ _initial_capacity =
+ static_cast<size_t>(static_cast<double>(_initial_capacity) *
adjust_weighted);
+ LOG(INFO) << fmt::format(
+ "[MemoryGC] {} reset initial capacity, new capacity {}, old
capacity {}, prune num "
+ "{}",
+ type_string(_type), _initial_capacity, old_capacity,
prune_num);
+ return prune_num;
+ };
+
protected:
void _init_mem_tracker(const std::string& type_name) {
if (std::find(CachePolicy::MetadataCache.begin(),
CachePolicy::MetadataCache.end(),
diff --git a/be/src/util/mem_info.cpp b/be/src/util/mem_info.cpp
index fe9cf84b2ae..f0efffd7c92 100644
--- a/be/src/util/mem_info.cpp
+++ b/be/src/util/mem_info.cpp
@@ -20,7 +20,7 @@
#include "mem_info.h"
-#include "gutil/strings/split.h"
+#include "runtime/memory/cache_manager.h"
#ifdef __APPLE__
#include <sys/sysctl.h>
@@ -254,6 +254,12 @@ void MemInfo::refresh_proc_meminfo() {
// 2. if physical_mem changed, refresh mem limit and gc size.
if (physical_mem > 0 && _s_physical_mem.load(std::memory_order_relaxed) !=
physical_mem) {
+ if (_s_physical_mem != std::numeric_limits<int64_t>::max()) {
+ // After MemInfo is initialized, if physical memory changed, reset
initial capacity of all caches.
+ CacheManager::instance()->for_each_cache_reset_initial_capacity(
+ physical_mem / (_s_physical_mem * 1.0));
+ }
+
_s_physical_mem.store(physical_mem);
bool is_percent = true;
diff --git a/be/test/olap/lru_cache_test.cpp b/be/test/olap/lru_cache_test.cpp
index 1acc38f2b9e..8c260d69755 100644
--- a/be/test/olap/lru_cache_test.cpp
+++ b/be/test/olap/lru_cache_test.cpp
@@ -704,4 +704,61 @@ TEST_F(CacheTest, SetCapacity) {
ASSERT_EQ(0, cache()->get_usage());
}
+TEST_F(CacheTest, ResetInitialCapacity) {
+ init_number_cache();
+ for (int i = 0; i < kCacheSize; i++) {
+ Insert(i, 1000 + i, 1);
+ EXPECT_EQ(1000 + i, Lookup(i));
+ }
+ ASSERT_EQ(kCacheSize, cache()->get_capacity());
+ ASSERT_EQ(kCacheSize, cache()->get_usage());
+
+ int64_t prune_num = cache()->adjust_capacity_weighted(0.5);
+ ASSERT_EQ(prune_num, kCacheSize / 2);
+ ASSERT_EQ(kCacheSize / 2, cache()->get_capacity());
+ ASSERT_EQ(kCacheSize / 2, cache()->get_usage());
+
+ prune_num = cache()->adjust_capacity_weighted(2);
+ ASSERT_EQ(prune_num, 0);
+ ASSERT_EQ(kCacheSize * 2, cache()->get_capacity());
+ ASSERT_EQ(kCacheSize / 2, cache()->get_usage());
+
+ prune_num = cache()->reset_initial_capacity(0.5);
+ ASSERT_EQ(prune_num, 0);
+ ASSERT_EQ(kCacheSize / 2, cache()->get_capacity());
+ ASSERT_EQ(kCacheSize / 2, cache()->get_usage());
+
+ prune_num = cache()->adjust_capacity_weighted(2);
+ ASSERT_EQ(prune_num, 0);
+ ASSERT_EQ(kCacheSize, cache()->get_capacity());
+ ASSERT_EQ(kCacheSize / 2, cache()->get_usage());
+
+ prune_num = cache()->adjust_capacity_weighted(1);
+ ASSERT_EQ(prune_num, 0);
+ ASSERT_EQ(kCacheSize / 2, cache()->get_capacity());
+ ASSERT_EQ(kCacheSize / 2, cache()->get_usage());
+
+ prune_num = cache()->reset_initial_capacity(4);
+ ASSERT_EQ(prune_num, 0);
+ ASSERT_EQ(kCacheSize * 2, cache()->get_capacity());
+ ASSERT_EQ(kCacheSize / 2, cache()->get_usage());
+
+ for (int i = kCacheSize; i < kCacheSize * 2; i++) {
+ Insert(i, 1000 + i, 1);
+ EXPECT_EQ(1000 + i, Lookup(i));
+ }
+ ASSERT_EQ(kCacheSize * 2, cache()->get_capacity());
+ ASSERT_EQ(kCacheSize + kCacheSize / 2, cache()->get_usage());
+
+ prune_num = cache()->adjust_capacity_weighted(0.5);
+ ASSERT_EQ(prune_num, kCacheSize / 2);
+ ASSERT_EQ(kCacheSize, cache()->get_capacity());
+ ASSERT_EQ(kCacheSize, cache()->get_usage());
+
+ prune_num = cache()->reset_initial_capacity(0.25);
+ ASSERT_EQ(prune_num, kCacheSize / 2);
+ ASSERT_EQ(kCacheSize / 2, cache()->get_capacity());
+ ASSERT_EQ(kCacheSize / 2, cache()->get_usage());
+}
+
} // namespace doris
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
