This is an automated email from the ASF dual-hosted git repository.
liaoxin01 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 011cff4f368 [fix](filecache) Disable file cache LRU dump by default
(#69099)
011cff4f368 is described below
commit 011cff4f3685ab2af5545e19100513eacd4a3f86
Author: Xin Liao <[email protected]>
AuthorDate: Thu Oct 15 21:34:01 2026 +0800
[fix](filecache) Disable file cache LRU dump by default (#69099)
File cache LRU dump may have some performance issues and other potential
problems, so disable it by default by changing the default value of
`file_cache_background_lru_dump_tail_record_num` from `5000000` to `0`.
It can still be enabled explicitly. Tests that cover LRU dump now enable
it explicitly.
### Release note
Disable file cache LRU dump by default.
---
be/src/common/config.cpp | 5 ++++-
be/src/common/config.h | 1 +
be/test/io/cache/block_file_cache_test_lru_dump.cpp | 10 +++++++++-
be/test/io/cache/cache_lru_dumper_test.cpp | 6 ++++++
regression-test/suites/demo_p0/test_lru_persist.groovy | 3 ++-
5 files changed, 22 insertions(+), 3 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index e8e31d06e5d..af6755e7f4d 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1300,7 +1300,10 @@
DEFINE_mInt64(file_cache_background_tablet_id_flush_interval_ms, "1000");
DEFINE_mInt64(file_cache_background_lru_dump_interval_ms, "60000");
// dump queue only if the queue update specific times through several dump
intervals
DEFINE_mInt64(file_cache_background_lru_dump_update_cnt_threshold, "1000");
-DEFINE_mInt64(file_cache_background_lru_dump_tail_record_num, "5000000");
+// Disabled by default. The restored LRU order can be worse than no order at
all: restored blocks
+// are put at the cold end while blocks missing from the dump are appended to
the hot end, and the
+// restore itself can overflow the bounded LRU log queue and drop the hottest
part of the dump.
+DEFINE_mInt64(file_cache_background_lru_dump_tail_record_num, "0");
DEFINE_mInt64(file_cache_background_lru_log_queue_max_size, "500000");
DEFINE_mInt64(file_cache_background_lru_log_replay_interval_ms, "1");
DEFINE_mBool(enable_evaluate_shadow_queue_diff, "false");
diff --git a/be/src/common/config.h b/be/src/common/config.h
index bd485388ae0..c32496630b8 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1355,6 +1355,7 @@ DECLARE_Int32(file_cache_downloader_thread_num_max);
DECLARE_mInt64(file_cache_background_lru_dump_interval_ms);
// dump queue only if the queue update specific times through several dump
intervals
DECLARE_mInt64(file_cache_background_lru_dump_update_cnt_threshold);
+// max LRU entries to dump for each queue, 0 disables LRU log recording,
dumping and restoring
DECLARE_mInt64(file_cache_background_lru_dump_tail_record_num);
DECLARE_mInt64(file_cache_background_lru_log_queue_max_size);
DECLARE_mInt64(file_cache_background_lru_log_replay_interval_ms);
diff --git a/be/test/io/cache/block_file_cache_test_lru_dump.cpp
b/be/test/io/cache/block_file_cache_test_lru_dump.cpp
index 3a581d77c76..a44e3f6585c 100644
--- a/be/test/io/cache/block_file_cache_test_lru_dump.cpp
+++ b/be/test/io/cache/block_file_cache_test_lru_dump.cpp
@@ -28,10 +28,13 @@ TEST_F(BlockFileCacheTest,
test_lru_log_record_replay_dump_restore) {
config::file_cache_background_lru_dump_interval_ms = 3000;
config::file_cache_background_lru_dump_update_cnt_threshold = 0;
const auto old_replay_interval_ms =
config::file_cache_background_lru_log_replay_interval_ms;
- Defer defer {[old_replay_interval_ms] {
+ const auto old_tail_record_num =
config::file_cache_background_lru_dump_tail_record_num;
+ Defer defer {[old_replay_interval_ms, old_tail_record_num] {
config::file_cache_background_lru_log_replay_interval_ms =
old_replay_interval_ms;
+ config::file_cache_background_lru_dump_tail_record_num =
old_tail_record_num;
}};
config::file_cache_background_lru_log_replay_interval_ms = 60 * 60 * 1000;
+ config::file_cache_background_lru_dump_tail_record_num = 5000000;
if (fs::exists(cache_base_path)) {
fs::remove_all(cache_base_path);
}
@@ -409,6 +412,11 @@ TEST_F(BlockFileCacheTest,
test_lru_duplicate_queue_entry_restore) {
config::file_cache_enter_disk_resource_limit_mode_percent = 99;
config::file_cache_background_lru_dump_interval_ms = 3000;
config::file_cache_background_lru_dump_update_cnt_threshold = 0;
+ const auto old_tail_record_num =
config::file_cache_background_lru_dump_tail_record_num;
+ Defer defer {[old_tail_record_num] {
+ config::file_cache_background_lru_dump_tail_record_num =
old_tail_record_num;
+ }};
+ config::file_cache_background_lru_dump_tail_record_num = 5000000;
if (fs::exists(cache_base_path)) {
fs::remove_all(cache_base_path);
}
diff --git a/be/test/io/cache/cache_lru_dumper_test.cpp
b/be/test/io/cache/cache_lru_dumper_test.cpp
index e49a142bb2c..98eb56d424f 100644
--- a/be/test/io/cache/cache_lru_dumper_test.cpp
+++ b/be/test/io/cache/cache_lru_dumper_test.cpp
@@ -131,6 +131,12 @@ TEST_F(CacheLRUDumperTest, test_remove_lru_dump_files) {
}
TEST_F(CacheLRUDumperTest, test_dump_and_restore_queue) {
+ const auto old_tail_record_num =
config::file_cache_background_lru_dump_tail_record_num;
+ Defer defer {[old_tail_record_num] {
+ config::file_cache_background_lru_dump_tail_record_num =
old_tail_record_num;
+ }};
+ config::file_cache_background_lru_dump_tail_record_num = 5000000;
+
LRUQueue src_queue;
std::string queue_name = "normal";
diff --git a/regression-test/suites/demo_p0/test_lru_persist.groovy
b/regression-test/suites/demo_p0/test_lru_persist.groovy
index 7d8c84e6f55..f9fe35c070c 100644
--- a/regression-test/suites/demo_p0/test_lru_persist.groovy
+++ b/regression-test/suites/demo_p0/test_lru_persist.groovy
@@ -54,7 +54,8 @@ suite('test_lru_persist', 'docker') {
options.beConfigs += ['enable_file_cache=true',
'enable_java_support=false',
'file_cache_enter_disk_resource_limit_mode_percent=99',
'file_cache_background_lru_dump_interval_ms=2000',
'file_cache_background_lru_log_replay_interval_ms=500',
'disable_auto_compaction=true',
'file_cache_enter_need_evict_cache_in_advance_percent=99',
-
'file_cache_background_lru_dump_update_cnt_threshold=0'
+
'file_cache_background_lru_dump_update_cnt_threshold=0',
+
'file_cache_background_lru_dump_tail_record_num=5000000'
]
// run another docker
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
