This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 5be25565031 [enhancement](compaction) replace hardcoded compaction
thresholds with config (#49662) (#50514)
5be25565031 is described below
commit 5be255650314af53241c34043f53455bceff7dee
Author: Luwei <[email protected]>
AuthorDate: Tue Apr 29 16:47:18 2025 +0800
[enhancement](compaction) replace hardcoded compaction thresholds with
config (#49662) (#50514)
pick master #49662
---
be/src/agent/task_worker_pool.cpp | 7 +++++--
be/src/common/config.cpp | 5 +++++
be/src/common/config.h | 5 +++++
be/src/olap/base_compaction.cpp | 2 +-
be/src/olap/olap_server.cpp | 3 ++-
be/src/olap/tablet_manager.cpp | 2 +-
6 files changed, 19 insertions(+), 5 deletions(-)
diff --git a/be/src/agent/task_worker_pool.cpp
b/be/src/agent/task_worker_pool.cpp
index f3302fc2f3e..c5239a1f7c6 100644
--- a/be/src/agent/task_worker_pool.cpp
+++ b/be/src/agent/task_worker_pool.cpp
@@ -1586,14 +1586,17 @@ void
PublishVersionWorkerPool::publish_version_callback(const TAgentTaskRequest&
.error(status);
} else {
if (!config::disable_auto_compaction &&
-
!GlobalMemoryArbitrator::is_exceed_soft_mem_limit(GB_EXCHANGE_BYTE)) {
+ (!config::enable_compaction_pause_on_high_memory ||
+
!GlobalMemoryArbitrator::is_exceed_soft_mem_limit(GB_EXCHANGE_BYTE))) {
for (auto [tablet_id, _] : succ_tablets) {
TabletSharedPtr tablet =
_engine.tablet_manager()->get_tablet(tablet_id);
if (tablet != nullptr) {
if
(!tablet->tablet_meta()->tablet_schema()->disable_auto_compaction()) {
tablet->published_count.fetch_add(1);
int64_t published_count =
tablet->published_count.load();
- if
(tablet->exceed_version_limit(config::max_tablet_version_num * 2 / 3) &&
+ if (tablet->exceed_version_limit(
+ config::max_tablet_version_num *
+
config::load_trigger_compaction_version_percent / 100) &&
published_count % 20 == 0) {
auto st = _engine.submit_compaction_task(
tablet,
CompactionType::CUMULATIVE_COMPACTION, true, false);
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index ff1bef18b91..8761047e533 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1368,6 +1368,11 @@ DEFINE_mInt32(compaction_num_per_round, "1");
// ATTN: this config is only for test
DEFINE_mBool(enable_prune_delete_sign_when_base_compaction, "true");
+DEFINE_mInt32(tablet_sched_delay_time_ms, "5000");
+DEFINE_mInt32(load_trigger_compaction_version_percent, "66");
+DEFINE_mInt64(base_compaction_interval_seconds_since_last_operation, "86400");
+DEFINE_mBool(enable_compaction_pause_on_high_memory, "true");
+
// clang-format off
#ifdef BE_TEST
// test s3
diff --git a/be/src/common/config.h b/be/src/common/config.h
index ccd6706863c..1bb1ea0ccc2 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1440,6 +1440,11 @@
DECLARE_mBool(enable_sleep_between_delete_cumu_compaction);
// ATTN: this config is only for test
DECLARE_mBool(enable_prune_delete_sign_when_base_compaction);
+DECLARE_mInt32(tablet_sched_delay_time_ms);
+DECLARE_mInt32(load_trigger_compaction_version_percent);
+DECLARE_mInt64(base_compaction_interval_seconds_since_last_operation);
+DECLARE_mBool(enable_compaction_pause_on_high_memory);
+
#ifdef BE_TEST
// test s3
DECLARE_String(test_s3_resource);
diff --git a/be/src/olap/base_compaction.cpp b/be/src/olap/base_compaction.cpp
index a9455d45381..9c787ee1f9a 100644
--- a/be/src/olap/base_compaction.cpp
+++ b/be/src/olap/base_compaction.cpp
@@ -200,7 +200,7 @@ Status BaseCompaction::pick_rowsets_to_compact() {
// 3. the interval since last base compaction reaches the threshold
int64_t base_creation_time = _input_rowsets[0]->creation_time();
- int64_t interval_threshold = 86400;
+ int64_t interval_threshold =
config::base_compaction_interval_seconds_since_last_operation;
int64_t interval_since_last_base_compaction = time(nullptr) -
base_creation_time;
if (interval_since_last_base_compaction > interval_threshold) {
VLOG_NOTICE << "satisfy the base compaction policy. tablet=" <<
_tablet->tablet_id()
diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp
index c4fe4a0e0b7..5b1cfb9ba59 100644
--- a/be/src/olap/olap_server.cpp
+++ b/be/src/olap/olap_server.cpp
@@ -622,7 +622,8 @@ void StorageEngine::_compaction_tasks_producer_callback() {
int64_t interval = config::generate_compaction_tasks_interval_ms;
do {
if (!config::disable_auto_compaction &&
-
!GlobalMemoryArbitrator::is_exceed_soft_mem_limit(GB_EXCHANGE_BYTE)) {
+ (!config::enable_compaction_pause_on_high_memory ||
+
!GlobalMemoryArbitrator::is_exceed_soft_mem_limit(GB_EXCHANGE_BYTE))) {
_adjust_compaction_thread_num();
bool check_score = false;
diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp
index 3a7aa303b2f..63c30b1f2ce 100644
--- a/be/src/olap/tablet_manager.cpp
+++ b/be/src/olap/tablet_manager.cpp
@@ -772,7 +772,7 @@ std::vector<TabletSharedPtr>
TabletManager::find_best_tablets_to_compaction(
if (compaction_type == CompactionType::BASE_COMPACTION) {
last_failure_ms = tablet_ptr->last_base_compaction_failure_time();
}
- if (now_ms - last_failure_ms <= 5000) {
+ if (now_ms - last_failure_ms <= config::tablet_sched_delay_time_ms) {
VLOG_DEBUG << "Too often to check compaction, skip it. "
<< "compaction_type=" << compaction_type_str
<< ", last_failure_time_ms=" << last_failure_ms
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
