This is an automated email from the ASF dual-hosted git repository.
zhangchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 9e1aa777ff4 [compaction](config) Add a config to control whether to
prune rows with delete sign=1 in base compaction (#48241)
9e1aa777ff4 is described below
commit 9e1aa777ff44a64f4064f0ab5e89c580bea64f39
Author: bobhan1 <[email protected]>
AuthorDate: Fri Feb 28 18:08:40 2025 +0800
[compaction](config) Add a config to control whether to prune rows with
delete sign=1 in base compaction (#48241)
Add a config to control whether to prune rows with delete sign=1 in base
compaction for correctness test
---
be/src/common/config.cpp | 4 +
be/src/common/config.h | 4 +
be/src/olap/tablet_reader.cpp | 3 +-
.../compaction/test_config_prune_delete_sign.out | Bin 0 -> 163 bytes
.../test_config_prune_delete_sign.groovy | 89 +++++++++++++++++++++
5 files changed, 99 insertions(+), 1 deletion(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 301a82336b7..c6d0ee8ed1d 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1439,6 +1439,10 @@ DEFINE_mInt32(check_tablet_delete_bitmap_score_top_n,
"10");
DEFINE_mBool(enable_check_tablet_delete_bitmap_score, "true");
DEFINE_mInt32(schema_dict_cache_capacity, "4096");
+// whether to prune rows with delete sign = 1 in base compaction
+// ATTN: this config is only for test
+DEFINE_mBool(enable_prune_delete_sign_when_base_compaction, "true");
+
// clang-format off
#ifdef BE_TEST
// test s3
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 845488139be..a1aab4a001d 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1514,6 +1514,10 @@ DECLARE_mInt32(check_tablet_delete_bitmap_score_top_n);
DECLARE_mBool(enable_check_tablet_delete_bitmap_score);
DECLARE_mInt32(schema_dict_cache_capacity);
+// whether to prune rows with delete sign = 1 in base compaction
+// ATTN: this config is only for test
+DECLARE_mBool(enable_prune_delete_sign_when_base_compaction);
+
#ifdef BE_TEST
// test s3
DECLARE_String(test_s3_resource);
diff --git a/be/src/olap/tablet_reader.cpp b/be/src/olap/tablet_reader.cpp
index 416d0fea476..585a787dc94 100644
--- a/be/src/olap/tablet_reader.cpp
+++ b/be/src/olap/tablet_reader.cpp
@@ -657,7 +657,8 @@ Status TabletReader::_init_delete_condition(const
ReaderParams& read_params) {
// Delete sign could not be applied when delete on cumu compaction is
enabled, bucause it is meant for delete with predicates.
// If delete design is applied on cumu compaction, it will lose effect
when doing base compaction.
// `_delete_sign_available` indicates the condition where we could apply
delete signs to data.
- _delete_sign_available = (read_params.reader_type ==
ReaderType::READER_BASE_COMPACTION ||
+ _delete_sign_available = ((read_params.reader_type ==
ReaderType::READER_BASE_COMPACTION &&
+
config::enable_prune_delete_sign_when_base_compaction) ||
read_params.reader_type ==
ReaderType::READER_COLD_DATA_COMPACTION ||
read_params.reader_type ==
ReaderType::READER_CHECKSUM);
diff --git a/regression-test/data/compaction/test_config_prune_delete_sign.out
b/regression-test/data/compaction/test_config_prune_delete_sign.out
new file mode 100644
index 00000000000..9bbb4251318
Binary files /dev/null and
b/regression-test/data/compaction/test_config_prune_delete_sign.out differ
diff --git
a/regression-test/suites/compaction/test_config_prune_delete_sign.groovy
b/regression-test/suites/compaction/test_config_prune_delete_sign.groovy
new file mode 100644
index 00000000000..06c56ce6405
--- /dev/null
+++ b/regression-test/suites/compaction/test_config_prune_delete_sign.groovy
@@ -0,0 +1,89 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_config_prune_delete_sign", "nonConcurrent") {
+
+ def inspectRows = { sqlStr ->
+ sql "set skip_delete_sign=true;"
+ sql "set skip_delete_bitmap=true;"
+ sql "sync"
+ qt_inspect sqlStr
+ sql "set skip_delete_sign=false;"
+ sql "set skip_delete_bitmap=false;"
+ sql "sync"
+ }
+
+ def custoBeConfig = [
+ enable_prune_delete_sign_when_base_compaction : false
+ ]
+
+ setBeConfigTemporary(custoBeConfig) {
+ def table1 = "test_config_prune_delete_sign"
+ sql "DROP TABLE IF EXISTS ${table1} FORCE;"
+ sql """ CREATE TABLE IF NOT EXISTS ${table1} (
+ `k1` int NOT NULL,
+ `c1` int,
+ `c2` int
+ )UNIQUE KEY(k1)
+ DISTRIBUTED BY HASH(k1) BUCKETS 1
+ PROPERTIES (
+ "enable_mow_light_delete" = "false",
+ "enable_unique_key_merge_on_write" = "true",
+ "disable_auto_compaction" = "true",
+ "replication_num" = "1"); """
+
+ def getDeleteSignCnt = {
+ sql "set skip_delete_sign=true;"
+ sql "set skip_delete_bitmap=true;"
+ sql "sync"
+ qt_del_cnt "select count() from ${table1} where
__DORIS_DELETE_SIGN__=1;"
+ sql "set skip_delete_sign=false;"
+ sql "set skip_delete_bitmap=false;"
+ sql "sync"
+ }
+
+ (1..30).each {
+ sql "insert into ${table1} values($it,$it,$it);"
+ }
+ trigger_and_wait_compaction(table1, "cumulative")
+
+ sql "delete from ${table1} where k1<=20;"
+ sql "sync;"
+ qt_sql "select count() from ${table1};"
+ getDeleteSignCnt()
+
+ (31..60).each {
+ sql "insert into ${table1} values($it,$it,$it);"
+ }
+ trigger_and_wait_compaction(table1, "cumulative")
+
+ trigger_and_wait_compaction(table1, "base")
+ qt_sql "select count() from ${table1};"
+ getDeleteSignCnt()
+
+ def tablets = sql_return_maparray """ show tablets from ${table1}; """
+ logger.info("tablets: ${tablets}")
+ assert 1 == tablets.size()
+ String compactionUrl = tablets[0]["CompactionStatus"]
+ def (code, out, err) = curl("GET", compactionUrl)
+ logger.info("Show tablets status: code=" + code + ", out=" + out + ",
err=" + err)
+ assert code == 0
+ def tabletJson = parseJson(out.trim())
+ assert tabletJson.rowsets.size() == 1
+ assert tabletJson.rowsets[0].contains("[0-62]")
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
