This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 48016a747e9 branch-3.0: [Opt](mow) Calc delete bitmaps between
segments concurrently in commit phase (#50106) (#50495)
48016a747e9 is described below
commit 48016a747e994ee1b13cbd04e499bc8b9f3f7cc0
Author: bobhan1 <[email protected]>
AuthorDate: Mon Apr 28 21:34:02 2025 +0800
branch-3.0: [Opt](mow) Calc delete bitmaps between segments concurrently in
commit phase (#50106) (#50495)
pick https://github.com/apache/doris/pull/50106
---
.../cloud/cloud_engine_calc_delete_bitmap_task.cpp | 4 ++--
be/src/common/config.cpp | 2 ++
be/src/common/config.h | 2 ++
be/src/olap/base_tablet.cpp | 6 +++---
be/src/olap/base_tablet.h | 2 +-
be/src/olap/calc_delete_bitmap_executor.cpp | 23 ++++++++++++++++++++++
be/src/olap/calc_delete_bitmap_executor.h | 6 ++++++
be/src/olap/rowset_builder.cpp | 9 +++++++--
be/src/olap/txn_manager.cpp | 2 +-
be/src/service/backend_service.cpp | 4 ++--
10 files changed, 49 insertions(+), 11 deletions(-)
diff --git a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp
b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp
index a55d4f42f66..07fcdf881c1 100644
--- a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp
+++ b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp
@@ -267,8 +267,8 @@ Status CloudTabletCalcDeleteBitmapTask::handle() const {
LOG_INFO("inject error when
CloudTabletCalcDeleteBitmapTask::_handle_rowset");
return Status::MemoryLimitExceeded("injected
MemoryLimitExceeded error");
});
- RETURN_IF_ERROR(
- tablet->calc_delete_bitmap_between_segments(rowset,
segments, delete_bitmap));
+
RETURN_IF_ERROR(tablet->calc_delete_bitmap_between_segments(rowset->rowset_id(),
+
segments, delete_bitmap));
}
status = CloudTablet::update_delete_bitmap(tablet, &txn_info,
_transaction_id,
txn_expiration);
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 92885c6cd89..3063e782dba 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1490,6 +1490,8 @@ DEFINE_mInt32(load_trigger_compaction_version_percent,
"66");
DEFINE_mInt64(base_compaction_interval_seconds_since_last_operation, "86400");
DEFINE_mBool(enable_compaction_pause_on_high_memory, "true");
+DEFINE_mBool(enable_calc_delete_bitmap_between_segments_concurrently, "false");
+
// clang-format off
#ifdef BE_TEST
// test s3
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 43662a04842..d9a93c1f7f6 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1568,6 +1568,8 @@ DECLARE_mInt32(load_trigger_compaction_version_percent);
DECLARE_mInt64(base_compaction_interval_seconds_since_last_operation);
DECLARE_mBool(enable_compaction_pause_on_high_memory);
+DECLARE_mBool(enable_calc_delete_bitmap_between_segments_concurrently);
+
#ifdef BE_TEST
// test s3
DECLARE_String(test_s3_resource);
diff --git a/be/src/olap/base_tablet.cpp b/be/src/olap/base_tablet.cpp
index c33043b3b64..41e8bbf16d8 100644
--- a/be/src/olap/base_tablet.cpp
+++ b/be/src/olap/base_tablet.cpp
@@ -359,7 +359,7 @@ void
BaseTablet::generate_tablet_meta_copy_unlocked(TabletMeta& new_tablet_meta)
}
Status BaseTablet::calc_delete_bitmap_between_segments(
- RowsetSharedPtr rowset, const
std::vector<segment_v2::SegmentSharedPtr>& segments,
+ const RowsetId& rowset_id, const
std::vector<segment_v2::SegmentSharedPtr>& segments,
DeleteBitmapPtr delete_bitmap) {
size_t const num_segments = segments.size();
if (num_segments < 2) {
@@ -367,7 +367,6 @@ Status BaseTablet::calc_delete_bitmap_between_segments(
}
OlapStopWatch watch;
- auto const rowset_id = rowset->rowset_id();
size_t seq_col_length = 0;
if (_tablet_meta->tablet_schema()->has_sequence_col()) {
auto seq_col_idx = _tablet_meta->tablet_schema()->sequence_col_idx();
@@ -1506,7 +1505,8 @@ Status BaseTablet::update_delete_bitmap_without_lock(
// calculate delete bitmap between segments if necessary.
DeleteBitmapPtr delete_bitmap =
std::make_shared<DeleteBitmap>(self->tablet_id());
- RETURN_IF_ERROR(self->calc_delete_bitmap_between_segments(rowset,
segments, delete_bitmap));
+
RETURN_IF_ERROR(self->calc_delete_bitmap_between_segments(rowset->rowset_id(),
segments,
+ delete_bitmap));
// get all base rowsets to calculate on
std::vector<RowsetSharedPtr> specified_rowsets;
diff --git a/be/src/olap/base_tablet.h b/be/src/olap/base_tablet.h
index 40928e63729..4f4b41f11ab 100644
--- a/be/src/olap/base_tablet.h
+++ b/be/src/olap/base_tablet.h
@@ -176,7 +176,7 @@ public:
RowsetWriter* rowset_writer);
Status calc_delete_bitmap_between_segments(
- RowsetSharedPtr rowset, const
std::vector<segment_v2::SegmentSharedPtr>& segments,
+ const RowsetId& rowset_id, const
std::vector<segment_v2::SegmentSharedPtr>& segments,
DeleteBitmapPtr delete_bitmap);
static Status commit_phase_update_delete_bitmap(
diff --git a/be/src/olap/calc_delete_bitmap_executor.cpp
b/be/src/olap/calc_delete_bitmap_executor.cpp
index 3983dc0a986..89e668c4c94 100644
--- a/be/src/olap/calc_delete_bitmap_executor.cpp
+++ b/be/src/olap/calc_delete_bitmap_executor.cpp
@@ -58,6 +58,29 @@ Status CalcDeleteBitmapToken::submit(BaseTabletSPtr tablet,
RowsetSharedPtr cur_
});
}
+Status CalcDeleteBitmapToken::submit(BaseTabletSPtr tablet, RowsetId rowset_id,
+ const
std::vector<segment_v2::SegmentSharedPtr>& segments,
+ DeleteBitmapPtr delete_bitmap) {
+ {
+ std::shared_lock rlock(_lock);
+ RETURN_IF_ERROR(_status);
+ _query_thread_context.init_unlocked();
+ }
+ return _thread_token->submit_func([=, this]() {
+ SCOPED_ATTACH_TASK(_query_thread_context);
+ auto st = tablet->calc_delete_bitmap_between_segments(rowset_id,
segments, delete_bitmap);
+ if (!st.ok()) {
+ LOG(WARNING) << "failed to calc delete bitmap between segments,
tablet_id: "
+ << tablet->tablet_id() << " rowset: " << rowset_id
+ << " segments num: " << segments.size() << " error: "
<< st;
+ std::lock_guard wlock(_lock);
+ if (_status.ok()) {
+ _status = st;
+ }
+ }
+ });
+}
+
Status CalcDeleteBitmapToken::wait() {
_thread_token->wait();
// all tasks complete here, don't need lock;
diff --git a/be/src/olap/calc_delete_bitmap_executor.h
b/be/src/olap/calc_delete_bitmap_executor.h
index fa1e79b7fea..5d471e2587e 100644
--- a/be/src/olap/calc_delete_bitmap_executor.h
+++ b/be/src/olap/calc_delete_bitmap_executor.h
@@ -49,11 +49,17 @@ public:
explicit CalcDeleteBitmapToken(std::unique_ptr<ThreadPoolToken>
thread_token)
: _thread_token(std::move(thread_token)), _status(Status::OK()) {}
+ // calculate delete bitmap of `cur_segment` to historical `target_rowsets`
Status submit(BaseTabletSPtr tablet, RowsetSharedPtr cur_rowset,
const segment_v2::SegmentSharedPtr& cur_segment,
const std::vector<RowsetSharedPtr>& target_rowsets, int64_t
end_version,
DeleteBitmapPtr delete_bitmap, RowsetWriter* rowset_writer);
+ // calculate delete bitmap between `segments`
+ Status submit(BaseTabletSPtr tablet, RowsetId rowset_id,
+ const std::vector<segment_v2::SegmentSharedPtr>& segments,
+ DeleteBitmapPtr delete_bitmap);
+
// wait all tasks in token to be completed.
Status wait();
diff --git a/be/src/olap/rowset_builder.cpp b/be/src/olap/rowset_builder.cpp
index c7fc1c38d77..eb855ab1316 100644
--- a/be/src/olap/rowset_builder.cpp
+++ b/be/src/olap/rowset_builder.cpp
@@ -268,8 +268,13 @@ Status BaseRowsetBuilder::submit_calc_delete_bitmap_task()
{
RETURN_IF_ERROR(beta_rowset->load_segments(&segments));
if (segments.size() > 1) {
// calculate delete bitmap between segments
- RETURN_IF_ERROR(
- _tablet->calc_delete_bitmap_between_segments(_rowset,
segments, _delete_bitmap));
+ if (config::enable_calc_delete_bitmap_between_segments_concurrently) {
+ RETURN_IF_ERROR(_calc_delete_bitmap_token->submit(_tablet,
_rowset->rowset_id(),
+ segments,
_delete_bitmap));
+ } else {
+
RETURN_IF_ERROR(_tablet->calc_delete_bitmap_between_segments(_rowset->rowset_id(),
+
segments, _delete_bitmap));
+ }
}
// tablet is under alter process. The delete bitmap will be calculated
after conversion.
diff --git a/be/src/olap/txn_manager.cpp b/be/src/olap/txn_manager.cpp
index 0973511080f..f47bd776fac 100644
--- a/be/src/olap/txn_manager.cpp
+++ b/be/src/olap/txn_manager.cpp
@@ -549,7 +549,7 @@ Status TxnManager::publish_txn(OlapMeta* meta, TPartitionId
partition_id,
std::vector<segment_v2::SegmentSharedPtr> segments;
RETURN_IF_ERROR(std::static_pointer_cast<BetaRowset>(rowset)->load_segments(&segments));
RETURN_IF_ERROR(tablet->calc_delete_bitmap_between_segments(
- rowset, segments, tablet_txn_info->delete_bitmap));
+ rowset->rowset_id(), segments,
tablet_txn_info->delete_bitmap));
}
RETURN_IF_ERROR(
diff --git a/be/src/service/backend_service.cpp
b/be/src/service/backend_service.cpp
index 61f276952c2..9f6413762f0 100644
--- a/be/src/service/backend_service.cpp
+++ b/be/src/service/backend_service.cpp
@@ -591,8 +591,8 @@ void _ingest_binlog(StorageEngine& engine, IngestBinlogArg*
arg) {
}
if (segments.size() > 1) {
// calculate delete bitmap between segments
- status = local_tablet->calc_delete_bitmap_between_segments(rowset,
segments,
-
delete_bitmap);
+ status =
local_tablet->calc_delete_bitmap_between_segments(rowset->rowset_id(),
+
segments, delete_bitmap);
if (!status) {
LOG(WARNING) << "failed to calculate delete bitmap"
<< ". tablet_id: " << local_tablet->tablet_id()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
