This is an automated email from the ASF dual-hosted git repository.
gavinchou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 64e8cf2c399 [improvement](be) Add queue time for delete bitmap tasks
(#65248)
64e8cf2c399 is described below
commit 64e8cf2c3992f7377e8d5af567c95b863d169166
Author: bobhan1 <[email protected]>
AuthorDate: Mon Jul 13 10:46:25 2026 +0800
[improvement](be) Add queue time for delete bitmap tasks (#65248)
Problem Summary: MOW memtable flush calculates delete bitmap
asynchronously, but its log did not show how long the async task waited
in the executor queue. The load delete-bitmap executor also used the
same thread-pool metric name as the normal delete-bitmap executor,
making their metrics hard to distinguish. This change records queue time
for flush, segment-level, between-segment, and cloud tablet-level async
delete-bitmap tasks, and gives the load delete-bitmap executor a
distinct thread-pool name.
---
.../cloud/cloud_engine_calc_delete_bitmap_task.cpp | 25 ++--
.../cloud/cloud_engine_calc_delete_bitmap_task.h | 2 +-
be/src/cloud/cloud_storage_engine.cpp | 4 +-
.../storage/delete/calc_delete_bitmap_executor.cpp | 12 +-
.../storage/delete/calc_delete_bitmap_executor.h | 3 +-
be/src/storage/rowset/beta_rowset_writer.cpp | 126 +++++++++++----------
be/src/storage/storage_engine.cpp | 4 +-
be/src/storage/tablet/base_tablet.cpp | 20 ++--
be/src/storage/tablet/base_tablet.h | 5 +-
9 files changed, 109 insertions(+), 92 deletions(-)
diff --git a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp
b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp
index c310cd6a0b6..869edca0b49 100644
--- a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp
+++ b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.cpp
@@ -93,15 +93,18 @@ Status CloudEngineCalcDeleteBitmapTask::execute() {
if (has_tablet_states) {
tablet_calc_delete_bitmap_ptr->set_tablet_state(partition.tablet_states[i]);
}
- auto submit_st = token->submit_func([tablet_id,
tablet_calc_delete_bitmap_ptr, this]() {
- auto st = tablet_calc_delete_bitmap_ptr->handle();
- if (st.ok()) {
- add_succ_tablet_id(tablet_id);
- } else {
- LOG(WARNING) << "handle calc delete bitmap fail, st=" <<
st.to_string();
- add_error_tablet_id(tablet_id, st);
- }
- });
+ const auto submit_time_us = MonotonicMicros();
+ auto submit_st = token->submit_func(
+ [tablet_id, tablet_calc_delete_bitmap_ptr, this,
submit_time_us]() {
+ const auto queue_time_us = MonotonicMicros() -
submit_time_us;
+ auto st =
tablet_calc_delete_bitmap_ptr->handle(queue_time_us);
+ if (st.ok()) {
+ add_succ_tablet_id(tablet_id);
+ } else {
+ LOG(WARNING) << "handle calc delete bitmap fail,
st=" << st.to_string();
+ add_error_tablet_id(tablet_id, st);
+ }
+ });
VLOG_DEBUG << "submit TabletCalcDeleteBitmapTask for tablet=" <<
tablet_id;
if (!submit_st.ok()) {
_res = submit_st;
@@ -143,7 +146,7 @@ void
CloudTabletCalcDeleteBitmapTask::set_tablet_state(int64_t tablet_state) {
_ms_tablet_state = tablet_state;
}
-Status CloudTabletCalcDeleteBitmapTask::handle() const {
+Status CloudTabletCalcDeleteBitmapTask::handle(int64_t queue_time_us) const {
VLOG_DEBUG << "start calculate delete bitmap on tablet " << _tablet_id
<< ", txn_id=" << _transaction_id;
SCOPED_ATTACH_TASK(_mem_tracker);
@@ -284,7 +287,7 @@ Status CloudTabletCalcDeleteBitmapTask::handle() const {
auto total_update_delete_bitmap_time_us = MonotonicMicros() - t3;
LOG(INFO) << "finish calculate delete bitmap on tablet"
<< ", table_id=" << tablet->table_id() << ", transaction_id=" <<
_transaction_id
- << ", tablet_id=" << tablet->tablet_id()
+ << ", tablet_id=" << tablet->tablet_id() << ", queue_time_us="
<< queue_time_us
<< ", get_tablet_time_us=" << get_tablet_time_us
<< ", sync_rowset_time_us=" << sync_rowset_time_us
<< ", total_update_delete_bitmap_time_us=" <<
total_update_delete_bitmap_time_us
diff --git a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.h
b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.h
index 5ea11e8d4c3..04b085b5ff6 100644
--- a/be/src/cloud/cloud_engine_calc_delete_bitmap_task.h
+++ b/be/src/cloud/cloud_engine_calc_delete_bitmap_task.h
@@ -43,7 +43,7 @@ public:
int64_t ms_cumulative_point);
void set_tablet_state(int64_t tablet_state);
- Status handle() const;
+ Status handle(int64_t queue_time_us) const;
private:
Status _handle_rowset(std::shared_ptr<CloudTablet> tablet, int64_t version,
diff --git a/be/src/cloud/cloud_storage_engine.cpp
b/be/src/cloud/cloud_storage_engine.cpp
index 803a738d6da..0dc8321c2c4 100644
--- a/be/src/cloud/cloud_storage_engine.cpp
+++ b/be/src/cloud/cloud_storage_engine.cpp
@@ -206,10 +206,12 @@ Status CloudStorageEngine::open() {
cast_set<int32_t>(io::FileCacheFactory::instance()->get_cache_instance_size()));
_calc_delete_bitmap_executor =
std::make_unique<CalcDeleteBitmapExecutor>();
- _calc_delete_bitmap_executor->init(config::calc_delete_bitmap_max_thread);
+ _calc_delete_bitmap_executor->init("TabletCalcDeleteBitmapThreadPool",
+ config::calc_delete_bitmap_max_thread);
_calc_delete_bitmap_executor_for_load =
std::make_unique<CalcDeleteBitmapExecutor>();
_calc_delete_bitmap_executor_for_load->init(
+ "LoadCalcDeleteBitmapThreadPool",
config::calc_delete_bitmap_for_load_max_thread > 0
? config::calc_delete_bitmap_for_load_max_thread
: std::max(1, CpuInfo::num_cores() / 2));
diff --git a/be/src/storage/delete/calc_delete_bitmap_executor.cpp
b/be/src/storage/delete/calc_delete_bitmap_executor.cpp
index 1867dc54a9e..af1b4bfb23a 100644
--- a/be/src/storage/delete/calc_delete_bitmap_executor.cpp
+++ b/be/src/storage/delete/calc_delete_bitmap_executor.cpp
@@ -41,11 +41,13 @@ Status CalcDeleteBitmapToken::submit(BaseTabletSPtr tablet,
RowsetSharedPtr cur_
_resource_ctx = thread_context()->resource_ctx();
}
+ const auto submit_time_us = MonotonicMicros();
return _thread_token->submit_func([=, this]() {
+ const auto queue_time_us = MonotonicMicros() - submit_time_us;
SCOPED_ATTACH_TASK(_resource_ctx);
auto st = tablet->calc_segment_delete_bitmap(cur_rowset, cur_segment,
target_rowsets,
delete_bitmap,
end_version, rowset_writer,
- tablet_delete_bitmap);
+ tablet_delete_bitmap,
queue_time_us);
if (!st.ok()) {
LOG(WARNING) << "failed to calc segment delete bitmap, tablet_id: "
<< tablet->tablet_id() << " rowset: " <<
cur_rowset->rowset_id()
@@ -68,10 +70,12 @@ Status CalcDeleteBitmapToken::submit(BaseTabletSPtr tablet,
TabletSchemaSPtr sch
RETURN_IF_ERROR(_status);
_resource_ctx = thread_context()->resource_ctx();
}
+ const auto submit_time_us = MonotonicMicros();
return _thread_token->submit_func([=, this]() {
+ const auto queue_time_us = MonotonicMicros() - submit_time_us;
SCOPED_ATTACH_TASK(_resource_ctx);
auto st = tablet->calc_delete_bitmap_between_segments(schema,
rowset_id, segments,
- delete_bitmap);
+ delete_bitmap,
queue_time_us);
if (!st.ok()) {
LOG(WARNING) << "failed to calc delete bitmap between segments,
tablet_id: "
<< tablet->tablet_id() << " rowset: " << rowset_id
@@ -90,8 +94,8 @@ Status CalcDeleteBitmapToken::wait() {
return _status;
}
-void CalcDeleteBitmapExecutor::init(int max_threads) {
- static_cast<void>(ThreadPoolBuilder("TabletCalcDeleteBitmapThreadPool")
+void CalcDeleteBitmapExecutor::init(const std::string& name, int max_threads) {
+ static_cast<void>(ThreadPoolBuilder(name)
.set_min_threads(1)
.set_max_threads(max_threads)
.build(&_thread_pool));
diff --git a/be/src/storage/delete/calc_delete_bitmap_executor.h
b/be/src/storage/delete/calc_delete_bitmap_executor.h
index be97505e038..4f0f0fd2c7f 100644
--- a/be/src/storage/delete/calc_delete_bitmap_executor.h
+++ b/be/src/storage/delete/calc_delete_bitmap_executor.h
@@ -23,6 +23,7 @@
#include <iosfwd>
#include <memory>
#include <shared_mutex>
+#include <string>
#include <utility>
#include <vector>
@@ -105,7 +106,7 @@ public:
~CalcDeleteBitmapExecutor() { _thread_pool->shutdown(); }
// init should be called after storage engine is opened,
- void init(int max_threads);
+ void init(const std::string& name, int max_threads);
std::unique_ptr<CalcDeleteBitmapToken> create_token();
diff --git a/be/src/storage/rowset/beta_rowset_writer.cpp
b/be/src/storage/rowset/beta_rowset_writer.cpp
index cf54ea07e1d..40aef241fcf 100644
--- a/be/src/storage/rowset/beta_rowset_writer.cpp
+++ b/be/src/storage/rowset/beta_rowset_writer.cpp
@@ -418,72 +418,74 @@ Status
BaseBetaRowsetWriter::_generate_delete_bitmap(int32_t segment_id) {
// Submit the entire delete bitmap calculation process to thread pool for
async execution
// This avoids blocking memtable flush thread while waiting for file
upload to complete
// The process includes: file_writer->close(), _build_tmp, load_segments,
and calc_delete_bitmap
- return _calc_delete_bitmap_token->submit_func(
- [this, segment_id, specified_rowsets =
std::move(specified_rowsets)]() -> Status {
- Status st = Status::OK();
- // Step 1: Close file_writer (must be done before
load_segments)
- auto* file_writer = _seg_files.get(segment_id);
- if (file_writer && file_writer->state() !=
io::FileWriter::State::CLOSED) {
- MonotonicStopWatch close_timer;
- close_timer.start();
- st = file_writer->close();
- close_timer.stop();
-
- auto close_time_ms =
close_timer.elapsed_time_milliseconds();
- if (close_time_ms > 1000) {
- LOG(INFO) << "file_writer->close() took " <<
close_time_ms
- << "ms for segment_id=" << segment_id
- << ", tablet_id=" << _context.tablet_id
- << ", rowset_id=" << _context.rowset_id;
- }
- if (!st.ok()) {
- return st;
- }
- }
+ const auto submit_time_us = MonotonicMicros();
+ return _calc_delete_bitmap_token->submit_func([this, segment_id,
+ specified_rowsets =
std::move(specified_rowsets),
+ submit_time_us]() -> Status
{
+ const auto queue_time_us = MonotonicMicros() - submit_time_us;
+ Status st = Status::OK();
+ // Step 1: Close file_writer (must be done before load_segments)
+ auto* file_writer = _seg_files.get(segment_id);
+ if (file_writer && file_writer->state() !=
io::FileWriter::State::CLOSED) {
+ MonotonicStopWatch close_timer;
+ close_timer.start();
+ st = file_writer->close();
+ close_timer.stop();
+
+ auto close_time_ms = close_timer.elapsed_time_milliseconds();
+ if (close_time_ms > 1000) {
+ LOG(INFO) << "file_writer->close() took " << close_time_ms
+ << "ms for segment_id=" << segment_id
+ << ", tablet_id=" << _context.tablet_id
+ << ", rowset_id=" << _context.rowset_id;
+ }
+ if (!st.ok()) {
+ return st;
+ }
+ }
- OlapStopWatch watch;
- // Step 2: Build tmp rowset (needs file_writer to be closed)
- RowsetSharedPtr rowset_ptr;
- st = _build_tmp(rowset_ptr);
- if (!st.ok()) {
- return st;
- }
+ OlapStopWatch watch;
+ // Step 2: Build tmp rowset (needs file_writer to be closed)
+ RowsetSharedPtr rowset_ptr;
+ st = _build_tmp(rowset_ptr);
+ if (!st.ok()) {
+ return st;
+ }
- // Step 3: Load segments (needs file_writer to be closed and
rowset to be built)
- auto* beta_rowset =
reinterpret_cast<BetaRowset*>(rowset_ptr.get());
- std::vector<segment_v2::SegmentSharedPtr> segments;
- st = beta_rowset->load_segments(segment_id, segment_id + 1,
&segments);
- if (!st.ok()) {
- return st;
- }
+ // Step 3: Load segments (needs file_writer to be closed and rowset to
be built)
+ auto* beta_rowset = reinterpret_cast<BetaRowset*>(rowset_ptr.get());
+ std::vector<segment_v2::SegmentSharedPtr> segments;
+ st = beta_rowset->load_segments(segment_id, segment_id + 1, &segments);
+ if (!st.ok()) {
+ return st;
+ }
- // Step 4: Calculate delete bitmap
- st = BaseTablet::calc_delete_bitmap(
- _context.tablet, rowset_ptr, segments,
specified_rowsets,
- _context.mow_context->delete_bitmap,
_context.mow_context->max_version,
- nullptr, nullptr, nullptr);
- if (!st.ok()) {
- return st;
- }
+ // Step 4: Calculate delete bitmap
+ st = BaseTablet::calc_delete_bitmap(_context.tablet, rowset_ptr,
segments,
+ specified_rowsets,
_context.mow_context->delete_bitmap,
+ _context.mow_context->max_version,
nullptr, nullptr,
+ nullptr);
+ if (!st.ok()) {
+ return st;
+ }
- size_t total_rows =
- std::accumulate(segments.begin(), segments.end(), 0,
- [](size_t sum, const
segment_v2::SegmentSharedPtr& s) {
- return sum += s->num_rows();
- });
- LOG(INFO) << "[Memtable Flush] construct delete bitmap tablet:
"
- << _context.tablet->tablet_id()
- << ", rowset_ids: " <<
_context.mow_context->rowset_ids->size()
- << ", cur max_version: " <<
_context.mow_context->max_version
- << ", transaction_id: " <<
_context.mow_context->txn_id
- << ", delete_bitmap_count: "
- <<
_context.mow_context->delete_bitmap->get_delete_bitmap_count()
- << ", delete_bitmap_cardinality: "
- << _context.mow_context->delete_bitmap->cardinality()
- << ", cost: " << watch.get_elapse_time_us()
- << "(us), total rows: " << total_rows;
- return Status::OK();
- });
+ size_t total_rows = std::accumulate(segments.begin(), segments.end(),
0,
+ [](size_t sum, const
segment_v2::SegmentSharedPtr& s) {
+ return sum += s->num_rows();
+ });
+ LOG(INFO) << "[Memtable Flush] construct delete bitmap tablet: "
+ << _context.tablet->tablet_id()
+ << ", rowset_ids: " <<
_context.mow_context->rowset_ids->size()
+ << ", cur max_version: " << _context.mow_context->max_version
+ << ", transaction_id: " << _context.mow_context->txn_id
+ << ", delete_bitmap_count: "
+ <<
_context.mow_context->delete_bitmap->get_delete_bitmap_count()
+ << ", delete_bitmap_cardinality: "
+ << _context.mow_context->delete_bitmap->cardinality()
+ << ", queue_time_us: " << queue_time_us
+ << ", cost: " << watch.get_elapse_time_us() << "(us), total
rows: " << total_rows;
+ return Status::OK();
+ });
}
Status BetaRowsetWriter::init(const RowsetWriterContext&
rowset_writer_context) {
diff --git a/be/src/storage/storage_engine.cpp
b/be/src/storage/storage_engine.cpp
index 2616742465a..2bc9e46b101 100644
--- a/be/src/storage/storage_engine.cpp
+++ b/be/src/storage/storage_engine.cpp
@@ -325,10 +325,12 @@ Status StorageEngine::_open() {
_memtable_flush_executor->init(_disk_num);
_calc_delete_bitmap_executor =
std::make_unique<CalcDeleteBitmapExecutor>();
- _calc_delete_bitmap_executor->init(config::calc_delete_bitmap_max_thread);
+ _calc_delete_bitmap_executor->init("TabletCalcDeleteBitmapThreadPool",
+ config::calc_delete_bitmap_max_thread);
_calc_delete_bitmap_executor_for_load =
std::make_unique<CalcDeleteBitmapExecutor>();
_calc_delete_bitmap_executor_for_load->init(
+ "LoadCalcDeleteBitmapThreadPool",
config::calc_delete_bitmap_for_load_max_thread > 0
? config::calc_delete_bitmap_for_load_max_thread
: std::max(1, CpuInfo::num_cores() / 2));
diff --git a/be/src/storage/tablet/base_tablet.cpp
b/be/src/storage/tablet/base_tablet.cpp
index 3cd8747ee79..92161a3476f 100644
--- a/be/src/storage/tablet/base_tablet.cpp
+++ b/be/src/storage/tablet/base_tablet.cpp
@@ -381,7 +381,8 @@ void
BaseTablet::generate_tablet_meta_copy_unlocked(TabletMeta& new_tablet_meta,
Status BaseTablet::calc_delete_bitmap_between_segments(
TabletSchemaSPtr schema, RowsetId rowset_id,
- const std::vector<segment_v2::SegmentSharedPtr>& segments,
DeleteBitmapPtr delete_bitmap) {
+ const std::vector<segment_v2::SegmentSharedPtr>& segments,
DeleteBitmapPtr delete_bitmap,
+ int64_t queue_time_us) {
size_t const num_segments = segments.size();
if (num_segments < 2) {
return Status::OK();
@@ -409,9 +410,9 @@ Status BaseTablet::calc_delete_bitmap_between_segments(
LOG(INFO) << fmt::format(
"construct delete bitmap between segments, "
"tablet: {}, rowset: {}, number of segments: {}, bitmap count: {},
bitmap cardinality: "
- "{}, cost {} (us)",
+ "{}, queue_time_us: {}, cost {} (us)",
tablet_id(), rowset_id.to_string(), num_segments,
- delete_bitmap->get_delete_bitmap_count(),
delete_bitmap->cardinality(),
+ delete_bitmap->get_delete_bitmap_count(),
delete_bitmap->cardinality(), queue_time_us,
watch.get_elapse_time_us());
return Status::OK();
}
@@ -600,7 +601,8 @@ Status
BaseTablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset,
const
std::vector<RowsetSharedPtr>& specified_rowsets,
DeleteBitmapPtr delete_bitmap,
int64_t end_version,
RowsetWriter* rowset_writer,
- DeleteBitmapPtr
tablet_delete_bitmap) {
+ DeleteBitmapPtr
tablet_delete_bitmap,
+ int64_t queue_time_us) {
OlapStopWatch watch;
auto rowset_id = rowset->rowset_id();
Version dummy_version(end_version + 1, end_version + 1);
@@ -869,7 +871,7 @@ Status
BaseTablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset,
}
RETURN_IF_ERROR(rowset_writer->flush_single_block(&ordered_block));
auto cost_us = watch.get_elapse_time_us();
- if (config::enable_mow_verbose_log || cost_us > 10 * 1000) {
+ if (config::enable_mow_verbose_log || cost_us > 10 * 1000 ||
queue_time_us > 10 * 1000) {
LOG(INFO) << "calc segment delete bitmap for "
<< partial_update_info->partial_update_mode_str()
<< ", tablet: " << tablet_id() << " rowset: " <<
rowset_id
@@ -878,19 +880,19 @@ Status
BaseTablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset,
<< " new generated rows: " << new_generated_rows
<< " bitmap num: " <<
delete_bitmap->get_delete_bitmap_count()
<< " bitmap cardinality: " <<
delete_bitmap->cardinality()
- << " cost: " << cost_us << "(us)";
+ << " queue_time_us: " << queue_time_us << ", cost: " <<
cost_us << "(us)";
}
return Status::OK();
}
auto cost_us = watch.get_elapse_time_us();
- if (config::enable_mow_verbose_log || cost_us > 10 * 1000) {
+ if (config::enable_mow_verbose_log || cost_us > 10 * 1000 || queue_time_us
> 10 * 1000) {
LOG(INFO) << "calc segment delete bitmap, tablet: " << tablet_id()
<< " rowset: " << rowset_id << " seg_id: " << seg->id()
<< " dummy_version: " << end_version + 1 << " rows: " <<
seg->num_rows()
<< " conflict rows: " << conflict_rows
<< " bitmap num: " <<
delete_bitmap->get_delete_bitmap_count()
- << " bitmap cardinality: " << delete_bitmap->cardinality()
<< " cost: " << cost_us
- << "(us)";
+ << " bitmap cardinality: " << delete_bitmap->cardinality()
+ << " queue_time_us: " << queue_time_us << ", cost: " <<
cost_us << "(us)";
}
return Status::OK();
}
diff --git a/be/src/storage/tablet/base_tablet.h
b/be/src/storage/tablet/base_tablet.h
index c26e944b01c..510e994e36c 100644
--- a/be/src/storage/tablet/base_tablet.h
+++ b/be/src/storage/tablet/base_tablet.h
@@ -202,12 +202,13 @@ public:
const std::vector<RowsetSharedPtr>&
specified_rowsets,
DeleteBitmapPtr delete_bitmap, int64_t
end_version,
RowsetWriter* rowset_writer,
- DeleteBitmapPtr tablet_delete_bitmap =
nullptr);
+ DeleteBitmapPtr tablet_delete_bitmap =
nullptr,
+ int64_t queue_time_us = 0);
Status calc_delete_bitmap_between_segments(
TabletSchemaSPtr schema, RowsetId rowset_id,
const std::vector<segment_v2::SegmentSharedPtr>& segments,
- DeleteBitmapPtr delete_bitmap);
+ DeleteBitmapPtr delete_bitmap, int64_t queue_time_us = 0);
static Status commit_phase_update_delete_bitmap(
const BaseTabletSPtr& tablet, const RowsetSharedPtr& rowset,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
