This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 64dd61bb1b6 branch-3.0: [chore](binlog) Log download binlog cost
#47360 (#47397)
64dd61bb1b6 is described below
commit 64dd61bb1b66603ba79d6e09264b032dee4a33d9
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Wed Jan 29 09:13:28 2025 +0800
branch-3.0: [chore](binlog) Log download binlog cost #47360 (#47397)
Cherry-picked from #47360
Co-authored-by: walter <maoch...@selectdb.com>
---
be/src/service/backend_service.cpp | 21 +++++++++++++++++++--
1 file changed, 19 insertions(+), 2 deletions(-)
diff --git a/be/src/service/backend_service.cpp
b/be/src/service/backend_service.cpp
index d74a9cd2e0b..6bb73f37b8c 100644
--- a/be/src/service/backend_service.cpp
+++ b/be/src/service/backend_service.cpp
@@ -113,10 +113,23 @@ void _ingest_binlog(StorageEngine& engine,
IngestBinlogArg* arg) {
auto& request = arg->request;
+ MonotonicStopWatch watch;
+ watch.start();
+ int64_t total_download_bytes = 0;
+ int64_t total_download_files = 0;
TStatus tstatus;
std::vector<std::string> download_success_files;
- Defer defer {[=, &engine, &tstatus, ingest_binlog_tstatus =
arg->tstatus]() {
- LOG(INFO) << "ingest binlog. result: " <<
apache::thrift::ThriftDebugString(tstatus);
+ Defer defer {[=, &engine, &tstatus, ingest_binlog_tstatus = arg->tstatus,
&watch,
+ &total_download_bytes, &total_download_files]() {
+ auto elapsed_time_ms = static_cast<int64_t>(watch.elapsed_time() /
1000000);
+ double copy_rate = 0.0;
+ if (elapsed_time_ms > 0) {
+ copy_rate = total_download_bytes / ((double)elapsed_time_ms) /
1000;
+ }
+ LOG(INFO) << "ingest binlog elapsed " << elapsed_time_ms << " ms,
download "
+ << total_download_files << " files, total " <<
total_download_bytes
+ << " bytes, avg rate " << copy_rate
+ << " MB/s. result: " <<
apache::thrift::ThriftDebugString(tstatus);
if (tstatus.status_code != TStatusCode::OK) {
// abort txn
engine.txn_manager()->abort_txn(partition_id, txn_id,
local_tablet_id,
@@ -269,6 +282,8 @@ void _ingest_binlog(StorageEngine& engine, IngestBinlogArg*
arg) {
status.to_thrift(&tstatus);
return;
}
+ total_download_bytes = total_size;
+ total_download_files = num_segments;
// Step 5.3: get all segment files
for (int64_t segment_index = 0; segment_index < num_segments;
++segment_index) {
@@ -442,6 +457,8 @@ void _ingest_binlog(StorageEngine& engine, IngestBinlogArg*
arg) {
status.to_thrift(&tstatus);
return;
}
+ total_download_bytes += total_index_size;
+ total_download_files += segment_index_file_urls.size();
// Step 6.3: get all segment index files
DCHECK(segment_index_file_sizes.size() == segment_index_file_names.size());
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
