This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new ca486cdfbc [Enhancement](storage) optimize segment compaction log
(#14448) (#14449)
ca486cdfbc is described below
commit ca486cdfbc4fd62d94559c86fd180618e6e6b49b
Author: zhengyu <freeman.zhang1...@gmail.com>
AuthorDate: Tue Nov 22 08:43:51 2022 +0800
[Enhancement](storage) optimize segment compaction log (#14448) (#14449)
Signed-off-by: freemandealer <freeman.zhang1...@gmail.com>
Signed-off-by: freemandealer <freeman.zhang1...@gmail.com>
---
be/src/olap/rowset/beta_rowset_writer.cpp | 24 +++++++++++++++---------
1 file changed, 15 insertions(+), 9 deletions(-)
diff --git a/be/src/olap/rowset/beta_rowset_writer.cpp
b/be/src/olap/rowset/beta_rowset_writer.cpp
index 65b78b288a..512e4d85b1 100644
--- a/be/src/olap/rowset/beta_rowset_writer.cpp
+++ b/be/src/olap/rowset/beta_rowset_writer.cpp
@@ -281,8 +281,6 @@ Status
BetaRowsetWriter::_do_compact_segments(SegCompactionCandidatesSharedPtr s
}
uint64_t begin = (*(segments->begin()))->id();
uint64_t end = (*(segments->end() - 1))->id();
- LOG(INFO) << "BetaRowsetWriter:" << this << " do segcompaction at " <<
segments->size()
- << " segments. Begin:" << begin << " End:" << end;
uint64_t begin_time = GetCurrentTimeMicros();
auto schema = std::make_shared<Schema>(_context.tablet_schema->columns(),
@@ -340,7 +338,8 @@ Status
BetaRowsetWriter::_do_compact_segments(SegCompactionCandidatesSharedPtr s
for (const auto& entry :
std::filesystem::directory_iterator(_context.rowset_dir)) {
fmt::format_to(vlog_buffer, "[{}]", string(entry.path()));
}
- VLOG_DEBUG << "_segcompacted_point:" << _segcompacted_point
+ VLOG_DEBUG << "tablet_id:" << _context.tablet_id << " rowset_id:" <<
_context.rowset_id
+ << "_segcompacted_point:" << _segcompacted_point
<< " _num_segment:" << _num_segment << "
_num_segcompacted:" << _num_segcompacted
<< " list directory:" << fmt::to_string(vlog_buffer);
}
@@ -348,8 +347,10 @@ Status
BetaRowsetWriter::_do_compact_segments(SegCompactionCandidatesSharedPtr s
_segcompacted_point += (end - begin + 1);
uint64_t elapsed = GetCurrentTimeMicros() - begin_time;
- LOG(INFO) << "BetaRowsetWriter:" << this << " segcompaction completed.
elapsed time:" << elapsed
- << "us. _segcompacted_point update:" << _segcompacted_point;
+ LOG(INFO) << "segcompaction completed. tablet_id:" << _context.tablet_id
+ << " rowset_id:" << _context.rowset_id << " elapsed time:" <<
elapsed
+ << "us. update segcompacted_point:" << _segcompacted_point
+ << " segment num:" << segments->size() << " begin:" << begin <<
" end:" << end;
return Status::OK();
}
@@ -365,7 +366,9 @@ void
BetaRowsetWriter::compact_segments(SegCompactionCandidatesSharedPtr segment
LOG(WARNING) << "segcompaction failed, try next time:" << status;
return;
default:
- LOG(WARNING) << "segcompaction fatal, terminating the write job:"
<< status;
+ LOG(WARNING) << "segcompaction fatal, terminating the write job."
+ << " tablet_id:" << _context.tablet_id
+ << " rowset_id:" << _context.rowset_id << " status:"
<< status;
// status will be checked by the next trigger of segcompaction or
the final wait
_segcompaction_status.store(OLAP_ERR_OTHER_ERROR);
}
@@ -503,7 +506,8 @@ Status BetaRowsetWriter::_segcompaction_if_necessary() {
SegCompactionCandidatesSharedPtr segments =
std::make_shared<SegCompactionCandidates>();
status = _get_segcompaction_candidates(segments, false);
if (LIKELY(status.ok()) && (segments->size() > 0)) {
- LOG(INFO) << "submit segcompaction task, segment num:" <<
_num_segment
+ LOG(INFO) << "submit segcompaction task, tablet_id:" <<
_context.tablet_id
+ << " rowset_id:" << _context.rowset_id << " segment
num:" << _num_segment
<< ", segcompacted_point:" << _segcompacted_point;
status =
StorageEngine::instance()->submit_seg_compaction_task(this, segments);
if (status.ok()) {
@@ -536,8 +540,9 @@ Status
BetaRowsetWriter::_segcompaction_ramaining_if_necessary() {
SegCompactionCandidatesSharedPtr segments =
std::make_shared<SegCompactionCandidates>();
status = _get_segcompaction_candidates(segments, true);
if (LIKELY(status.ok()) && (segments->size() > 0)) {
- LOG(INFO) << "submit segcompaction remaining task, segment num:" <<
_num_segment
- << ", segcompacted_point:" << _segcompacted_point;
+ LOG(INFO) << "submit segcompaction remaining task, tablet_id:" <<
_context.tablet_id
+ << " rowset_id:" << _context.rowset_id << " segment num:" <<
_num_segment
+ << " segcompacted_point:" << _segcompacted_point;
status = StorageEngine::instance()->submit_seg_compaction_task(this,
segments);
if (status.ok()) {
return status;
@@ -885,6 +890,7 @@ Status BetaRowsetWriter::_create_segment_writer(
size_t total_segment_num = _num_segment - _segcompacted_point + 1 +
_num_segcompacted;
if (UNLIKELY(total_segment_num > config::max_segment_num_per_rowset)) {
LOG(ERROR) << "too many segments in rowset."
+ << " tablet_id:" << _context.tablet_id << " rowset_id:" <<
_context.rowset_id
<< " max:" << config::max_segment_num_per_rowset
<< " _num_segment:" << _num_segment
<< " _segcompacted_point:" << _segcompacted_point
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
