This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 0396ac9d38 fix(compaction) release the block and segment iterator
after reading to the end of the segment file (#22082)
0396ac9d38 is described below
commit 0396ac9d38ca2c2501f44aea7ecb1b5f2073b440
Author: Chenyang Sun <csun5...@gmail.com>
AuthorDate: Mon Jul 24 08:47:19 2023 +0800
fix(compaction) release the block and segment iterator after reading to the
end of the segment file (#22082)
When reading to the end of the segment file, clearing the block did not
release the memory, leading to high memory usage during compaction.
When reading through segment file for columns that are dictionary encoded,
the column iterator in the segment iterator will hold the dictionary. Release
the segment iterator to free up the dictionary.
---
be/src/vec/olap/vertical_merge_iterator.cpp | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
diff --git a/be/src/vec/olap/vertical_merge_iterator.cpp
b/be/src/vec/olap/vertical_merge_iterator.cpp
index 7285531d71..7ace9b457c 100644
--- a/be/src/vec/olap/vertical_merge_iterator.cpp
+++ b/be/src/vec/olap/vertical_merge_iterator.cpp
@@ -356,6 +356,13 @@ Status VerticalMergeIteratorContext::_load_next_block() {
if (!st.ok()) {
_valid = false;
if (st.is<END_OF_FILE>()) {
+ // When reading to the end of the segment file, clearing the
block did not release the memory.
+ // Directly releasing the block to free up memory.
+ _block.reset();
+ // When reading through segment file for columns that are
dictionary encoded,
+ // the column iterator in the segment iterator will hold the
dictionary.
+ // Release the segment iterator to free up the dictionary.
+ _iter.reset();
return Status::OK();
} else {
return st;
@@ -601,7 +608,9 @@ Status VerticalFifoMergeIterator::init(const
StorageReadOptions& opts) {
Status VerticalMaskMergeIterator::check_all_iter_finished() {
for (auto iter : _origin_iter_ctx) {
if (iter->inited()) {
- RETURN_IF_ERROR(iter->advance());
+ if (iter->valid()) {
+ RETURN_IF_ERROR(iter->advance());
+ }
DCHECK(!iter->valid());
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
