This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 4f16bf6399b [fix] (inverted index) remove tmp columns in block (#39369)
4f16bf6399b is described below
commit 4f16bf6399b3f4260e0174e9628708c2df355b2b
Author: Sun Chenyang <csun5...@gmail.com>
AuthorDate: Thu Aug 15 22:10:34 2024 +0800
[fix] (inverted index) remove tmp columns in block (#39369)
---
be/src/pipeline/exec/scan_operator.cpp | 7 +------
be/src/vec/core/block.cpp | 9 +++++++++
be/src/vec/core/block.h | 5 +++++
be/src/vec/exec/scan/vscanner.cpp | 11 +++--------
be/src/vec/olap/vcollect_iterator.cpp | 19 +++++--------------
.../suites/inverted_index_p0/topn_clear_block.groovy | 10 ++++++++++
6 files changed, 33 insertions(+), 28 deletions(-)
diff --git a/be/src/pipeline/exec/scan_operator.cpp
b/be/src/pipeline/exec/scan_operator.cpp
index 021f18544db..14b5ab2acd5 100644
--- a/be/src/pipeline/exec/scan_operator.cpp
+++ b/be/src/pipeline/exec/scan_operator.cpp
@@ -1479,12 +1479,7 @@ Status
ScanOperatorX<LocalStateType>::get_block(RuntimeState* state, vectorized:
// remove them when query leave scan node to avoid other nodes use
block->columns() to make a wrong decision
Defer drop_block_temp_column {[&]() {
std::unique_lock l(local_state._block_lock);
- auto all_column_names = block->get_names();
- for (auto& name : all_column_names) {
- if (name.rfind(BeConsts::BLOCK_TEMP_COLUMN_PREFIX, 0) == 0) {
- block->erase(name);
- }
- }
+ block->erase_tmp_columns();
}};
if (state->is_cancelled()) {
diff --git a/be/src/vec/core/block.cpp b/be/src/vec/core/block.cpp
index fabd468ca4c..8bcce65f229 100644
--- a/be/src/vec/core/block.cpp
+++ b/be/src/vec/core/block.cpp
@@ -746,6 +746,15 @@ void Block::clear_column_data(int column_size) noexcept {
row_same_bit.clear();
}
+void Block::erase_tmp_columns() noexcept {
+ auto all_column_names = get_names();
+ for (auto& name : all_column_names) {
+ if (name.rfind(BeConsts::BLOCK_TEMP_COLUMN_PREFIX, 0) == 0) {
+ erase(name);
+ }
+ }
+}
+
void Block::swap(Block& other) noexcept {
SCOPED_SKIP_MEMORY_CHECK();
data.swap(other.data);
diff --git a/be/src/vec/core/block.h b/be/src/vec/core/block.h
index 741039e1465..108cb5e1c9f 100644
--- a/be/src/vec/core/block.h
+++ b/be/src/vec/core/block.h
@@ -399,6 +399,11 @@ public:
// for debug purpose.
std::string print_use_count();
+ // remove tmp columns in block
+ // in inverted index apply logic, in order to optimize query performance,
+ // we built some temporary columns into block
+ void erase_tmp_columns() noexcept;
+
private:
void erase_impl(size_t position);
};
diff --git a/be/src/vec/exec/scan/vscanner.cpp
b/be/src/vec/exec/scan/vscanner.cpp
index 6232be47302..43d791caffa 100644
--- a/be/src/vec/exec/scan/vscanner.cpp
+++ b/be/src/vec/exec/scan/vscanner.cpp
@@ -117,6 +117,8 @@ Status VScanner::get_block(RuntimeState* state, Block*
block, bool* eof) {
RETURN_IF_ERROR(_get_block_impl(state, block, eof));
if (*eof) {
DCHECK(block->rows() == 0);
+ // clear TEMP columns to avoid column align problem
+ block->erase_tmp_columns();
break;
}
_num_rows_read += block->rows();
@@ -152,14 +154,7 @@ Status VScanner::get_block(RuntimeState* state, Block*
block, bool* eof) {
}
Status VScanner::_filter_output_block(Block* block) {
- Defer clear_tmp_block([&]() {
- auto all_column_names = block->get_names();
- for (auto& name : all_column_names) {
- if (name.rfind(BeConsts::BLOCK_TEMP_COLUMN_PREFIX, 0) == 0) {
- block->erase(name);
- }
- }
- });
+ Defer clear_tmp_block([&]() { block->erase_tmp_columns(); });
if (block->has(BeConsts::BLOCK_TEMP_COLUMN_SCANNER_FILTERED)) {
// scanner filter_block is already done (only by _topn_next
currently), just skip it
return Status::OK();
diff --git a/be/src/vec/olap/vcollect_iterator.cpp
b/be/src/vec/olap/vcollect_iterator.cpp
index 0d5e595b404..3eb768ff803 100644
--- a/be/src/vec/olap/vcollect_iterator.cpp
+++ b/be/src/vec/olap/vcollect_iterator.cpp
@@ -257,18 +257,7 @@ Status VCollectIterator::_topn_next(Block* block) {
}
// clear TEMP columns to avoid column align problem
- auto clear_temp_columns = [](Block* block) {
- auto all_column_names = block->get_names();
- for (auto& name : all_column_names) {
- if (name.rfind(BeConsts::BLOCK_TEMP_COLUMN_PREFIX, 0) == 0) {
- // clear TEMP columns from block to prevent from storage
engine merge with this
- // fake column
- block->erase(name);
- }
- }
- };
-
- clear_temp_columns(block);
+ block->erase_tmp_columns();
auto clone_block = block->clone_empty();
MutableBlock mutable_block =
vectorized::MutableBlock::build_mutable_block(&clone_block);
@@ -305,7 +294,7 @@ Status VCollectIterator::_topn_next(Block* block) {
eof = true;
if (block->rows() == 0) {
// clear TEMP columns to avoid column align problem in
segment iterator
- clear_temp_columns(block);
+ block->erase_tmp_columns();
break;
}
} else {
@@ -317,7 +306,7 @@ Status VCollectIterator::_topn_next(Block* block) {
RETURN_IF_ERROR(VExprContext::filter_block(
_reader->_reader_context.filter_block_conjuncts, block,
block->columns()));
// clear TMPE columns to avoid column align problem in
mutable_block.add_rows bellow
- clear_temp_columns(block);
+ block->erase_tmp_columns();
// update read rows
read_rows += block->rows();
@@ -861,6 +850,8 @@ Status
VCollectIterator::Level1Iterator::_normal_next(Block* block) {
while (res.is<END_OF_FILE>() && !_children.empty()) {
_cur_child = std::move(*(_children.begin()));
_children.pop_front();
+ // clear TEMP columns to avoid column align problem
+ block->erase_tmp_columns();
res = _cur_child->next(block);
}
diff --git a/regression-test/suites/inverted_index_p0/topn_clear_block.groovy
b/regression-test/suites/inverted_index_p0/topn_clear_block.groovy
index 7486a658d60..586173e0f3e 100644
--- a/regression-test/suites/inverted_index_p0/topn_clear_block.groovy
+++ b/regression-test/suites/inverted_index_p0/topn_clear_block.groovy
@@ -78,7 +78,17 @@ suite("test_clear_block") {
sql """ delete from dup_httplogs where size = 24736; """
sql """ delete from dup_httplogs where request = 'GET /images/hm_bg.jpg
HTTP/1.0'; """
+ sql """ set enable_match_without_inverted_index = false """
sql """ sync """
qt_sql """ SELECT clientip from ${dupTableName} WHERE clientip NOT IN
(NULL, '') or clientip IN ('17.0.0.0') ORDER BY id LIMIT 2 """
+
+ def result1 = sql """ SELECT clientip from ${dupTableName} WHERE clientip
NOT IN (NULL, '') or clientip IN ('17.0.0.0') ORDER BY id LIMIT 5000 """
+ def result2 = sql """ SELECT clientip from ${dupTableName} WHERE clientip
NOT IN (NULL, '') or clientip IN ('17.0.0.0') ORDER BY id LIMIT 5000 """
+ if (result1 != result2) {
+ logger.info("result1 is: {}", result1)
+ logger.info("result2 is: {}", result2)
+ assertTrue(false)
+ }
+
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
