This is an automated email from the ASF dual-hosted git repository.
panxiaolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 8c6c113c2d2 [Performance](sort) change merge sort read logic to avoid
only one line (#53678)
8c6c113c2d2 is described below
commit 8c6c113c2d2863fc3da329a33a6aa4065a06acde
Author: HappenLee <[email protected]>
AuthorDate: Wed Jul 23 21:58:10 2025 +0800
[Performance](sort) change merge sort read logic to avoid only one line
(#53678)
Remove some logic to make sure batch block read：
before：
```
select * from lineitem order by l_partkey limit 10; // 17.56 sec
```
after:
```
select * from lineitem order by l_partkey limit 10; // 4.27 sec
```
---
be/src/vec/common/sort/sorter.cpp | 23 ++++-------------------
be/src/vec/common/sort/sorter.h | 2 +-
be/test/pipeline/operator/sort_operator_test.cpp | 13 ++-----------
be/test/vec/exec/sort/heap_sorter_test.cpp | 21 +++++----------------
4 files changed, 12 insertions(+), 47 deletions(-)
diff --git a/be/src/vec/common/sort/sorter.cpp
b/be/src/vec/common/sort/sorter.cpp
index a2a69552192..1bae830b090 100644
--- a/be/src/vec/common/sort/sorter.cpp
+++ b/be/src/vec/common/sort/sorter.cpp
@@ -91,12 +91,12 @@ Status
MergeSorterState::merge_sort_read(doris::vectorized::Block* block, int ba
bool* eos) {
DCHECK(_sorted_blocks.empty());
DCHECK(unsorted_block()->empty());
- RETURN_IF_ERROR(_merge_sort_read_impl(batch_size, block, eos));
+ _merge_sort_read_impl(batch_size, block, eos);
return Status::OK();
}
-Status MergeSorterState::_merge_sort_read_impl(int batch_size,
doris::vectorized::Block* block,
- bool* eos) {
+void MergeSorterState::_merge_sort_read_impl(int batch_size,
doris::vectorized::Block* block,
+ bool* eos) {
size_t num_columns = unsorted_block()->columns();
MutableBlock m_block =
VectorizedUtils::build_mutable_mem_reuse_block(block, *unsorted_block());
@@ -113,17 +113,6 @@ Status MergeSorterState::_merge_sort_read_impl(int
batch_size, doris::vectorized
_offset -= step;
current_rows -= step;
- if (current->impl->is_last(current_rows + step) && current->impl->pos
== 0 && step == 0) {
- if (merged_rows != 0) {
- // return directly for next time's read swap whole block
- return Status::OK();
- }
- // swap and return block directly when we should get all data from
cursor
- block->swap(*current->impl->block);
- _queue.remove_top();
- return Status::OK();
- }
-
if (current_rows) {
for (size_t i = 0; i < num_columns; ++i) {
merged_columns[i]->insert_range_from(*current->impl->columns[i],
@@ -140,11 +129,7 @@ Status MergeSorterState::_merge_sort_read_impl(int
batch_size, doris::vectorized
}
block->set_columns(std::move(merged_columns));
-
- if (merged_rows == 0) {
- *eos = true;
- }
- return Status::OK();
+ *eos = merged_rows == 0;
}
Status Sorter::merge_sort_read_for_spill(RuntimeState* state,
doris::vectorized::Block* block,
diff --git a/be/src/vec/common/sort/sorter.h b/be/src/vec/common/sort/sorter.h
index 146b7523077..ca33a9eacfa 100644
--- a/be/src/vec/common/sort/sorter.h
+++ b/be/src/vec/common/sort/sorter.h
@@ -84,7 +84,7 @@ public:
std::unique_ptr<Block>& unsorted_block() { return _unsorted_block; }
private:
- Status _merge_sort_read_impl(int batch_size, doris::vectorized::Block*
block, bool* eos);
+ void _merge_sort_read_impl(int batch_size, doris::vectorized::Block*
block, bool* eos);
std::unique_ptr<Block> _unsorted_block;
MergeSorterQueue _queue;
diff --git a/be/test/pipeline/operator/sort_operator_test.cpp
b/be/test/pipeline/operator/sort_operator_test.cpp
index 0cab5a95bd9..6cbcdafe8d7 100644
--- a/be/test/pipeline/operator/sort_operator_test.cpp
+++ b/be/test/pipeline/operator/sort_operator_test.cpp
@@ -207,19 +207,10 @@ TEST_F(SortOperatorTest, test_dep) {
auto st = source->get_block(state.get(), &block, &eos);
EXPECT_TRUE(st.ok()) << st.msg();
EXPECT_FALSE(eos);
- EXPECT_EQ(block.rows(), 3);
- std::cout << block.dump_data() << std::endl;
- EXPECT_TRUE(ColumnHelper::block_equal(
- block, ColumnHelper::create_block<DataTypeInt64>({1, 2, 3})));
-
- block.clear();
- st = source->get_block(state.get(), &block, &eos);
- EXPECT_TRUE(st.ok()) << st.msg();
- EXPECT_FALSE(eos);
- EXPECT_EQ(block.rows(), 3);
+ EXPECT_EQ(block.rows(), 6);
std::cout << block.dump_data() << std::endl;
EXPECT_TRUE(ColumnHelper::block_equal(
- block, ColumnHelper::create_block<DataTypeInt64>({4, 5, 6})));
+ block, ColumnHelper::create_block<DataTypeInt64>({1, 2, 3, 4,
5, 6})));
block.clear();
st = source->get_block(state.get(), &block, &eos);
diff --git a/be/test/vec/exec/sort/heap_sorter_test.cpp
b/be/test/vec/exec/sort/heap_sorter_test.cpp
index 503207b62d2..a195ab3478b 100644
--- a/be/test/vec/exec/sort/heap_sorter_test.cpp
+++ b/be/test/vec/exec/sort/heap_sorter_test.cpp
@@ -84,7 +84,7 @@ TEST_F(HeapSorterTest, test_topn_sorter1) {
sort_exec_exprs._sort_tuple_slot_expr_ctxs =
MockSlotRef::create_mock_contexts(data_types);
- sort_exec_exprs._need_convert_to_nullable_flags = {true, false};
+ sort_exec_exprs._need_convert_to_nullable_flags = {false, false};
sorter = HeapSorter::create_unique(sort_exec_exprs, 6, 0, &pool,
is_asc_order, nulls_first,
*row_desc);
@@ -119,24 +119,13 @@ TEST_F(HeapSorterTest, test_topn_sorter1) {
Block block;
bool eos = false;
EXPECT_TRUE(sorter->get_next(&_state, &block, &eos));
- EXPECT_EQ(block.rows(), 5);
- EXPECT_EQ(eos, false);
+ EXPECT_EQ(block.rows(), 6);
EXPECT_TRUE(ColumnHelper::block_equal(
block,
- Block
{ColumnHelper::create_nullable_column_with_name<DataTypeInt64>(
- {1, 2, 3, 4, 5}, {false, false, false, false,
false}),
-
ColumnHelper::create_column_with_name<DataTypeInt64>({1, 2, 3, 4, 5})}));
+ Block
{ColumnHelper::create_column_with_name<DataTypeInt64>({1, 2, 3, 4, 5, 6}),
+
ColumnHelper::create_column_with_name<DataTypeInt64>({1, 2, 3, 4, 5, 6})}));
- block.clear();
- EXPECT_TRUE(sorter->get_next(&_state, &block, &eos));
- EXPECT_EQ(block.rows(), 1);
- EXPECT_EQ(eos, false);
- EXPECT_TRUE(ColumnHelper::block_equal(
- block,
- Block
{ColumnHelper::create_nullable_column_with_name<DataTypeInt64>({6}, {false}),
-
ColumnHelper::create_column_with_name<DataTypeInt64>({6})}));
-
- block.clear();
+ block.clear_column_data();
EXPECT_TRUE(sorter->get_next(&_state, &block, &eos));
EXPECT_EQ(block.rows(), 0);
EXPECT_EQ(eos, true);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
