This is an automated email from the ASF dual-hosted git repository.
liaoxin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 288fef58c09 [performance](load) do not copy input_block in memtable
(#36939)
288fef58c09 is described below
commit 288fef58c0944ea967e2d0a2b6d8ac5dcb524302
Author: Kaijie Chen <[email protected]>
AuthorDate: Sat Jun 29 16:06:35 2024 +0800
[performance](load) do not copy input_block in memtable (#36939)
Pass `column_offset` to `Block::clone_without_columns()` and
`MutableBlock::add_rows()`,
so we do not need to copy `input_block` in `MemTable::insert()`.
To optimize performance in cases with many (e.g. 12k+) tablets:
Before:
```
- DeltaWriterWriteTime: 1m58s
- MemTableWriterFlushTime: 0ns
- MemTableWriterLockTime: 1s41ms
- MemTableWriterShrinkTime: 0ns
- MemTableWriterWriteTime: 1m53s
- MemTableCopyCount: 12.044003M (12044003)
- MemTableCopyTime: 42s857ms
- MemTableCopyTime0: 11s866ms
- MemTableInitTime: 13.384ms
- MemTableInsertTime: 51s543ms
```
After:
```
- DeltaWriterWriteTime: 1m
- MemTableWriterFlushTime: 0ns
- MemTableWriterLockTime: 971.481ms
- MemTableWriterShrinkTime: 0ns
- MemTableWriterWriteTime: 55s274ms
- MemTableCopyCount: 0
- MemTableCopyTime: 0ns
- MemTableCopyTime0: 0ns
- MemTableInitTime: 18.746ms
- MemTableInsertTime: 53s772ms
```
---
be/src/olap/memtable.cpp | 14 ++++++--------
be/src/vec/core/block.cpp | 25 ++++++++++++++++++-------
be/src/vec/core/block.h | 5 +++--
3 files changed, 27 insertions(+), 17 deletions(-)
diff --git a/be/src/olap/memtable.cpp b/be/src/olap/memtable.cpp
index 750026c289d..e55fd678bd2 100644
--- a/be/src/olap/memtable.cpp
+++ b/be/src/olap/memtable.cpp
@@ -180,16 +180,14 @@ int RowInBlockComparator::operator()(const RowInBlock*
left, const RowInBlock* r
Status MemTable::insert(const vectorized::Block* input_block,
const std::vector<uint32_t>& row_idxs) {
- vectorized::Block target_block = *input_block;
- target_block = input_block->copy_block(_column_offset);
if (_is_first_insertion) {
_is_first_insertion = false;
- auto cloneBlock = target_block.clone_without_columns();
+ auto cloneBlock = input_block->clone_without_columns(&_column_offset);
_input_mutable_block =
vectorized::MutableBlock::build_mutable_block(&cloneBlock);
_vec_row_comparator->set_block(&_input_mutable_block);
_output_mutable_block =
vectorized::MutableBlock::build_mutable_block(&cloneBlock);
if (_keys_type != KeysType::DUP_KEYS) {
- _init_agg_functions(&target_block);
+ _init_agg_functions(input_block);
}
if (_tablet_schema->has_sequence_col()) {
if (_is_partial_update) {
@@ -210,11 +208,11 @@ Status MemTable::insert(const vectorized::Block*
input_block,
auto num_rows = row_idxs.size();
size_t cursor_in_mutableblock = _input_mutable_block.rows();
auto block_size0 = _input_mutable_block.allocated_bytes();
- RETURN_IF_ERROR(_input_mutable_block.add_rows(&target_block,
row_idxs.data(),
- row_idxs.data() + num_rows));
+ RETURN_IF_ERROR(_input_mutable_block.add_rows(input_block, row_idxs.data(),
+ row_idxs.data() + num_rows,
&_column_offset));
auto block_size1 = _input_mutable_block.allocated_bytes();
g_memtable_input_block_allocated_size << block_size1 - block_size0;
- auto input_size = size_t(target_block.bytes() * num_rows /
target_block.rows() *
+ auto input_size = size_t(input_block->bytes() * num_rows /
input_block->rows() *
config::memtable_insert_memory_ratio);
_mem_usage += input_size;
_insert_mem_tracker->consume(input_size);
@@ -348,7 +346,7 @@ Status MemTable::_sort_by_cluster_keys() {
row_pos_vec.emplace_back(row_in_blocks[i]->_row_pos);
}
return _output_mutable_block.add_rows(&in_block, row_pos_vec.data(),
- row_pos_vec.data() +
in_block.rows());
+ row_pos_vec.data() +
in_block.rows(), &_column_offset);
}
void MemTable::_sort_one_column(std::vector<RowInBlock*>& row_in_blocks, Tie&
tie,
diff --git a/be/src/vec/core/block.cpp b/be/src/vec/core/block.cpp
index 676674e8ec0..22062dc5310 100644
--- a/be/src/vec/core/block.cpp
+++ b/be/src/vec/core/block.cpp
@@ -653,12 +653,19 @@ Block Block::clone_with_columns(const Columns& columns)
const {
return res;
}
-Block Block::clone_without_columns() const {
+Block Block::clone_without_columns(const std::vector<int>* column_offset)
const {
Block res;
- size_t num_columns = data.size();
- for (size_t i = 0; i < num_columns; ++i) {
- res.insert({nullptr, data[i].type, data[i].name});
+ if (column_offset != nullptr) {
+ size_t num_columns = column_offset->size();
+ for (size_t i = 0; i < num_columns; ++i) {
+ res.insert({nullptr, data[(*column_offset)[i]].type,
data[(*column_offset)[i]].name});
+ }
+ } else {
+ size_t num_columns = data.size();
+ for (size_t i = 0; i < num_columns; ++i) {
+ res.insert({nullptr, data[i].type, data[i].name});
+ }
}
return res;
}
@@ -1008,14 +1015,18 @@ void MutableBlock::add_row(const Block* block, int row)
{
}
Status MutableBlock::add_rows(const Block* block, const uint32_t* row_begin,
- const uint32_t* row_end) {
+ const uint32_t* row_end, const std::vector<int>*
column_offset) {
RETURN_IF_CATCH_EXCEPTION({
DCHECK_LE(columns(), block->columns());
+ if (column_offset != nullptr) {
+ DCHECK_EQ(columns(), column_offset->size());
+ }
const auto& block_data = block->get_columns_with_type_and_name();
for (size_t i = 0; i < _columns.size(); ++i) {
- DCHECK_EQ(_data_types[i]->get_name(),
block_data[i].type->get_name());
+ const auto& src_col = column_offset ?
block_data[(*column_offset)[i]] : block_data[i];
+ DCHECK_EQ(_data_types[i]->get_name(), src_col.type->get_name());
auto& dst = _columns[i];
- const auto& src = *block_data[i].column.get();
+ const auto& src = *src_col.column.get();
DCHECK_GE(src.size(), row_end - row_begin);
dst->insert_indices_from(src, row_begin, row_end);
}
diff --git a/be/src/vec/core/block.h b/be/src/vec/core/block.h
index 3611252ea59..6f50ff0035a 100644
--- a/be/src/vec/core/block.h
+++ b/be/src/vec/core/block.h
@@ -223,7 +223,7 @@ public:
void set_columns(const Columns& columns);
Block clone_with_columns(const Columns& columns) const;
- Block clone_without_columns() const;
+ Block clone_without_columns(const std::vector<int>* column_offset =
nullptr) const;
/** Get empty columns with the same types as in block. */
MutableColumns clone_empty_columns() const;
@@ -607,7 +607,8 @@ public:
void add_row(const Block* block, int row);
// Batch add row should return error status if allocate memory failed.
- Status add_rows(const Block* block, const uint32_t* row_begin, const
uint32_t* row_end);
+ Status add_rows(const Block* block, const uint32_t* row_begin, const
uint32_t* row_end,
+ const std::vector<int>* column_offset = nullptr);
Status add_rows(const Block* block, size_t row_begin, size_t length);
Status add_rows(const Block* block, std::vector<int64_t> rows);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
