This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new dc287d521f5 [enhancement](memory) return error if allocate memory
failed during page builder allocate memory failed (#39964)
dc287d521f5 is described below
commit dc287d521f5bb80c517b7886689516608cdb6d5e
Author: yiguolei <676222...@qq.com>
AuthorDate: Tue Aug 27 20:45:49 2024 +0800
[enhancement](memory) return error if allocate memory failed during page
builder allocate memory failed (#39964)
## Proposed changes
Issue Number: close #xxx
<!--Describe your changes.-->
---------
Co-authored-by: yiguolei <yiguo...@gmail.com>
---
be/src/olap/rowset/segment_v2/binary_dict_page.cpp | 14 ++++++------
be/src/olap/rowset/segment_v2/binary_dict_page.h | 2 +-
be/src/olap/rowset/segment_v2/binary_plain_page.h | 25 ++++++++++++----------
.../olap/rowset/segment_v2/binary_prefix_page.cpp | 23 +++++++++++---------
be/src/olap/rowset/segment_v2/binary_prefix_page.h | 2 +-
be/src/olap/rowset/segment_v2/bitshuffle_page.h | 5 +++--
be/src/olap/rowset/segment_v2/column_writer.cpp | 10 +++++----
.../rowset/segment_v2/frame_of_reference_page.h | 5 +++--
.../rowset/segment_v2/indexed_column_writer.cpp | 3 ++-
be/src/olap/rowset/segment_v2/page_builder.h | 3 ++-
be/src/olap/rowset/segment_v2/plain_page.h | 18 ++++++++++------
be/src/olap/rowset/segment_v2/rle_page.h | 5 +++--
12 files changed, 67 insertions(+), 48 deletions(-)
diff --git a/be/src/olap/rowset/segment_v2/binary_dict_page.cpp
b/be/src/olap/rowset/segment_v2/binary_dict_page.cpp
index dd7ad59ba1d..d7b8cb0d387 100644
--- a/be/src/olap/rowset/segment_v2/binary_dict_page.cpp
+++ b/be/src/olap/rowset/segment_v2/binary_dict_page.cpp
@@ -142,7 +142,7 @@ Status BinaryDictPageBuilder::add(const uint8_t* vals,
size_t* count) {
}
}
-OwnedSlice BinaryDictPageBuilder::finish() {
+Status BinaryDictPageBuilder::finish(OwnedSlice* slice) {
if (VLOG_DEBUG_IS_ON && _encoding_type == DICT_ENCODING) {
VLOG_DEBUG << "dict page size:" << _dict_builder->size();
}
@@ -150,11 +150,14 @@ OwnedSlice BinaryDictPageBuilder::finish() {
DCHECK(!_finished);
_finished = true;
- OwnedSlice data_slice = _data_page_builder->finish();
+ OwnedSlice data_slice;
+ RETURN_IF_ERROR(_data_page_builder->finish(&data_slice));
// TODO(gaodayue) separate page header and content to avoid this copy
- _buffer.append(data_slice.slice().data, data_slice.slice().size);
+ RETURN_IF_CATCH_EXCEPTION(
+ { _buffer.append(data_slice.slice().data,
data_slice.slice().size); });
encode_fixed32_le(&_buffer[0], _encoding_type);
- return _buffer.build();
+ *slice = _buffer.build();
+ return Status::OK();
}
Status BinaryDictPageBuilder::reset() {
@@ -185,8 +188,7 @@ uint64_t BinaryDictPageBuilder::size() const {
}
Status BinaryDictPageBuilder::get_dictionary_page(OwnedSlice* dictionary_page)
{
- *dictionary_page = _dict_builder->finish();
- return Status::OK();
+ return _dict_builder->finish(dictionary_page);
}
Status BinaryDictPageBuilder::get_first_value(void* value) const {
diff --git a/be/src/olap/rowset/segment_v2/binary_dict_page.h
b/be/src/olap/rowset/segment_v2/binary_dict_page.h
index b098d76b0c0..31af5df2e77 100644
--- a/be/src/olap/rowset/segment_v2/binary_dict_page.h
+++ b/be/src/olap/rowset/segment_v2/binary_dict_page.h
@@ -68,7 +68,7 @@ public:
Status add(const uint8_t* vals, size_t* count) override;
- OwnedSlice finish() override;
+ Status finish(OwnedSlice* slice) override;
Status reset() override;
diff --git a/be/src/olap/rowset/segment_v2/binary_plain_page.h
b/be/src/olap/rowset/segment_v2/binary_plain_page.h
index b05ab4906d1..e043164ef28 100644
--- a/be/src/olap/rowset/segment_v2/binary_plain_page.h
+++ b/be/src/olap/rowset/segment_v2/binary_plain_page.h
@@ -93,19 +93,22 @@ public:
return Status::OK();
}
- OwnedSlice finish() override {
+ Status finish(OwnedSlice* slice) override {
DCHECK(!_finished);
_finished = true;
- // Set up trailer
- for (uint32_t _offset : _offsets) {
- put_fixed32_le(&_buffer, _offset);
- }
- put_fixed32_le(&_buffer, _offsets.size());
- if (_offsets.size() > 0) {
- _copy_value_at(0, &_first_value);
- _copy_value_at(_offsets.size() - 1, &_last_value);
- }
- return _buffer.build();
+ RETURN_IF_CATCH_EXCEPTION({
+ // Set up trailer
+ for (uint32_t _offset : _offsets) {
+ put_fixed32_le(&_buffer, _offset);
+ }
+ put_fixed32_le(&_buffer, _offsets.size());
+ if (_offsets.size() > 0) {
+ _copy_value_at(0, &_first_value);
+ _copy_value_at(_offsets.size() - 1, &_last_value);
+ }
+ *slice = _buffer.build();
+ });
+ return Status::OK();
}
Status reset() override {
diff --git a/be/src/olap/rowset/segment_v2/binary_prefix_page.cpp
b/be/src/olap/rowset/segment_v2/binary_prefix_page.cpp
index 9d1ecdb9470..34eb14951ae 100644
--- a/be/src/olap/rowset/segment_v2/binary_prefix_page.cpp
+++ b/be/src/olap/rowset/segment_v2/binary_prefix_page.cpp
@@ -88,18 +88,21 @@ Status BinaryPrefixPageBuilder::add(const uint8_t* vals,
size_t* add_count) {
return Status::OK();
}
-OwnedSlice BinaryPrefixPageBuilder::finish() {
+Status BinaryPrefixPageBuilder::finish(OwnedSlice* slice) {
DCHECK(!_finished);
_finished = true;
- put_fixed32_le(&_buffer, (uint32_t)_count);
- uint8_t restart_point_internal = RESTART_POINT_INTERVAL;
- _buffer.append(&restart_point_internal, 1);
- auto restart_point_size = _restart_points_offset.size();
- for (uint32_t i = 0; i < restart_point_size; ++i) {
- put_fixed32_le(&_buffer, _restart_points_offset[i]);
- }
- put_fixed32_le(&_buffer, restart_point_size);
- return _buffer.build();
+ RETURN_IF_CATCH_EXCEPTION({
+ put_fixed32_le(&_buffer, (uint32_t)_count);
+ uint8_t restart_point_internal = RESTART_POINT_INTERVAL;
+ _buffer.append(&restart_point_internal, 1);
+ auto restart_point_size = _restart_points_offset.size();
+ for (uint32_t i = 0; i < restart_point_size; ++i) {
+ put_fixed32_le(&_buffer, _restart_points_offset[i]);
+ }
+ put_fixed32_le(&_buffer, restart_point_size);
+ *slice = _buffer.build();
+ });
+ return Status::OK();
}
const uint8_t* BinaryPrefixPageDecoder::_decode_value_lengths(const uint8_t*
ptr, uint32_t* shared,
diff --git a/be/src/olap/rowset/segment_v2/binary_prefix_page.h
b/be/src/olap/rowset/segment_v2/binary_prefix_page.h
index de4ec60070b..41deb4e6c1f 100644
--- a/be/src/olap/rowset/segment_v2/binary_prefix_page.h
+++ b/be/src/olap/rowset/segment_v2/binary_prefix_page.h
@@ -52,7 +52,7 @@ public:
Status add(const uint8_t* vals, size_t* add_count) override;
- OwnedSlice finish() override;
+ Status finish(OwnedSlice* slice) override;
Status reset() override {
_restart_points_offset.clear();
diff --git a/be/src/olap/rowset/segment_v2/bitshuffle_page.h
b/be/src/olap/rowset/segment_v2/bitshuffle_page.h
index a079c569b31..a435558dcd4 100644
--- a/be/src/olap/rowset/segment_v2/bitshuffle_page.h
+++ b/be/src/olap/rowset/segment_v2/bitshuffle_page.h
@@ -142,12 +142,13 @@ public:
return Status::OK();
}
- OwnedSlice finish() override {
+ Status finish(OwnedSlice* slice) override {
if (_count > 0) {
_first_value = cell(0);
_last_value = cell(_count - 1);
}
- return _finish(SIZE_OF_TYPE);
+ RETURN_IF_CATCH_EXCEPTION({ *slice = _finish(SIZE_OF_TYPE); });
+ return Status::OK();
}
Status reset() override {
diff --git a/be/src/olap/rowset/segment_v2/column_writer.cpp
b/be/src/olap/rowset/segment_v2/column_writer.cpp
index 9e748a01db2..75acd9f1f95 100644
--- a/be/src/olap/rowset/segment_v2/column_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/column_writer.cpp
@@ -69,9 +69,10 @@ public:
// Returns whether the building nullmap contains nullptr
bool has_null() const { return _has_null; }
- OwnedSlice finish() {
+ Status finish(OwnedSlice* slice) {
_rle_encoder.Flush();
- return _bitmap_buf.build();
+ RETURN_IF_CATCH_EXCEPTION({ *slice = _bitmap_buf.build(); });
+ return Status::OK();
}
void reset() {
@@ -676,14 +677,15 @@ Status ScalarColumnWriter::finish_current_page() {
// build data page body : encoded values + [nullmap]
std::vector<Slice> body;
- OwnedSlice encoded_values = _page_builder->finish();
+ OwnedSlice encoded_values;
+ RETURN_IF_ERROR(_page_builder->finish(&encoded_values));
RETURN_IF_ERROR(_page_builder->reset());
body.push_back(encoded_values.slice());
OwnedSlice nullmap;
if (_null_bitmap_builder != nullptr) {
if (is_nullable() && _null_bitmap_builder->has_null()) {
- nullmap = _null_bitmap_builder->finish();
+ RETURN_IF_ERROR(_null_bitmap_builder->finish(&nullmap));
body.push_back(nullmap.slice());
}
_null_bitmap_builder->reset();
diff --git a/be/src/olap/rowset/segment_v2/frame_of_reference_page.h
b/be/src/olap/rowset/segment_v2/frame_of_reference_page.h
index 4477912803b..5aedf126b55 100644
--- a/be/src/olap/rowset/segment_v2/frame_of_reference_page.h
+++ b/be/src/olap/rowset/segment_v2/frame_of_reference_page.h
@@ -54,11 +54,12 @@ public:
return Status::OK();
}
- OwnedSlice finish() override {
+ Status finish(OwnedSlice* slice) override {
DCHECK(!_finished);
_finished = true;
_encoder->flush();
- return _buf.build();
+ RETURN_IF_CATCH_EXCEPTION({ *slice = _buf.build(); });
+ return Status::OK();
}
Status reset() override {
diff --git a/be/src/olap/rowset/segment_v2/indexed_column_writer.cpp
b/be/src/olap/rowset/segment_v2/indexed_column_writer.cpp
index e1b238084a9..51606d81889 100644
--- a/be/src/olap/rowset/segment_v2/indexed_column_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/indexed_column_writer.cpp
@@ -117,7 +117,8 @@ Status
IndexedColumnWriter::_finish_current_data_page(size_t& num_val) {
ordinal_t first_ordinal = _num_values - num_values_in_page;
// IndexedColumn doesn't have NULLs, thus data page body only contains
encoded values
- OwnedSlice page_body = _data_page_builder->finish();
+ OwnedSlice page_body;
+ RETURN_IF_ERROR(_data_page_builder->finish(&page_body));
RETURN_IF_ERROR(_data_page_builder->reset());
PageFooterPB footer;
diff --git a/be/src/olap/rowset/segment_v2/page_builder.h
b/be/src/olap/rowset/segment_v2/page_builder.h
index 61fa2eaf8e1..7e24c56796c 100644
--- a/be/src/olap/rowset/segment_v2/page_builder.h
+++ b/be/src/olap/rowset/segment_v2/page_builder.h
@@ -63,7 +63,8 @@ public:
// Finish building the current page, return the encoded data.
// This api should be followed by reset() before reusing the builder
- virtual OwnedSlice finish() = 0;
+ // It will return error status when memory allocated failed during finish
+ virtual Status finish(OwnedSlice* owned_slice) = 0;
// Get the dictionary page for dictionary encoding mode column.
virtual Status get_dictionary_page(OwnedSlice* dictionary_page) {
diff --git a/be/src/olap/rowset/segment_v2/plain_page.h
b/be/src/olap/rowset/segment_v2/plain_page.h
index 067dce5adc1..4e108755b1a 100644
--- a/be/src/olap/rowset/segment_v2/plain_page.h
+++ b/be/src/olap/rowset/segment_v2/plain_page.h
@@ -59,14 +59,18 @@ public:
return Status::OK();
}
- OwnedSlice finish() override {
+ Status finish(OwnedSlice* slice) override {
encode_fixed32_le((uint8_t*)&_buffer[0], _count);
- if (_count > 0) {
- _first_value.assign_copy(&_buffer[PLAIN_PAGE_HEADER_SIZE],
SIZE_OF_TYPE);
- _last_value.assign_copy(&_buffer[PLAIN_PAGE_HEADER_SIZE + (_count
- 1) * SIZE_OF_TYPE],
- SIZE_OF_TYPE);
- }
- return _buffer.build();
+ RETURN_IF_CATCH_EXCEPTION({
+ if (_count > 0) {
+ _first_value.assign_copy(&_buffer[PLAIN_PAGE_HEADER_SIZE],
SIZE_OF_TYPE);
+ _last_value.assign_copy(
+ &_buffer[PLAIN_PAGE_HEADER_SIZE + (_count - 1) *
SIZE_OF_TYPE],
+ SIZE_OF_TYPE);
+ }
+ *slice = _buffer.build();
+ });
+ return Status::OK();
}
Status reset() override {
diff --git a/be/src/olap/rowset/segment_v2/rle_page.h
b/be/src/olap/rowset/segment_v2/rle_page.h
index 2a0bea846b3..eb7b0872f4d 100644
--- a/be/src/olap/rowset/segment_v2/rle_page.h
+++ b/be/src/olap/rowset/segment_v2/rle_page.h
@@ -94,14 +94,15 @@ public:
return Status::OK();
}
- OwnedSlice finish() override {
+ Status finish(OwnedSlice* slice) override {
DCHECK(!_finished);
_finished = true;
// here should Flush first and then encode the count header
// or it will lead to a bug if the header is less than 8 byte and the
data is small
_rle_encoder->Flush();
encode_fixed32_le(&_buf[0], _count);
- return _buf.build();
+ *slice = _buf.build();
+ return Status::OK();
}
Status reset() override {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
