This is an automated email from the ASF dual-hosted git repository.
Gabriel39 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 17867b8d184 [opt](serde)Build Arrow serde error context lazily.
(#64660)
17867b8d184 is described below
commit 17867b8d184d77802dc782f676005a25361af0b4
Author: daidai <[email protected]>
AuthorDate: Tue Jun 23 09:52:31 2026 +0800
[opt](serde)Build Arrow serde error context lazily. (#64660)
Optimize the arrow serde `checkArrowStatus` method.
---
.../core/data_type_serde/data_type_array_serde.cpp | 6 ++---
.../data_type_serde/data_type_bitmap_serde.cpp | 5 ++--
.../data_type_date_or_datetime_serde.cpp | 5 ++--
.../data_type_serde/data_type_datetimev2_serde.cpp | 8 +++---
.../data_type_serde/data_type_datev2_serde.cpp | 5 ++--
.../data_type_serde/data_type_decimal_serde.cpp | 30 ++++++++--------------
.../core/data_type_serde/data_type_hll_serde.cpp | 5 ++--
.../core/data_type_serde/data_type_ipv4_serde.cpp | 2 +-
.../core/data_type_serde/data_type_ipv6_serde.cpp | 5 ++--
.../core/data_type_serde/data_type_jsonb_serde.cpp | 5 ++--
.../core/data_type_serde/data_type_map_serde.cpp | 9 +++----
.../data_type_serde/data_type_number_serde.cpp | 20 +++++++--------
.../data_type_quantilestate_serde.h | 5 ++--
be/src/core/data_type_serde/data_type_serde.h | 7 ++---
.../data_type_serde/data_type_string_serde.cpp | 5 ++--
.../data_type_serde/data_type_struct_serde.cpp | 6 ++---
.../data_type_timestamptz_serde.cpp | 8 +++---
.../data_type_serde/data_type_varbinary_serde.cpp | 5 ++--
.../data_type_serde/data_type_variant_serde.cpp | 5 ++--
19 files changed, 60 insertions(+), 86 deletions(-)
diff --git a/be/src/core/data_type_serde/data_type_array_serde.cpp
b/be/src/core/data_type_serde/data_type_array_serde.cpp
index 42cd2e9df6b..c54f3e1b46a 100644
--- a/be/src/core/data_type_serde/data_type_array_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_array_serde.cpp
@@ -304,12 +304,10 @@ Status DataTypeArraySerDe::write_column_to_arrow(const
IColumn& column, const Nu
auto* nested_builder = builder.value_builder();
for (size_t array_idx = start; array_idx < end; ++array_idx) {
if (null_map && (*null_map)[array_idx]) {
- RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(),
column.get_name(),
- array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(), column,
*array_builder));
continue;
}
- RETURN_IF_ERROR(checkArrowStatus(builder.Append(), column.get_name(),
- array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.Append(), column,
*array_builder));
RETURN_IF_ERROR(nested_serde->write_column_to_arrow(nested_data,
nullptr, nested_builder,
offsets[array_idx
- 1],
offsets[array_idx], ctz));
diff --git a/be/src/core/data_type_serde/data_type_bitmap_serde.cpp
b/be/src/core/data_type_serde/data_type_bitmap_serde.cpp
index cb11d03e303..21cf5ae6fde 100644
--- a/be/src/core/data_type_serde/data_type_bitmap_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_bitmap_serde.cpp
@@ -127,15 +127,14 @@ Status DataTypeBitMapSerDe::write_column_to_arrow(const
IColumn& column, const N
auto& builder = assert_cast<arrow::BinaryBuilder&>(*array_builder);
for (size_t string_i = start; string_i < end; ++string_i) {
if (null_map && (*null_map)[string_i]) {
- RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(),
column.get_name(),
- array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(), column,
*array_builder));
} else {
auto& bitmap_value = col.get_element(string_i);
std::string memory_buffer(bitmap_value.getSizeInBytes(), '0');
bitmap_value.write_to(memory_buffer.data());
RETURN_IF_ERROR(checkArrowStatus(
builder.Append(memory_buffer.data(),
static_cast<int>(memory_buffer.size())),
- column.get_name(), array_builder->type()->name()));
+ column, *array_builder));
}
}
return Status::OK();
diff --git a/be/src/core/data_type_serde/data_type_date_or_datetime_serde.cpp
b/be/src/core/data_type_serde/data_type_date_or_datetime_serde.cpp
index b4564e98c41..e948529fac3 100644
--- a/be/src/core/data_type_serde/data_type_date_or_datetime_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_date_or_datetime_serde.cpp
@@ -156,11 +156,10 @@ Status DataTypeDateSerDe<T>::write_column_to_arrow(const
IColumn& column, const
const auto* time_val = (const VecDateTimeValue*)(&col_data[i]);
size_t len = time_val->to_buffer(buf);
if (null_map && (*null_map)[i]) {
- RETURN_IF_ERROR(checkArrowStatus(string_builder.AppendNull(),
column.get_name(),
- array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(string_builder.AppendNull(),
column, *array_builder));
} else {
RETURN_IF_ERROR(checkArrowStatus(string_builder.Append(buf,
cast_set<int32_t>(len)),
- column.get_name(),
array_builder->type()->name()));
+ column, *array_builder));
}
}
return Status::OK();
diff --git a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp
b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp
index 92a5106b481..deb956714d1 100644
--- a/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_datetimev2_serde.cpp
@@ -372,8 +372,8 @@ Status DataTypeDateTimeV2SerDe::write_column_to_arrow(const
IColumn& column,
const cctz::time_zone& real_ctz = timezone.empty() ? cctz::utc_time_zone()
: ctz;
for (size_t i = start; i < end; ++i) {
if (null_map && (*null_map)[i]) {
- RETURN_IF_ERROR(checkArrowStatus(timestamp_builder.AppendNull(),
column.get_name(),
- array_builder->type()->name()));
+ RETURN_IF_ERROR(
+ checkArrowStatus(timestamp_builder.AppendNull(), column,
*array_builder));
} else {
int64_t timestamp = 0;
DateV2Value<DateTimeV2ValueType> datetime_val = col_data[i];
@@ -386,8 +386,8 @@ Status DataTypeDateTimeV2SerDe::write_column_to_arrow(const
IColumn& column,
uint32_t millisecond = datetime_val.microsecond() / 1000;
timestamp = (timestamp * 1000) + millisecond;
}
-
RETURN_IF_ERROR(checkArrowStatus(timestamp_builder.Append(timestamp),
column.get_name(),
- array_builder->type()->name()));
+ RETURN_IF_ERROR(
+ checkArrowStatus(timestamp_builder.Append(timestamp),
column, *array_builder));
}
}
return Status::OK();
diff --git a/be/src/core/data_type_serde/data_type_datev2_serde.cpp
b/be/src/core/data_type_serde/data_type_datev2_serde.cpp
index d72f3664ae6..1a503af38ef 100644
--- a/be/src/core/data_type_serde/data_type_datev2_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_datev2_serde.cpp
@@ -96,12 +96,11 @@ Status DataTypeDateV2SerDe::write_column_to_arrow(const
IColumn& column, const N
for (size_t i = start; i < end; ++i) {
auto daynr = col_data[i].daynr() - date_threshold;
if (null_map && (*null_map)[i]) {
- RETURN_IF_ERROR(checkArrowStatus(date32_builder.AppendNull(),
column.get_name(),
- array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(date32_builder.AppendNull(),
column, *array_builder));
} else {
RETURN_IF_ERROR(
checkArrowStatus(date32_builder.Append(cast_set<int,
int64_t, false>(daynr)),
- column.get_name(),
array_builder->type()->name()));
+ column, *array_builder));
}
}
return Status::OK();
diff --git a/be/src/core/data_type_serde/data_type_decimal_serde.cpp
b/be/src/core/data_type_serde/data_type_decimal_serde.cpp
index ba1d6dd08fa..fa21498d9fe 100644
--- a/be/src/core/data_type_serde/data_type_decimal_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_decimal_serde.cpp
@@ -238,8 +238,7 @@ Status DataTypeDecimalSerDe<T>::write_column_to_arrow(const
IColumn& column,
std::make_shared<arrow::Decimal128Type>(27, 9);
for (size_t i = start; i < end; ++i) {
if (null_map && (*null_map)[i]) {
- RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(),
column.get_name(),
-
array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(), column,
*array_builder));
continue;
}
const auto& data_ref = col.get_data_at(i);
@@ -247,8 +246,7 @@ Status DataTypeDecimalSerDe<T>::write_column_to_arrow(const
IColumn& column,
int64_t high = (p_value->value) >> 64;
uint64_t low = cast_set<uint64_t>((p_value->value) &
0xFFFFFFFFFFFFFFFF);
arrow::Decimal128 value(high, low);
- RETURN_IF_ERROR(checkArrowStatus(builder.Append(value),
column.get_name(),
- array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.Append(value), column,
*array_builder));
}
} else if constexpr (T == TYPE_DECIMAL128I) {
auto& builder =
reinterpret_cast<arrow::Decimal128Builder&>(*array_builder);
@@ -256,8 +254,7 @@ Status DataTypeDecimalSerDe<T>::write_column_to_arrow(const
IColumn& column,
std::make_shared<arrow::Decimal128Type>(38, col.get_scale());
for (size_t i = start; i < end; ++i) {
if (null_map && (*null_map)[i]) {
- RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(),
column.get_name(),
-
array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(), column,
*array_builder));
continue;
}
const auto& data_ref = col.get_data_at(i);
@@ -265,8 +262,7 @@ Status DataTypeDecimalSerDe<T>::write_column_to_arrow(const
IColumn& column,
int64_t high = (p_value->value) >> 64;
uint64_t low = cast_set<uint64_t>((p_value->value) &
0xFFFFFFFFFFFFFFFF);
arrow::Decimal128 value(high, low);
- RETURN_IF_ERROR(checkArrowStatus(builder.Append(value),
column.get_name(),
- array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.Append(value), column,
*array_builder));
}
} else if constexpr (T == TYPE_DECIMAL32) {
auto& builder =
reinterpret_cast<arrow::Decimal128Builder&>(*array_builder);
@@ -274,14 +270,12 @@ Status
DataTypeDecimalSerDe<T>::write_column_to_arrow(const IColumn& column,
std::make_shared<arrow::Decimal128Type>(8, col.get_scale());
for (size_t i = start; i < end; ++i) {
if (null_map && (*null_map)[i]) {
- RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(),
column.get_name(),
-
array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(), column,
*array_builder));
continue;
}
Int128 p_value = col.get_element(i).value;
arrow::Decimal128 value(reinterpret_cast<const
uint8_t*>(&p_value));
- RETURN_IF_ERROR(checkArrowStatus(builder.Append(value),
column.get_name(),
- array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.Append(value), column,
*array_builder));
}
} else if constexpr (T == TYPE_DECIMAL64) {
auto& builder =
reinterpret_cast<arrow::Decimal128Builder&>(*array_builder);
@@ -289,14 +283,12 @@ Status
DataTypeDecimalSerDe<T>::write_column_to_arrow(const IColumn& column,
std::make_shared<arrow::Decimal128Type>(18, col.get_scale());
for (size_t i = start; i < end; ++i) {
if (null_map && (*null_map)[i]) {
- RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(),
column.get_name(),
-
array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(), column,
*array_builder));
continue;
}
Int128 p_value = col.get_element(i).value;
arrow::Decimal128 value(reinterpret_cast<const
uint8_t*>(&p_value));
- RETURN_IF_ERROR(checkArrowStatus(builder.Append(value),
column.get_name(),
- array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.Append(value), column,
*array_builder));
}
} else if constexpr (T == TYPE_DECIMAL256) {
auto& builder =
reinterpret_cast<arrow::Decimal256Builder&>(*array_builder);
@@ -304,8 +296,7 @@ Status DataTypeDecimalSerDe<T>::write_column_to_arrow(const
IColumn& column,
std::make_shared<arrow::Decimal256Type>(76, col.get_scale());
for (size_t i = start; i < end; ++i) {
if (null_map && (*null_map)[i]) {
- RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(),
column.get_name(),
-
array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(), column,
*array_builder));
continue;
}
auto p_value = wide::Int256(col.get_element(i));
@@ -317,8 +308,7 @@ Status DataTypeDecimalSerDe<T>::write_column_to_arrow(const
IColumn& column,
std::array<uint64_t, 4> word_array = {a0, a1, a2, a3};
arrow::Decimal256 value(arrow::Decimal256::LittleEndianArray,
word_array);
- RETURN_IF_ERROR(checkArrowStatus(builder.Append(value),
column.get_name(),
- array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.Append(value), column,
*array_builder));
}
} else {
return Status::InvalidArgument("write_column_to_arrow with type " +
column.get_name());
diff --git a/be/src/core/data_type_serde/data_type_hll_serde.cpp
b/be/src/core/data_type_serde/data_type_hll_serde.cpp
index 630d42629bc..46f832d10ae 100644
--- a/be/src/core/data_type_serde/data_type_hll_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_hll_serde.cpp
@@ -131,15 +131,14 @@ Status DataTypeHLLSerDe::write_column_to_arrow(const
IColumn& column, const Null
auto& builder = assert_cast<arrow::BinaryBuilder&>(*array_builder);
for (size_t string_i = start; string_i < end; ++string_i) {
if (null_map && (*null_map)[string_i]) {
- RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(),
column.get_name(),
- array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(), column,
*array_builder));
} else {
auto& hll_value = col.get_element(string_i);
std::string memory_buffer(hll_value.max_serialized_size(), '0');
hll_value.serialize((uint8_t*)memory_buffer.data());
RETURN_IF_ERROR(checkArrowStatus(
builder.Append(memory_buffer.data(),
static_cast<int>(memory_buffer.size())),
- column.get_name(), array_builder->type()->name()));
+ column, *array_builder));
}
}
return Status::OK();
diff --git a/be/src/core/data_type_serde/data_type_ipv4_serde.cpp
b/be/src/core/data_type_serde/data_type_ipv4_serde.cpp
index 0255504b08a..df6174ec319 100644
--- a/be/src/core/data_type_serde/data_type_ipv4_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_ipv4_serde.cpp
@@ -106,7 +106,7 @@ Status DataTypeIPv4SerDe::write_column_to_arrow(const
IColumn& column, const Nul
int32_builder.AppendValues(reinterpret_cast<const
Int32*>(col_data.data()) + start,
end - start,
reinterpret_cast<const
uint8_t*>(arrow_null_map_data)),
- column.get_name(), array_builder->type()->name()));
+ column, *array_builder));
return Status::OK();
}
diff --git a/be/src/core/data_type_serde/data_type_ipv6_serde.cpp
b/be/src/core/data_type_serde/data_type_ipv6_serde.cpp
index 54624521b04..9c9ba1a2972 100644
--- a/be/src/core/data_type_serde/data_type_ipv6_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_ipv6_serde.cpp
@@ -127,14 +127,13 @@ Status DataTypeIPv6SerDe::write_column_to_arrow(const
IColumn& column, const Nul
auto& string_builder = assert_cast<arrow::StringBuilder&>(*array_builder);
for (size_t i = start; i < end; ++i) {
if (null_map && (*null_map)[i]) {
- RETURN_IF_ERROR(checkArrowStatus(string_builder.AppendNull(),
column.get_name(),
- array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(string_builder.AppendNull(),
column, *array_builder));
} else {
std::string ipv6_str = IPv6Value::to_string(col_data[i]);
RETURN_IF_ERROR(checkArrowStatus(
string_builder.Append(ipv6_str.c_str(),
cast_set<int, size_t,
false>(ipv6_str.size())),
- column.get_name(), array_builder->type()->name()));
+ column, *array_builder));
}
}
return Status::OK();
diff --git a/be/src/core/data_type_serde/data_type_jsonb_serde.cpp
b/be/src/core/data_type_serde/data_type_jsonb_serde.cpp
index bd8dfdfd312..75268e2f769 100644
--- a/be/src/core/data_type_serde/data_type_jsonb_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_jsonb_serde.cpp
@@ -99,8 +99,7 @@ Status DataTypeJsonbSerDe::write_column_to_arrow(const
IColumn& column, const Nu
auto& builder = assert_cast<arrow::StringBuilder&>(*array_builder);
for (size_t string_i = start; string_i < end; ++string_i) {
if (null_map && (*null_map)[string_i]) {
- RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(),
column.get_name(),
- array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(), column,
*array_builder));
continue;
}
std::string_view string_ref =
string_column.get_data_at(string_i).to_string_view();
@@ -109,7 +108,7 @@ Status DataTypeJsonbSerDe::write_column_to_arrow(const
IColumn& column, const Nu
RETURN_IF_ERROR(
checkArrowStatus(builder.Append(json_string.data(),
cast_set<int, size_t,
false>(json_string.size())),
- column.get_name(),
array_builder->type()->name()));
+ column, *array_builder));
}
return Status::OK();
}
diff --git a/be/src/core/data_type_serde/data_type_map_serde.cpp
b/be/src/core/data_type_serde/data_type_map_serde.cpp
index 89a2b2955df..9721e6eed99 100644
--- a/be/src/core/data_type_serde/data_type_map_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_map_serde.cpp
@@ -348,8 +348,7 @@ Status DataTypeMapSerDe::write_column_to_arrow(const
IColumn& column, const Null
for (size_t r = start; r < end; ++r) {
if ((null_map && (*null_map)[r])) {
- RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(),
column.get_name(),
- array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(), column,
*array_builder));
} else if (simd::contain_one(keys_nullmap_data + offsets[r - 1],
offsets[r] - offsets[r - 1])) {
// arrow do not support key is null, so we ignore the null
key-value
@@ -363,8 +362,7 @@ Status DataTypeMapSerDe::write_column_to_arrow(const
IColumn& column, const Null
key_mutable_data->insert_from(nested_keys_column, i);
value_mutable_data->insert_from(nested_values_column, i);
}
- RETURN_IF_ERROR(checkArrowStatus(builder.Append(),
column.get_name(),
- array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.Append(), column,
*array_builder));
RETURN_IF_ERROR(key_serde->write_column_to_arrow(
*key_mutable_data, nullptr, key_builder, 0,
key_mutable_data->size(), ctz));
@@ -372,8 +370,7 @@ Status DataTypeMapSerDe::write_column_to_arrow(const
IColumn& column, const Null
value_builder,
0,
value_mutable_data->size(), ctz));
} else {
- RETURN_IF_ERROR(checkArrowStatus(builder.Append(),
column.get_name(),
- array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.Append(), column,
*array_builder));
RETURN_IF_ERROR(key_serde->write_column_to_arrow(
nested_keys_column, nullptr, key_builder, offsets[r - 1],
offsets[r], ctz));
RETURN_IF_ERROR(value_serde->write_column_to_arrow(
diff --git a/be/src/core/data_type_serde/data_type_number_serde.cpp
b/be/src/core/data_type_serde/data_type_number_serde.cpp
index 5f518aa6bbd..2124547c2f8 100644
--- a/be/src/core/data_type_serde/data_type_number_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_number_serde.cpp
@@ -101,8 +101,8 @@ Status DataTypeNumberSerDe<T>::write_column_to_arrow(const
IColumn& column, cons
auto* null_builder = dynamic_cast<arrow::NullBuilder*>(array_builder);
if (null_builder) {
for (size_t i = start; i < end; ++i) {
- RETURN_IF_ERROR(checkArrowStatus(null_builder->AppendNull(),
column.get_name(),
-
null_builder->type()->name()));
+ RETURN_IF_ERROR(
+ checkArrowStatus(null_builder->AppendNull(), column,
*null_builder));
}
} else {
auto& builder = assert_cast<ARROW_BUILDER_TYPE&>(*array_builder);
@@ -110,7 +110,7 @@ Status DataTypeNumberSerDe<T>::write_column_to_arrow(const
IColumn& column, cons
builder.AppendValues(reinterpret_cast<const
uint8_t*>(col_data.data() + start),
end - start,
reinterpret_cast<const
uint8_t*>(arrow_null_map_data)),
- column.get_name(), array_builder->type()->name()));
+ column, *array_builder));
}
} else if constexpr (T == TYPE_LARGEINT) {
@@ -119,13 +119,13 @@ Status
DataTypeNumberSerDe<T>::write_column_to_arrow(const IColumn& column, cons
auto& data_value = col_data[i];
std::string value_str = fmt::format("{}", data_value);
if (null_map && (*null_map)[i]) {
- RETURN_IF_ERROR(checkArrowStatus(string_builder.AppendNull(),
column.get_name(),
-
array_builder->type()->name()));
+ RETURN_IF_ERROR(
+ checkArrowStatus(string_builder.AppendNull(), column,
*array_builder));
} else {
RETURN_IF_ERROR(checkArrowStatus(
string_builder.Append(value_str.data(),
cast_set<int, size_t,
false>(value_str.length())),
- column.get_name(), array_builder->type()->name()));
+ column, *array_builder));
}
}
} else if constexpr (T == TYPE_IPV6) {
@@ -134,25 +134,25 @@ Status
DataTypeNumberSerDe<T>::write_column_to_arrow(const IColumn& column, cons
RETURN_IF_ERROR(checkArrowStatus(
builder.AppendValues((int64_t*)col_data.data() + start, end -
start,
reinterpret_cast<const
uint8_t*>(arrow_null_map_data)),
- column.get_name(), array_builder->type()->name()));
+ column, *array_builder));
} else if constexpr (T == TYPE_DATEV2) {
auto& builder = assert_cast<ARROW_BUILDER_TYPE&>(*array_builder);
RETURN_IF_ERROR(checkArrowStatus(
builder.AppendValues((uint32_t*)col_data.data() + start, end -
start,
reinterpret_cast<const
uint8_t*>(arrow_null_map_data)),
- column.get_name(), array_builder->type()->name()));
+ column, *array_builder));
} else if constexpr (T == TYPE_DATETIMEV2 || T == TYPE_TIMESTAMPTZ) {
auto& builder = assert_cast<ARROW_BUILDER_TYPE&>(*array_builder);
RETURN_IF_ERROR(checkArrowStatus(
builder.AppendValues((uint64_t*)col_data.data() + start, end -
start,
reinterpret_cast<const
uint8_t*>(arrow_null_map_data)),
- column.get_name(), array_builder->type()->name()));
+ column, *array_builder));
} else {
auto& builder = assert_cast<ARROW_BUILDER_TYPE&>(*array_builder);
RETURN_IF_ERROR(checkArrowStatus(
builder.AppendValues(col_data.data() + start, end - start,
reinterpret_cast<const
uint8_t*>(arrow_null_map_data)),
- column.get_name(), array_builder->type()->name()));
+ column, *array_builder));
}
return Status::OK();
}
diff --git a/be/src/core/data_type_serde/data_type_quantilestate_serde.h
b/be/src/core/data_type_serde/data_type_quantilestate_serde.h
index 606a1c2116e..4f3faffb3e2 100644
--- a/be/src/core/data_type_serde/data_type_quantilestate_serde.h
+++ b/be/src/core/data_type_serde/data_type_quantilestate_serde.h
@@ -106,8 +106,7 @@ public:
auto& builder = assert_cast<arrow::BinaryBuilder&>(*array_builder);
for (size_t string_i = start; string_i < end; ++string_i) {
if (null_map && (*null_map)[string_i]) {
- RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(),
column.get_name(),
-
array_builder->type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(), column,
*array_builder));
} else {
auto& quantile_state_value = col.get_element(string_i);
std::string
memory_buffer(quantile_state_value.get_serialized_size(), '0');
@@ -115,7 +114,7 @@ public:
RETURN_IF_ERROR(
checkArrowStatus(builder.Append(memory_buffer.data(),
static_cast<int>(memory_buffer.size())),
- column.get_name(),
array_builder->type()->name()));
+ column, *array_builder));
}
}
return Status::OK();
diff --git a/be/src/core/data_type_serde/data_type_serde.h
b/be/src/core/data_type_serde/data_type_serde.h
index 7c007c6558d..eb7ce74fbe7 100644
--- a/be/src/core/data_type_serde/data_type_serde.h
+++ b/be/src/core/data_type_serde/data_type_serde.h
@@ -545,11 +545,12 @@ inline static NullMap revert_null_map(const NullMap*
null_bytemap, size_t start,
return res;
}
-inline Status checkArrowStatus(const arrow::Status& status, const std::string&
column,
- const std::string& format_name) {
+template <typename ColumnType, typename BuilderType>
+inline Status checkArrowStatus(const arrow::Status& status, const ColumnType&
column,
+ const BuilderType& builder) {
if (!status.ok()) {
return Status::FatalError("arrow serde with arrow: {} with column : {}
with error msg: {}",
- format_name, column, status.ToString());
+ builder.type()->name(), column.get_name(),
status.ToString());
}
return Status::OK();
}
diff --git a/be/src/core/data_type_serde/data_type_string_serde.cpp
b/be/src/core/data_type_serde/data_type_string_serde.cpp
index 08d87cc38ca..8c5bf1664da 100644
--- a/be/src/core/data_type_serde/data_type_string_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_string_serde.cpp
@@ -225,14 +225,13 @@ Status
DataTypeStringSerDeBase<ColumnType>::write_column_to_arrow_impl(const ICo
const auto& string_column = assert_cast<const ColumnType&>(column);
for (size_t string_i = start; string_i < end; ++string_i) {
if (null_map && (*null_map)[string_i]) {
- RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(),
column.get_name(),
- builder.type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(), column,
builder));
continue;
}
auto string_ref = string_column.get_data_at(string_i);
RETURN_IF_ERROR(checkArrowStatus(
builder.Append(string_ref.data, cast_set<int, size_t,
false>(string_ref.size)),
- column.get_name(), builder.type()->name()));
+ column, builder));
}
return Status::OK();
}
diff --git a/be/src/core/data_type_serde/data_type_struct_serde.cpp
b/be/src/core/data_type_serde/data_type_struct_serde.cpp
index 7514089ea0d..0530e1fce66 100644
--- a/be/src/core/data_type_serde/data_type_struct_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_struct_serde.cpp
@@ -401,12 +401,10 @@ Status DataTypeStructSerDe::write_column_to_arrow(const
IColumn& column, const N
const auto& struct_column = assert_cast<const ColumnStruct&>(column);
for (auto r = start; r < end; ++r) {
if (null_map != nullptr && (*null_map)[r]) {
- RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(),
struct_column.get_name(),
- builder.type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(),
struct_column, builder));
continue;
}
- RETURN_IF_ERROR(checkArrowStatus(builder.Append(),
struct_column.get_name(),
- builder.type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.Append(), struct_column,
builder));
for (auto ei = 0; ei < struct_column.tuple_size(); ++ei) {
auto* elem_builder = builder.field_builder(ei);
RETURN_IF_ERROR(elem_serdes_ptrs[ei]->write_column_to_arrow(
diff --git a/be/src/core/data_type_serde/data_type_timestamptz_serde.cpp
b/be/src/core/data_type_serde/data_type_timestamptz_serde.cpp
index c22c65dbd5a..e8c26f6db68 100644
--- a/be/src/core/data_type_serde/data_type_timestamptz_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_timestamptz_serde.cpp
@@ -201,8 +201,8 @@ Status
DataTypeTimeStampTzSerDe::write_column_to_arrow(const IColumn& column,
static const auto& UTC = cctz::utc_time_zone();
for (size_t i = start; i < end; ++i) {
if (null_map && (*null_map)[i]) {
- RETURN_IF_ERROR(checkArrowStatus(timestamp_builder.AppendNull(),
column.get_name(),
- array_builder->type()->name()));
+ RETURN_IF_ERROR(
+ checkArrowStatus(timestamp_builder.AppendNull(), column,
*array_builder));
} else {
int64_t timestamp = 0;
const auto& tz = col_data[i];
@@ -215,8 +215,8 @@ Status
DataTypeTimeStampTzSerDe::write_column_to_arrow(const IColumn& column,
uint32_t millisecond = tz.microsecond() / 1000;
timestamp = (timestamp * 1000) + millisecond;
}
-
RETURN_IF_ERROR(checkArrowStatus(timestamp_builder.Append(timestamp),
column.get_name(),
- array_builder->type()->name()));
+ RETURN_IF_ERROR(
+ checkArrowStatus(timestamp_builder.Append(timestamp),
column, *array_builder));
}
}
return Status::OK();
diff --git a/be/src/core/data_type_serde/data_type_varbinary_serde.cpp
b/be/src/core/data_type_serde/data_type_varbinary_serde.cpp
index 76ac43d535c..2b1069d3bbb 100644
--- a/be/src/core/data_type_serde/data_type_varbinary_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_varbinary_serde.cpp
@@ -58,13 +58,12 @@ Status DataTypeVarbinarySerDe::write_column_to_arrow(const
IColumn& column, cons
const auto& varbinary_column_data = assert_cast<const
ColumnVarbinary&>(column).get_data();
for (size_t i = start; i < end; ++i) {
if (null_map && (*null_map)[i]) {
- RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(),
column.get_name(),
- builder.type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(), column,
builder));
continue;
}
const auto& string_view = varbinary_column_data[i];
RETURN_IF_ERROR(checkArrowStatus(builder.Append(string_view.data(),
string_view.size()),
- column.get_name(),
builder.type()->name()));
+ column, builder));
}
return Status::OK();
};
diff --git a/be/src/core/data_type_serde/data_type_variant_serde.cpp
b/be/src/core/data_type_serde/data_type_variant_serde.cpp
index 22896483787..6cd0126291e 100644
--- a/be/src/core/data_type_serde/data_type_variant_serde.cpp
+++ b/be/src/core/data_type_serde/data_type_variant_serde.cpp
@@ -49,8 +49,7 @@ Status write_variant_column_to_arrow_impl(const IColumn&
column, const ColumnVar
options.timezone = &ctz;
for (int64_t i = start; i < end; ++i) {
if (null_map && (*null_map)[cast_set<size_t>(i)]) {
- RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(),
column.get_name(),
- builder.type()->name()));
+ RETURN_IF_ERROR(checkArrowStatus(builder.AppendNull(), column,
builder));
continue;
}
@@ -59,7 +58,7 @@ Status write_variant_column_to_arrow_impl(const IColumn&
column, const ColumnVar
const auto serialized_size =
cast_set<typename
BuilderType::offset_type>(serialized_value.size());
RETURN_IF_ERROR(checkArrowStatus(builder.Append(serialized_value.data(),
serialized_size),
- column.get_name(),
builder.type()->name()));
+ column, builder));
}
return Status::OK();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
