Gabriel39 commented on code in PR #68667:
URL: https://github.com/apache/doris/pull/68667#discussion_r4141475999
##########
be/src/core/data_type_serde/data_type_variant_serde.cpp:
##########
@@ -157,6 +163,64 @@ Status DataTypeVariantSerDe::write_column_to_arrow(const
IColumn& column, const
int64_t start, int64_t end,
const cctz::time_zone& ctz)
const {
const auto* var = check_and_get_column<ColumnVariant>(column);
+ if (array_builder->type()->id() == arrow::Type::STRUCT) {
+ // Legacy documents need JSON conversion; typed scalar roots can keep
their type.
+ // The outer null map must remain SQL NULL on the wire.
+ if (start < 0 || end < start || end > column.size() ||
+ (null_map != nullptr && end > null_map->size())) {
+ return Status::InvalidArgument("Invalid Variant Arrow row range
[{}, {})", start, end);
+ }
+ if (var->is_scalar_variant()) {
+ auto scalar_type = remove_nullable(var->get_root_type());
+ if (scalar_type->get_primitive_type() == TYPE_DECIMAL256) {
+ return Status::NotSupported(
+ "Native Arrow Variant does not support Decimal256
roots");
+ }
+ if
(is_supported_variant_typed_identity(scalar_type->get_primitive_type())) {
+ // Avoid a JSON round trip that would turn exact decimal roots
into doubles.
+ auto typed =
+
ColumnVariantV2::create_typed(make_nullable(var->get_root()), scalar_type);
+ return DataTypeVariantV2SerDe().write_column_to_arrow(
+ *typed, null_map, array_builder, start, end, ctz);
+ }
+ }
+ JsonToVariantOptions parse_options;
+ parse_options.throw_on_invalid_json = true;
+ // Stored keys were already accepted at ingestion; mutable parse
limits must not reject reads.
+ parse_options.max_json_key_length =
std::numeric_limits<uint32_t>::max();
+ parse_options.check_duplicate_json_path = false;
+ JsonStringToVariantEncoder encoder(parse_options);
+ FormatOptions options;
+ options.timezone = &ctz;
+ NullMap selected_nulls;
+ if (null_map != nullptr) {
+ selected_nulls.assign(null_map->begin() + start, null_map->begin()
+ end);
+ }
+ for (int64_t row = start; row < end; ++row) {
+ std::string json;
+ if (null_map != nullptr && (*null_map)[row]) {
+ json = "null";
+ } else {
+ var->serialize_one_row_to_string(row, &json, options);
+ if (var->get_root_type()->get_primitive_type() == TYPE_STRING
&&
Review Comment:
Fixed in ff92c38643d. Mixed-batch visible roots now use typed encoding
rather than JSON string quoting/reparsing, covering CHAR, VARCHAR and STRING
uniformly. The new test combines ordinary text and JSON-looking text (hello,
true, 42) with an object-path row and verifies that the root values remain
strings.
##########
be/src/format/arrow/arrow_block_convertor.cpp:
##########
@@ -472,6 +472,26 @@ Status ArrowBlockConvertor::init() {
return Status::OK();
}
+Status ArrowFlightArrowBlockConvertor::write_column(const
std::shared_ptr<const IDataType>& type,
+ const DataTypeSerDe& serde,
+ const IColumn& column,
const NullMap* null_map,
+ const
std::shared_ptr<arrow::Field>& field,
+ arrow::ArrayBuilder*
array_builder,
+ int64_t start, int64_t end,
+ const cctz::time_zone&
ctz) const {
+ if (contains_extension_type(field->type())) {
+ std::shared_ptr<arrow::DataType> native_type;
+ RETURN_IF_ERROR(convert_to_arrow_type(type, &native_type, ctz.name(),
true, true));
Review Comment:
Fixed in ff92c38643d. Native schema binding now reuses the recursive
declared-type binding check, which recognizes equivalent timestamp timezone
labels while validating the Variant extension identity and nested field shape.
Added STRUCT<VARIANT, TIMESTAMPTZ> tests with +08:00, +05:45 and -03:30
published labels. All 37 related BE tests pass.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
