This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 9b5028785d13a588ab4307e626a5ef26b60ad145 Author: xueweizhang <zxw520bl...@163.com> AuthorDate: Fri May 17 14:19:04 2024 +0800 [fix](prepare) fix datetimev2 return err when binary_row_format (#34662) fix datetimev2 return err when binary_row_format. before pr, Backend return datetimev2 alwary by to_string. fix datatimev2 return metadata loss scale. --- .../exec/schema_scanner/schema_columns_scanner.cpp | 27 +++++++++++-- be/src/util/mysql_row_buffer.cpp | 46 ++++++++++++---------- be/src/util/mysql_row_buffer.h | 4 +- .../serde/data_type_datetimev2_serde.cpp | 4 +- .../org/apache/doris/mysql/MysqlSerializer.java | 2 + .../test_compaction_uniq_keys_row_store.out | 32 +++++++-------- .../scalar_types/sql/infomation_schema.out | 2 +- 7 files changed, 71 insertions(+), 46 deletions(-) diff --git a/be/src/exec/schema_scanner/schema_columns_scanner.cpp b/be/src/exec/schema_scanner/schema_columns_scanner.cpp index 56a6c5f256e..763f24b9e53 100644 --- a/be/src/exec/schema_scanner/schema_columns_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_columns_scanner.cpp @@ -512,7 +512,9 @@ Status SchemaColumnsScanner::_fill_block_impl(vectorized::Block* block) { { int64_t srcs[columns_num]; for (int i = 0; i < columns_num; ++i) { - if (_desc_result.columns[i].columnDesc.__isset.columnPrecision) { + int data_type = _desc_result.columns[i].columnDesc.columnType; + if (_desc_result.columns[i].columnDesc.__isset.columnPrecision && + data_type != TPrimitiveType::DATETIMEV2) { srcs[i] = _desc_result.columns[i].columnDesc.columnPrecision; datas[i] = srcs + i; } else { @@ -525,7 +527,9 @@ Status SchemaColumnsScanner::_fill_block_impl(vectorized::Block* block) { { int64_t srcs[columns_num]; for (int i = 0; i < columns_num; ++i) { - if (_desc_result.columns[i].columnDesc.__isset.columnScale) { + int data_type = _desc_result.columns[i].columnDesc.columnType; + if (_desc_result.columns[i].columnDesc.__isset.columnScale && + data_type != TPrimitiveType::DATETIMEV2) { srcs[i] = _desc_result.columns[i].columnDesc.columnScale; datas[i] = srcs + i; } else { @@ -535,7 +539,20 @@ Status SchemaColumnsScanner::_fill_block_impl(vectorized::Block* block) { RETURN_IF_ERROR(fill_dest_column_for_range(block, 11, datas)); } // DATETIME_PRECISION - { RETURN_IF_ERROR(fill_dest_column_for_range(block, 12, null_datas)); } + { + std::vector<int64_t> srcs(columns_num); + for (int i = 0; i < columns_num; ++i) { + int data_type = _desc_result.columns[i].columnDesc.columnType; + if (_desc_result.columns[i].columnDesc.__isset.columnScale && + data_type == TPrimitiveType::DATETIMEV2) { + srcs[i] = _desc_result.columns[i].columnDesc.columnScale; + datas[i] = srcs.data() + i; + } else { + datas[i] = nullptr; + } + } + RETURN_IF_ERROR(fill_dest_column_for_range(block, 12, datas)); + } // CHARACTER_SET_NAME { RETURN_IF_ERROR(fill_dest_column_for_range(block, 13, null_datas)); } // COLLATION_NAME @@ -605,7 +622,9 @@ Status SchemaColumnsScanner::_fill_block_impl(vectorized::Block* block) { { int64_t srcs[columns_num]; for (int i = 0; i < columns_num; ++i) { - if (_desc_result.columns[i].columnDesc.__isset.columnScale) { + int data_type = _desc_result.columns[i].columnDesc.columnType; + if (_desc_result.columns[i].columnDesc.__isset.columnScale && + data_type != TPrimitiveType::DATETIMEV2) { srcs[i] = _desc_result.columns[i].columnDesc.columnScale; datas[i] = srcs + i; } else { diff --git a/be/src/util/mysql_row_buffer.cpp b/be/src/util/mysql_row_buffer.cpp index a15fa37a0f3..3e4aa332cea 100644 --- a/be/src/util/mysql_row_buffer.cpp +++ b/be/src/util/mysql_row_buffer.cpp @@ -385,19 +385,25 @@ int MysqlRowBuffer<is_binary_format>::push_timev2(double data, int scale) { template <bool is_binary_format> template <typename DateType> -int MysqlRowBuffer<is_binary_format>::push_vec_datetime(DateType& data) { +int MysqlRowBuffer<is_binary_format>::push_vec_datetime(DateType& data, int scale) { if (is_binary_format && !_dynamic_mode) { - return push_datetime(data); + return push_datetime(data, scale); } char buf[64]; - char* pos = data.to_string(buf); + char* pos = nullptr; + if constexpr (std::is_same_v<DateType, DateV2Value<DateV2ValueType>> || + std::is_same_v<DateType, DateV2Value<DateTimeV2ValueType>>) { + pos = data.to_string(buf, scale); + } else { + pos = data.to_string(buf); + } return push_string(buf, pos - buf - 1); } template <bool is_binary_format> template <typename DateType> -int MysqlRowBuffer<is_binary_format>::push_datetime(const DateType& data) { +int MysqlRowBuffer<is_binary_format>::push_datetime(const DateType& data, int scale) { if (is_binary_format && !_dynamic_mode) { char buff[12], *pos; size_t length; @@ -410,16 +416,6 @@ int MysqlRowBuffer<is_binary_format>::push_datetime(const DateType& data) { pos[4] = (uchar)data.hour(); pos[5] = (uchar)data.minute(); pos[6] = (uchar)data.second(); - if constexpr (std::is_same_v<DateType, DateV2Value<DateV2ValueType>> || - std::is_same_v<DateType, DateV2Value<DateTimeV2ValueType>>) { - int4store(pos + 7, data.microsecond()); - if (data.microsecond()) { - length = 11; - } - } else { - int4store(pos + 7, 0); - } - if (data.hour() || data.minute() || data.second()) { length = 7; } else if (data.year() || data.month() || data.day()) { @@ -427,6 +423,14 @@ int MysqlRowBuffer<is_binary_format>::push_datetime(const DateType& data) { } else { length = 0; } + if constexpr (std::is_same_v<DateType, DateV2Value<DateV2ValueType>> || + std::is_same_v<DateType, DateV2Value<DateTimeV2ValueType>>) { + if (scale > 0 || data.microsecond()) { + int4store(pos + 7, data.microsecond()); + length = 11; + } + } + buff[0] = (char)length; // Length is stored first return append(buff, length + 1); } @@ -511,14 +515,16 @@ template class MysqlRowBuffer<true>; template class MysqlRowBuffer<false>; template int MysqlRowBuffer<true>::push_vec_datetime<DateV2Value<DateV2ValueType>>( - DateV2Value<DateV2ValueType>& value); + DateV2Value<DateV2ValueType>& value, int scale); template int MysqlRowBuffer<true>::push_vec_datetime<DateV2Value<DateTimeV2ValueType>>( - DateV2Value<DateTimeV2ValueType>& value); -template int MysqlRowBuffer<true>::push_vec_datetime<VecDateTimeValue>(VecDateTimeValue& value); + DateV2Value<DateTimeV2ValueType>& value, int scale); +template int MysqlRowBuffer<true>::push_vec_datetime<VecDateTimeValue>(VecDateTimeValue& value, + int scale); template int MysqlRowBuffer<false>::push_vec_datetime<DateV2Value<DateV2ValueType>>( - DateV2Value<DateV2ValueType>& value); + DateV2Value<DateV2ValueType>& value, int scale); template int MysqlRowBuffer<false>::push_vec_datetime<DateV2Value<DateTimeV2ValueType>>( - DateV2Value<DateTimeV2ValueType>& value); -template int MysqlRowBuffer<false>::push_vec_datetime<VecDateTimeValue>(VecDateTimeValue& value); + DateV2Value<DateTimeV2ValueType>& value, int scale); +template int MysqlRowBuffer<false>::push_vec_datetime<VecDateTimeValue>(VecDateTimeValue& value, + int scale); } // namespace doris diff --git a/be/src/util/mysql_row_buffer.h b/be/src/util/mysql_row_buffer.h index 6f12fda1ecb..b740efa7764 100644 --- a/be/src/util/mysql_row_buffer.h +++ b/be/src/util/mysql_row_buffer.h @@ -76,7 +76,7 @@ public: int push_time(double data); int push_timev2(double data, int scale); template <typename DateType> - int push_datetime(const DateType& data); + int push_datetime(const DateType& data, int scale); int push_decimal(const DecimalV2Value& data, int round_scale); int push_ipv4(const IPv4Value& ipv4_val); int push_ipv6(const IPv6Value& ipv6_val); @@ -84,7 +84,7 @@ public: int push_null(); template <typename DateType> - int push_vec_datetime(DateType& data); + int push_vec_datetime(DateType& data, int scale = -1); // this function reserved size, change the pos step size, return old pos // Becareful when use the returned pointer. diff --git a/be/src/vec/data_types/serde/data_type_datetimev2_serde.cpp b/be/src/vec/data_types/serde/data_type_datetimev2_serde.cpp index 56aeb3c29ca..73e859f985a 100644 --- a/be/src/vec/data_types/serde/data_type_datetimev2_serde.cpp +++ b/be/src/vec/data_types/serde/data_type_datetimev2_serde.cpp @@ -178,10 +178,8 @@ Status DataTypeDateTimeV2SerDe::_write_column_to_mysql(const IColumn& column, int row_idx, bool col_const) const { auto& data = assert_cast<const ColumnVector<UInt64>&>(column).get_data(); const auto col_index = index_check_const(row_idx, col_const); - char buf[64]; DateV2Value<DateTimeV2ValueType> date_val = binary_cast<UInt64, DateV2Value<DateTimeV2ValueType>>(data[col_index]); - char* pos = date_val.to_string(buf, scale); // _nesting_level >= 2 means this datetimev2 is in complex type // and we should add double quotes if (_nesting_level >= 2) { @@ -189,7 +187,7 @@ Status DataTypeDateTimeV2SerDe::_write_column_to_mysql(const IColumn& column, return Status::InternalError("pack mysql buffer failed."); } } - if (UNLIKELY(0 != result.push_string(buf, pos - buf - 1))) { + if (UNLIKELY(0 != result.push_vec_datetime(date_val, scale))) { return Status::InternalError("pack mysql buffer failed."); } if (_nesting_level >= 2) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlSerializer.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlSerializer.java index 228f3891ce9..1454dca3c3d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlSerializer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlSerializer.java @@ -298,6 +298,8 @@ public class MysqlSerializer { case DECIMAL64: case DECIMAL128: case DECIMAL256: + case TIMEV2: + case DATETIMEV2: return ((ScalarType) type).decimalScale(); case FLOAT: case DOUBLE: diff --git a/regression-test/data/compaction/test_compaction_uniq_keys_row_store.out b/regression-test/data/compaction/test_compaction_uniq_keys_row_store.out index e49aea97d90..cedf0dbe9bd 100644 --- a/regression-test/data/compaction/test_compaction_uniq_keys_row_store.out +++ b/regression-test/data/compaction/test_compaction_uniq_keys_row_store.out @@ -1,49 +1,49 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !point_select -- -1 2017-10-01 2017-10-01 2017-10-01 11:11:11.021 2017-10-01 11:11:11.011000 Beijing 10 1 2020-01-01 00:00:00 2020-01-01 00:00:00 2017-10-01 11:11:11.170 2017-10-01 11:11:11.110111 2020-01-01 00:00:00 1 30 20 +1 2017-10-01 2017-10-01 2017-10-01T11:11:11.021 2017-10-01T11:11:11.011 Beijing 10 1 2020-01-01T00:00 2020-01-01T00:00 2017-10-01T11:11:11.170 2017-10-01T11:11:11.110111 2020-01-01T00:00 1 30 20 -- !point_select -- -1 2017-10-01 2017-10-01 2017-10-01 11:11:11.022 2017-10-01 11:11:11.012000 Beijing 10 1 2020-01-02 00:00:00 2020-01-02 00:00:00 2017-10-01 11:11:11.160 2017-10-01 11:11:11.100111 2020-01-02 00:00:00 1 31 19 +1 2017-10-01 2017-10-01 2017-10-01T11:11:11.022 2017-10-01T11:11:11.012 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2017-10-01T11:11:11.160 2017-10-01T11:11:11.100111 2020-01-02T00:00 1 31 19 -- !point_select -- -2 2017-10-01 2017-10-01 2017-10-01 11:11:11.023 2017-10-01 11:11:11.013000 Beijing 10 1 2020-01-02 00:00:00 2020-01-02 00:00:00 2017-10-01 11:11:11.150 2017-10-01 11:11:11.130111 2020-01-02 00:00:00 1 31 21 +2 2017-10-01 2017-10-01 2017-10-01T11:11:11.023 2017-10-01T11:11:11.013 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2017-10-01T11:11:11.150 2017-10-01T11:11:11.130111 2020-01-02T00:00 1 31 21 -- !point_select -- -2 2017-10-01 2017-10-01 2017-10-01 11:11:11.024 2017-10-01 11:11:11.014000 Beijing 10 1 2020-01-03 00:00:00 2020-01-03 00:00:00 2017-10-01 11:11:11.140 2017-10-01 11:11:11.120111 2020-01-03 00:00:00 1 32 20 +2 2017-10-01 2017-10-01 2017-10-01T11:11:11.024 2017-10-01T11:11:11.014 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 2017-10-01T11:11:11.140 2017-10-01T11:11:11.120111 2020-01-03T00:00 1 32 20 -- !point_select -- -3 2017-10-01 2017-10-01 2017-10-01 11:11:11.025 2017-10-01 11:11:11.015000 Beijing 10 1 2020-01-03 00:00:00 2020-01-03 00:00:00 2017-10-01 11:11:11.100 2017-10-01 11:11:11.140111 2020-01-03 00:00:00 1 32 22 +3 2017-10-01 2017-10-01 2017-10-01T11:11:11.025 2017-10-01T11:11:11.015 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 2017-10-01T11:11:11.100 2017-10-01T11:11:11.140111 2020-01-03T00:00 1 32 22 -- !point_select -- -3 2017-10-01 2017-10-01 2017-10-01 11:11:11.026 2017-10-01 11:11:11.016000 Beijing 10 1 2020-01-04 00:00:00 2020-01-04 00:00:00 2017-10-01 11:11:11.110 2017-10-01 11:11:11.150111 2020-01-04 00:00:00 1 33 21 +3 2017-10-01 2017-10-01 2017-10-01T11:11:11.026 2017-10-01T11:11:11.016 Beijing 10 1 2020-01-04T00:00 2020-01-04T00:00 2017-10-01T11:11:11.110 2017-10-01T11:11:11.150111 2020-01-04T00:00 1 33 21 -- !point_select -- -3 2017-10-01 2017-10-01 2017-10-01 11:11:11.027 2017-10-01 11:11:11.017000 Beijing 10 1 1970-01-01 00:00:00 1970-01-01 00:00:00 1970-01-01 00:00:00.111 1970-01-01 00:00:00.000000 2020-01-05 00:00:00 1 34 20 +3 2017-10-01 2017-10-01 2017-10-01T11:11:11.027 2017-10-01T11:11:11.017 Beijing 10 1 1970-01-01T00:00 1970-01-01T00:00 1970-01-01T00:00:00.111 1970-01-01T00:00 2020-01-05T00:00 1 34 20 -- !point_select -- -4 2017-10-01 2017-10-01 2017-10-01 11:11:11.028 2017-10-01 11:11:11.018000 Beijing 10 1 1970-01-01 00:00:00 1970-01-01 00:00:00 1970-01-01 00:00:00.111 1970-01-01 00:00:00.000000 2020-01-05 00:00:00 1 34 20 +4 2017-10-01 2017-10-01 2017-10-01T11:11:11.028 2017-10-01T11:11:11.018 Beijing 10 1 1970-01-01T00:00 1970-01-01T00:00 1970-01-01T00:00:00.111 1970-01-01T00:00 2020-01-05T00:00 1 34 20 -- !point_select -- -1 2017-10-01 2017-10-01 2017-10-01 11:11:11.021 2017-10-01 11:11:11.011000 Beijing 10 1 2020-01-01 00:00:00 2020-01-01 00:00:00 2017-10-01 11:11:11.170 2017-10-01 11:11:11.110111 2020-01-01 00:00:00 1 30 20 +1 2017-10-01 2017-10-01 2017-10-01T11:11:11.021 2017-10-01T11:11:11.011 Beijing 10 1 2020-01-01T00:00 2020-01-01T00:00 2017-10-01T11:11:11.170 2017-10-01T11:11:11.110111 2020-01-01T00:00 1 30 20 -- !point_select -- -1 2017-10-01 2017-10-01 2017-10-01 11:11:11.022 2017-10-01 11:11:11.012000 Beijing 10 1 2020-01-02 00:00:00 2020-01-02 00:00:00 2017-10-01 11:11:11.160 2017-10-01 11:11:11.100111 2020-01-02 00:00:00 1 31 19 +1 2017-10-01 2017-10-01 2017-10-01T11:11:11.022 2017-10-01T11:11:11.012 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2017-10-01T11:11:11.160 2017-10-01T11:11:11.100111 2020-01-02T00:00 1 31 19 -- !point_select -- -2 2017-10-01 2017-10-01 2017-10-01 11:11:11.023 2017-10-01 11:11:11.013000 Beijing 10 1 2020-01-02 00:00:00 2020-01-02 00:00:00 2017-10-01 11:11:11.150 2017-10-01 11:11:11.130111 2020-01-02 00:00:00 1 31 21 +2 2017-10-01 2017-10-01 2017-10-01T11:11:11.023 2017-10-01T11:11:11.013 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2017-10-01T11:11:11.150 2017-10-01T11:11:11.130111 2020-01-02T00:00 1 31 21 -- !point_select -- -2 2017-10-01 2017-10-01 2017-10-01 11:11:11.024 2017-10-01 11:11:11.014000 Beijing 10 1 2020-01-03 00:00:00 2020-01-03 00:00:00 2017-10-01 11:11:11.140 2017-10-01 11:11:11.120111 2020-01-03 00:00:00 1 32 20 +2 2017-10-01 2017-10-01 2017-10-01T11:11:11.024 2017-10-01T11:11:11.014 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 2017-10-01T11:11:11.140 2017-10-01T11:11:11.120111 2020-01-03T00:00 1 32 20 -- !point_select -- -3 2017-10-01 2017-10-01 2017-10-01 11:11:11.025 2017-10-01 11:11:11.015000 Beijing 10 1 2020-01-03 00:00:00 2020-01-03 00:00:00 2017-10-01 11:11:11.100 2017-10-01 11:11:11.140111 2020-01-03 00:00:00 1 32 22 +3 2017-10-01 2017-10-01 2017-10-01T11:11:11.025 2017-10-01T11:11:11.015 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 2017-10-01T11:11:11.100 2017-10-01T11:11:11.140111 2020-01-03T00:00 1 32 22 -- !point_select -- -3 2017-10-01 2017-10-01 2017-10-01 11:11:11.026 2017-10-01 11:11:11.016000 Beijing 10 1 2020-01-04 00:00:00 2020-01-04 00:00:00 2017-10-01 11:11:11.110 2017-10-01 11:11:11.150111 2020-01-04 00:00:00 1 33 21 +3 2017-10-01 2017-10-01 2017-10-01T11:11:11.026 2017-10-01T11:11:11.016 Beijing 10 1 2020-01-04T00:00 2020-01-04T00:00 2017-10-01T11:11:11.110 2017-10-01T11:11:11.150111 2020-01-04T00:00 1 33 21 -- !point_select -- -3 2017-10-01 2017-10-01 2017-10-01 11:11:11.027 2017-10-01 11:11:11.017000 Beijing 10 1 1970-01-01 00:00:00 1970-01-01 00:00:00 1970-01-01 00:00:00.111 1970-01-01 00:00:00.000000 2020-01-05 00:00:00 1 34 20 +3 2017-10-01 2017-10-01 2017-10-01T11:11:11.027 2017-10-01T11:11:11.017 Beijing 10 1 1970-01-01T00:00 1970-01-01T00:00 1970-01-01T00:00:00.111 1970-01-01T00:00 2020-01-05T00:00 1 34 20 -- !point_select -- -4 2017-10-01 2017-10-01 2017-10-01 11:11:11.028 2017-10-01 11:11:11.018000 Beijing 10 1 1970-01-01 00:00:00 1970-01-01 00:00:00 1970-01-01 00:00:00.111 1970-01-01 00:00:00.000000 2020-01-05 00:00:00 1 34 20 +4 2017-10-01 2017-10-01 2017-10-01T11:11:11.028 2017-10-01T11:11:11.018 Beijing 10 1 1970-01-01T00:00 1970-01-01T00:00 1970-01-01T00:00:00.111 1970-01-01T00:00 2020-01-05T00:00 1 34 20 diff --git a/regression-test/data/datatype_p0/scalar_types/sql/infomation_schema.out b/regression-test/data/datatype_p0/scalar_types/sql/infomation_schema.out index a5c5eaad199..97e08bc652c 100644 --- a/regression-test/data/datatype_p0/scalar_types/sql/infomation_schema.out +++ b/regression-test/data/datatype_p0/scalar_types/sql/infomation_schema.out @@ -14,7 +14,7 @@ internal regression_test_datatype_p0_scalar_types tbl_scalar_types_dup c_decimal internal regression_test_datatype_p0_scalar_types tbl_scalar_types_dup c_date 12 \N YES date \N \N \N \N \N \N \N date \N \N \N \N internal regression_test_datatype_p0_scalar_types tbl_scalar_types_dup c_datetime 13 \N YES datetime \N \N \N \N \N \N \N datetime \N \N \N \N internal regression_test_datatype_p0_scalar_types tbl_scalar_types_dup c_datev2 14 \N YES date \N \N \N \N \N \N \N date \N \N \N \N -internal regression_test_datatype_p0_scalar_types tbl_scalar_types_dup c_datetimev2 15 \N YES datetime \N \N 18 0 \N \N \N datetime \N 0 \N \N +internal regression_test_datatype_p0_scalar_types tbl_scalar_types_dup c_datetimev2 15 \N YES datetime \N \N \N \N 0 \N \N datetime \N \N \N \N internal regression_test_datatype_p0_scalar_types tbl_scalar_types_dup c_char 16 \N YES char 15 60 \N \N \N \N \N char(15) 15 \N \N \N internal regression_test_datatype_p0_scalar_types tbl_scalar_types_dup c_varchar 17 \N YES varchar 100 400 \N \N \N \N \N varchar(100) 100 \N \N \N internal regression_test_datatype_p0_scalar_types tbl_scalar_types_dup c_string 18 \N YES varchar 2147483643 8589934572 \N \N \N \N \N string 2147483643 \N \N \N --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org