This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/doris.git
commit e3e5f18f26af29edd0ab7363789d33b60fbc9adf Author: lihangyu <15605149...@163.com> AuthorDate: Fri May 17 17:16:14 2024 +0800 [Fix](Json type) correct cast result for json type (#34764) --- be/src/util/jsonb_document.h | 2 +- be/src/vec/functions/function_cast.h | 79 +++++++++------------- be/test/vec/function/function_jsonb_test.cpp | 15 ++-- .../data/datatype_p0/json/json_cast.out | 43 ++++++++++++ .../data/json_p0/test_json_load_and_function.out | 52 +++++++------- .../test_json_load_unique_key_and_function.out | 12 ++-- .../data/jsonb_p0/test_jsonb_load_and_function.out | 52 +++++++------- .../test_jsonb_load_unique_key_and_function.out | 12 ++-- .../data/nereids_function_p0/scalar_function/J.out | 52 +++++++------- .../json_p0/test_json_load_and_function.out | 52 +++++++------- .../test_json_load_unique_key_and_function.out | 12 ++-- .../jsonb_p0/test_jsonb_load_and_function.out | 52 +++++++------- .../test_jsonb_load_unique_key_and_function.out | 12 ++-- regression-test/data/variant_p0/load.out | 17 ++++- .../suites/datatype_p0/json/json_cast.groovy | 34 ++++++++++ regression-test/suites/variant_p0/load.groovy | 25 +++++-- 16 files changed, 307 insertions(+), 216 deletions(-) diff --git a/be/src/util/jsonb_document.h b/be/src/util/jsonb_document.h index 73b6a44b38a..4d71cc9f6b3 100644 --- a/be/src/util/jsonb_document.h +++ b/be/src/util/jsonb_document.h @@ -498,7 +498,7 @@ public: bool isNull() const { return (type_ == JsonbType::T_Null); } bool isTrue() const { return (type_ == JsonbType::T_True); } bool isFalse() const { return (type_ == JsonbType::T_False); } - bool isInt() const { return isInt8() || isInt16() || isInt32() || isInt64(); } + bool isInt() const { return isInt8() || isInt16() || isInt32() || isInt64() || isInt128(); } bool isInt8() const { return (type_ == JsonbType::T_Int8); } bool isInt16() const { return (type_ == JsonbType::T_Int16); } bool isInt32() const { return (type_ == JsonbType::T_Int32); } diff --git a/be/src/vec/functions/function_cast.h b/be/src/vec/functions/function_cast.h index d19cb4d8574..7f3f74217b4 100644 --- a/be/src/vec/functions/function_cast.h +++ b/be/src/vec/functions/function_cast.h @@ -89,6 +89,7 @@ #include "vec/data_types/data_type_struct.h" #include "vec/data_types/data_type_time.h" #include "vec/data_types/data_type_time_v2.h" +#include "vec/data_types/serde/data_type_serde.h" #include "vec/functions/function.h" #include "vec/functions/function_convert_tz.h" #include "vec/functions/function_helpers.h" @@ -762,29 +763,43 @@ struct ConvertImplGenericToJsonb { auto column_string = ColumnString::create(); JsonbWriter writer; + ColumnUInt8::MutablePtr col_null_map_to = ColumnUInt8::create(col_from.size()); + ColumnUInt8::Container* vec_null_map_to = &col_null_map_to->get_data(); + DataTypeSerDe::FormatOptions format_options; + format_options.converted_from_string = true; + DataTypeSerDeSPtr from_serde = type.get_serde(); + DataTypeSerDeSPtr to_serde = data_type_to->get_serde(); + auto col_to = data_type_to->create_column(); + auto tmp_col = ColumnString::create(); + vectorized::DataTypeSerDe::FormatOptions options; for (size_t i = 0; i < input_rows_count; i++) { // convert to string tmp_col->clear(); VectorBufferWriter write_buffer(*tmp_col.get()); - type.to_string(col_from, i, write_buffer); + Status st = + from_serde->serialize_column_to_json(col_from, i, i + 1, write_buffer, options); + // if serialized failed, will return null + (*vec_null_map_to)[i] = !st.ok(); + if (!st.ok()) { + col_to->insert_default(); + continue; + } write_buffer.commit(); writer.reset(); auto str_ref = tmp_col->get_data_at(0); - ReadBuffer read_buffer((char*)(str_ref.data), str_ref.size); + Slice data((char*)(str_ref.data), str_ref.size); // first try to parse string - Status st = data_type_to->from_string(read_buffer, column_string.get()); + st = to_serde->deserialize_one_cell_from_json(*col_to, data, format_options); + // if parsing failed, will return null + (*vec_null_map_to)[i] = !st.ok(); if (!st.ok()) { - // write raw string to jsonb - writer.writeStartString(); - writer.writeString(str_ref.data, str_ref.size); - writer.writeEndString(); - column_string->insert_data(writer.getOutput()->getBuffer(), - writer.getOutput()->getSize()); + col_to->insert_default(); } } - block.replace_by_position(result, std::move(column_string)); + block.replace_by_position( + result, ColumnNullable::create(std::move(col_to), std::move(col_null_map_to))); return Status::OK(); } }; @@ -845,39 +860,13 @@ struct ConvertImplFromJsonb { null_map[i] = 1; res[i] = 0; } - } else if constexpr (type_index == TypeIndex::Int8) { - if (value->isInt8()) { - res[i] = (int8_t)((const JsonbIntVal*)value)->val(); - } else { - null_map[i] = 1; - res[i] = 0; - } - } else if constexpr (type_index == TypeIndex::Int16) { - if (value->isInt8() || value->isInt16()) { - res[i] = (int16_t)((const JsonbIntVal*)value)->val(); - } else { - null_map[i] = 1; - res[i] = 0; - } - } else if constexpr (type_index == TypeIndex::Int32) { - if (value->isInt8() || value->isInt16() || value->isInt32()) { - res[i] = (int32_t)((const JsonbIntVal*)value)->val(); - } else { - null_map[i] = 1; - res[i] = 0; - } - } else if constexpr (type_index == TypeIndex::Int64) { - if (value->isInt8() || value->isInt16() || value->isInt32() || - value->isInt64()) { - res[i] = (int64_t)((const JsonbIntVal*)value)->val(); - } else { - null_map[i] = 1; - res[i] = 0; - } - } else if constexpr (type_index == TypeIndex::Int128) { - if (value->isInt8() || value->isInt16() || value->isInt32() || - value->isInt64() || value->isInt128()) { - res[i] = (int128_t)((const JsonbIntVal*)value)->val(); + } else if constexpr (type_index == TypeIndex::Int8 || + type_index == TypeIndex::Int16 || + type_index == TypeIndex::Int32 || + type_index == TypeIndex::Int64 || + type_index == TypeIndex::Int128) { + if (value->isInt()) { + res[i] = ((const JsonbIntVal*)value)->val(); } else { null_map[i] = 1; res[i] = 0; @@ -885,8 +874,7 @@ struct ConvertImplFromJsonb { } else if constexpr (type_index == TypeIndex::Float64) { if (value->isDouble()) { res[i] = ((const JsonbDoubleVal*)value)->val(); - } else if (value->isInt8() || value->isInt16() || value->isInt32() || - value->isInt64()) { + } else if (value->isInt()) { res[i] = ((const JsonbIntVal*)value)->val(); } else { null_map[i] = 1; @@ -2082,7 +2070,6 @@ private: const auto& col_with_type_and_name = block.get_by_position(arguments[0]); auto& from_type = col_with_type_and_name.type; auto& col_from = col_with_type_and_name.column; - // set variant root column/type to from column/type auto variant = ColumnObject::create(true /*always nullable*/); variant->create_root(from_type, col_from->assume_mutable()); diff --git a/be/test/vec/function/function_jsonb_test.cpp b/be/test/vec/function/function_jsonb_test.cpp index d1c7ef121a4..91206028ef1 100644 --- a/be/test/vec/function/function_jsonb_test.cpp +++ b/be/test/vec/function/function_jsonb_test.cpp @@ -1271,11 +1271,11 @@ TEST(FunctionJsonbTEST, JsonbCastToOtherTest) { {{STRING("null"), static_cast<int8_t>(TypeIndex::Int8)}, Null()}, {{STRING("true"), static_cast<int8_t>(TypeIndex::Int8)}, Null()}, {{STRING("false"), static_cast<int8_t>(TypeIndex::Int8)}, Null()}, - {{STRING("100"), static_cast<int8_t>(TypeIndex::Int8)}, TINYINT(100)}, //int8 - {{STRING("10000"), static_cast<int8_t>(TypeIndex::Int8)}, Null()}, // int16 - {{STRING("1000000000"), static_cast<int8_t>(TypeIndex::Int8)}, Null()}, // int32 + {{STRING("100"), static_cast<int8_t>(TypeIndex::Int8)}, TINYINT(100)}, //int8 + {{STRING("10000"), static_cast<int8_t>(TypeIndex::Int8)}, TINYINT(16)}, // int16 + {{STRING("1000000000"), static_cast<int8_t>(TypeIndex::Int8)}, TINYINT(0)}, // int32 {{STRING("1152921504606846976"), static_cast<int8_t>(TypeIndex::Int8)}, - Null()}, // int64 + TINYINT(0)}, // int64 {{STRING("6.18"), static_cast<int8_t>(TypeIndex::Int8)}, Null()}, // double {{STRING(R"("abcd")"), static_cast<int8_t>(TypeIndex::Int8)}, Null()}, // string {{STRING("{}"), static_cast<int8_t>(TypeIndex::Int8)}, Null()}, // empty object @@ -1306,9 +1306,10 @@ TEST(FunctionJsonbTEST, JsonbCastToOtherTest) { {{STRING("false"), static_cast<int16_t>(TypeIndex::Int16)}, Null()}, {{STRING("100"), static_cast<int16_t>(TypeIndex::Int16)}, SMALLINT(100)}, //int8 {{STRING("10000"), static_cast<int16_t>(TypeIndex::Int16)}, SMALLINT(10000)}, // int16 - {{STRING("1000000000"), static_cast<int16_t>(TypeIndex::Int16)}, Null()}, // int32 + {{STRING("1000000000"), static_cast<int16_t>(TypeIndex::Int16)}, + SMALLINT(-13824)}, // int32 {{STRING("1152921504606846976"), static_cast<int16_t>(TypeIndex::Int16)}, - Null()}, // int64 + SMALLINT(0)}, // int64 {{STRING("6.18"), static_cast<int16_t>(TypeIndex::Int16)}, Null()}, // double {{STRING(R"("abcd")"), static_cast<int16_t>(TypeIndex::Int16)}, Null()}, // string {{STRING("{}"), static_cast<int16_t>(TypeIndex::Int16)}, Null()}, // empty object @@ -1342,7 +1343,7 @@ TEST(FunctionJsonbTEST, JsonbCastToOtherTest) { {{STRING("1000000000"), static_cast<int32_t>(TypeIndex::Int32)}, INT(1000000000)}, // int32 {{STRING("1152921504606846976"), static_cast<int32_t>(TypeIndex::Int32)}, - Null()}, // int64 + INT(0)}, // int64 {{STRING("6.18"), static_cast<int32_t>(TypeIndex::Int32)}, Null()}, // double {{STRING(R"("abcd")"), static_cast<int32_t>(TypeIndex::Int32)}, Null()}, // string {{STRING("{}"), static_cast<int32_t>(TypeIndex::Int32)}, Null()}, // empty object diff --git a/regression-test/data/datatype_p0/json/json_cast.out b/regression-test/data/datatype_p0/json/json_cast.out new file mode 100644 index 00000000000..73809a89024 --- /dev/null +++ b/regression-test/data/datatype_p0/json/json_cast.out @@ -0,0 +1,43 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql1 -- +10 + +-- !sql2 -- +23 + +-- !sql3 -- +-28649 + +-- !sql4 -- +-679213870 + +-- !sql5 -- +1234 + +-- !sql6 -- +-46 + +-- !sql7 -- +true + +-- !sql8 -- +\N + +-- !sql9 -- +1000.1111 + +-- !sql10 -- +\N + +-- !sql11 -- +["CXO0N: 1045901740","HMkTa: 1348450505","44 HHD: 915015173","j9WoJ: -1517316688"] + +-- !sql12 -- +111111 + +-- !sql13 -- +111111 + +-- !sql14 -- +1.1111 + diff --git a/regression-test/data/json_p0/test_json_load_and_function.out b/regression-test/data/json_p0/test_json_load_and_function.out index 40e195544ea..d8c03d5c3ef 100644 --- a/regression-test/data/json_p0/test_json_load_and_function.out +++ b/regression-test/data/json_p0/test_json_load_and_function.out @@ -5151,8 +5151,8 @@ 4 false \N 5 100 100 6 10000 10000 -7 1000000000 \N -8 1152921504606846976 \N +7 1000000000 -13824 +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -5166,9 +5166,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N -30 -9223372036854775808 \N -31 18446744073709551615 \N +29 12524337771678448270 -17778 +30 -9223372036854775808 0 +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -5178,7 +5178,7 @@ 5 100 100 6 10000 10000 7 1000000000 1000000000 -8 1152921504606846976 \N +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -5192,9 +5192,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N -30 -9223372036854775808 \N -31 18446744073709551615 \N +29 12524337771678448270 -1209615730 +30 -9223372036854775808 0 +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -5218,9 +5218,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N +29 12524337771678448270 -5922406302031103346 30 -9223372036854775808 -9223372036854775808 -31 18446744073709551615 \N +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -5244,9 +5244,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N +29 12524337771678448270 1.2524337771678448E19 30 -9223372036854775808 -9.223372036854776E18 -31 18446744073709551615 \N +31 18446744073709551615 1.8446744073709552E19 -- !select -- 1 \N \N @@ -5307,8 +5307,8 @@ 4 false \N 5 100 100 6 10000 10000 -7 1000000000 \N -8 1152921504606846976 \N +7 1000000000 -13824 +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -5322,9 +5322,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N -30 -9223372036854775808 \N -31 18446744073709551615 \N +29 12524337771678448270 -17778 +30 -9223372036854775808 0 +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -5334,7 +5334,7 @@ 5 100 100 6 10000 10000 7 1000000000 1000000000 -8 1152921504606846976 \N +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -5348,9 +5348,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N -30 -9223372036854775808 \N -31 18446744073709551615 \N +29 12524337771678448270 -1209615730 +30 -9223372036854775808 0 +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -5374,9 +5374,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N +29 12524337771678448270 -5922406302031103346 30 -9223372036854775808 -9223372036854775808 -31 18446744073709551615 \N +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -5400,9 +5400,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N +29 12524337771678448270 1.25243377716784e+19 30 -9223372036854775808 -9.22337203685478e+18 -31 18446744073709551615 \N +31 18446744073709551615 1.84467440737096e+19 -- !select -- 1 \N \N diff --git a/regression-test/data/json_p0/test_json_load_unique_key_and_function.out b/regression-test/data/json_p0/test_json_load_unique_key_and_function.out index f63959c2e02..3cb34304af4 100644 --- a/regression-test/data/json_p0/test_json_load_unique_key_and_function.out +++ b/regression-test/data/json_p0/test_json_load_unique_key_and_function.out @@ -3959,8 +3959,8 @@ 4 false \N 5 100 100 6 10000 10000 -7 1000000000 \N -8 1152921504606846976 \N +7 1000000000 -13824 +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -3983,7 +3983,7 @@ 5 100 100 6 10000 10000 7 1000000000 1000000000 -8 1152921504606846976 \N +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -4097,8 +4097,8 @@ 4 false \N 5 100 100 6 10000 10000 -7 1000000000 \N -8 1152921504606846976 \N +7 1000000000 -13824 +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -4121,7 +4121,7 @@ 5 100 100 6 10000 10000 7 1000000000 1000000000 -8 1152921504606846976 \N +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N diff --git a/regression-test/data/jsonb_p0/test_jsonb_load_and_function.out b/regression-test/data/jsonb_p0/test_jsonb_load_and_function.out index 8a3b573cebd..9e7eddb936e 100644 --- a/regression-test/data/jsonb_p0/test_jsonb_load_and_function.out +++ b/regression-test/data/jsonb_p0/test_jsonb_load_and_function.out @@ -6942,8 +6942,8 @@ 4 false \N 5 100 100 6 10000 10000 -7 1000000000 \N -8 1152921504606846976 \N +7 1000000000 -13824 +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -6957,9 +6957,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N -30 -9223372036854775808 \N -31 18446744073709551615 \N +29 12524337771678448270 -17778 +30 -9223372036854775808 0 +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -6969,7 +6969,7 @@ 5 100 100 6 10000 10000 7 1000000000 1000000000 -8 1152921504606846976 \N +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -6983,9 +6983,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N -30 -9223372036854775808 \N -31 18446744073709551615 \N +29 12524337771678448270 -1209615730 +30 -9223372036854775808 0 +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -7009,9 +7009,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N +29 12524337771678448270 -5922406302031103346 30 -9223372036854775808 -9223372036854775808 -31 18446744073709551615 \N +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -7035,9 +7035,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N +29 12524337771678448270 1.2524337771678448E19 30 -9223372036854775808 -9.223372036854776E18 -31 18446744073709551615 \N +31 18446744073709551615 1.8446744073709552E19 -- !select -- 1 \N \N @@ -7098,8 +7098,8 @@ 4 false \N 5 100 100 6 10000 10000 -7 1000000000 \N -8 1152921504606846976 \N +7 1000000000 -13824 +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -7113,9 +7113,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N -30 -9223372036854775808 \N -31 18446744073709551615 \N +29 12524337771678448270 -17778 +30 -9223372036854775808 0 +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -7125,7 +7125,7 @@ 5 100 100 6 10000 10000 7 1000000000 1000000000 -8 1152921504606846976 \N +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -7139,9 +7139,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N -30 -9223372036854775808 \N -31 18446744073709551615 \N +29 12524337771678448270 -1209615730 +30 -9223372036854775808 0 +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -7165,9 +7165,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N +29 12524337771678448270 -5922406302031103346 30 -9223372036854775808 -9223372036854775808 -31 18446744073709551615 \N +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -7191,9 +7191,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N +29 12524337771678448270 1.25243377716784e+19 30 -9223372036854775808 -9.22337203685478e+18 -31 18446744073709551615 \N +31 18446744073709551615 1.84467440737096e+19 -- !select -- 1 \N \N diff --git a/regression-test/data/jsonb_p0/test_jsonb_load_unique_key_and_function.out b/regression-test/data/jsonb_p0/test_jsonb_load_unique_key_and_function.out index eca0a75867a..a9bec25de09 100644 --- a/regression-test/data/jsonb_p0/test_jsonb_load_unique_key_and_function.out +++ b/regression-test/data/jsonb_p0/test_jsonb_load_unique_key_and_function.out @@ -5546,8 +5546,8 @@ 4 false \N 5 100 100 6 10000 10000 -7 1000000000 \N -8 1152921504606846976 \N +7 1000000000 -13824 +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -5570,7 +5570,7 @@ 5 100 100 6 10000 10000 7 1000000000 1000000000 -8 1152921504606846976 \N +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -5684,8 +5684,8 @@ 4 false \N 5 100 100 6 10000 10000 -7 1000000000 \N -8 1152921504606846976 \N +7 1000000000 -13824 +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -5708,7 +5708,7 @@ 5 100 100 6 10000 10000 7 1000000000 1000000000 -8 1152921504606846976 \N +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N diff --git a/regression-test/data/nereids_function_p0/scalar_function/J.out b/regression-test/data/nereids_function_p0/scalar_function/J.out index 5785ef0c483..25f57c51c16 100644 --- a/regression-test/data/nereids_function_p0/scalar_function/J.out +++ b/regression-test/data/nereids_function_p0/scalar_function/J.out @@ -6942,8 +6942,8 @@ 4 false \N 5 100 100 6 10000 10000 -7 1000000000 \N -8 1152921504606846976 \N +7 1000000000 -13824 +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -6957,9 +6957,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N -30 -9223372036854775808 \N -31 18446744073709551615 \N +29 12524337771678448270 -17778 +30 -9223372036854775808 0 +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -6969,7 +6969,7 @@ 5 100 100 6 10000 10000 7 1000000000 1000000000 -8 1152921504606846976 \N +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -6983,9 +6983,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N -30 -9223372036854775808 \N -31 18446744073709551615 \N +29 12524337771678448270 -1209615730 +30 -9223372036854775808 0 +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -7009,9 +7009,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N +29 12524337771678448270 -5922406302031103346 30 -9223372036854775808 -9223372036854775808 -31 18446744073709551615 \N +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -7035,9 +7035,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N +29 12524337771678448270 1.2524337771678448E19 30 -9223372036854775808 -9.223372036854776E18 -31 18446744073709551615 \N +31 18446744073709551615 1.8446744073709552E19 -- !select -- 1 \N \N @@ -7098,8 +7098,8 @@ 4 false \N 5 100 100 6 10000 10000 -7 1000000000 \N -8 1152921504606846976 \N +7 1000000000 -13824 +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -7113,9 +7113,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N -30 -9223372036854775808 \N -31 18446744073709551615 \N +29 12524337771678448270 -17778 +30 -9223372036854775808 0 +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -7125,7 +7125,7 @@ 5 100 100 6 10000 10000 7 1000000000 1000000000 -8 1152921504606846976 \N +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -7139,9 +7139,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N -30 -9223372036854775808 \N -31 18446744073709551615 \N +29 12524337771678448270 -1209615730 +30 -9223372036854775808 0 +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -7165,9 +7165,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N +29 12524337771678448270 -5922406302031103346 30 -9223372036854775808 -9223372036854775808 -31 18446744073709551615 \N +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -7191,9 +7191,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N +29 12524337771678448270 1.25243377716784e+19 30 -9223372036854775808 -9.22337203685478e+18 -31 18446744073709551615 \N +31 18446744073709551615 1.84467440737096e+19 -- !select -- 1 \N \N diff --git a/regression-test/data/nereids_p0/json_p0/test_json_load_and_function.out b/regression-test/data/nereids_p0/json_p0/test_json_load_and_function.out index e12fb79e380..b474baa8102 100644 --- a/regression-test/data/nereids_p0/json_p0/test_json_load_and_function.out +++ b/regression-test/data/nereids_p0/json_p0/test_json_load_and_function.out @@ -5148,8 +5148,8 @@ 4 false \N 5 100 100 6 10000 10000 -7 1000000000 \N -8 1152921504606846976 \N +7 1000000000 -13824 +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -5163,9 +5163,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N -30 -9223372036854775808 \N -31 18446744073709551615 \N +29 12524337771678448270 -17778 +30 -9223372036854775808 0 +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -5175,7 +5175,7 @@ 5 100 100 6 10000 10000 7 1000000000 1000000000 -8 1152921504606846976 \N +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -5189,9 +5189,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N -30 -9223372036854775808 \N -31 18446744073709551615 \N +29 12524337771678448270 -1209615730 +30 -9223372036854775808 0 +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -5215,9 +5215,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N +29 12524337771678448270 -5922406302031103346 30 -9223372036854775808 -9223372036854775808 -31 18446744073709551615 \N +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -5241,9 +5241,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N +29 12524337771678448270 1.2524337771678448E19 30 -9223372036854775808 -9.223372036854776E18 -31 18446744073709551615 \N +31 18446744073709551615 1.8446744073709552E19 -- !select -- 1 \N \N @@ -5304,8 +5304,8 @@ 4 false \N 5 100 100 6 10000 10000 -7 1000000000 \N -8 1152921504606846976 \N +7 1000000000 -13824 +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -5319,9 +5319,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N -30 -9223372036854775808 \N -31 18446744073709551615 \N +29 12524337771678448270 -17778 +30 -9223372036854775808 0 +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -5331,7 +5331,7 @@ 5 100 100 6 10000 10000 7 1000000000 1000000000 -8 1152921504606846976 \N +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -5345,9 +5345,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N -30 -9223372036854775808 \N -31 18446744073709551615 \N +29 12524337771678448270 -1209615730 +30 -9223372036854775808 0 +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -5371,9 +5371,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N +29 12524337771678448270 -5922406302031103346 30 -9223372036854775808 -9223372036854775808 -31 18446744073709551615 \N +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -5397,9 +5397,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N +29 12524337771678448270 1.25243377716784e+19 30 -9223372036854775808 -9.22337203685478e+18 -31 18446744073709551615 \N +31 18446744073709551615 1.84467440737096e+19 -- !select -- 1 \N \N diff --git a/regression-test/data/nereids_p0/json_p0/test_json_load_unique_key_and_function.out b/regression-test/data/nereids_p0/json_p0/test_json_load_unique_key_and_function.out index f63959c2e02..3cb34304af4 100644 --- a/regression-test/data/nereids_p0/json_p0/test_json_load_unique_key_and_function.out +++ b/regression-test/data/nereids_p0/json_p0/test_json_load_unique_key_and_function.out @@ -3959,8 +3959,8 @@ 4 false \N 5 100 100 6 10000 10000 -7 1000000000 \N -8 1152921504606846976 \N +7 1000000000 -13824 +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -3983,7 +3983,7 @@ 5 100 100 6 10000 10000 7 1000000000 1000000000 -8 1152921504606846976 \N +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -4097,8 +4097,8 @@ 4 false \N 5 100 100 6 10000 10000 -7 1000000000 \N -8 1152921504606846976 \N +7 1000000000 -13824 +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -4121,7 +4121,7 @@ 5 100 100 6 10000 10000 7 1000000000 1000000000 -8 1152921504606846976 \N +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N diff --git a/regression-test/data/nereids_p0/jsonb_p0/test_jsonb_load_and_function.out b/regression-test/data/nereids_p0/jsonb_p0/test_jsonb_load_and_function.out index 8e77ddb11c1..442e68cadb2 100644 --- a/regression-test/data/nereids_p0/jsonb_p0/test_jsonb_load_and_function.out +++ b/regression-test/data/nereids_p0/jsonb_p0/test_jsonb_load_and_function.out @@ -6942,8 +6942,8 @@ 4 false \N 5 100 100 6 10000 10000 -7 1000000000 \N -8 1152921504606846976 \N +7 1000000000 -13824 +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -6957,9 +6957,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N -30 -9223372036854775808 \N -31 18446744073709551615 \N +29 12524337771678448270 -17778 +30 -9223372036854775808 0 +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -6969,7 +6969,7 @@ 5 100 100 6 10000 10000 7 1000000000 1000000000 -8 1152921504606846976 \N +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -6983,9 +6983,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N -30 -9223372036854775808 \N -31 18446744073709551615 \N +29 12524337771678448270 -1209615730 +30 -9223372036854775808 0 +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -7009,9 +7009,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N +29 12524337771678448270 -5922406302031103346 30 -9223372036854775808 -9223372036854775808 -31 18446744073709551615 \N +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -7035,9 +7035,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N +29 12524337771678448270 1.2524337771678448E19 30 -9223372036854775808 -9.223372036854776E18 -31 18446744073709551615 \N +31 18446744073709551615 1.8446744073709552E19 -- !select -- 1 \N \N @@ -7098,8 +7098,8 @@ 4 false \N 5 100 100 6 10000 10000 -7 1000000000 \N -8 1152921504606846976 \N +7 1000000000 -13824 +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -7113,9 +7113,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N -30 -9223372036854775808 \N -31 18446744073709551615 \N +29 12524337771678448270 -17778 +30 -9223372036854775808 0 +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -7125,7 +7125,7 @@ 5 100 100 6 10000 10000 7 1000000000 1000000000 -8 1152921504606846976 \N +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -7139,9 +7139,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N -30 -9223372036854775808 \N -31 18446744073709551615 \N +29 12524337771678448270 -1209615730 +30 -9223372036854775808 0 +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -7165,9 +7165,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N +29 12524337771678448270 -5922406302031103346 30 -9223372036854775808 -9223372036854775808 -31 18446744073709551615 \N +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -7191,9 +7191,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N +29 12524337771678448270 1.25243377716784e+19 30 -9223372036854775808 -9.22337203685478e+18 -31 18446744073709551615 \N +31 18446744073709551615 1.84467440737096e+19 -- !select -- 1 \N \N diff --git a/regression-test/data/nereids_p0/jsonb_p0/test_jsonb_load_unique_key_and_function.out b/regression-test/data/nereids_p0/jsonb_p0/test_jsonb_load_unique_key_and_function.out index 632badf32e1..f69d695d455 100644 --- a/regression-test/data/nereids_p0/jsonb_p0/test_jsonb_load_unique_key_and_function.out +++ b/regression-test/data/nereids_p0/jsonb_p0/test_jsonb_load_unique_key_and_function.out @@ -5546,8 +5546,8 @@ 4 false \N 5 100 100 6 10000 10000 -7 1000000000 \N -8 1152921504606846976 \N +7 1000000000 -13824 +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -5570,7 +5570,7 @@ 5 100 100 6 10000 10000 7 1000000000 1000000000 -8 1152921504606846976 \N +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -5684,8 +5684,8 @@ 4 false \N 5 100 100 6 10000 10000 -7 1000000000 \N -8 1152921504606846976 \N +7 1000000000 -13824 +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -5708,7 +5708,7 @@ 5 100 100 6 10000 10000 7 1000000000 1000000000 -8 1152921504606846976 \N +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N diff --git a/regression-test/data/variant_p0/load.out b/regression-test/data/variant_p0/load.out index cc1a74c7041..d8f145d5fc4 100644 --- a/regression-test/data/variant_p0/load.out +++ b/regression-test/data/variant_p0/load.out @@ -212,11 +212,14 @@ [123] -- !sql_25 -- -50000 55000.00000000545 6150000 +50000 54999.9999999998 6150000 -- !sql_26 -- 5000 +-- !sql_27 -- +16 + -- !sql_29_1 -- 1 {"kxxxx":123} {"xxxxyyyy":123} 1 {"kyyyy":"123"} {"kxkxkxkx":[123]} @@ -322,3 +325,15 @@ -- !sql_31 -- kaana +-- !sql_39 -- +1 array +2 string +3 bigint +4 double + +-- !sql_39 -- +["CXO0N: 1045901740", "HMkTa: 1348450505", "44 HHD: 915015173", "j9WoJ: -1517316688"] + +-- !sql_39 -- +[1] + diff --git a/regression-test/suites/datatype_p0/json/json_cast.groovy b/regression-test/suites/datatype_p0/json/json_cast.groovy new file mode 100644 index 00000000000..7646b4f7ccb --- /dev/null +++ b/regression-test/suites/datatype_p0/json/json_cast.groovy @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_json_type_cast", "p0") { + qt_sql1 "SELECT CAST(CAST(10 AS JSON) as INT)" + qt_sql2 "SELECT CAST(CAST(102423 AS JSON) as TINYINT)" + qt_sql3 "SELECT CAST(CAST(102423 AS JSON) as SMALLINT)" + qt_sql4 "SELECT CAST(CAST(102400001234 AS JSON) as INT)" + qt_sql5 "SELECT CAST(CAST(102400001234 AS JSON) as SMALLINT)" + qt_sql6 "SELECT CAST(CAST(102400001234 AS JSON) as TINYINT)" + qt_sql7 "SELECT CAST(CAST(102400001234 AS JSON) as BOOLEAN)" + qt_sql8 "SELECT CAST(CAST(1000.1111 AS JSON) as INT)" + qt_sql9 "SELECT CAST(CAST(1000.1111 AS JSON) as DOUBLE)" + qt_sql10 "SELECT CAST(CAST(1000.1111 AS JSON) as BOOLEAN)" + + qt_sql11 """select cast('["CXO0N: 1045901740", "HMkTa: 1348450505", "44 HHD: 915015173", "j9WoJ: -1517316688"]' as json);""" + qt_sql12 """select cast("111111" as json)""" + qt_sql13 """select cast(111111 as json)""" + qt_sql14 """select cast(1.1111 as json)""" +} \ No newline at end of file diff --git a/regression-test/suites/variant_p0/load.groovy b/regression-test/suites/variant_p0/load.groovy index b85dd686f84..77a975687ee 100644 --- a/regression-test/suites/variant_p0/load.groovy +++ b/regression-test/suites/variant_p0/load.groovy @@ -206,13 +206,14 @@ suite("regression_test_variant", "nonConcurrent"){ load_json_data.call(table_name, """${getS3Url() + '/load/ghdata_sample.json'}""") qt_sql_26 "select count() from ${table_name}" - // FIXME: this case it not passed - // // 8. json empty string - // // table_name = "empty_string" - // // create_table table_name - // // sql """INSERT INTO empty_string VALUES (1, ''), (2, '{"k1": 1, "k2": "v1"}'), (3, '{}'), (4, '{"k1": 2}');""" - // // sql """INSERT INTO empty_string VALUES (3, null), (4, '{"k1": 1, "k2": "v1"}'), (3, '{}'), (4, '{"k1": 2}');""" - // // qt_sql_27 "SELECT * FROM ${table_name} ORDER BY k;" + // 8. json empty string + table_name = "empty_string" + create_table table_name + sql """INSERT INTO empty_string VALUES (1, ''), (2, '{"k1": 1, "k2": "v1"}'), (3, '{}'), (4, '{"k1": 2}');""" + sql """INSERT INTO empty_string VALUES (3, null), (4, '{"k1": 1, "k2": "v1"}'), (3, '{}'), (4, '{"k1": 2}');""" + sql """INSERT INTO empty_string VALUES (3, null), (4, null), (3, '{}'), (4, '{"k1": 2}');""" + sql """INSERT INTO empty_string VALUES (3, ''), (4, null), (3, '{}'), (4, null);""" + qt_sql_27 "SELECT count() FROM ${table_name};" // // // 9. btc data // // table_name = "btcdata" @@ -393,6 +394,16 @@ suite("regression_test_variant", "nonConcurrent"){ qt_sql_31 """select cast(v['xxxx'] as string) from sparse_columns where cast(v['xxxx'] as string) != 'null' order by k limit 1;""" sql "truncate table sparse_columns" set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "0.95") + + // test cast + table_name = "variant_cast" + create_table.call(table_name, "DUPLICATE", "1") + sql """ + insert into variant_cast values(1,'["CXO0N: 1045901740", "HMkTa: 1348450505", "44 HHD: 915015173", "j9WoJ: -1517316688"]'),(2,'"[1]"'),(3,'123456'),(4,'1.11111') + """ + qt_sql_39 "select k, json_type(cast(v as json), '\$') from variant_cast order by k" + qt_sql_39 "select cast(v as array<text>) from variant_cast where k = 1 order by k" + qt_sql_39 "select cast(v as string) from variant_cast where k = 2 order by k" } finally { // reset flags set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "0.95") --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org