This is an automated email from the ASF dual-hosted git repository.
panxiaolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new dc470875603 [fix](function) fix str_to_date default return type scale
for nereids (#24932)
dc470875603 is described below
commit dc4708756036a48bda1f0c2159f64a50de8eb1e2
Author: zclllyybb <zhaochan...@selectdb.com>
AuthorDate: Fri Oct 20 12:55:49 2023 +0800
[fix](function) fix str_to_date default return type scale for nereids
(#24932)
fix str_to_date default return type scale for nereids
---
be/src/vec/functions/function_timestamp.cpp | 3 +++
be/src/vec/runtime/vdatetime_value.cpp | 24 +++++++++++++++++-----
.../expressions/functions/scalar/StrToDate.java | 10 ++++++---
gensrc/script/doris_builtins_functions.py | 4 ++--
.../datetime_functions/test_date_function.out | 5 ++++-
.../datetime_functions/test_date_function.out | 3 +++
.../datetime_functions/test_date_function.groovy | 1 +
.../datetime_functions/test_date_function.groovy | 1 +
8 files changed, 40 insertions(+), 11 deletions(-)
diff --git a/be/src/vec/functions/function_timestamp.cpp
b/be/src/vec/functions/function_timestamp.cpp
index 7d36c7c5040..927d0d81035 100644
--- a/be/src/vec/functions/function_timestamp.cpp
+++ b/be/src/vec/functions/function_timestamp.cpp
@@ -72,6 +72,7 @@ struct StrToDate {
static DataTypes get_variadic_argument_types() { return {}; }
static DataTypePtr get_return_type_impl(const DataTypes& arguments) {
+ //TODO: it doesn't matter now. maybe sometime we should find the
function signature with return_type together
return make_nullable(std::make_shared<DataTypeDateTime>());
}
@@ -115,6 +116,8 @@ struct StrToDate {
auto& rdata = specific_char_column->get_chars();
auto& roffsets = specific_char_column->get_offsets();
+ // Because of we cant distinguish by return_type when we find
function. so the return_type may NOT be same with real return type
+ // which decided by FE. that's found by which.
ColumnPtr res = nullptr;
WhichDataType
which(remove_nullable(block.get_by_position(result).type));
if (which.is_date_time_v2()) {
diff --git a/be/src/vec/runtime/vdatetime_value.cpp
b/be/src/vec/runtime/vdatetime_value.cpp
index 44e2cc0a69e..b7f30d6e024 100644
--- a/be/src/vec/runtime/vdatetime_value.cpp
+++ b/be/src/vec/runtime/vdatetime_value.cpp
@@ -2300,14 +2300,28 @@ bool DateV2Value<T>::from_date_format_str(const char*
format, int format_len, co
break;
// Micro second
case 'f':
- tmp = val + min(6, val_end - val);
- if (!str_to_int64(val, &tmp, &int_value)) {
- return false;
+ tmp = val;
+ // when there's still something to the end, fix the scale of
ms.
+ while (tmp < val_end && isdigit(*tmp)) {
+ tmp++;
+ }
+
+ if (tmp - val > 6) {
+ const char* tmp2 = val + 6;
+ if (!str_to_int64(val, &tmp2, &int_value)) {
+ return false;
+ }
+ } else {
+ if (!str_to_int64(val, &tmp, &int_value)) {
+ return false;
+ }
+ }
+ if constexpr (is_datetime) {
+ microsecond = int_value * int_exp10(6 - min(6, tmp - val));
+ frac_part_used = true;
}
- microsecond = int_value * int_exp10(6 - min(6, val_end - val));
val = tmp;
time_part_used = true;
- frac_part_used = true;
break;
// AM/PM
case 'p':
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrToDate.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrToDate.java
index eb17439df8e..4f512ec09fe 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrToDate.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StrToDate.java
@@ -28,7 +28,7 @@ import
org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral;
import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
import org.apache.doris.nereids.types.DataType;
-import org.apache.doris.nereids.types.DateTimeType;
+import org.apache.doris.nereids.types.DateTimeV2Type;
import org.apache.doris.nereids.types.StringType;
import org.apache.doris.nereids.types.VarcharType;
@@ -44,8 +44,9 @@ public class StrToDate extends ScalarFunction
implements BinaryExpression, ExplicitlyCastableSignature,
AlwaysNullable {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
-
FunctionSignature.ret(DateTimeType.INSTANCE).args(VarcharType.SYSTEM_DEFAULT,
VarcharType.SYSTEM_DEFAULT),
-
FunctionSignature.ret(DateTimeType.INSTANCE).args(StringType.INSTANCE,
StringType.INSTANCE)
+
FunctionSignature.ret(DateTimeV2Type.SYSTEM_DEFAULT).args(VarcharType.SYSTEM_DEFAULT,
+ VarcharType.SYSTEM_DEFAULT),
+
FunctionSignature.ret(DateTimeV2Type.SYSTEM_DEFAULT).args(StringType.INSTANCE,
StringType.INSTANCE)
);
/**
@@ -86,6 +87,9 @@ public class StrToDate extends ScalarFunction
if (getArgument(1) instanceof StringLikeLiteral) {
if (DateLiteral.hasTimePart(((StringLikeLiteral)
getArgument(1)).getStringValue())) {
returnType =
DataType.fromCatalogType(ScalarType.getDefaultDateType(Type.DATETIME));
+ if (returnType.isDateTimeV2Type()) {
+ returnType =
DataType.fromCatalogType(Type.DATETIMEV2_WITH_MAX_SCALAR);
+ }
} else {
returnType =
DataType.fromCatalogType(ScalarType.getDefaultDateType(Type.DATE));
}
diff --git a/gensrc/script/doris_builtins_functions.py
b/gensrc/script/doris_builtins_functions.py
index bdddde58036..d8bab55de02 100644
--- a/gensrc/script/doris_builtins_functions.py
+++ b/gensrc/script/doris_builtins_functions.py
@@ -939,8 +939,6 @@ visible_functions = {
[['datediff'], 'INT', ['DATETIME', 'DATETIME'], 'ALWAYS_NULLABLE'],
[['timediff'], 'TIME', ['DATETIME', 'DATETIME'], 'ALWAYS_NULLABLE'],
- [['str_to_date'], 'DATETIME', ['VARCHAR', 'VARCHAR'],
'ALWAYS_NULLABLE'],
- [['str_to_date'], 'DATETIME', ['STRING', 'STRING'], 'ALWAYS_NULLABLE'],
[['date_format'], 'VARCHAR', ['DATETIME', 'VARCHAR'],
'ALWAYS_NULLABLE'],
[['date_format'], 'VARCHAR', ['DATE', 'VARCHAR'], 'ALWAYS_NULLABLE'],
[['date', 'to_date'], 'DATE', ['DATETIME'], 'ALWAYS_NULLABLE'],
@@ -1104,6 +1102,8 @@ visible_functions = {
[['datediff'], 'INT', ['DATEV2', 'DATEV2'], ''],
[['timediff'], 'TIMEV2', ['DATEV2', 'DATEV2'], ''],
+ [['str_to_date'], 'DATETIMEV2', ['VARCHAR', 'VARCHAR'],
'ALWAYS_NULLABLE'],
+ [['str_to_date'], 'DATETIMEV2', ['STRING', 'STRING'],
'ALWAYS_NULLABLE'],
[['date_format'], 'VARCHAR', ['DATETIMEV2', 'VARCHAR'],
'ALWAYS_NULLABLE'],
[['date_format'], 'VARCHAR', ['DATEV2', 'VARCHAR'], 'ALWAYS_NULLABLE'],
[['date', 'to_date', 'datev2', 'to_datev2'], 'DATEV2', ['DATETIMEV2'],
'ALWAYS_NULLABLE'],
diff --git
a/regression-test/data/nereids_p0/sql_functions/datetime_functions/test_date_function.out
b/regression-test/data/nereids_p0/sql_functions/datetime_functions/test_date_function.out
index 43eff1b302c..34c466cb71b 100644
---
a/regression-test/data/nereids_p0/sql_functions/datetime_functions/test_date_function.out
+++
b/regression-test/data/nereids_p0/sql_functions/datetime_functions/test_date_function.out
@@ -235,7 +235,10 @@ February
2014-12-21T12:34:56
-- !sql --
-2014-12-21T12:34:56
+2014-12-21T12:34:56.789
+
+-- !sql --
+2023-07-05T02:09:55.880
-- !sql --
2004-10-18
diff --git
a/regression-test/data/query_p0/sql_functions/datetime_functions/test_date_function.out
b/regression-test/data/query_p0/sql_functions/datetime_functions/test_date_function.out
index 6e9ca29f3f9..5c61bfc31d2 100644
---
a/regression-test/data/query_p0/sql_functions/datetime_functions/test_date_function.out
+++
b/regression-test/data/query_p0/sql_functions/datetime_functions/test_date_function.out
@@ -275,6 +275,9 @@ February
-- !sql --
2014-12-21T12:34:56
+-- !sql --
+2023-07-05T02:09:55
+
-- !sql --
2004-10-18
diff --git
a/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function.groovy
b/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function.groovy
index 073c5dc725d..13add305b65 100644
---
a/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function.groovy
+++
b/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function.groovy
@@ -304,6 +304,7 @@ suite("test_date_function") {
qt_sql """ select str_to_date(test_datetime, '%Y-%m-%d %H:%i:%s') from
${tableName}; """
qt_sql """ select str_to_date("2014-12-21 12:34%3A56", '%Y-%m-%d
%H:%i%%3A%s'); """
qt_sql """ select str_to_date("2014-12-21 12:34:56.789 PM", '%Y-%m-%d
%h:%i:%s.%f %p'); """
+ qt_sql """ select
str_to_date('2023-07-05T02:09:55.880Z','%Y-%m-%dT%H:%i:%s.%fZ') """
qt_sql """ select str_to_date('200442 Monday', '%X%V %W') """
sql """ truncate table ${tableName} """
sql """ insert into ${tableName} values ("2020-09-01") """
diff --git
a/regression-test/suites/query_p0/sql_functions/datetime_functions/test_date_function.groovy
b/regression-test/suites/query_p0/sql_functions/datetime_functions/test_date_function.groovy
index ae031a97fa9..2dc1834c717 100644
---
a/regression-test/suites/query_p0/sql_functions/datetime_functions/test_date_function.groovy
+++
b/regression-test/suites/query_p0/sql_functions/datetime_functions/test_date_function.groovy
@@ -333,6 +333,7 @@ suite("test_date_function") {
qt_sql """ select str_to_date(test_datetime, '%Y-%m-%d %H:%i:%s') from
${tableName}; """
qt_sql """ select str_to_date("2014-12-21 12:34%3A56", '%Y-%m-%d
%H:%i%%3A%s'); """
qt_sql """ select str_to_date("2014-12-21 12:34:56.789 PM", '%Y-%m-%d
%h:%i:%s.%f %p'); """
+ qt_sql """ select
str_to_date('2023-07-05T02:09:55.880Z','%Y-%m-%dT%H:%i:%s.%fZ') """
qt_sql """ select str_to_date('200442 Monday', '%X%V %W') """
sql """ truncate table ${tableName} """
sql """ insert into ${tableName} values ("2020-09-01") """
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
