This is an automated email from the ASF dual-hosted git repository.
Mryange pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 5c147002568 [fix](expr) Fix datetime array range tail omission (#68566)
5c147002568 is described below
commit 5c1470025689742045b083c66e652a88ee874fe5
Author: Mryange <[email protected]>
AuthorDate: Tue Sep 29 18:36:14 2026 +0800
[fix](expr) Fix datetime array range tail omission (#68566)
Date-based array ranges skipped a final value when the remaining
interval was less than one complete unit. The loop used datetime_diff to
decide whether
to continue, so valid values before the exclusive end were omitted. This
change uses
exact range comparison and safely stops when calculating the successor
exceeds the
representable date range.
### Release note
None
### Check List (For Author)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No code files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
.../exprs/function/array/function_array_range.cpp | 29 ++++++++++-------
.../timestamp_ns/test_timestamp_ns_functions.out | 4 +--
...est_timestamp_ns_mixed_datetime_expressions.out | 2 +-
.../nereids_function_p0/scalar_function/Array.out | 2 +-
.../test_array_range_datetime_tail.out | 33 ++++++++++++++++++++
.../array_functions/test_array_functions.out | 14 ++++-----
.../test_array_range_datetime_tail.groovy | 36 ++++++++++++++++++++++
7 files changed, 98 insertions(+), 22 deletions(-)
diff --git a/be/src/exprs/function/array/function_array_range.cpp
b/be/src/exprs/function/array/function_array_range.cpp
index bc5c24948b2..ab8bea14302 100644
--- a/be/src/exprs/function/array/function_array_range.cpp
+++ b/be/src/exprs/function/array/function_array_range.cpp
@@ -228,8 +228,9 @@ private:
std::integral_constant<TimeUnit, TimeUnit::DAY>,
TimeUnitOrVoid>;
int move = 0;
- while (doris::datetime_diff<UNIT::value>(idx, end_row) >
0) {
- if (move > max_array_size_as_field) {
+ // A value below end belongs to the range even if less
than one unit remains.
+ while (idx < end_row) {
+ if (move >= max_array_size_as_field) {
return Status::InvalidArgument("Array size exceeds
the limit {}",
max_array_size_as_field);
}
@@ -237,17 +238,23 @@ private:
dest_nested_null_map.push_back(0);
offset++;
move++;
- if constexpr (SourceDataPType == TYPE_TIMESTAMP_NS) {
- auto next = idx;
- if (!next.template date_add_interval<UNIT::value>(
- TimeInterval(UNIT::value, step_row,
false))) {
- break;
- }
- idx = next;
+ auto next = idx;
+ bool advanced;
+ if constexpr (SourceDataPType == TYPE_DATETIMEV2 &&
+ (UNIT::value == TimeUnit::DAY ||
+ UNIT::value == TimeUnit::WEEK)) {
+ const Int64 days = static_cast<Int64>(step_row) *
+ (UNIT::value == TimeUnit::WEEK
? 7 : 1);
+ advanced = next.template
date_add_days<false>(days);
} else {
- idx = doris::date_time_add<UNIT::value,
SourceDataPType, Int32>(
- idx, step_row);
+ advanced = next.template
date_add_interval<UNIT::value>(
+ TimeInterval(UNIT::value, step_row,
false));
}
+ // The successor can exceed the date range after
emitting a valid value.
+ if (!advanced) {
+ break;
+ }
+ idx = next;
}
dest_offsets.push_back(offset);
}
diff --git
a/regression-test/data/datatype_p0/timestamp_ns/test_timestamp_ns_functions.out
b/regression-test/data/datatype_p0/timestamp_ns/test_timestamp_ns_functions.out
index 383e08f8911..b9a2a1c28b0 100644
---
a/regression-test/data/datatype_p0/timestamp_ns/test_timestamp_ns_functions.out
+++
b/regression-test/data/datatype_p0/timestamp_ns/test_timestamp_ns_functions.out
@@ -171,7 +171,7 @@ false 2
["1969-12-31 23:59:58.000000001", "1969-12-31 23:59:59.000000001", "1970-01-01
00:00:00.000000001", "1970-01-01 00:00:01.000000001"]
-- !sequence_month --
-["2024-01-31 12:34:56.123456789", "2024-02-29 12:34:56.123456789", "2024-03-29
12:34:56.123456789"]
+["2024-01-31 12:34:56.123456789", "2024-02-29 12:34:56.123456789", "2024-03-29
12:34:56.123456789", "2024-04-29 12:34:56.123456789"]
-- !sequence_lower_bound_overshoot --
["1677-09-21 00:12:43.145224192"]
@@ -180,7 +180,7 @@ false 2
["2262-04-11 23:47:14.854775807"]
-- !sequence_all_units --
-["2024-02-29 12:34:56.123456789", "2025-02-28 12:34:56.123456789"]
["2024-02-29 12:34:56.123456789", "2024-05-29 12:34:56.123456789", "2024-08-29
12:34:56.123456789"] ["2024-02-29 12:34:56.123456789", "2024-03-07
12:34:56.123456789", "2024-03-14 12:34:56.123456789"] ["2024-02-29
12:34:56.123456789", "2024-03-01 12:34:56.123456789", "2024-03-02
12:34:56.123456789"] ["2024-02-29 22:34:56.123456789", "2024-02-29
23:34:56.123456789", "2024-03-01 00:34:56.123456789", "2024-03-01 01:34:56.1234
[...]
+["2024-02-29 12:34:56.123456789", "2025-02-28 12:34:56.123456789", "2026-02-28
12:34:56.123456789"] ["2024-02-29 12:34:56.123456789", "2024-05-29
12:34:56.123456789", "2024-08-29 12:34:56.123456789", "2024-11-29
12:34:56.123456789"] ["2024-02-29 12:34:56.123456789", "2024-03-07
12:34:56.123456789", "2024-03-14 12:34:56.123456789", "2024-03-21
12:34:56.123456789"] ["2024-02-29 12:34:56.123456789", "2024-03-01
12:34:56.123456789", "2024-03-02 12:34:56.123456789"] ["2024-02-29
22:34:56.1234 [...]
-- !convert_tz --
2024-02-29 04:34:56.123456789
diff --git
a/regression-test/data/datatype_p0/timestamp_ns/test_timestamp_ns_mixed_datetime_expressions.out
b/regression-test/data/datatype_p0/timestamp_ns/test_timestamp_ns_mixed_datetime_expressions.out
index 896563171bb..7c28051fdbc 100644
---
a/regression-test/data/datatype_p0/timestamp_ns/test_timestamp_ns_mixed_datetime_expressions.out
+++
b/regression-test/data/datatype_p0/timestamp_ns/test_timestamp_ns_mixed_datetime_expressions.out
@@ -64,7 +64,7 @@ true true true true
2024-02-29 12:34:56.123456000 2024-02-29 12:34:56.123456000
-- !mixed_range_and_field --
-["2024-02-29 12:34:56.123456789", "2024-02-29 12:34:57.123456789"]
["2024-02-29 12:34:56.123456789", "2024-02-29 12:34:57.123456789"] 0
+["2024-02-29 12:34:56.123456789", "2024-02-29 12:34:57.123456789", "2024-02-29
12:34:58.123456789"] ["2024-02-29 12:34:56.123456789", "2024-02-29
12:34:57.123456789", "2024-02-29 12:34:58.123456789"] 0
-- !mixed_literals_fold --
true 1 2024-02-29 12:34:56.123456789 0 ["2024-02-29
12:34:56.123456789", "2024-02-29 12:34:56.123456000"]
diff --git a/regression-test/data/nereids_function_p0/scalar_function/Array.out
b/regression-test/data/nereids_function_p0/scalar_function/Array.out
index 31fa9067102..36e7429f611 100644
--- a/regression-test/data/nereids_function_p0/scalar_function/Array.out
+++ b/regression-test/data/nereids_function_p0/scalar_function/Array.out
@@ -14158,7 +14158,7 @@ true
["2012-03-12 12:11:12", "2015-03-12 12:11:12"]
-- !array_range_datetime3_timestamp_ns --
-["1969-12-31 23:59:59.999999999", "1970-06-30 23:59:59.999999999"]
+["1969-12-31 23:59:59.999999999", "1970-06-30 23:59:59.999999999", "1970-12-30
23:59:59.999999999"]
["1970-01-01 00:00:00.000000000", "1970-10-01 00:00:00.000000000"]
["2012-03-07 07:06:07.000000001", "2013-12-07 07:06:07.000000001"]
["2012-03-08 08:07:08.000001000", "2014-03-08 08:07:08.000001000"]
diff --git
a/regression-test/data/nereids_function_p0/scalar_function/test_array_range_datetime_tail.out
b/regression-test/data/nereids_function_p0/scalar_function/test_array_range_datetime_tail.out
new file mode 100644
index 00000000000..39bef215e3e
--- /dev/null
+++
b/regression-test/data/nereids_function_p0/scalar_function/test_array_range_datetime_tail.out
@@ -0,0 +1,33 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !day_partial --
+["2024-01-01 00:00:00.000000", "2024-01-02 00:00:00.000000"]
+
+-- !day_exact --
+["2024-01-01 00:00:00.000000"]
+
+-- !day_empty --
+[]
+
+-- !day_vector --
+["2024-01-01 00:00:00.000000", "2024-01-02 00:00:00.000000"]
+["2024-01-02 00:00:00.000000"]
+
+-- !second_partial --
+["2024-01-01 00:00:00.000000", "2024-01-01 00:00:01.000000"]
+
+-- !second_exact --
+["2024-01-01 00:00:00.000000"]
+
+-- !second_vector --
+["2024-01-01 00:00:00.000000", "2024-01-01 00:00:01.000000"]
+["2024-01-01 00:00:01.000000"]
+
+-- !month_end --
+["2020-01-31 00:00:00.000000", "2020-02-29 00:00:00.000000", "2020-03-29
00:00:00.000000"]
+
+-- !upper_bound --
+["9999-12-31 23:59:59.999998"]
+
+-- !timestamp_ns --
+["2024-01-01 00:00:00.000000001", "2024-01-01 00:00:01.000000001"]
+
diff --git
a/regression-test/data/query_p0/sql_functions/array_functions/test_array_functions.out
b/regression-test/data/query_p0/sql_functions/array_functions/test_array_functions.out
index d604e6acb20..ca0ae11a101 100644
---
a/regression-test/data/query_p0/sql_functions/array_functions/test_array_functions.out
+++
b/regression-test/data/query_p0/sql_functions/array_functions/test_array_functions.out
@@ -1877,19 +1877,19 @@
9 [9]
-- !table_select --
-2022-05-15T12:00 2022-05-18T12:00:00.123 ["2022-05-15 12:00:00.000",
"2022-05-16 12:00:00.000", "2022-05-17 12:00:00.000"]
+2022-05-15T12:00 2022-05-18T12:00:00.123 ["2022-05-15 12:00:00.000",
"2022-05-16 12:00:00.000", "2022-05-17 12:00:00.000", "2022-05-18 12:00:00.000"]
-- !table_select --
-2022-05-15T12:00 2022-05-18T12:00:00.123456 ["2022-05-15
12:00:00.000000", "2022-05-16 12:00:00.000000", "2022-05-17 12:00:00.000000"]
+2022-05-15T12:00 2022-05-18T12:00:00.123456 ["2022-05-15
12:00:00.000000", "2022-05-16 12:00:00.000000", "2022-05-17 12:00:00.000000",
"2022-05-18 12:00:00.000000"]
-- !table_select --
-2022-05-15T12:00 2022-05-18T12:00:00.123 1 ["2022-05-15
12:00:00.000", "2022-05-16 12:00:00.000", "2022-05-17 12:00:00.000"]
+2022-05-15T12:00 2022-05-18T12:00:00.123 1 ["2022-05-15
12:00:00.000", "2022-05-16 12:00:00.000", "2022-05-17 12:00:00.000",
"2022-05-18 12:00:00.000"]
-- !table_select --
-2022-04-22T12:00:00.123 2022-05-08T12:00:00.123456 1
["2022-04-22 12:00:00.123000", "2022-04-29 12:00:00.123000"]
+2022-04-22T12:00:00.123 2022-05-08T12:00:00.123456 1
["2022-04-22 12:00:00.123000", "2022-04-29 12:00:00.123000", "2022-05-06
12:00:00.123000"]
-- !table_select --
-2022-01-15T12:00:00.123 2022-05-18T12:00:00.123456 2
["2022-01-15 12:00:00.123000", "2022-03-15 12:00:00.123000"]
+2022-01-15T12:00:00.123 2022-05-18T12:00:00.123456 2
["2022-01-15 12:00:00.123000", "2022-03-15 12:00:00.123000", "2022-05-15
12:00:00.123000"]
-- !table_select --
2015-05-15T12:00:00.123 2022-05-18T12:00:00.123456 3
["2015-05-15 12:00:00.123000", "2018-05-15 12:00:00.123000", "2021-05-15
12:00:00.123000"]
@@ -1898,7 +1898,7 @@
2022-05-18T12:00:00.123 2022-05-18T23:10:00.123456 4
["2022-05-18 12:00:00.123000", "2022-05-18 16:00:00.123000", "2022-05-18
20:00:00.123000"]
-- !table_select --
-2022-05-18T12:00:00.123 2022-05-18T12:16 5 ["2022-05-18
12:00:00.123000", "2022-05-18 12:05:00.123000", "2022-05-18 12:10:00.123000"]
+2022-05-18T12:00:00.123 2022-05-18T12:16 5 ["2022-05-18
12:00:00.123000", "2022-05-18 12:05:00.123000", "2022-05-18 12:10:00.123000",
"2022-05-18 12:15:00.123000"]
-- !table_select --
2022-05-18T12:00:10 2022-05-18T12:00:30 6 ["2022-05-18
12:00:10.000000", "2022-05-18 12:00:16.000000", "2022-05-18 12:00:22.000000",
"2022-05-18 12:00:28.000000"]
@@ -1931,7 +1931,7 @@
["2022-05-15 12:00:00", "2022-05-16 12:00:00"]
-- !const_select --
-["2022-05-01 12:00:00"]
+["2022-05-01 12:00:00", "2022-05-15 12:00:00"]
-- !const_select --
["2022-01-15 12:00:00", "2022-04-15 12:00:00"]
diff --git
a/regression-test/suites/nereids_function_p0/scalar_function/test_array_range_datetime_tail.groovy
b/regression-test/suites/nereids_function_p0/scalar_function/test_array_range_datetime_tail.groovy
new file mode 100644
index 00000000000..6999e1be5a3
--- /dev/null
+++
b/regression-test/suites/nereids_function_p0/scalar_function/test_array_range_datetime_tail.groovy
@@ -0,0 +1,36 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_array_range_datetime_tail") {
+ qt_day_partial """select array_range(cast('2024-01-01 00:00:00.000000' as
datetimev2(6)), cast('2024-01-02 12:00:00.000000' as datetimev2(6)))"""
+ qt_day_exact """select array_range(cast('2024-01-01 00:00:00.000000' as
datetimev2(6)), cast('2024-01-02 00:00:00.000000' as datetimev2(6)))"""
+ qt_day_empty """select array_range(cast('2024-01-02 00:00:00.000000' as
datetimev2(6)), cast('2024-01-01 00:00:00.000000' as datetimev2(6)))"""
+ order_qt_day_vector """select array_range(date_add(cast('2024-01-01
00:00:00.000000' as datetimev2(6)), interval number day), cast('2024-01-02
12:00:00.000000' as datetimev2(6))) from numbers("number" = "2") order by
number"""
+
+ qt_second_partial """select sequence(cast('2024-01-01 00:00:00.000000' as
datetimev2(6)), cast('2024-01-01 00:00:01.500000' as datetimev2(6)), interval 1
second)"""
+ qt_second_exact """select sequence(cast('2024-01-01 00:00:00.000000' as
datetimev2(6)), cast('2024-01-01 00:00:01.000000' as datetimev2(6)), interval 1
second)"""
+ order_qt_second_vector """select sequence(date_add(cast('2024-01-01
00:00:00.000000' as datetimev2(6)), interval number second), cast('2024-01-01
00:00:01.500000' as datetimev2(6)), interval 1 second) from numbers("number" =
"2") order by number"""
+
+ qt_month_end """select array_range(cast('2020-01-31 00:00:00' as
datetimev2(6)), cast('2020-03-30 00:00:00' as datetimev2(6)), interval 1
month)"""
+ qt_upper_bound """select array_range(cast('9999-12-31 23:59:59.999998' as
datetimev2(6)), cast('9999-12-31 23:59:59.999999' as datetimev2(6)), interval 1
second)"""
+ qt_timestamp_ns """select sequence(cast('2024-01-01 00:00:00.000000001' as
timestamp_ns), cast('2024-01-01 00:00:01.000000002' as timestamp_ns), interval
1 second)"""
+
+ test {
+ sql """select array_range(date_add(cast('2024-01-01 00:00:00' as
datetimev2(6)), interval number second), cast('2024-01-12 13:46:41' as
datetimev2(6)), interval 1 second) from numbers("number" = "1")"""
+ exception "Array size exceeds the limit 1000000"
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
