This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
The following commit(s) were added to refs/heads/main by this push:
new 73bce15f29 fix: Fix scalar broadcast for to_timestamp() (#20224)
73bce15f29 is described below
commit 73bce15f292e754585e4f00a09de9be4169d84c2
Author: Neil Conway <[email protected]>
AuthorDate: Thu Feb 12 15:26:55 2026 -0500
fix: Fix scalar broadcast for to_timestamp() (#20224)
When to_timestamp() was invoked with a scalar Float64 and an array of
strings, the previous coding neglected to broadcast the scalar to the
array properly when producing the return value. That is, a query like
`SELECT to_timestamp(123.5, t.x) FROM t` would result in:
Internal error: UDF to_timestamp returned a different number of rows
than expected
## Which issue does this PR close?
<!--
We generally require a GitHub issue to be filed for all bug fixes and
enhancements and this helps us generate change logs for our releases.
You can link an issue to this PR using the GitHub syntax. For example
`Closes #123` indicates that this PR will close issue #123.
-->
- Closes #20223
## Rationale for this change
<!--
Why are you proposing this change? If this is already explained clearly
in the issue then this section is not needed.
Explaining clearly why changes are proposed helps reviewers understand
your changes and offer better suggestions for fixes.
-->
## What changes are included in this PR?
<!--
There is no need to duplicate the description in the issue here but it
is sometimes worth providing a summary of the individual changes in this
PR.
-->
## Are these changes tested?
Yes, added SLT.
<!--
We typically require tests for all PRs in order to:
1. Prevent the code from being accidentally broken by subsequent changes
2. Serve as another way to document the expected behavior of the code
If tests are not included in your PR, please explain why (for example,
are they covered by existing tests)?
-->
## Are there any user-facing changes?
<!--
If there are user-facing changes then we may require documentation to be
updated before approving the PR.
-->
<!--
If there are any breaking changes to public APIs, please add the `api
change` label.
-->
---
datafusion/functions/src/datetime/to_timestamp.rs | 71 +++++++++++++++-------
.../test_files/datetime/arith_date_time.slt | 1 -
.../datetime/arith_timestamp_duration.slt | 2 +-
.../test_files/datetime/timestamps.slt | 30 +++++++++
datafusion/sqllogictest/test_files/limit.slt | 2 +-
.../test_files/limit_single_row_batches.slt | 2 +-
.../test_files/spark/collection/size.slt | 1 -
.../test_files/spark/datetime/time_trunc.slt | 1 -
.../test_files/spark/datetime/trunc.slt | 1 -
datafusion/sqllogictest/test_files/struct.slt | 2 +-
datafusion/sqllogictest/test_files/truncate.slt | 2 +-
11 files changed, 85 insertions(+), 30 deletions(-)
diff --git a/datafusion/functions/src/datetime/to_timestamp.rs
b/datafusion/functions/src/datetime/to_timestamp.rs
index 738c4acee7..6d40133bd2 100644
--- a/datafusion/functions/src/datetime/to_timestamp.rs
+++ b/datafusion/functions/src/datetime/to_timestamp.rs
@@ -430,27 +430,56 @@ impl ScalarUDFImpl for ToTimestampFunc {
.cast_to(&Timestamp(Second, None), None)?
.cast_to(&Timestamp(Nanosecond, tz), None),
Null | Timestamp(_, _) => args[0].cast_to(&Timestamp(Nanosecond,
tz), None),
- Float16 => {
- let arr = args[0].to_array(1)?;
- let f16_arr = downcast_arg!(&arr, Float16Array);
- let result: TimestampNanosecondArray =
- f16_arr.unary(|x| (x.to_f64() * 1_000_000_000.0) as i64);
-
Ok(ColumnarValue::Array(Arc::new(result.with_timezone_opt(tz))))
- }
- Float32 => {
- let arr = args[0].to_array(1)?;
- let f32_arr = downcast_arg!(&arr, Float32Array);
- let result: TimestampNanosecondArray =
- f32_arr.unary(|x| (x as f64 * 1_000_000_000.0) as i64);
-
Ok(ColumnarValue::Array(Arc::new(result.with_timezone_opt(tz))))
- }
- Float64 => {
- let arr = args[0].to_array(1)?;
- let f64_arr = downcast_arg!(&arr, Float64Array);
- let result: TimestampNanosecondArray =
- f64_arr.unary(|x| (x * 1_000_000_000.0) as i64);
-
Ok(ColumnarValue::Array(Arc::new(result.with_timezone_opt(tz))))
- }
+ Float16 => match &args[0] {
+ ColumnarValue::Scalar(ScalarValue::Float16(value)) => {
+ let timestamp_nanos =
+ value.map(|v| (v.to_f64() * 1_000_000_000.0) as i64);
+ Ok(ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(
+ timestamp_nanos,
+ tz,
+ )))
+ }
+ ColumnarValue::Array(arr) => {
+ let f16_arr = downcast_arg!(arr, Float16Array);
+ let result: TimestampNanosecondArray =
+ f16_arr.unary(|x| (x.to_f64() * 1_000_000_000.0) as
i64);
+
Ok(ColumnarValue::Array(Arc::new(result.with_timezone_opt(tz))))
+ }
+ _ => exec_err!("Invalid Float16 value for to_timestamp"),
+ },
+ Float32 => match &args[0] {
+ ColumnarValue::Scalar(ScalarValue::Float32(value)) => {
+ let timestamp_nanos =
+ value.map(|v| (v as f64 * 1_000_000_000.0) as i64);
+ Ok(ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(
+ timestamp_nanos,
+ tz,
+ )))
+ }
+ ColumnarValue::Array(arr) => {
+ let f32_arr = downcast_arg!(arr, Float32Array);
+ let result: TimestampNanosecondArray =
+ f32_arr.unary(|x| (x as f64 * 1_000_000_000.0) as i64);
+
Ok(ColumnarValue::Array(Arc::new(result.with_timezone_opt(tz))))
+ }
+ _ => exec_err!("Invalid Float32 value for to_timestamp"),
+ },
+ Float64 => match &args[0] {
+ ColumnarValue::Scalar(ScalarValue::Float64(value)) => {
+ let timestamp_nanos = value.map(|v| (v * 1_000_000_000.0)
as i64);
+ Ok(ColumnarValue::Scalar(ScalarValue::TimestampNanosecond(
+ timestamp_nanos,
+ tz,
+ )))
+ }
+ ColumnarValue::Array(arr) => {
+ let f64_arr = downcast_arg!(arr, Float64Array);
+ let result: TimestampNanosecondArray =
+ f64_arr.unary(|x| (x * 1_000_000_000.0) as i64);
+
Ok(ColumnarValue::Array(Arc::new(result.with_timezone_opt(tz))))
+ }
+ _ => exec_err!("Invalid Float64 value for to_timestamp"),
+ },
Decimal32(_, _) | Decimal64(_, _) | Decimal256(_, _) => {
let arg = args[0].cast_to(&Decimal128(38, 9), None)?;
decimal128_to_timestamp_nanos(&arg, tz)
diff --git a/datafusion/sqllogictest/test_files/datetime/arith_date_time.slt
b/datafusion/sqllogictest/test_files/datetime/arith_date_time.slt
index bc796a51ff..8e85c8f905 100644
--- a/datafusion/sqllogictest/test_files/datetime/arith_date_time.slt
+++ b/datafusion/sqllogictest/test_files/datetime/arith_date_time.slt
@@ -113,4 +113,3 @@ SELECT '2001-09-28'::date / '03:00'::time
query error Invalid timestamp arithmetic operation
SELECT '2001-09-28'::date % '03:00'::time
-
diff --git
a/datafusion/sqllogictest/test_files/datetime/arith_timestamp_duration.slt
b/datafusion/sqllogictest/test_files/datetime/arith_timestamp_duration.slt
index 10381346f8..aeeebe73db 100644
--- a/datafusion/sqllogictest/test_files/datetime/arith_timestamp_duration.slt
+++ b/datafusion/sqllogictest/test_files/datetime/arith_timestamp_duration.slt
@@ -144,4 +144,4 @@ query error Invalid timestamp arithmetic operation
SELECT '2001-09-28T01:00:00'::timestamp % arrow_cast(12345,
'Duration(Second)');
query error Invalid timestamp arithmetic operation
-SELECT '2001-09-28T01:00:00'::timestamp / arrow_cast(12345,
'Duration(Second)');
\ No newline at end of file
+SELECT '2001-09-28T01:00:00'::timestamp / arrow_cast(12345,
'Duration(Second)');
diff --git a/datafusion/sqllogictest/test_files/datetime/timestamps.slt
b/datafusion/sqllogictest/test_files/datetime/timestamps.slt
index fa25994ed7..8ed32940e8 100644
--- a/datafusion/sqllogictest/test_files/datetime/timestamps.slt
+++ b/datafusion/sqllogictest/test_files/datetime/timestamps.slt
@@ -5328,3 +5328,33 @@ drop table ts_data_secs
statement ok
drop table ts_data_micros_kolkata
+
+##########
+## Test to_timestamp with scalar float inputs
+##########
+
+statement ok
+create table test_to_timestamp_scalar(id int, name varchar) as values
+ (1, 'foo'),
+ (2, 'bar');
+
+query P
+SELECT to_timestamp(123.5, name) FROM test_to_timestamp_scalar ORDER BY id;
+----
+1970-01-01T00:02:03.500
+1970-01-01T00:02:03.500
+
+query P
+SELECT to_timestamp(456.789::float, name) FROM test_to_timestamp_scalar ORDER
BY id;
+----
+1970-01-01T00:07:36.789001464
+1970-01-01T00:07:36.789001464
+
+query P
+SELECT to_timestamp(arrow_cast(100.5, 'Float16'), name) FROM
test_to_timestamp_scalar ORDER BY id;
+----
+1970-01-01T00:01:40.500
+1970-01-01T00:01:40.500
+
+statement ok
+drop table test_to_timestamp_scalar
diff --git a/datafusion/sqllogictest/test_files/limit.slt
b/datafusion/sqllogictest/test_files/limit.slt
index 429181a2d3..ec8363f51a 100644
--- a/datafusion/sqllogictest/test_files/limit.slt
+++ b/datafusion/sqllogictest/test_files/limit.slt
@@ -871,4 +871,4 @@ DROP TABLE test_limit_with_partitions;
# Tear down src_table table:
statement ok
-DROP TABLE src_table;
\ No newline at end of file
+DROP TABLE src_table;
diff --git a/datafusion/sqllogictest/test_files/limit_single_row_batches.slt
b/datafusion/sqllogictest/test_files/limit_single_row_batches.slt
index fbdb0140e0..9f626816e2 100644
--- a/datafusion/sqllogictest/test_files/limit_single_row_batches.slt
+++ b/datafusion/sqllogictest/test_files/limit_single_row_batches.slt
@@ -19,4 +19,4 @@ SELECT COUNT(*) FROM (SELECT i FROM filter_limit WHERE i <> 0
LIMIT 1);
1
statement ok
-DROP TABLE filter_limit;
\ No newline at end of file
+DROP TABLE filter_limit;
diff --git a/datafusion/sqllogictest/test_files/spark/collection/size.slt
b/datafusion/sqllogictest/test_files/spark/collection/size.slt
index dabcfd069b..106760eebf 100644
--- a/datafusion/sqllogictest/test_files/spark/collection/size.slt
+++ b/datafusion/sqllogictest/test_files/spark/collection/size.slt
@@ -129,4 +129,3 @@ SELECT size(column1) FROM VALUES (map(['a'], [1])),
(map(['a','b'], [1,2])), (NU
1
2
-1
-
diff --git a/datafusion/sqllogictest/test_files/spark/datetime/time_trunc.slt
b/datafusion/sqllogictest/test_files/spark/datetime/time_trunc.slt
index f00c40f0a9..35ffa483bb 100644
--- a/datafusion/sqllogictest/test_files/spark/datetime/time_trunc.slt
+++ b/datafusion/sqllogictest/test_files/spark/datetime/time_trunc.slt
@@ -71,4 +71,3 @@ NULL
# incorrect format
query error DataFusion error: Optimizer rule 'simplify_expressions'
failed\ncaused by\nError during planning: The format argument of `TIME_TRUNC`
must be one of: hour, minute, second, millisecond, microsecond
SELECT time_trunc('test', '09:32:05.123456'::time);
-
diff --git a/datafusion/sqllogictest/test_files/spark/datetime/trunc.slt
b/datafusion/sqllogictest/test_files/spark/datetime/trunc.slt
index f6bf6b5751..aa26d7bd0e 100644
--- a/datafusion/sqllogictest/test_files/spark/datetime/trunc.slt
+++ b/datafusion/sqllogictest/test_files/spark/datetime/trunc.slt
@@ -90,4 +90,3 @@ SELECT trunc('2009-02-12'::date, NULL::string);
# incorrect format
query error DataFusion error: Optimizer rule 'simplify_expressions'
failed\ncaused by\nError during planning: The format argument of `TRUNC` must
be one of: year, yy, yyyy, month, mm, mon, day, week, quarter.
SELECT trunc('2009-02-12'::date, 'test'::string);
-
diff --git a/datafusion/sqllogictest/test_files/struct.slt
b/datafusion/sqllogictest/test_files/struct.slt
index e20815a58c..53a1bb4ec6 100644
--- a/datafusion/sqllogictest/test_files/struct.slt
+++ b/datafusion/sqllogictest/test_files/struct.slt
@@ -1666,4 +1666,4 @@ order by id;
3 2 150
statement ok
-drop table t_agg_window;
\ No newline at end of file
+drop table t_agg_window;
diff --git a/datafusion/sqllogictest/test_files/truncate.slt
b/datafusion/sqllogictest/test_files/truncate.slt
index 5a5d47760d..ad3ccbb1a7 100644
--- a/datafusion/sqllogictest/test_files/truncate.slt
+++ b/datafusion/sqllogictest/test_files/truncate.slt
@@ -82,4 +82,4 @@ logical_plan
physical_plan_error
01)TRUNCATE operation on table 't1'
02)caused by
-03)This feature is not implemented: TRUNCATE not supported for Base table
\ No newline at end of file
+03)This feature is not implemented: TRUNCATE not supported for Base table
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
