This is an automated email from the ASF dual-hosted git repository. jacktengg pushed a commit to branch dev-timestamp-ns in repository https://gitbox.apache.org/repos/asf/doris.git
commit cb7e82a8cf0f6d312e2fcf1b43b2b3b01abc0cef Author: jacktengg <[email protected]> AuthorDate: Thu Aug 27 19:23:34 2026 +0800 [test](regression) Add Arrow coverage for timestamp_ns ### What problem does this PR solve? Issue Number: None Related PR: #66761 Problem Summary: TIMESTAMP_NS had basic Arrow round-trip coverage but lacked complete time-unit boundary checks, nested FE schema checks, and an end-to-end Arrow Flight SQL regression. Add exact conversion and overflow tests, nested schema assertions, and Flight queries for nanosecond boundary values under multiple session time zones. ### Release note None ### Check List (For Author) - Test: Unit Test and Regression test - `./run-fe-ut.sh --run org.apache.doris.service.arrowflight.FlightSqlSchemaHelperArrowTypeTest` - `./run-be-ut.sh --run --filter=DataTypeSerDeArrowTest.TimeStampNsArrowRoundTrip:DataTypeSerDeArrowTest.TimeStampNsArrowUnitConversion -j 16` - `./run-regression-test.sh --run -d arrow_flight_sql_p0 -s test_select` - Behavior changed: No - Does this need documentation: No --- .../data_type_serde/data_type_serde_arrow_test.cpp | 91 ++++++++++++++-------- .../FlightSqlSchemaHelperArrowTypeTest.java | 11 ++- .../data/arrow_flight_sql_p0/test_select.out | 16 ++++ .../suites/arrow_flight_sql_p0/test_select.groovy | 62 +++++++++++++++ 4 files changed, 146 insertions(+), 34 deletions(-) diff --git a/be/test/core/data_type_serde/data_type_serde_arrow_test.cpp b/be/test/core/data_type_serde/data_type_serde_arrow_test.cpp index cd89cf6e309..07c6820217c 100644 --- a/be/test/core/data_type_serde/data_type_serde_arrow_test.cpp +++ b/be/test/core/data_type_serde/data_type_serde_arrow_test.cpp @@ -802,42 +802,67 @@ TEST(DataTypeSerDeArrowTest, TimeStampNsArrowRoundTrip) { TEST(DataTypeSerDeArrowTest, TimeStampNsArrowUnitConversion) { const auto timestamp_ns_type = std::make_shared<DataTypeTimeStampNs>(); const auto serde = timestamp_ns_type->get_serde(); - - auto source_column = ColumnTimeStampNs::create(); - source_column->insert_value(TimeStampNsValue(-1000)); - source_column->insert_value(TimeStampNsValue(1)); - - arrow::TimestampBuilder micro_builder(arrow::timestamp(arrow::TimeUnit::MICRO), - arrow::default_memory_pool()); - auto status = serde->write_column_to_arrow(*source_column, nullptr, µ_builder, 0, 1, + const std::vector<std::pair<arrow::TimeUnit::type, int64_t>> unit_cases { + {arrow::TimeUnit::SECOND, TimeStampNsValue::NANOS_PER_SECOND}, + {arrow::TimeUnit::MILLI, TimeStampNsValue::NANOS_PER_MILLISECOND}, + {arrow::TimeUnit::MICRO, TimeStampNsValue::NANOS_PER_MICROSECOND}, + {arrow::TimeUnit::NANO, 1}}; + + for (const auto& [unit, nanos_per_unit] : unit_cases) { + auto source_column = ColumnTimeStampNs::create(); + source_column->insert_value(TimeStampNsValue(-2 * nanos_per_unit)); + source_column->insert_value(TimeStampNsValue(0)); + source_column->insert_value(TimeStampNsValue(3 * nanos_per_unit)); + + arrow::TimestampBuilder builder(arrow::timestamp(unit), arrow::default_memory_pool()); + auto status = serde->write_column_to_arrow(*source_column, nullptr, &builder, 0, + source_column->size(), cctz::utc_time_zone()); + ASSERT_TRUE(status.ok()) << status; + + std::shared_ptr<arrow::Array> array; + ASSERT_TRUE(builder.Finish(&array).ok()); + const auto* timestamp_array = assert_cast<const arrow::TimestampArray*>(array.get()); + EXPECT_EQ(-2, timestamp_array->Value(0)); + EXPECT_EQ(0, timestamp_array->Value(1)); + EXPECT_EQ(3, timestamp_array->Value(2)); + + auto target_column = ColumnTimeStampNs::create(); + status = serde->read_column_from_arrow(*target_column, array.get(), 0, array->length(), cctz::utc_time_zone()); - ASSERT_TRUE(status.ok()) << status; - status = serde->write_column_to_arrow(*source_column, nullptr, µ_builder, 1, 2, - cctz::utc_time_zone()); - EXPECT_FALSE(status.ok()); + ASSERT_TRUE(status.ok()) << status; + ASSERT_EQ(source_column->size(), target_column->size()); + for (size_t i = 0; i < source_column->size(); ++i) { + EXPECT_EQ(source_column->get_data()[i].epoch_nanos(), + target_column->get_data()[i].epoch_nanos()); + } - std::shared_ptr<arrow::Array> micro_array; - ASSERT_TRUE(micro_builder.Finish(µ_array).ok()); - ASSERT_EQ(-1, assert_cast<const arrow::TimestampArray*>(micro_array.get())->Value(0)); + if (unit != arrow::TimeUnit::NANO) { + auto lossy_column = ColumnTimeStampNs::create(); + lossy_column->insert_value(TimeStampNsValue(1)); + arrow::TimestampBuilder lossy_builder(arrow::timestamp(unit), + arrow::default_memory_pool()); + status = serde->write_column_to_arrow(*lossy_column, nullptr, &lossy_builder, 0, + lossy_column->size(), cctz::utc_time_zone()); + EXPECT_FALSE(status.ok()); + } + } - auto micro_column = ColumnTimeStampNs::create(); - status = serde->read_column_from_arrow(*micro_column, micro_array.get(), 0, - micro_array->length(), cctz::utc_time_zone()); - ASSERT_TRUE(status.ok()) << status; - ASSERT_EQ(1, micro_column->size()); - EXPECT_EQ(-1000, micro_column->get_data()[0].epoch_nanos()); - - arrow::TimestampBuilder seconds_builder(arrow::timestamp(arrow::TimeUnit::SECOND), - arrow::default_memory_pool()); - ASSERT_TRUE(seconds_builder.Append(std::numeric_limits<int64_t>::max()).ok()); - std::shared_ptr<arrow::Array> seconds_array; - ASSERT_TRUE(seconds_builder.Finish(&seconds_array).ok()); - - auto overflow_column = ColumnTimeStampNs::create(); - status = serde->read_column_from_arrow(*overflow_column, seconds_array.get(), 0, - seconds_array->length(), cctz::utc_time_zone()); - EXPECT_FALSE(status.ok()); - EXPECT_EQ(0, overflow_column->size()); + for (const auto unit : + {arrow::TimeUnit::SECOND, arrow::TimeUnit::MILLI, arrow::TimeUnit::MICRO}) { + for (const auto value : {std::numeric_limits<int64_t>::min(), + std::numeric_limits<int64_t>::max()}) { + arrow::TimestampBuilder builder(arrow::timestamp(unit), arrow::default_memory_pool()); + ASSERT_TRUE(builder.Append(value).ok()); + std::shared_ptr<arrow::Array> array; + ASSERT_TRUE(builder.Finish(&array).ok()); + + auto overflow_column = ColumnTimeStampNs::create(); + const auto status = serde->read_column_from_arrow( + *overflow_column, array.get(), 0, array->length(), cctz::utc_time_zone()); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(0, overflow_column->size()); + } + } } TEST(DataTypeSerDeArrowTest, DateTimeV2ArrowEncodingFollowsSchemaTimezone) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlSchemaHelperArrowTypeTest.java b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlSchemaHelperArrowTypeTest.java index dadc26dffae..cb30f9ca23a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlSchemaHelperArrowTypeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/FlightSqlSchemaHelperArrowTypeTest.java @@ -213,7 +213,12 @@ public class FlightSqlSchemaHelperArrowTypeTest { byte[] serialized = FlightSqlSchemaHelper.getSerializedSchema(Arrays.asList( buildField(desc("ts", TPrimitiveType.TIMESTAMP_NS)), buildField(desc("items", TPrimitiveType.ARRAY, - desc("item", TPrimitiveType.TIMESTAMP_NS))))); + desc("item", TPrimitiveType.TIMESTAMP_NS))), + buildField(desc("by_name", TPrimitiveType.MAP, + desc("key", TPrimitiveType.VARCHAR), + desc("value", TPrimitiveType.TIMESTAMP_NS))), + buildField(desc("record", TPrimitiveType.STRUCT, + desc("ts", TPrimitiveType.TIMESTAMP_NS))))); Schema schema = MessageSerializer.deserializeSchema( new ReadChannel(Channels.newChannel(new ByteArrayInputStream(serialized)))); @@ -221,5 +226,9 @@ public class FlightSqlSchemaHelperArrowTypeTest { Assertions.assertEquals(timestampNs, schema.getFields().get(0).getType()); Assertions.assertEquals(timestampNs, schema.getFields().get(1).getChildren().get(0).getType()); + Assertions.assertEquals(timestampNs, + schema.getFields().get(2).getChildren().get(0).getChildren().get(1).getType()); + Assertions.assertEquals(timestampNs, + schema.getFields().get(3).getChildren().get(0).getType()); } } diff --git a/regression-test/data/arrow_flight_sql_p0/test_select.out b/regression-test/data/arrow_flight_sql_p0/test_select.out index fb6bea38e2c..0e2e646b668 100644 --- a/regression-test/data/arrow_flight_sql_p0/test_select.out +++ b/regression-test/data/arrow_flight_sql_p0/test_select.out @@ -7,6 +7,22 @@ 222 plsql222 2024-07-20 12:00:00.123456 2024-07-20 12:00:00 111 plsql111 2024-07-19 12:00:00.123456 2024-07-19 12:00:00 +-- !arrow_flight_sql_timestamp_ns_text_utc -- +1 1677-09-21 00:12:43.145224192 +2 1969-12-31 23:59:59.999999999 +3 1970-01-01 00:00:00.000000001 +4 2024-02-29 12:34:56.123456789 +5 2262-04-11 23:47:16.854775807 +6 \N + +-- !arrow_flight_sql_timestamp_ns_text_plus_eight -- +1 1677-09-21 00:12:43.145224192 +2 1969-12-31 23:59:59.999999999 +3 1970-01-01 00:00:00.000000001 +4 2024-02-29 12:34:56.123456789 +5 2262-04-11 23:47:16.854775807 +6 \N + -- !arrow_flight_sql_jsonb -- 1 {"k1":1,"k2":"v2"} 2 [1,2,{"nested":true}] diff --git a/regression-test/suites/arrow_flight_sql_p0/test_select.groovy b/regression-test/suites/arrow_flight_sql_p0/test_select.groovy index 2a129f20abb..4d58db7c0cb 100644 --- a/regression-test/suites/arrow_flight_sql_p0/test_select.groovy +++ b/regression-test/suites/arrow_flight_sql_p0/test_select.groovy @@ -96,6 +96,68 @@ suite("test_select", "arrow_flight_sql") { } } + sql "DROP TABLE IF EXISTS test_select_timestamp_ns" + sql """ + CREATE TABLE test_select_timestamp_ns ( + id INT, + ts TIMESTAMP_NS NULL + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES ("replication_num" = "1") + """ + sql """ + INSERT INTO test_select_timestamp_ns VALUES + (1, '1677-09-21 00:12:43.145224192'), + (2, '1969-12-31 23:59:59.999999999'), + (3, '1970-01-01 00:00:00.000000001'), + (4, '2024-02-29 12:34:56.123456789'), + (5, '2262-04-11 23:47:16.854775807'), + (6, NULL) + """ + + arrow_flight_sql "SET time_zone = '+00:00'" + order_qt_arrow_flight_sql_timestamp_ns_text_utc """ + SELECT id, CAST(ts AS STRING) FROM test_select_timestamp_ns ORDER BY id + """ + arrow_flight_sql "SET time_zone = '+08:00'" + order_qt_arrow_flight_sql_timestamp_ns_text_plus_eight """ + SELECT id, CAST(ts AS STRING) FROM test_select_timestamp_ns ORDER BY id + """ + + def expectedTimestampNs = [ + LocalDateTime.parse("1677-09-21T00:12:43.145224192"), + LocalDateTime.parse("1969-12-31T23:59:59.999999999"), + LocalDateTime.parse("1970-01-01T00:00:00.000000001"), + LocalDateTime.parse("2024-02-29T12:34:56.123456789"), + LocalDateTime.parse("2262-04-11T23:47:16.854775807"), + null + ] + for (def sessionTimeZone : ["+00:00", "+08:00"]) { + arrow_flight_sql "SET time_zone = '${sessionTimeZone}'" + context.getArrowFlightSqlConnection().createStatement().withCloseable { statement -> + statement.executeQuery(""" + USE ${context.dbName}; + SELECT ts FROM test_select_timestamp_ns ORDER BY id + """).withCloseable { resultSet -> + def metadata = resultSet.getMetaData() + assertEquals(1, metadata.getColumnCount()) + assertEquals(Types.TIMESTAMP, metadata.getColumnType(1)) + assertEquals("TIMESTAMP", metadata.getColumnTypeName(1)) + + for (def expected : expectedTimestampNs) { + assertTrue(resultSet.next()) + if (expected == null) { + assertNull(resultSet.getObject(1)) + } else { + assertEquals(expected, resultSet.getObject(1, LocalDateTime.class)) + } + } + assertFalse(resultSet.next()) + } + } + } + tableName = "test_select_jsonb" sql "DROP TABLE IF EXISTS ${tableName}" sql """ --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
