This is an automated email from the ASF dual-hosted git repository. jacktengg pushed a commit to branch dev-timestamp-ns in repository https://gitbox.apache.org/repos/asf/doris.git
commit 2f5da8b41fe2ee28a72e39a4ce64855f7a39f355 Author: jacktengg <[email protected]> AuthorDate: Thu Aug 27 22:58:01 2026 +0800 [refactor](be) Parse TIMESTAMP_NS directly into value objects ### What problem does this PR solve? Issue Number: None Related PR: #66761 Problem Summary: The TIMESTAMP_NS string parser already produced a TimeStampNsValue internally, but parse_timestamp_ns_impl converted it to a raw epoch-nanosecond integer and callers immediately reconstructed the value object. Make both the internal parser and the public parse_timestamp_ns interface output TimeStampNsValue directly, then update SerDe, literal, storage, and test callers to remove redundant conversions. ### Release note None ### Check List (For Author) - Test: Unit Test - ./build.sh --be - ./run-be-ut.sh --run --filter=DataTypeTimeStampNsTest.*:CastTimeStampTzTest.timestamp_ns_and_timestamptz_round_trip -j 16 - Behavior changed: No - Does this need documentation: No --- be/src/core/data_type/data_type_timestamp_ns.cpp | 5 ++- .../data_type_timestamp_ns_serde.cpp | 39 ++++++++++----------- .../data_type_serde/data_type_timestamp_ns_serde.h | 2 +- be/src/storage/delete/delete_handler.cpp | 4 +-- be/src/storage/tablet_info.cpp | 5 ++- .../core/data_type/data_type_timestamp_ns_test.cpp | 40 ++++++++++------------ .../function/cast/cast_to_timestamptz_test.cpp | 6 ++-- 7 files changed, 47 insertions(+), 54 deletions(-) diff --git a/be/src/core/data_type/data_type_timestamp_ns.cpp b/be/src/core/data_type/data_type_timestamp_ns.cpp index 61eb62bae5f..5bcfa3aa383 100644 --- a/be/src/core/data_type/data_type_timestamp_ns.cpp +++ b/be/src/core/data_type/data_type_timestamp_ns.cpp @@ -17,7 +17,6 @@ #include "core/data_type/data_type_timestamp_ns.h" -#include <cstdint> #include <typeinfo> #include "common/exception.h" @@ -31,7 +30,7 @@ DataTypeSerDeSPtr DataTypeTimeStampNs::get_serde(int nesting_level) const { } Field DataTypeTimeStampNs::get_field(const TExprNode& node) const { - int64_t value = 0; + TimeStampNsValue value; const StringRef string_value(node.date_literal.value.data(), node.date_literal.value.size()); const auto status = parse_timestamp_ns(string_value, &value); if (!status.ok()) { @@ -39,7 +38,7 @@ Field DataTypeTimeStampNs::get_field(const TExprNode& node) const { "Invalid value: {} for type TimeStampNs: {}", node.date_literal.value, status.to_string()); } - return Field::create_field<TYPE_TIMESTAMP_NS>(TimeStampNsValue(value)); + return Field::create_field<TYPE_TIMESTAMP_NS>(value); } bool DataTypeTimeStampNs::equals(const IDataType& rhs) const { diff --git a/be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp b/be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp index 992a848c767..83786c27fec 100644 --- a/be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp +++ b/be/src/core/data_type_serde/data_type_timestamp_ns_serde.cpp @@ -59,11 +59,10 @@ Status get_nanos_per_arrow_timestamp_unit(arrow::TimeUnit::type unit, int64_t* n } template <DatelikeParseMode ParseMode> -Status parse_timestamp_ns_impl(StringRef str, int64_t& epoch_nanos, +Status parse_timestamp_ns_impl(StringRef str, TimeStampNsValue& value, const cctz::time_zone* local_time_zone) { constexpr bool IsStrict = is_datelike_parse_strict(ParseMode); CastParameters params {.status = Status::OK(), .is_strict = IsStrict}; - TimeStampNsValue value; bool parsed = false; if constexpr (IsStrict) { parsed = CastToTimestampNs::from_string_strict_mode<ParseMode>( @@ -82,21 +81,19 @@ Status parse_timestamp_ns_impl(StringRef str, int64_t& epoch_nanos, std::string(str.data, str.size)); } - epoch_nanos = value.epoch_nanos(); return Status::OK(); } -Status parse_timestamp_ns_non_strict(StringRef str, int64_t* epoch_nanos, +Status parse_timestamp_ns_non_strict(StringRef str, TimeStampNsValue& value, const cctz::time_zone* local_time_zone) { - return parse_timestamp_ns_impl<DatelikeParseMode::NON_STRICT>(str, *epoch_nanos, - local_time_zone); + return parse_timestamp_ns_impl<DatelikeParseMode::NON_STRICT>(str, value, local_time_zone); } } // namespace -Status parse_timestamp_ns(StringRef str, int64_t* epoch_nanos, +Status parse_timestamp_ns(StringRef str, TimeStampNsValue* value, const cctz::time_zone* local_time_zone) { - return parse_timestamp_ns_impl<DatelikeParseMode::STRICT>(str, *epoch_nanos, local_time_zone); + return parse_timestamp_ns_impl<DatelikeParseMode::STRICT>(str, *value, local_time_zone); } Status DataTypeTimeStampNsSerDe::from_string_batch(const ColumnString& strings, @@ -106,11 +103,11 @@ Status DataTypeTimeStampNsSerDe::from_string_batch(const ColumnString& strings, auto& null_map = result.get_null_map_column().get_data(); result.resize(strings.size()); for (size_t i = 0; i < strings.size(); ++i) { - int64_t value = 0; - const auto status = - parse_timestamp_ns_non_strict(strings.get_data_at(i), &value, options.timezone); + TimeStampNsValue value; + const auto status = parse_timestamp_ns_non_strict(strings.get_data_at(i), value, + options.timezone); null_map[i] = !status.ok(); - data[i] = TimeStampNsValue(value); + data[i] = value; } return Status::OK(); } @@ -124,26 +121,26 @@ Status DataTypeTimeStampNsSerDe::from_string_strict_mode_batch( if (null_map != nullptr && null_map[i]) { continue; } - int64_t value = 0; + TimeStampNsValue value; RETURN_IF_ERROR(parse_timestamp_ns(strings.get_data_at(i), &value, options.timezone)); - data[i] = TimeStampNsValue(value); + data[i] = value; } return Status::OK(); } Status DataTypeTimeStampNsSerDe::from_string(StringRef& str, IColumn& column, const FormatOptions& options) const { - int64_t value = 0; - RETURN_IF_ERROR(parse_timestamp_ns_non_strict(str, &value, options.timezone)); - assert_cast<ColumnTimeStampNs&>(column).insert_value(TimeStampNsValue(value)); + TimeStampNsValue value; + RETURN_IF_ERROR(parse_timestamp_ns_non_strict(str, value, options.timezone)); + assert_cast<ColumnTimeStampNs&>(column).insert_value(value); return Status::OK(); } Status DataTypeTimeStampNsSerDe::from_string_strict_mode(StringRef& str, IColumn& column, const FormatOptions& options) const { - int64_t value = 0; + TimeStampNsValue value; RETURN_IF_ERROR(parse_timestamp_ns(str, &value, options.timezone)); - assert_cast<ColumnTimeStampNs&>(column).insert_value(TimeStampNsValue(value)); + assert_cast<ColumnTimeStampNs&>(column).insert_value(value); return Status::OK(); } @@ -315,9 +312,9 @@ std::string DataTypeTimeStampNsSerDe::to_olap_string(const Field& field) const { Status DataTypeTimeStampNsSerDe::from_olap_string(const std::string& str, Field& field, const FormatOptions& options) const { - int64_t value = 0; + TimeStampNsValue value; RETURN_IF_ERROR(parse_timestamp_ns(StringRef(str.data(), str.size()), &value)); - field = Field::create_field<TYPE_TIMESTAMP_NS>(TimeStampNsValue(value)); + field = Field::create_field<TYPE_TIMESTAMP_NS>(value); return Status::OK(); } diff --git a/be/src/core/data_type_serde/data_type_timestamp_ns_serde.h b/be/src/core/data_type_serde/data_type_timestamp_ns_serde.h index 519245ea418..266192c3552 100644 --- a/be/src/core/data_type_serde/data_type_timestamp_ns_serde.h +++ b/be/src/core/data_type_serde/data_type_timestamp_ns_serde.h @@ -22,7 +22,7 @@ namespace doris { -Status parse_timestamp_ns(StringRef str, int64_t* epoch_nanos, +Status parse_timestamp_ns(StringRef str, TimeStampNsValue* value, const cctz::time_zone* local_time_zone = nullptr); class DataTypeTimeStampNsSerDe final diff --git a/be/src/storage/delete/delete_handler.cpp b/be/src/storage/delete/delete_handler.cpp index f4ccbe4652c..faa432867bb 100644 --- a/be/src/storage/delete/delete_handler.cpp +++ b/be/src/storage/delete/delete_handler.cpp @@ -364,8 +364,8 @@ bool DeleteHandler::is_condition_value_valid(const TabletColumn& column, case FieldType::OLAP_FIELD_TYPE_TIMESTAMPTZ: return valid_datetime(value_str, column.frac()); case FieldType::OLAP_FIELD_TYPE_TIMESTAMP_NS: { - int64_t epoch_nanos = 0; - return parse_timestamp_ns(StringRef(value_str), &epoch_nanos).ok(); + TimeStampNsValue value; + return parse_timestamp_ns(StringRef(value_str), &value).ok(); } case FieldType::OLAP_FIELD_TYPE_BOOL: return valid_bool(value_str); diff --git a/be/src/storage/tablet_info.cpp b/be/src/storage/tablet_info.cpp index 9b6a3fe6ed7..9352651935b 100644 --- a/be/src/storage/tablet_info.cpp +++ b/be/src/storage/tablet_info.cpp @@ -649,11 +649,10 @@ static Status _create_partition_key(const TExprNode& t_expr, BlockRow* part_key, } column->insert_data(reinterpret_cast<const char*>(&dt), 0); } else if (primitive_type == TYPE_TIMESTAMP_NS) { - int64_t epoch_nanos = 0; + TimeStampNsValue dt; RETURN_IF_ERROR(parse_timestamp_ns( {t_expr.date_literal.value.data(), t_expr.date_literal.value.size()}, - &epoch_nanos)); - const TimeStampNsValue dt(epoch_nanos); + &dt)); column->insert_data(reinterpret_cast<const char*>(&dt), 0); } else if (primitive_type == TYPE_TIMESTAMPTZ) { TimestampTzValue res; diff --git a/be/test/core/data_type/data_type_timestamp_ns_test.cpp b/be/test/core/data_type/data_type_timestamp_ns_test.cpp index ff8c0ba0e31..a75ccae6567 100644 --- a/be/test/core/data_type/data_type_timestamp_ns_test.cpp +++ b/be/test/core/data_type/data_type_timestamp_ns_test.cpp @@ -118,9 +118,8 @@ TEST(DataTypeTimeStampNsTest, EpochDerivedAndCivilAccessorsAgreeAtBoundaries) { }; for (const auto& test_case : cases) { - int64_t raw = 0; - ASSERT_TRUE(parse_timestamp_ns(StringRef(test_case.input), &raw).ok()) << test_case.input; - const TimeStampNsValue value(raw); + TimeStampNsValue value; + ASSERT_TRUE(parse_timestamp_ns(StringRef(test_case.input), &value).ok()) << test_case.input; const auto civil_date = value.to_date(); const auto civil_value = value.to_datetime(); const int64_t expected_daynr = calc_daynr(test_case.year, test_case.month, test_case.day); @@ -158,19 +157,19 @@ TEST(DataTypeTimeStampNsTest, EpochDerivedAndCivilAccessorsAgreeAtBoundaries) { } TEST(DataTypeTimeStampNsTest, ParseAtFixedNanosecondPrecision) { - int64_t value = 0; + TimeStampNsValue value; ASSERT_TRUE(parse_timestamp_ns(StringRef("1970-01-01 00:00:00.12345675"), &value).ok()); - EXPECT_EQ(TimeStampNsValue(value).to_string(), "1970-01-01 00:00:00.123456750"); + EXPECT_EQ(value.to_string(), "1970-01-01 00:00:00.123456750"); ASSERT_TRUE(parse_timestamp_ns(StringRef("1969-12-31 23:59:59.999999999"), &value).ok()); - EXPECT_EQ(value, -1); + EXPECT_EQ(value.epoch_nanos(), -1); ASSERT_TRUE(parse_timestamp_ns(StringRef("1970-01-01 00:00:00.999999995"), &value).ok()); - EXPECT_EQ(TimeStampNsValue(value).to_string(), "1970-01-01 00:00:00.999999995"); + EXPECT_EQ(value.to_string(), "1970-01-01 00:00:00.999999995"); ASSERT_TRUE(parse_timestamp_ns(StringRef("1970-01-01 00:00:00.9999999995"), &value).ok()); - EXPECT_EQ(TimeStampNsValue(value).to_string(), "1970-01-01 00:00:01.000000000"); + EXPECT_EQ(value.to_string(), "1970-01-01 00:00:01.000000000"); } TEST(DataTypeTimeStampNsTest, ParseTimezoneSuffixInSessionTimezone) { @@ -178,16 +177,16 @@ TEST(DataTypeTimeStampNsTest, ParseTimezoneSuffixInSessionTimezone) { cctz::time_zone shanghai; ASSERT_TRUE(cctz::load_time_zone("Asia/Shanghai", &shanghai)); - int64_t value = 0; + TimeStampNsValue value; auto status = parse_timestamp_ns(StringRef("2023-08-17T01:41:18.123456789Z"), &value, &shanghai); ASSERT_TRUE(status.ok()) << status.to_string(); - EXPECT_EQ(TimeStampNsValue(value).to_string(), "2023-08-17 09:41:18.123456789"); + EXPECT_EQ(value.to_string(), "2023-08-17 09:41:18.123456789"); ASSERT_TRUE(parse_timestamp_ns(StringRef("2023-08-17T01:41:18.123456789America/Los_Angeles"), &value, &shanghai) .ok()); - EXPECT_EQ(TimeStampNsValue(value).to_string(), "2023-08-17 16:41:18.123456789"); + EXPECT_EQ(value.to_string(), "2023-08-17 16:41:18.123456789"); EXPECT_FALSE( parse_timestamp_ns(StringRef("1677-09-21T00:12:43.145224192+14:00"), &value, &shanghai) @@ -202,14 +201,14 @@ TEST(DataTypeTimeStampNsTest, RoundZonedInputBeforeSessionTimezoneConversion) { cctz::time_zone new_york; ASSERT_TRUE(cctz::load_time_zone("America/New_York", &new_york)); - int64_t value = 0; + TimeStampNsValue value; ASSERT_TRUE(parse_timestamp_ns(StringRef("2024-03-10T06:59:59.9999999995Z"), &value, &new_york) .ok()); - EXPECT_EQ(TimeStampNsValue(value).to_string(), "2024-03-10 03:00:00.000000000"); + EXPECT_EQ(value.to_string(), "2024-03-10 03:00:00.000000000"); ASSERT_TRUE(parse_timestamp_ns(StringRef("2024-11-03T05:59:59.9999999995Z"), &value, &new_york) .ok()); - EXPECT_EQ(TimeStampNsValue(value).to_string(), "2024-11-03 01:00:00.000000000"); + EXPECT_EQ(value.to_string(), "2024-11-03 01:00:00.000000000"); } TEST(DataTypeTimeStampNsTest, ParseAcceptsFractionalWidthsAndRejectsMalformedValues) { @@ -227,9 +226,9 @@ TEST(DataTypeTimeStampNsTest, ParseAcceptsFractionalWidthsAndRejectsMalformedVal }; for (const auto& test_case : valid_cases) { - int64_t value = 0; + TimeStampNsValue value; ASSERT_TRUE(parse_timestamp_ns(StringRef(test_case.input), &value).ok()) << test_case.input; - EXPECT_EQ(TimeStampNsValue(value).to_string(), test_case.expected); + EXPECT_EQ(value.to_string(), test_case.expected); } const std::vector<const char*> invalid_values = { @@ -244,13 +243,13 @@ TEST(DataTypeTimeStampNsTest, ParseAcceptsFractionalWidthsAndRejectsMalformedVal "2024.01-01 00:00:00", }; for (const char* input : invalid_values) { - int64_t value = 0; + TimeStampNsValue value; EXPECT_FALSE(parse_timestamp_ns(StringRef(input), &value).ok()) << input; } } TEST(DataTypeTimeStampNsTest, RejectValuesOutsideEpochRange) { - int64_t value = 0; + TimeStampNsValue value; EXPECT_FALSE(parse_timestamp_ns(StringRef("0000-01-01 00:00:00.000000000"), &value).ok()); EXPECT_FALSE(parse_timestamp_ns(StringRef("1677-09-21 00:12:43.145224191"), &value).ok()); EXPECT_FALSE(parse_timestamp_ns(StringRef("2262-04-11 23:47:16.854775808"), &value).ok()); @@ -388,9 +387,8 @@ TEST(DataTypeTimeStampNsTest, DataTypeLiteralField) { } TEST(DataTypeTimeStampNsTest, FormattingAndHash) { - int64_t raw = 0; - ASSERT_TRUE(parse_timestamp_ns(StringRef("2024-02-29 12:34:56.123456789"), &raw).ok()); - TimeStampNsValue value(raw); + TimeStampNsValue value; + ASSERT_TRUE(parse_timestamp_ns(StringRef("2024-02-29 12:34:56.123456789"), &value).ok()); EXPECT_TRUE(value.is_valid_date()); diff --git a/be/test/exprs/function/cast/cast_to_timestamptz_test.cpp b/be/test/exprs/function/cast/cast_to_timestamptz_test.cpp index 9ca1912ba23..cc08b4a69e2 100644 --- a/be/test/exprs/function/cast/cast_to_timestamptz_test.cpp +++ b/be/test/exprs/function/cast/cast_to_timestamptz_test.cpp @@ -269,9 +269,9 @@ TEST_F(CastTimeStampTzTest, from_datetime_non_strict_mode_to_timestamptz) { TEST_F(CastTimeStampTzTest, timestamp_ns_and_timestamptz_round_trip) { const auto make_timestamp_ns = [](std::string_view text) { - int64_t epoch_nanos = 0; - EXPECT_TRUE(parse_timestamp_ns(StringRef {text.data(), text.size()}, &epoch_nanos).ok()); - return TimeStampNsValue(epoch_nanos); + TimeStampNsValue value; + EXPECT_TRUE(parse_timestamp_ns(StringRef {text.data(), text.size()}, &value).ok()); + return value; }; auto timestamp_ns_block = ColumnHelper::create_block<DataTypeTimeStampNs>( --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
