LuciferYang commented on code in PR #68148:
URL: https://github.com/apache/doris/pull/68148#discussion_r4049430084
##########
be/src/format/parquet/parquet_predicate.h:
##########
@@ -211,6 +212,55 @@ class ParquetPredicate {
RowRange row_group_range;
};
+ // An adjusted-to-UTC INT64 timestamp mapped to DATETIMEV2 is displayed in
local civil time, so
+ // its converted min/max are only a usable bound when the UTC interval
contains no backward
+ // clock transition. Mirror the unit/adjust derivation in
TimestampConverter::init and defer the
+ // transition check to the shared v2 helper. Returns true (usable) for
anything that is not such
+ // a timestamp: a non-adjusted timestamp is shown in UTC (no transitions),
and INT96 is handled
+ // by its own singleton rule before this is reached.
+ static bool adjusted_utc_timestamp_range_is_monotonic(const FieldSchema*
col_schema,
+ const std::string&
encoded_min,
+ const std::string&
encoded_max,
+ const
cctz::time_zone& ctz) {
+ if (col_schema->parquet_schema.type != tparquet::Type::type::INT64) {
+ return true;
+ }
+ const auto& schema = col_schema->parquet_schema;
+ bool adjusted = false;
+ int64_t units_per_second = 0;
+ if (schema.__isset.logicalType &&
schema.logicalType.__isset.TIMESTAMP) {
+ const auto& ts = schema.logicalType.TIMESTAMP;
+ adjusted = ts.isAdjustedToUTC;
+ if (ts.unit.__isset.MILLIS) {
+ units_per_second = 1000;
+ } else if (ts.unit.__isset.MICROS) {
+ units_per_second = 1000000;
+ } else if (ts.unit.__isset.NANOS) {
+ units_per_second = 1000000000;
+ }
+ } else if (schema.__isset.converted_type) {
+ // Legacy TIMESTAMP_MILLIS / TIMESTAMP_MICROS carry instant
(UTC-normalized) semantics.
+ if (schema.converted_type ==
tparquet::ConvertedType::TIMESTAMP_MILLIS) {
+ adjusted = true;
+ units_per_second = 1000;
+ } else if (schema.converted_type ==
tparquet::ConvertedType::TIMESTAMP_MICROS) {
+ adjusted = true;
+ units_per_second = 1000000;
+ }
+ }
+ if (!adjusted || units_per_second == 0) {
+ return true;
+ }
+ if (encoded_min.size() < sizeof(int64_t) || encoded_max.size() <
sizeof(int64_t)) {
Review Comment:
Fixed in db0b0e958b7. Added `physical_stat_width_ok`, called at the top of
`parse_min_max_value` before any load or copy; a wrong-width bound returns
`DataQualityError` up front, so a short INT64 cannot be read out of bounds and
an oversized INT96 cannot overrun the destination. Covered by
`ParseMinMaxRejectsMalformedStatWidth` (4-byte INT64 and 11-byte INT96).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
