This is an automated email from the ASF dual-hosted git repository. joemcdonnell pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit 9a2496dd288ea23857c67cade25f96807c5e61dc Author: Steve Carlin <[email protected]> AuthorDate: Tue Nov 5 06:35:16 2024 -0800 IMPALA-13511: Calcite planner: support sub-second datetime parts This commit allows support for milliseconds, microseconds, and nanoseconds for Calcite. The ImpalaSqlIntervalQualifier class extends the Calcite SqlIntervalQualifier class which handles most datetime parts. Some code was copied from the base class since some of the methods were private, but in general, the code handling the parts is exactly the same. Change-Id: I392c3900c70e7754a35ef25fc720ba4a2f2e5dd6 Reviewed-on: http://gerrit.cloudera.org:8080/22029 Reviewed-by: Michael Smith <[email protected]> Reviewed-by: Joe McDonnell <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- java/calcite-planner/src/main/codegen/config.fmpp | 1 + .../src/main/codegen/templates/Parser.jj | 150 +++++++++++++------ .../calcite/coercenodes/CoerceOperandShuttle.java | 6 + .../calcite/operators/ImpalaOperatorTable.java | 3 + .../calcite/type/ImpalaSqlIntervalQualifier.java | 159 +++++++++++++++++++++ 5 files changed, 273 insertions(+), 46 deletions(-) diff --git a/java/calcite-planner/src/main/codegen/config.fmpp b/java/calcite-planner/src/main/codegen/config.fmpp index 18886790a..9cd56e6c3 100644 --- a/java/calcite-planner/src/main/codegen/config.fmpp +++ b/java/calcite-planner/src/main/codegen/config.fmpp @@ -24,6 +24,7 @@ parser: { # List of additional classes and packages to import. # Example: "org.apache.calcite.sql.*", "java.util.List". imports: [ + "org.apache.impala.calcite.type.ImpalaSqlIntervalQualifier" ] # List of new keywords. Example: "DATABASES", "TABLES". If the keyword is diff --git a/java/calcite-planner/src/main/codegen/templates/Parser.jj b/java/calcite-planner/src/main/codegen/templates/Parser.jj index f21f912c1..128750f30 100644 --- a/java/calcite-planner/src/main/codegen/templates/Parser.jj +++ b/java/calcite-planner/src/main/codegen/templates/Parser.jj @@ -121,6 +121,7 @@ import org.apache.calcite.util.Pair; import org.apache.calcite.util.SourceStringReader; import org.apache.calcite.util.Util; import org.apache.calcite.util.trace.CalciteTrace; +import org.apache.impala.calcite.type.ImpalaSqlIntervalQualifier; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -4890,7 +4891,7 @@ SqlNode PeriodConstructor() : SqlLiteral IntervalLiteral() : { final String p; - final SqlIntervalQualifier intervalQualifier; + final ImpalaSqlIntervalQualifier intervalQualifier; int sign = 1; final Span s; } @@ -4914,7 +4915,7 @@ SqlLiteral IntervalLiteral() : SqlNode IntervalLiteralOrExpression() : { final String p; - final SqlIntervalQualifier intervalQualifier; + final ImpalaSqlIntervalQualifier intervalQualifier; int sign = 1; final Span s; SqlNode e; @@ -5027,7 +5028,34 @@ TimeUnit Second() : <SECONDS> { return warn(TimeUnit.SECOND); } } -SqlIntervalQualifier IntervalQualifier() : +TimeUnit MilliSecond() : +{ +} +{ + <MILLISECOND> { return TimeUnit.MILLISECOND; } +| + <MILLISECONDS> { return warn(TimeUnit.MILLISECOND); } +} + +TimeUnit MicroSecond() : +{ +} +{ + <MICROSECOND> { return TimeUnit.MICROSECOND; } +| + <MICROSECONDS> { return warn(TimeUnit.MICROSECOND); } +} + +TimeUnit NanoSecond() : +{ +} +{ + <NANOSECOND> { return TimeUnit.NANOSECOND; } +| + <NANOSECONDS> { return warn(TimeUnit.NANOSECOND); } +} + +ImpalaSqlIntervalQualifier IntervalQualifier() : { final Span s; final TimeUnit start; @@ -5093,15 +5121,39 @@ SqlIntervalQualifier IntervalQualifier() : | { startPrec = -1; } ) { end = null; } + | + start = MilliSecond() { s = span(); } + ( + <LPAREN> startPrec = UnsignedIntLiteral() + <RPAREN> + | { startPrec = -1; } + ) + { end = null; } + | + start = MicroSecond() { s = span(); } + ( + <LPAREN> startPrec = UnsignedIntLiteral() + <RPAREN> + | { startPrec = -1; } + ) + { end = null; } + | + start = NanoSecond() { s = span(); } + ( + <LPAREN> startPrec = UnsignedIntLiteral() + <RPAREN> + | { startPrec = -1; } + ) + { end = null; } ) { - return new SqlIntervalQualifier(start, startPrec, end, secondFracPrec, + return new ImpalaSqlIntervalQualifier(start, startPrec, end, secondFracPrec, s.end(this)); } } /** Interval qualifier without 'TO unit'. */ -SqlIntervalQualifier IntervalQualifierStart() : +ImpalaSqlIntervalQualifier IntervalQualifierStart() : { final Span s; final TimeUnit start; @@ -5118,6 +5170,9 @@ SqlIntervalQualifier IntervalQualifierStart() : | start = Day() | start = Hour() | start = Minute() + | start = MilliSecond() + | start = MicroSecond() + | start = NanoSecond() ) { s = span(); } startPrec = PrecisionOpt() @@ -5129,14 +5184,14 @@ SqlIntervalQualifier IntervalQualifierStart() : ] ) { - return new SqlIntervalQualifier(start, startPrec, null, secondFracPrec, + return new ImpalaSqlIntervalQualifier(start, startPrec, null, secondFracPrec, s.end(this)); } } /** Parses a built-in time unit (e.g. "YEAR") * or user-defined time frame (e.g. "MINUTE15") - * and in each case returns a {@link SqlIntervalQualifier}. + * and in each case returns a {@link ImpalaSqlIntervalQualifier}. * * <p>The units are used in several functions, incuding CEIL, FLOOR, EXTRACT. * Includes NANOSECOND, MILLISECOND, which were previously allowed in EXTRACT @@ -5149,9 +5204,9 @@ SqlIntervalQualifier IntervalQualifierStart() : * parsed as identifiers and can be resolved in the validator if they are * registered as abbreviations in your time frame set. */ -SqlIntervalQualifier TimeUnitOrName() : { +ImpalaSqlIntervalQualifier TimeUnitOrName() : { final SqlIdentifier unitName; - final SqlIntervalQualifier intervalQualifier; + final ImpalaSqlIntervalQualifier intervalQualifier; } { // When we see a time unit that is also a non-reserved keyword, such as @@ -5166,13 +5221,13 @@ SqlIntervalQualifier TimeUnitOrName() : { return intervalQualifier; } | unitName = SimpleIdentifier() { - return new SqlIntervalQualifier(unitName.getSimple(), + return new ImpalaSqlIntervalQualifier(unitName.getSimple(), unitName.getParserPosition()); } } /** Parses a built-in time unit (e.g. "YEAR") - * and returns a {@link SqlIntervalQualifier}. + * and returns a {@link ImpalaSqlIntervalQualifier}. * * <p>Includes {@code WEEK} and {@code WEEK(SUNDAY)} through {@code WEEK(SATURDAY)}. @@ -5181,42 +5236,42 @@ SqlIntervalQualifier TimeUnitOrName() : { * parsed as identifiers and can be resolved in the validator if they are * registered as abbreviations in your time frame set. */ -SqlIntervalQualifier TimeUnit() : { +ImpalaSqlIntervalQualifier TimeUnit() : { final Span span; final String w; } { - <NANOSECOND> { return new SqlIntervalQualifier(TimeUnit.NANOSECOND, null, getPos()); } -| <MICROSECOND> { return new SqlIntervalQualifier(TimeUnit.MICROSECOND, null, getPos()); } -| <MILLISECOND> { return new SqlIntervalQualifier(TimeUnit.MILLISECOND, null, getPos()); } -| <SECOND> { return new SqlIntervalQualifier(TimeUnit.SECOND, null, getPos()); } -| <MINUTE> { return new SqlIntervalQualifier(TimeUnit.MINUTE, null, getPos()); } -| <HOUR> { return new SqlIntervalQualifier(TimeUnit.HOUR, null, getPos()); } -| <DAY> { return new SqlIntervalQualifier(TimeUnit.DAY, null, getPos()); } -| <DAYOFWEEK> { return new SqlIntervalQualifier(TimeUnit.DOW, null, getPos()); } -| <DAYOFYEAR> { return new SqlIntervalQualifier(TimeUnit.DOY, null, getPos()); } -| <DOW> { return new SqlIntervalQualifier(TimeUnit.DOW, null, getPos()); } -| <DOY> { return new SqlIntervalQualifier(TimeUnit.DOY, null, getPos()); } -| <ISODOW> { return new SqlIntervalQualifier(TimeUnit.ISODOW, null, getPos()); } -| <ISOYEAR> { return new SqlIntervalQualifier(TimeUnit.ISOYEAR, null, getPos()); } + <NANOSECOND> { return new ImpalaSqlIntervalQualifier(TimeUnit.NANOSECOND, null, getPos()); } +| <MICROSECOND> { return new ImpalaSqlIntervalQualifier(TimeUnit.MICROSECOND, null, getPos()); } +| <MILLISECOND> { return new ImpalaSqlIntervalQualifier(TimeUnit.MILLISECOND, null, getPos()); } +| <SECOND> { return new ImpalaSqlIntervalQualifier(TimeUnit.SECOND, null, getPos()); } +| <MINUTE> { return new ImpalaSqlIntervalQualifier(TimeUnit.MINUTE, null, getPos()); } +| <HOUR> { return new ImpalaSqlIntervalQualifier(TimeUnit.HOUR, null, getPos()); } +| <DAY> { return new ImpalaSqlIntervalQualifier(TimeUnit.DAY, null, getPos()); } +| <DAYOFWEEK> { return new ImpalaSqlIntervalQualifier(TimeUnit.DOW, null, getPos()); } +| <DAYOFYEAR> { return new ImpalaSqlIntervalQualifier(TimeUnit.DOY, null, getPos()); } +| <DOW> { return new ImpalaSqlIntervalQualifier(TimeUnit.DOW, null, getPos()); } +| <DOY> { return new ImpalaSqlIntervalQualifier(TimeUnit.DOY, null, getPos()); } +| <ISODOW> { return new ImpalaSqlIntervalQualifier(TimeUnit.ISODOW, null, getPos()); } +| <ISOYEAR> { return new ImpalaSqlIntervalQualifier(TimeUnit.ISOYEAR, null, getPos()); } | <WEEK> { span = span(); } ( // There is a choice between "WEEK(weekday)" and "WEEK". We prefer // the former, and the parser will look ahead for '('. LOOKAHEAD(2) <LPAREN> w = weekdayName() <RPAREN> { - return new SqlIntervalQualifier(w, span.end(this)); + return new ImpalaSqlIntervalQualifier(w, span.end(this)); } | - { return new SqlIntervalQualifier(TimeUnit.WEEK, null, getPos()); } + { return new ImpalaSqlIntervalQualifier(TimeUnit.WEEK, null, getPos()); } ) -| <MONTH> { return new SqlIntervalQualifier(TimeUnit.MONTH, null, getPos()); } -| <QUARTER> { return new SqlIntervalQualifier(TimeUnit.QUARTER, null, getPos()); } -| <YEAR> { return new SqlIntervalQualifier(TimeUnit.YEAR, null, getPos()); } -| <EPOCH> { return new SqlIntervalQualifier(TimeUnit.EPOCH, null, getPos()); } -| <DECADE> { return new SqlIntervalQualifier(TimeUnit.DECADE, null, getPos()); } -| <CENTURY> { return new SqlIntervalQualifier(TimeUnit.CENTURY, null, getPos()); } -| <MILLENNIUM> { return new SqlIntervalQualifier(TimeUnit.MILLENNIUM, null, getPos()); } +| <MONTH> { return new ImpalaSqlIntervalQualifier(TimeUnit.MONTH, null, getPos()); } +| <QUARTER> { return new ImpalaSqlIntervalQualifier(TimeUnit.QUARTER, null, getPos()); } +| <YEAR> { return new ImpalaSqlIntervalQualifier(TimeUnit.YEAR, null, getPos()); } +| <EPOCH> { return new ImpalaSqlIntervalQualifier(TimeUnit.EPOCH, null, getPos()); } +| <DECADE> { return new ImpalaSqlIntervalQualifier(TimeUnit.DECADE, null, getPos()); } +| <CENTURY> { return new ImpalaSqlIntervalQualifier(TimeUnit.CENTURY, null, getPos()); } +| <MILLENNIUM> { return new ImpalaSqlIntervalQualifier(TimeUnit.MILLENNIUM, null, getPos()); } } String weekdayName() : @@ -6098,7 +6153,7 @@ SqlNode BuiltinFunctionCall() : SqlNode e; final Span s; SqlDataTypeSpec dt; - final SqlIntervalQualifier unit; + final ImpalaSqlIntervalQualifier unit; final SqlNode node; final SqlLiteral style; // mssql convert 'style' operand final SqlFunction f; @@ -6853,7 +6908,7 @@ SqlCall DateDiffFunctionCall() : { final List<SqlNode> args = new ArrayList<SqlNode>(); final Span s; - final SqlIntervalQualifier unit; + final ImpalaSqlIntervalQualifier unit; } { <DATE_DIFF> { s = span(); } @@ -6875,7 +6930,7 @@ SqlCall TimestampAddFunctionCall() : { final List<SqlNode> args = new ArrayList<SqlNode>(); final Span s; - final SqlIntervalQualifier unit; + final ImpalaSqlIntervalQualifier unit; } { <TIMESTAMPADD> { s = span(); } @@ -6898,7 +6953,7 @@ SqlCall TimestampDiffFunctionCall() : { final List<SqlNode> args = new ArrayList<SqlNode>(); final Span s; - final SqlIntervalQualifier unit; + final ImpalaSqlIntervalQualifier unit; } { <TIMESTAMPDIFF> { s = span(); } @@ -6926,7 +6981,7 @@ SqlCall TimestampDiff3FunctionCall() : { final List<SqlNode> args = new ArrayList<SqlNode>(); final Span s; - final SqlIntervalQualifier unit; + final ImpalaSqlIntervalQualifier unit; } { <TIMESTAMP_DIFF> { s = span(); } @@ -6948,7 +7003,7 @@ SqlCall DatetimeDiffFunctionCall() : { final List<SqlNode> args = new ArrayList<SqlNode>(); final Span s; - final SqlIntervalQualifier unit; + final ImpalaSqlIntervalQualifier unit; } { <DATETIME_DIFF> { s = span(); } @@ -6970,7 +7025,7 @@ SqlCall DateTruncFunctionCall() : { final List<SqlNode> args = new ArrayList<SqlNode>(); final Span s; - final SqlIntervalQualifier unit; + final ImpalaSqlIntervalQualifier unit; } { <DATE_TRUNC> { s = span(); } @@ -6998,7 +7053,7 @@ SqlCall TimestampTruncFunctionCall() : { final List<SqlNode> args = new ArrayList<SqlNode>(); final Span s; - final SqlIntervalQualifier unit; + final ImpalaSqlIntervalQualifier unit; } { <TIMESTAMP_TRUNC> { s = span(); } @@ -7018,7 +7073,7 @@ SqlCall TimeDiffFunctionCall() : { final List<SqlNode> args = new ArrayList<SqlNode>(); final Span s; - final SqlIntervalQualifier unit; + final ImpalaSqlIntervalQualifier unit; } { <TIME_DIFF> { s = span(); } @@ -7040,7 +7095,7 @@ SqlNode DatetimeTruncFunctionCall() : { final List<SqlNode> args = new ArrayList<SqlNode>(); final Span s; - final SqlIntervalQualifier unit; + final ImpalaSqlIntervalQualifier unit; final SqlNode literal; } { @@ -7061,7 +7116,7 @@ SqlCall TimeTruncFunctionCall() : { final List<SqlNode> args = new ArrayList<SqlNode>(); final Span s; - final SqlIntervalQualifier unit; + final ImpalaSqlIntervalQualifier unit; } { <TIME_TRUNC> { s = span(); } @@ -7376,7 +7431,7 @@ SqlNode StandardFloorCeilOptions(Span s, boolean floorFlag) : { SqlNode e; final List<SqlNode> args = new ArrayList<SqlNode>(); - final SqlIntervalQualifier unit; + final ImpalaSqlIntervalQualifier unit; SqlCall function; final Span s1; } @@ -8187,7 +8242,9 @@ SqlPostfixOperator PostfixRowOperator() : | < MESSAGE_TEXT: "MESSAGE_TEXT" > | < METHOD: "METHOD" > | < MICROSECOND: "MICROSECOND" > +| < MICROSECONDS: "MICROSECONDS" > | < MILLISECOND: "MILLISECOND" > +| < MILLISECONDS: "MILLISECONDS" > | < MILLENNIUM: "MILLENNIUM" > | < MIN: "MIN" > | < MINUTE: "MINUTE" > @@ -8205,6 +8262,7 @@ SqlPostfixOperator PostfixRowOperator() : | < NAME: "NAME" > | < NAMES: "NAMES" > | < NANOSECOND: "NANOSECOND" > +| < NANOSECONDS: "NANOSECONDS" > | < NATIONAL: "NATIONAL" > | < NATURAL: "NATURAL" > | < NCHAR: "NCHAR" > diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/coercenodes/CoerceOperandShuttle.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/coercenodes/CoerceOperandShuttle.java index 67aac1060..8c845e0f5 100644 --- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/coercenodes/CoerceOperandShuttle.java +++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/coercenodes/CoerceOperandShuttle.java @@ -349,6 +349,12 @@ public class CoerceOperandShuttle extends RexShuttle { RelDataType fromType = node.getType(); + // no need to cast if type is interval, this will get handled when + // changing to the physical Expr object. + if (SqlTypeUtil.isInterval(fromType)) { + return node; + } + // No need to cast if types are the same if (fromType.getSqlTypeName().equals(toType.getSqlTypeName())) { return node; diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaOperatorTable.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaOperatorTable.java index 8e9e28059..354e33b19 100644 --- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaOperatorTable.java +++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaOperatorTable.java @@ -71,6 +71,9 @@ public class ImpalaOperatorTable extends ReflectiveSqlOperatorTable { .add("minute") .add("second") .add("millisecond") + .add("dayofmonth") + .add("dayofyear") + .add("weekofyear") .add("corr") .add("covar_pop") .add("covar_samp") diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/type/ImpalaSqlIntervalQualifier.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/type/ImpalaSqlIntervalQualifier.java new file mode 100644 index 000000000..da10640d8 --- /dev/null +++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/type/ImpalaSqlIntervalQualifier.java @@ -0,0 +1,159 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.impala.calcite.type; + +import org.apache.calcite.avatica.util.TimeUnit; +import org.apache.calcite.avatica.util.TimeUnitRange; +import org.apache.calcite.rel.type.RelDataTypeSystem; +import org.apache.calcite.runtime.CalciteContextException; +import org.apache.calcite.sql.SqlUtil; +import org.apache.calcite.sql.SqlIntervalQualifier; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.util.Util; + +import static org.apache.calcite.linq4j.Nullness.castNonNull; +import static org.apache.calcite.util.Static.RESOURCE; +import static java.util.Objects.requireNonNull; + +import java.math.BigDecimal; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * ImpalaSqlIntervalQualifier is an extension of the Calcite SqlIntervalQualifier + * class. The Calcite version does not handle sub-millisecond parts (the other parts + * are still processed in the super class). In order to handle these parts, some + * private methods had to be copied from the base class. + */ +public class ImpalaSqlIntervalQualifier extends SqlIntervalQualifier { + + public ImpalaSqlIntervalQualifier( + TimeUnit startUnit, + int startPrecision, + TimeUnit endUnit, + int fractionalSecondPrecision, + SqlParserPos pos) { + super(startUnit, startPrecision, endUnit, fractionalSecondPrecision, pos); + } + + public ImpalaSqlIntervalQualifier( + TimeUnit startUnit, + TimeUnit endUnit, + SqlParserPos pos) { + super(startUnit, endUnit, pos); + } + + public ImpalaSqlIntervalQualifier(String timeFrameName, + SqlParserPos pos) { + super(timeFrameName, pos); + } + + @Override + public int[] evaluateIntervalLiteral(String value, SqlParserPos pos, + RelDataTypeSystem typeSystem) { + + switch(timeUnitRange) { + case MILLISECOND: + case MICROSECOND: + case NANOSECOND: + return super.evaluateIntervalLiteral(value, pos, typeSystem); + } + + // save original value for if we have to throw + final String value0 = value; + + // First strip off any leading whitespace + value = value.trim(); + + // check if the sign was explicitly specified. Record + // the explicit or implicit sign, and strip it off to + // simplify pattern matching later. + final int sign = getIntervalSign(value); + value = stripLeadingSign(value); + + // If we have an empty or null literal at this point, + // it's illegal. Complain and bail out. + if (Util.isNullOrEmpty(value)) { + throw invalidValueException(pos, value0); + } + + return evaluateIntervalLiteralAsSubSeconds(typeSystem, sign, value, value0, + pos); + } + + private int[] evaluateIntervalLiteralAsSubSeconds( + RelDataTypeSystem typeSystem, int sign, + String value, + String originalValue, + SqlParserPos pos) { + BigDecimal subseconds; + + // validate the integer decimal pattern used for subseconds + String intervalPattern = "(\\d+)"; + + Matcher m = Pattern.compile(intervalPattern).matcher(value); + if (m.matches()) { + // Break out field values + try { + subseconds = parseField(m, 1); + } catch (NumberFormatException e) { + throw invalidValueException(pos, originalValue); + } + + // package values up for return + return fillIntervalValueArray(sign, subseconds); + } else { + throw invalidValueException(pos, originalValue); + } + } + + private static String stripLeadingSign(String value) { + String unsignedValue = value; + + if (!Util.isNullOrEmpty(value)) { + if (('-' == value.charAt(0)) || ('+' == value.charAt(0))) { + unsignedValue = value.substring(1); + } + } + + return unsignedValue; + } + + private static BigDecimal parseField(Matcher m, int i) { + return new BigDecimal(castNonNull(m.group(i))); + } + + private CalciteContextException invalidValueException(SqlParserPos pos, + String value) { + return SqlUtil.newContextException(pos, + RESOURCE.unsupportedIntervalLiteral( + "'" + value + "'", "INTERVAL " + toString())); + } + + private static int[] fillIntervalValueArray( + int sign, + BigDecimal subseconds) { + int[] ret = new int[3]; + + ret[0] = sign; + ret[1] = subseconds.intValue(); + ret[2] = 0; + + return ret; + } +}
