This is an automated email from the ASF dual-hosted git repository. dbecker pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit ff7a553324891d067a81a2dbf0089cf95866a48d Author: Steve Carlin <[email protected]> AuthorDate: Wed Oct 16 16:56:57 2024 -0700 IMPALA-13462: Added support for functions used in tpcds The tpcds queries contain some functions that require some modifications that the general function resolver cannot handle. These include: - Some functions don't have the same name within Calcite. An example of this is "is_not_null" which is "is_not_null_pred" in Impala. - The grouping function returns a tinyint in Impala which is different from Calcite. - The params for functions that adjust the scale (e.g. ROUND) need to handle casting of parameters in the Impala way which is different from Calcite. Also handled in this commit is turning on the identifier expansion in the Calcite validator. This is needed to fix some of the tpcds queries as well. Change-Id: Id451357f2fb92d35e09b100751f0f4a49760a51c Reviewed-on: http://gerrit.cloudera.org:8080/21947 Reviewed-by: Impala Public Jenkins <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- .../impala/calcite/functions/FunctionResolver.java | 65 +++++++++++++-------- .../operators/ImpalaAdjustScaleFunction.java | 68 ++++++++++++++++++++++ .../operators/ImpalaCustomOperatorTable.java | 29 +++++++++ .../calcite/operators/ImpalaGroupingFunction.java | 53 +++++++++++++++++ .../impala/calcite/operators/ImpalaOperator.java | 9 +-- .../impala/calcite/service/CalciteValidator.java | 3 + 6 files changed, 196 insertions(+), 31 deletions(-) diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/FunctionResolver.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/FunctionResolver.java index 014107630..887ec9194 100644 --- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/FunctionResolver.java +++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/FunctionResolver.java @@ -56,6 +56,8 @@ public class FunctionResolver { public static Map<SqlKind, String> CALCITE_KIND_TO_IMPALA_FUNC = ImmutableMap.<SqlKind, String> builder() .put(SqlKind.EQUALS, "eq") + .put(SqlKind.IS_NOT_NULL, "is_not_null_pred") + .put(SqlKind.IS_NULL, "is_null_pred") .put(SqlKind.GREATER_THAN, "gt") .put(SqlKind.GREATER_THAN_OR_EQUAL, "ge") .put(SqlKind.LESS_THAN, "lt") @@ -68,6 +70,12 @@ public class FunctionResolver { .put(SqlKind.SUM0, "sum_init_zero") .build(); + // Map of Calcite names to an Impala function name when the names are different + public static Map<String, String> CALCITE_NAME_TO_IMPALA_FUNC = + ImmutableMap.<String, String> builder() + .put("||", "concat") + .build(); + public static Set<SqlKind> ARITHMETIC_TYPES = ImmutableSet.<SqlKind> builder() .add(SqlKind.PLUS) @@ -113,39 +121,16 @@ public class FunctionResolver { List<RelDataType> argTypes, boolean exactMatch) { // Some names in Calcite don't map exactly to their corresponding Impala - // functions, so we get the right mapping from the HashMap table. + // functions, so we check to see if the mapping exists return getFunction(getMappedName(name, kind, argTypes), argTypes, exactMatch); } - private static Function getFunction(String name, List<RelDataType> argTypes, - boolean exactMatch) { - String lowercaseName = name.toLowerCase(); - - List<Type> impalaArgTypes = getArgTypes(lowercaseName, argTypes, exactMatch); - - Function searchDesc = new Function(new FunctionName(BuiltinsDb.NAME, lowercaseName), - impalaArgTypes, Type.INVALID, false); - - Function.CompareMode compareMode = exactMatch - ? Function.CompareMode.IS_INDISTINGUISHABLE - : Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF; - - Function fn = BuiltinsDb.getInstance().getFunction(searchDesc, compareMode); - - if (fn == null) { - LOG.debug("Failed to find function " + lowercaseName); - } - - return fn; - } - /** * For most Calcite operators, the function name within Calcite matches the * Impala function name. This method handles the exceptions to that rule. */ private static String getMappedName(String name, SqlKind kind, List<RelDataType> argTypes) { - // First check if any special mappings exist from Calcite SqlKinds to // Impala functions. String mappedName = CALCITE_KIND_TO_IMPALA_FUNC.get(kind); @@ -160,8 +145,38 @@ public class FunctionResolver { return mappedName; } + String lowercaseName = name.toLowerCase(); + // Next check if there are any names in Calcite that do not match + // the Impala function name. + mappedName = CALCITE_NAME_TO_IMPALA_FUNC.get(lowercaseName); + if (mappedName != null) { + return mappedName; + } + // If reached here, use the function name as given, no special mapping needed. - return name.toLowerCase(); + return lowercaseName; + } + + private static Function getFunction(String name, List<RelDataType> argTypes, + boolean exactMatch) { + String lowercaseName = name.toLowerCase(); + + List<Type> impalaArgTypes = getArgTypes(lowercaseName, argTypes, exactMatch); + + Function searchDesc = new Function(new FunctionName(BuiltinsDb.NAME, lowercaseName), + impalaArgTypes, Type.INVALID, false); + + Function.CompareMode compareMode = exactMatch + ? Function.CompareMode.IS_INDISTINGUISHABLE + : Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF; + + Function fn = BuiltinsDb.getInstance().getFunction(searchDesc, compareMode); + + if (fn == null) { + LOG.debug("Failed to find function " + lowercaseName); + } + + return fn; } private static List<Type> getArgTypes(String name, List<RelDataType> argTypes, diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaAdjustScaleFunction.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaAdjustScaleFunction.java new file mode 100644 index 000000000..778b424cf --- /dev/null +++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaAdjustScaleFunction.java @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.impala.calcite.operators; + +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlOperatorBinding; +import org.apache.calcite.sql.type.OperandTypes; +import org.apache.calcite.sql.type.SqlReturnTypeInference; +import org.apache.calcite.sql.SqlFunctionCategory; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.impala.catalog.ScalarType; +import org.apache.impala.catalog.Type; +import org.apache.impala.calcite.type.ImpalaTypeConverter; + +/** + * The ImpalaAdjustScaleFunction operator contains the special Impala rules + * needed for inferring the return type from its parameters. + */ +public class ImpalaAdjustScaleFunction extends SqlFunction { + + public ImpalaAdjustScaleFunction(String name) { + super(name, null, SqlKind.OTHER_FUNCTION, + null, null, OperandTypes.NUMERIC_OPTIONAL_INTEGER, + SqlFunctionCategory.NUMERIC); + } + + @Override + public RelDataType inferReturnType(SqlOperatorBinding opBinding) { + if (opBinding.getOperandType(0).getSqlTypeName().equals(SqlTypeName.DOUBLE) || + opBinding.getOperandType(0).getSqlTypeName().equals(SqlTypeName.FLOAT)) { + return ImpalaTypeConverter.getRelDataType(Type.DOUBLE); + } + + ScalarType impalaType = + (ScalarType) ImpalaTypeConverter.createImpalaType(opBinding.getOperandType(0)); + ScalarType decimalType = impalaType.getMinResolutionDecimal(); + + int precision = getName().equals("ROUND") || getName().equals("DROUND") + ? Math.min(decimalType.decimalPrecision() + 1, + ScalarType.MAX_PRECISION) + : decimalType.decimalPrecision(); + + Integer scale = opBinding.getOperandCount() > 1 + ? opBinding.getOperandLiteralValue(1, Integer.class) + : 0; + + Type newDecimalType = ScalarType.createDecimalType(precision, scale); + + return ImpalaTypeConverter.createRelDataType(newDecimalType); + } +} diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaCustomOperatorTable.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaCustomOperatorTable.java index a3f39cf78..6e3398617 100644 --- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaCustomOperatorTable.java +++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaCustomOperatorTable.java @@ -24,6 +24,7 @@ import org.apache.calcite.sql.SqlAggFunction; import org.apache.calcite.sql.SqlBinaryOperator; import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.SqlOperatorBinding; +import org.apache.calcite.sql.fun.ImpalaGroupingFunction; import org.apache.calcite.sql.fun.SqlMonotonicBinaryOperator; import org.apache.calcite.sql.fun.SqlCountAggFunction; import org.apache.calcite.sql.type.InferTypes; @@ -200,6 +201,34 @@ public class ImpalaCustomOperatorTable extends ReflectiveSqlOperatorTable { public static final SqlAggFunction COUNT = new SqlCountAggFunction("COUNT", OperandTypes.VARIADIC); + public static final ImpalaAdjustScaleFunction ROUND = + new ImpalaAdjustScaleFunction("ROUND"); + + public static final ImpalaAdjustScaleFunction DROUND = + new ImpalaAdjustScaleFunction("DROUND"); + + public static final ImpalaAdjustScaleFunction TRUNCATE = + new ImpalaAdjustScaleFunction("TRUNCATE"); + + public static final ImpalaAdjustScaleFunction TRUNC = + new ImpalaAdjustScaleFunction("TRUNC"); + + public static final ImpalaAdjustScaleFunction DTRUNC = + new ImpalaAdjustScaleFunction("DTRUNC"); + + public static final ImpalaGroupingFunction GROUPING = + new ImpalaGroupingFunction(); + + public static final SqlBinaryOperator CONCAT = + new SqlBinaryOperator( + "||", + SqlKind.OTHER, + 60, + true, + STRING_TYPE, + null, + OperandTypes.STRING_SAME_SAME_OR_ARRAY_SAME_SAME); + public static ImpalaCustomOperatorTable instance() { return INSTANCE.get(); } diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaGroupingFunction.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaGroupingFunction.java new file mode 100644 index 000000000..fa09b1f39 --- /dev/null +++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaGroupingFunction.java @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.calcite.sql.fun; + +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.sql.SqlOperatorBinding; +import org.apache.calcite.sql.type.ReturnTypes; +import org.apache.calcite.sql.type.SqlReturnTypeInference; + +/** + * Grouping operator for Impala + * + * We use the Calcite operator for the grouping function (deriving from + * SqlGroupingFunction) since Calcite understands known operator types. + * However, we can't use the SqlGroupingFunction directly because + * Calcite's version returns a BIGINT type. We override this in the + * getReturnTypeInference. + * + * One unfortunate hack is that this had to be declared in the same package + * as the SqlGroupingFunction package (org.apache.calcite.sql.fun) since the + * constructor there is protected. + */ +public class ImpalaGroupingFunction extends SqlGroupingFunction { + + public ImpalaGroupingFunction() { + super("GROUPING"); + } + + @Override + public SqlReturnTypeInference getReturnTypeInference() { + return ReturnTypes.TINYINT; + } + + @Override + public RelDataType inferReturnType(SqlOperatorBinding opBinding) { + return getReturnTypeInference().inferReturnType(opBinding); + } +} diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaOperator.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaOperator.java index 538d32ee4..e68081a60 100644 --- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaOperator.java +++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaOperator.java @@ -115,15 +115,12 @@ public class ImpalaOperator extends SqlFunction { } private List<RelDataType> getOperandTypes(SqlOperatorBinding opBinding) { - Preconditions.checkState(opBinding instanceof SqlCallBinding); - SqlCallBinding callBinding = (SqlCallBinding) opBinding; - List<RelDataType> operandTypes = new ArrayList<>(); - for (int i = 0; i < callBinding.getOperandCount(); ++i) { - if (callBinding.isOperandNull(i, false)) { + for (int i = 0; i < opBinding.getOperandCount(); ++i) { + if (opBinding.isOperandNull(i, false)) { operandTypes.add(ImpalaTypeConverter.getRelDataType(Type.NULL)); } else { - operandTypes.add(callBinding.getOperandType(i)); + operandTypes.add(opBinding.getOperandType(i)); } } return operandTypes; diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteValidator.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteValidator.java index 7ca3ddbb4..8bd6c7e17 100644 --- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteValidator.java +++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteValidator.java @@ -56,6 +56,9 @@ public class CalciteValidator implements CompilerStep { ImpalaOperatorTable.getInstance(), catalogReader, typeFactory, SqlValidator.Config.DEFAULT + // Impala requires identifier expansion (tpcds test queries fail + // without this) + .withIdentifierExpansion(true) .withConformance(ImpalaConformance.INSTANCE) ); }
