This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new 33631047f IMPALA-13468: Fix various aggregation issues in
aggregation.test
33631047f is described below
commit 33631047f804e6ed41644c1294e277fc1e9ba358
Author: Steve Carlin <[email protected]>
AuthorDate: Sat Oct 19 11:59:22 2024 -0700
IMPALA-13468: Fix various aggregation issues in aggregation.test
Various small issues fixed including:
- There is a special operator dedicated to scalar functions not handled in
Calcite. The special agg operator equivalent was created
(ImpalaAggOperator)
- Grouping_id function needs to be handled in a special way, calling
AggregateFunction.createRewrittenFunction
- A custom Avg operator was created to handle avg(TIMESTAMP) which isn't
allowed
in Calcite.
- A custom Min/Max operator was created to handle min(NULL) and min(char
types).
- The corr, covar_pop, and covar_samp functions use the default Impala
function
resolver rather than the Calcite resolver.
Change-Id: I038127d6a2f228ae8d263e983b1906e99ae05f77
Reviewed-on: http://gerrit.cloudera.org:8080/21961
Reviewed-by: Michael Smith <[email protected]>
Tested-by: Impala Public Jenkins <[email protected]>
Reviewed-by: Joe McDonnell <[email protected]>
---
.../impala/calcite/functions/FunctionResolver.java | 23 ++++++
...aOperator.java => CommonOperatorFunctions.java} | 91 ++++++++++------------
.../calcite/operators/ImpalaAggOperator.java | 73 +++++++++++++++++
.../calcite/operators/ImpalaAvgAggFunction.java | 56 +++++++++++++
.../operators/ImpalaCustomOperatorTable.java | 41 ++++------
.../operators/ImpalaGroupingIdFunction.java | 39 ++++++++++
.../calcite/operators/ImpalaMinMaxAggFunction.java | 64 +++++++++++++++
.../impala/calcite/operators/ImpalaOperator.java | 72 +----------------
.../calcite/operators/ImpalaOperatorTable.java | 14 +++-
9 files changed, 325 insertions(+), 148 deletions(-)
diff --git
a/java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/FunctionResolver.java
b/java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/FunctionResolver.java
index 887ec9194..ef6936ae4 100644
---
a/java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/FunctionResolver.java
+++
b/java/calcite-planner/src/main/java/org/apache/impala/calcite/functions/FunctionResolver.java
@@ -27,6 +27,7 @@ import org.apache.calcite.rex.RexNode;
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.impala.analysis.FunctionName;
+import org.apache.impala.catalog.AggregateFunction;
import org.apache.impala.calcite.type.ImpalaTypeConverter;
import org.apache.impala.catalog.BuiltinsDb;
import org.apache.impala.catalog.Function;
@@ -85,6 +86,11 @@ public class FunctionResolver {
.add(SqlKind.MOD)
.build();
+ public static Set<String> SPECIAL_PROCESSING_FUNCTIONS =
+ ImmutableSet.<String> builder()
+ .add("grouping_id")
+ .build();
+
public static Function getSupertypeFunction(RexCall call) {
// For arithmetic types, we use separate logic. Calcite has already
// calculated the proper return type at validation time. So an
@@ -163,6 +169,23 @@ public class FunctionResolver {
List<Type> impalaArgTypes = getArgTypes(lowercaseName, argTypes,
exactMatch);
+ return SPECIAL_PROCESSING_FUNCTIONS.contains(lowercaseName)
+ ? getSpecialProcessingFunction(lowercaseName, impalaArgTypes,
exactMatch)
+ : getImpalaFunction(lowercaseName, impalaArgTypes, exactMatch);
+ }
+
+ private static Function getSpecialProcessingFunction(String lowercaseName,
+ List<Type> impalaArgTypes, boolean exactMatch) {
+ if (lowercaseName.equals("grouping_id")) {
+ return AggregateFunction.createRewrittenBuiltin(BuiltinsDb.getInstance(),
+ lowercaseName, impalaArgTypes, Type.BIGINT, true, false, true);
+ }
+
+ throw new RuntimeException("Special function not found: " + lowercaseName);
+ }
+
+ private static Function getImpalaFunction(String lowercaseName,
+ List<Type> impalaArgTypes, boolean exactMatch) {
Function searchDesc = new Function(new FunctionName(BuiltinsDb.NAME,
lowercaseName),
impalaArgTypes, Type.INVALID, false);
diff --git
a/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaOperator.java
b/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/CommonOperatorFunctions.java
similarity index 58%
copy from
java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaOperator.java
copy to
java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/CommonOperatorFunctions.java
index e68081a60..bd8fcac33 100644
---
a/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaOperator.java
+++
b/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/CommonOperatorFunctions.java
@@ -21,9 +21,11 @@ package org.apache.impala.calcite.operators;
import com.google.common.base.Preconditions;
import org.apache.impala.calcite.type.ImpalaTypeSystemImpl;
import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
+import org.apache.calcite.rel.core.Aggregate.AggCallBinding;
import org.apache.calcite.rel.type.RelDataType;
import org.apache.calcite.rel.type.RelDataTypeFactory;
import org.apache.calcite.rex.RexBuilder;
+import org.apache.calcite.rex.RexCallBinding;
import org.apache.calcite.sql.SqlCallBinding;
import org.apache.calcite.sql.SqlFunction;
import org.apache.calcite.sql.SqlFunctionCategory;
@@ -32,38 +34,28 @@ import org.apache.calcite.sql.SqlOperandCountRange;
import org.apache.calcite.sql.SqlOperatorBinding;
import org.apache.calcite.sql.SqlSyntax;
import org.apache.calcite.sql.type.SqlOperandCountRanges;
+import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.impala.calcite.functions.FunctionResolver;
import org.apache.impala.calcite.type.ImpalaTypeConverter;
import org.apache.impala.catalog.Function;
import org.apache.impala.catalog.Type;
+import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.List;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
/**
- * ImpalaOperator is a custom Calcite operator that handles all generic
functions
- * that are not defined by Calcite. It is preferable to use a Calcite operator
- * if possible because Calcite has optimizations that are based on the operator
- * class.
+ * ImpalaOperatorCommon contains common functionality across Agg and nonAgg
+ * operators.
*/
-public class ImpalaOperator extends SqlFunction {
- protected static final Logger LOG =
- LoggerFactory.getLogger(ImpalaOperator.class.getName());
+public class CommonOperatorFunctions {
// Allow any count because this is used for all functions. Validation for
specific
// number of parameters will be done when Impala function resolving is done.
public static SqlOperandCountRange ANY_COUNT_RANGE =
SqlOperandCountRanges.any();
- public ImpalaOperator(String name) {
- super(name.toUpperCase(), SqlKind.OTHER, null, null, null,
- SqlFunctionCategory.USER_DEFINED_FUNCTION);
- }
-
- @Override
- public RelDataType inferReturnType(SqlOperatorBinding opBinding) {
+ public static RelDataType inferReturnType(SqlOperatorBinding opBinding,
+ String name) {
final List<RelDataType> operandTypes = getOperandTypes(opBinding);
RexBuilder rexBuilder =
@@ -71,11 +63,11 @@ public class ImpalaOperator extends SqlFunction {
RelDataTypeFactory factory = rexBuilder.getTypeFactory();
// Resolve Impala function through Impala method.
- Function fn = FunctionResolver.getSupertypeFunction(getName(),
operandTypes);
+ Function fn = FunctionResolver.getSupertypeFunction(name, operandTypes);
if (fn == null) {
throw new IllegalArgumentException("Cannot infer return type for "
- + getName() + "; operand types: " + operandTypes);
+ + name + "; operand types: " + operandTypes);
}
RelDataType returnType =
@@ -85,15 +77,7 @@ public class ImpalaOperator extends SqlFunction {
: factory.createTypeWithNullability(returnType, true);
}
- @Override
- public boolean checkOperandTypes(SqlCallBinding callBinding, boolean
throwOnFailure) {
- // Validation for operand types are done when checking for signature in
- // the inferReturnType method.
- return true;
- }
-
- @Override
- public SqlOperandCountRange getOperandCountRange() {
+ public static SqlOperandCountRange getOperandCountRange() {
// Validation for operand count ranges are done when checking for
signature in
// the inferReturnType method.
return ANY_COUNT_RANGE;
@@ -102,37 +86,46 @@ public class ImpalaOperator extends SqlFunction {
/**
* getAllowedSignatures used for error messages.
*/
- @Override
- public String getAllowedSignatures(String opNameToUse) {
+ public static String getAllowedSignatures(String opNameToUse) {
// TODO: IMPALA-13099 leave blank for now since this might be hard to
derive because
// of implicit type support.
return "";
}
- @Override
- public SqlSyntax getSyntax() {
- return SqlSyntax.FUNCTION;
- }
-
- private List<RelDataType> getOperandTypes(SqlOperatorBinding opBinding) {
- List<RelDataType> operandTypes = new ArrayList<>();
+ public static List<RelDataType> getOperandTypes(SqlOperatorBinding
opBinding) {
+ List<RelDataType> operandTypes = new
ArrayList<>(opBinding.getOperandCount());
for (int i = 0; i < opBinding.getOperandCount(); ++i) {
- if (opBinding.isOperandNull(i, false)) {
- operandTypes.add(ImpalaTypeConverter.getRelDataType(Type.NULL));
- } else {
- operandTypes.add(opBinding.getOperandType(i));
- }
+ operandTypes.add(getOperandType(opBinding, i));
}
return operandTypes;
}
- // return false if all operand types are not nullable. Else return true.
- private boolean isNullable(List<RelDataType> operandTypes) {
- for (RelDataType rdt : operandTypes) {
- if (rdt.isNullable()) {
- return true;
- }
+ public static RelDataType getOperandType(SqlOperatorBinding opBinding, int
operand) {
+ if (opBinding instanceof AggCallBinding) {
+ return opBinding.getOperandType(operand);
+ }
+
+ if (opBinding.isOperandNull(operand, false)) {
+ return ImpalaTypeConverter.getRelDataType(Type.NULL);
}
- return false;
+
+ RelDataType opType = opBinding.getOperandType(operand);
+ if (opType.getSqlTypeName().equals(SqlTypeName.INTEGER) &&
+ opBinding.isOperandLiteral(operand, true)) {
+ // For literal types, currently Calcite treats all of them as INTEGERs.
Impala
+ // requires the smallest possible type (e.g. 2 should be a TINYINT), and
needs
+ // this to infer the proper return type. Note though: this method is
only used
+ // for inferring the return type and not coercing the operand, which
will stay
+ // an INTEGER for now. The operand will be coerced later in the
compilation,
+ // under the coercenodes modules.
+ BigDecimal bd0 = opBinding.getOperandLiteralValue(operand,
BigDecimal.class);
+ return ImpalaTypeConverter.getLiteralDataType(bd0, opType);
+ }
+ return opType;
+ }
+
+ // return true if any operand type is nullable.
+ private static boolean isNullable(List<RelDataType> operandTypes) {
+ return operandTypes.stream().anyMatch(rdt -> rdt.isNullable());
}
}
diff --git
a/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaAggOperator.java
b/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaAggOperator.java
new file mode 100644
index 000000000..751fb2d7f
--- /dev/null
+++
b/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaAggOperator.java
@@ -0,0 +1,73 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.calcite.operators;
+
+import com.google.common.base.Preconditions;
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.sql.SqlAggFunction;
+import org.apache.calcite.sql.SqlCallBinding;
+import org.apache.calcite.sql.SqlFunction;
+import org.apache.calcite.sql.SqlFunctionCategory;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.SqlOperandCountRange;
+import org.apache.calcite.sql.SqlOperatorBinding;
+import org.apache.calcite.sql.SqlSyntax;
+
+/**
+ * ImpalaAggOperator is a custom Calcite operator that handles all generic
functions
+ * that are not defined by Calcite. It is preferable to use a Calcite operator
+ * if possible because Calcite has optimizations that are based on the operator
+ * class or SqlKind.
+ */
+public class ImpalaAggOperator extends SqlAggFunction {
+
+ public ImpalaAggOperator(String name) {
+ super(name.toUpperCase(), SqlKind.OTHER, null, null, null,
+ SqlFunctionCategory.USER_DEFINED_FUNCTION);
+ }
+
+ @Override
+ public RelDataType inferReturnType(SqlOperatorBinding opBinding) {
+ return CommonOperatorFunctions.inferReturnType(opBinding, getName());
+ }
+
+ @Override
+ public boolean checkOperandTypes(SqlCallBinding callBinding, boolean
throwOnFailure) {
+ // Validation for operand types are done when checking for signature in
+ // the inferReturnType method.
+ return true;
+ }
+
+ @Override
+ public SqlOperandCountRange getOperandCountRange() {
+ return CommonOperatorFunctions.getOperandCountRange();
+ }
+
+ /**
+ * getAllowedSignatures used for error messages.
+ */
+ @Override
+ public String getAllowedSignatures(String opNameToUse) {
+ return CommonOperatorFunctions.getAllowedSignatures(opNameToUse);
+ }
+
+ @Override
+ public SqlSyntax getSyntax() {
+ return SqlSyntax.FUNCTION;
+ }
+}
diff --git
a/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaAvgAggFunction.java
b/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaAvgAggFunction.java
new file mode 100644
index 000000000..7d41cb98e
--- /dev/null
+++
b/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaAvgAggFunction.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.impala.calcite.operators;
+
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.sql.SqlCallBinding;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.SqlStaticAggFunction;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.SqlOperatorBinding;
+import org.apache.calcite.sql.fun.SqlAvgAggFunction;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.impala.calcite.type.ImpalaTypeConverter;
+import org.apache.impala.catalog.Type;
+
+/**
+ * Special implementation of Calcite's SqlAvgAggFunction which allows a
TIMESTAMP
+ * as a parameter.
+ */
+public class ImpalaAvgAggFunction extends SqlAvgAggFunction {
+
+ public ImpalaAvgAggFunction() {
+ super(SqlKind.AVG);
+ }
+
+ @Override
+ public RelDataType inferReturnType(SqlOperatorBinding opBinding) {
+ final RelDataType operandType =
CommonOperatorFunctions.getOperandType(opBinding, 0);
+ return operandType.getSqlTypeName().equals(SqlTypeName.TIMESTAMP)
+ ? operandType
+ : super.inferReturnType(opBinding);
+ }
+
+ @Override
+ public boolean checkOperandTypes(
+ SqlCallBinding callBinding,
+ boolean throwOnFailure) {
+ return
callBinding.getOperandType(0).getSqlTypeName().equals(SqlTypeName.TIMESTAMP)
+ ? true
+ : super.checkOperandTypes(callBinding, throwOnFailure);
+ }
+}
diff --git
a/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaCustomOperatorTable.java
b/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaCustomOperatorTable.java
index 6e3398617..fc8741fc5 100644
---
a/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaCustomOperatorTable.java
+++
b/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaCustomOperatorTable.java
@@ -70,10 +70,8 @@ public class ImpalaCustomOperatorTable extends
ReflectiveSqlOperatorTable {
return ImpalaTypeConverter.getRelDataType(Type.TIMESTAMP);
}
- RelDataType type0 =
- getOperandForArithmeticOpSqlOperatorBinding(opBinding, operandTypes,
0);
- RelDataType type1 =
- getOperandForArithmeticOpSqlOperatorBinding(opBinding, operandTypes,
1);
+ RelDataType type0 = CommonOperatorFunctions.getOperandType(opBinding, 0);
+ RelDataType type1 = CommonOperatorFunctions.getOperandType(opBinding, 1);
RelDataTypeFactory typeFactory = opBinding.getTypeFactory();
ImpalaTypeSystemImpl typeSystemImpl =
@@ -81,34 +79,12 @@ public class ImpalaCustomOperatorTable extends
ReflectiveSqlOperatorTable {
return typeSystemImpl.deriveArithmeticType(typeFactory, type0, type1, op);
};
- private static RelDataType getOperandForArithmeticOpSqlOperatorBinding(
- SqlOperatorBinding opBinding, List<RelDataType> operandTypes, int index)
{
- if (opBinding instanceof ExplicitOperatorBinding) {
- return operandTypes.get(index);
- } else if (opBinding.isOperandNull(index, true)) {
- // for nulltypes, just assume smallest possible number type
- return ImpalaTypeConverter.getRelDataType(Type.TINYINT);
- } else if
(operandTypes.get(index).getSqlTypeName().equals(SqlTypeName.INTEGER) &&
- opBinding.isOperandLiteral(index, true)) {
- // For literal types, currently Calcite treats all of them as INTEGERs.
Impala
- // requires the smallest possible type (e.g. 2 should be a TINYINT), and
needs
- // this to infer the proper return type. Note though: this method is
only used
- // for inferring the return type and not coercing the operand, which
will stay
- // an INTEGER for now. The operand will be coerced later in the
compilation,
- // under the coercenodes modules.
- BigDecimal bd0 = opBinding.getOperandLiteralValue(index,
BigDecimal.class);
- RelDataType rdt =
- ImpalaTypeConverter.getLiteralDataType(bd0,
opBinding.getOperandType(index));
- return rdt;
- }
-
- return opBinding.getOperandType(index);
- }
-
private static final Supplier<ImpalaCustomOperatorTable> INSTANCE =
Suppliers.memoize(() ->
(ImpalaCustomOperatorTable) new ImpalaCustomOperatorTable().init());
+ public static final SqlAggFunction AVG = new ImpalaAvgAggFunction();
+
public static final SqlReturnTypeInference ADD_ADJUSTED_RETURN_TYPE =
opBinding -> {
return inferReturnTypeForArithmeticOps(opBinding,
ArithmeticExpr.Operator.ADD);
};
@@ -219,6 +195,15 @@ public class ImpalaCustomOperatorTable extends
ReflectiveSqlOperatorTable {
public static final ImpalaGroupingFunction GROUPING =
new ImpalaGroupingFunction();
+ public static final SqlAggFunction MIN =
+ new ImpalaMinMaxAggFunction(SqlKind.MIN);
+
+ public static final SqlAggFunction MAX =
+ new ImpalaMinMaxAggFunction(SqlKind.MAX);
+
+ public static final SqlAggFunction GROUPING_ID =
+ new ImpalaGroupingIdFunction();
+
public static final SqlBinaryOperator CONCAT =
new SqlBinaryOperator(
"||",
diff --git
a/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaGroupingIdFunction.java
b/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaGroupingIdFunction.java
new file mode 100644
index 000000000..d3598831e
--- /dev/null
+++
b/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaGroupingIdFunction.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.impala.calcite.operators;
+
+
+import org.apache.calcite.sql.SqlAggFunction;
+import org.apache.calcite.sql.SqlFunctionCategory;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.SqlSyntax;
+import org.apache.calcite.sql.type.OperandTypes;
+import org.apache.calcite.sql.type.ReturnTypes;
+import org.apache.calcite.util.Optionality;
+
+/**
+ * ImpalaGroupingIdFunction replaces the grouping id function used
+ * in Calcite. This version returns a BIGINT.
+ */
+public class ImpalaGroupingIdFunction extends SqlAggFunction {
+ public ImpalaGroupingIdFunction() {
+ super("GROUPING_ID", null, SqlKind.GROUPING_ID, ReturnTypes.BIGINT, null,
+ OperandTypes.VARIADIC, SqlFunctionCategory.SYSTEM, false, false,
+ Optionality.FORBIDDEN);
+ }
+}
+
diff --git
a/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaMinMaxAggFunction.java
b/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaMinMaxAggFunction.java
new file mode 100644
index 000000000..03bdc5fb0
--- /dev/null
+++
b/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaMinMaxAggFunction.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.impala.calcite.operators;
+
+import org.apache.calcite.rel.type.RelDataType;
+import org.apache.calcite.sql.SqlCallBinding;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.SqlStaticAggFunction;
+import org.apache.calcite.sql.SqlOperator;
+import org.apache.calcite.sql.SqlOperatorBinding;
+import org.apache.calcite.sql.fun.SqlMinMaxAggFunction;
+import org.apache.calcite.sql.type.SqlTypeName;
+import org.apache.impala.calcite.type.ImpalaTypeConverter;
+import org.apache.impala.catalog.Type;
+
+/**
+ * Special implementation of Calcite's SqlMinMaxAggFunction which allows a
TIMESTAMP
+ * as a parameter.
+ */
+public class ImpalaMinMaxAggFunction extends SqlMinMaxAggFunction {
+
+ public ImpalaMinMaxAggFunction(SqlKind kind) {
+ super(kind);
+ }
+
+ @Override
+ public RelDataType inferReturnType(SqlOperatorBinding opBinding) {
+ final RelDataType operandType =
CommonOperatorFunctions.getOperandType(opBinding, 0);
+ switch (operandType.getSqlTypeName()) {
+ case NULL:
+ return ImpalaTypeConverter.getRelDataType(Type.BOOLEAN);
+ case CHAR:
+ case VARCHAR:
+ return ImpalaTypeConverter.getRelDataType(Type.STRING);
+ case TIMESTAMP:
+ return ImpalaTypeConverter.getRelDataType(Type.TIMESTAMP);
+ default:
+ return super.inferReturnType(opBinding);
+ }
+ }
+
+ @Override
+ public boolean checkOperandTypes(
+ SqlCallBinding callBinding,
+ boolean throwOnFailure) {
+ return
callBinding.getOperandType(0).getSqlTypeName().equals(SqlTypeName.TIMESTAMP)
+ ? true
+ : super.checkOperandTypes(callBinding, throwOnFailure);
+ }
+}
diff --git
a/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaOperator.java
b/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaOperator.java
index e68081a60..641e24b89 100644
---
a/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaOperator.java
+++
b/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaOperator.java
@@ -17,13 +17,8 @@
package org.apache.impala.calcite.operators;
-
import com.google.common.base.Preconditions;
-import org.apache.impala.calcite.type.ImpalaTypeSystemImpl;
-import org.apache.calcite.jdbc.JavaTypeFactoryImpl;
import org.apache.calcite.rel.type.RelDataType;
-import org.apache.calcite.rel.type.RelDataTypeFactory;
-import org.apache.calcite.rex.RexBuilder;
import org.apache.calcite.sql.SqlCallBinding;
import org.apache.calcite.sql.SqlFunction;
import org.apache.calcite.sql.SqlFunctionCategory;
@@ -31,17 +26,6 @@ import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.SqlOperandCountRange;
import org.apache.calcite.sql.SqlOperatorBinding;
import org.apache.calcite.sql.SqlSyntax;
-import org.apache.calcite.sql.type.SqlOperandCountRanges;
-import org.apache.impala.calcite.functions.FunctionResolver;
-import org.apache.impala.calcite.type.ImpalaTypeConverter;
-import org.apache.impala.catalog.Function;
-import org.apache.impala.catalog.Type;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* ImpalaOperator is a custom Calcite operator that handles all generic
functions
@@ -50,12 +34,6 @@ import org.slf4j.LoggerFactory;
* class.
*/
public class ImpalaOperator extends SqlFunction {
- protected static final Logger LOG =
- LoggerFactory.getLogger(ImpalaOperator.class.getName());
-
- // Allow any count because this is used for all functions. Validation for
specific
- // number of parameters will be done when Impala function resolving is done.
- public static SqlOperandCountRange ANY_COUNT_RANGE =
SqlOperandCountRanges.any();
public ImpalaOperator(String name) {
super(name.toUpperCase(), SqlKind.OTHER, null, null, null,
@@ -64,25 +42,7 @@ public class ImpalaOperator extends SqlFunction {
@Override
public RelDataType inferReturnType(SqlOperatorBinding opBinding) {
- final List<RelDataType> operandTypes = getOperandTypes(opBinding);
-
- RexBuilder rexBuilder =
- new RexBuilder(new JavaTypeFactoryImpl(new ImpalaTypeSystemImpl()));
- RelDataTypeFactory factory = rexBuilder.getTypeFactory();
-
- // Resolve Impala function through Impala method.
- Function fn = FunctionResolver.getSupertypeFunction(getName(),
operandTypes);
-
- if (fn == null) {
- throw new IllegalArgumentException("Cannot infer return type for "
- + getName() + "; operand types: " + operandTypes);
- }
-
- RelDataType returnType =
- ImpalaTypeConverter.getRelDataType(fn.getReturnType());
- return isNullable(operandTypes)
- ? returnType
- : factory.createTypeWithNullability(returnType, true);
+ return CommonOperatorFunctions.inferReturnType(opBinding, getName());
}
@Override
@@ -94,9 +54,7 @@ public class ImpalaOperator extends SqlFunction {
@Override
public SqlOperandCountRange getOperandCountRange() {
- // Validation for operand count ranges are done when checking for
signature in
- // the inferReturnType method.
- return ANY_COUNT_RANGE;
+ return CommonOperatorFunctions.getOperandCountRange();
}
/**
@@ -104,35 +62,11 @@ public class ImpalaOperator extends SqlFunction {
*/
@Override
public String getAllowedSignatures(String opNameToUse) {
- // TODO: IMPALA-13099 leave blank for now since this might be hard to
derive because
- // of implicit type support.
- return "";
+ return CommonOperatorFunctions.getAllowedSignatures(opNameToUse);
}
@Override
public SqlSyntax getSyntax() {
return SqlSyntax.FUNCTION;
}
-
- private List<RelDataType> getOperandTypes(SqlOperatorBinding opBinding) {
- List<RelDataType> operandTypes = new ArrayList<>();
- for (int i = 0; i < opBinding.getOperandCount(); ++i) {
- if (opBinding.isOperandNull(i, false)) {
- operandTypes.add(ImpalaTypeConverter.getRelDataType(Type.NULL));
- } else {
- operandTypes.add(opBinding.getOperandType(i));
- }
- }
- return operandTypes;
- }
-
- // return false if all operand types are not nullable. Else return true.
- private boolean isNullable(List<RelDataType> operandTypes) {
- for (RelDataType rdt : operandTypes) {
- if (rdt.isNullable()) {
- return true;
- }
- }
- return false;
- }
}
diff --git
a/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaOperatorTable.java
b/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaOperatorTable.java
index 323198440..f9dbab1ee 100644
---
a/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaOperatorTable.java
+++
b/java/calcite-planner/src/main/java/org/apache/impala/calcite/operators/ImpalaOperatorTable.java
@@ -26,8 +26,10 @@ import org.apache.calcite.sql.SqlOperator;
import org.apache.calcite.sql.SqlSyntax;
import org.apache.calcite.sql.fun.SqlStdOperatorTable;
import org.apache.calcite.sql.util.ReflectiveSqlOperatorTable;
+import org.apache.impala.catalog.AggregateFunction;
import org.apache.impala.catalog.BuiltinsDb;
import org.apache.impala.catalog.Db;
+import org.apache.impala.catalog.Function;
import java.util.List;
import java.util.Set;
@@ -69,6 +71,9 @@ public class ImpalaOperatorTable extends
ReflectiveSqlOperatorTable {
.add("minute")
.add("second")
.add("millisecond")
+ .add("corr")
+ .add("covar_pop")
+ .add("covar_samp")
.build();
private static ImpalaOperatorTable INSTANCE;
@@ -107,11 +112,16 @@ public class ImpalaOperatorTable extends
ReflectiveSqlOperatorTable {
}
// Check Impala Builtins for existence: TODO: IMPALA-13095: handle UDFs
- if (!BuiltinsDb.getInstance().containsFunction(lowercaseOpName)) {
+ List<Function> functions =
BuiltinsDb.getInstance().getFunctions(lowercaseOpName);
+ if (functions.size() == 0) {
return;
}
- operatorList.add(new ImpalaOperator(opName.getSimple()));
+ SqlOperator impalaOp = (functions.get(0) instanceof AggregateFunction)
+ ? new ImpalaAggOperator(opName.getSimple())
+ : new ImpalaOperator(opName.getSimple());
+
+ operatorList.add(impalaOp);
}
public static synchronized void create(Db db) {
