This is an automated email from the ASF dual-hosted git repository. joemcdonnell pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit b028da18c926efacac11c466d316dc64656dd689 Author: Steve Carlin <[email protected]> AuthorDate: Wed Mar 27 08:15:54 2024 -0700 IMPALA-12947: Implement Calcite Planner Union and Value RelNodes This commit handles Union and Value RelNode operators The Union RelNode is created within Calcite when there is a "union" clause. The Values RelNode is created when the lowest level does not come from a table, but instead comes from constant values. For example, the query "select 3" would create a Values RelNode with one literal value of 3. The PlanNode creation simulates what already exists within the Impala planner. There is no corresponding Values PlanNode. Instead, a Union is created with the values expression serving as inputs expressions (hence the reason of combining these 2 RelNodes in the same commit). Other plan nodes used are the "SelectNode" and the "EmptySetNode". The EmptySetNode is used where there are no rows coming from the value node. While this cannot be simulated at this point, this will be needed when we start introducing optimization rules, and will be tested when we turn on the Impala test framework queries. The SelectNode is used for functions that are applied on top of the UnionNode. There is a major issue with this iteration of Union and Value nodes due to a Calcite issue. Calcite currently treats all string literals as "CHAR" type. This causes problems in the union operator if one tries to implement the following query: "select 'a' union select 'ab'", since the 2 types in the value clauses are CHAR(1) and CHAR(2) which do not match. This would cause an exception on the server. A future commit will fix this issue. Also of concern is that Calcite treats non-bigint constant as integers only. That is, 3, 257, 65539 are all considered of type INT. This will also be fixed in a later commit. Change-Id: Ibd989dbb5cf0df0fcc88f72dd579ce4fd713f547 Reviewed-on: http://gerrit.cloudera.org:8080/21211 Reviewed-by: Joe McDonnell <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- .../java/org/apache/impala/planner/SelectNode.java | 10 ++ .../calcite/rel/node/ConvertToImpalaRelRules.java | 29 ++++- .../impala/calcite/rel/node/ImpalaFilterRel.java | 5 + .../impala/calcite/rel/node/ImpalaHdfsScanRel.java | 5 + .../impala/calcite/rel/node/ImpalaPlanRel.java | 23 +++- .../impala/calcite/rel/node/ImpalaProjectRel.java | 5 + .../impala/calcite/rel/node/ImpalaUnionRel.java | 89 +++++++++++++++ .../impala/calcite/rel/node/ImpalaValuesRel.java | 119 +++++++++++++++++++++ .../impala/calcite/rel/node/NodeCreationUtils.java | 115 ++++++++++++++++++++ .../calcite/rel/node/ParentPlanRelContext.java | 25 ++++- .../impala/calcite/rel/phys/ImpalaUnionNode.java | 45 ++++++++ .../calcite/rel/util/TupleDescriptorFactory.java | 96 +++++++++++++++++ .../impala/calcite/service/CalciteOptimizer.java | 4 +- .../calcite/service/CalcitePhysPlanCreator.java | 7 +- .../queries/QueryTest/calcite.test | 38 +++++++ 15 files changed, 606 insertions(+), 9 deletions(-) diff --git a/fe/src/main/java/org/apache/impala/planner/SelectNode.java b/fe/src/main/java/org/apache/impala/planner/SelectNode.java index 1bde0d41a..99353789f 100644 --- a/fe/src/main/java/org/apache/impala/planner/SelectNode.java +++ b/fe/src/main/java/org/apache/impala/planner/SelectNode.java @@ -92,6 +92,16 @@ public class SelectNode extends PlanNode { return selectNode; } + /** + * Create a SelectNode directly from the Calcite module. The Calcite module + * has done some pre-analysis, so it can bypass some of the steps in the + * "create". + */ + public static SelectNode createFromCalcite(PlanNodeId id, PlanNode child, + List<Expr> conjuncts) { + return new SelectNode(id, child, conjuncts); + } + @Override public void computeStats(Analyzer analyzer) { super.computeStats(analyzer); diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ConvertToImpalaRelRules.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ConvertToImpalaRelRules.java index 9965152ac..32f376199 100644 --- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ConvertToImpalaRelRules.java +++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ConvertToImpalaRelRules.java @@ -19,10 +19,11 @@ package org.apache.impala.calcite.rel.node; import org.apache.calcite.plan.RelOptRule; import org.apache.calcite.plan.RelOptRuleCall; -import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.logical.LogicalFilter; import org.apache.calcite.rel.logical.LogicalProject; import org.apache.calcite.rel.logical.LogicalTableScan; +import org.apache.calcite.rel.logical.LogicalUnion; +import org.apache.calcite.rel.logical.LogicalValues; /** * ConvertToImpalaRelRules. Contains the rules used to change the Calcite RelNodes @@ -69,4 +70,30 @@ public class ConvertToImpalaRelRules { } } + public static class ImpalaUnionRule extends RelOptRule { + + public ImpalaUnionRule() { + super(operand(LogicalUnion.class, any())); + } + + @Override + public void onMatch(RelOptRuleCall call) { + final LogicalUnion union = call.rel(0); + call.transformTo(new ImpalaUnionRel(union)); + } + } + + public static class ImpalaValuesRule extends RelOptRule { + + public ImpalaValuesRule() { + super(operand(LogicalValues.class, none())); + } + + @Override + public void onMatch(RelOptRuleCall call) { + final LogicalValues values = call.rel(0); + call.transformTo(new ImpalaValuesRel(values)); + } + } + } diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ImpalaFilterRel.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ImpalaFilterRel.java index 1a3eaddc9..a247ee17a 100644 --- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ImpalaFilterRel.java +++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ImpalaFilterRel.java @@ -99,4 +99,9 @@ public class ImpalaFilterRel extends Filter List<RexNode> conditions = ImmutableList.of(previousCondition, newCondition); return RexUtil.composeConjunction(getCluster().getRexBuilder(), conditions); } + + @Override + public RelNodeType relNodeType() { + return RelNodeType.FILTER; + } } diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ImpalaHdfsScanRel.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ImpalaHdfsScanRel.java index 608b066ef..3a294cf2d 100644 --- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ImpalaHdfsScanRel.java +++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ImpalaHdfsScanRel.java @@ -129,4 +129,9 @@ public class ImpalaHdfsScanRel extends TableScan } return inputRefFieldNames; } + + @Override + public RelNodeType relNodeType() { + return RelNodeType.HDFSSCAN; + } } diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ImpalaPlanRel.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ImpalaPlanRel.java index aca02da40..1e11b7d13 100644 --- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ImpalaPlanRel.java +++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ImpalaPlanRel.java @@ -17,18 +17,39 @@ package org.apache.impala.calcite.rel.node; -import org.apache.impala.common.ImpalaException; +import com.google.common.collect.ImmutableList; +import org.apache.impala.common.ImpalaException; /** * ImpalaPlanRel. Interface used for all Impala intermediary RelNodes */ public interface ImpalaPlanRel { + /** + * Enum representing the type of class used in the RelNode + * Using an enum here so that Impala Plan RelNodes can be used in + * a "case" statement rather than using multiple "instanceof" statements. + * Also, the PROJECT enum is used for two different RelNodes, the normal + * project and a Project containing analytical functions which needs to + * be handled differently. + */ + public enum RelNodeType { + FILTER, + HDFSSCAN, + PROJECT, + UNION, + VALUES + } + /** * getPlanNode returns a NodeWithExprs object, a thin structure containing * the plan node along with the output expressions generated by the plan node. */ public NodeWithExprs getPlanNode(ParentPlanRelContext context) throws ImpalaException; + /** + * Return the RelNodeType enum of the implementing class of ImpalaPlanRel. + */ + public RelNodeType relNodeType(); } diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ImpalaProjectRel.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ImpalaProjectRel.java index bc30436d5..d8f571351 100644 --- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ImpalaProjectRel.java +++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ImpalaProjectRel.java @@ -110,4 +110,9 @@ public class ImpalaProjectRel extends Project builder.setInputRefs(RelOptUtil.InputFinder.bits(getProjects(), null)); return relInput.getPlanNode(builder.build()); } + + @Override + public RelNodeType relNodeType() { + return RelNodeType.PROJECT; + } } diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ImpalaUnionRel.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ImpalaUnionRel.java new file mode 100644 index 000000000..9d087c893 --- /dev/null +++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ImpalaUnionRel.java @@ -0,0 +1,89 @@ +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.apache.impala.calcite.rel.node; + +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Union; +import org.apache.calcite.rel.type.RelDataType; + +import org.apache.impala.common.ImpalaException; +import org.apache.impala.planner.PlanNodeId; + +import java.util.ArrayList; +import java.util.List; + +/** + * Impala RelNode class for Union. + * One note: Both "union distinct" and "union all" are handled by this class. + * Calcite handles the "union distinct" by changing it to a "union all" plus + * aggregation. + */ +public class ImpalaUnionRel extends Union + implements ImpalaPlanRel { + + public ImpalaUnionRel(Union union) { + super(union.getCluster(), union.getTraitSet(), union.getInputs(), union.all); + } + + private ImpalaUnionRel(RelOptCluster cluster, RelTraitSet traits, + List<RelNode> inputs, boolean all) { + super(cluster, traits, inputs, all); + } + + @Override + public ImpalaUnionRel copy(RelTraitSet traitSet, List<RelNode> inputs, boolean all) { + return new ImpalaUnionRel(getCluster(), traitSet, inputs, all); + } + + @Override + public NodeWithExprs getPlanNode(ParentPlanRelContext context) throws ImpalaException { + PlanNodeId nodeId = context.ctx_.getNextNodeId(); + + RelDataType rowType = getRowType(); + + List<NodeWithExprs> nodeWithExprsList = getChildrenPlanNodes(getInputs(), context); + + NodeWithExprs retNode = NodeCreationUtils.createUnionPlanNode(nodeId, + context.ctx_.getRootAnalyzer(), rowType, nodeWithExprsList); + + // If there is a filter condition, a SelectNode will get added on top + // of the retNode. + return NodeCreationUtils.wrapInSelectNodeIfNeeded(context, retNode, + getCluster().getRexBuilder()); + } + + private List<NodeWithExprs> getChildrenPlanNodes(List<RelNode> relInputs, + ParentPlanRelContext context) throws ImpalaException { + List<NodeWithExprs> childrenNodes = new ArrayList<>(); + for (RelNode input : relInputs) { + ImpalaPlanRel inputRel = (ImpalaPlanRel) input; + ParentPlanRelContext.Builder builder = + new ParentPlanRelContext.Builder(context, this); + builder.setFilterCondition(null); + childrenNodes.add(inputRel.getPlanNode(builder.build())); + } + return childrenNodes; + } + + @Override + public RelNodeType relNodeType() { + return RelNodeType.UNION; + } +} diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ImpalaValuesRel.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ImpalaValuesRel.java new file mode 100644 index 000000000..6a39eb573 --- /dev/null +++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ImpalaValuesRel.java @@ -0,0 +1,119 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.apache.impala.calcite.rel.node; + +import com.google.common.base.Preconditions; +import org.apache.calcite.rel.core.Values; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.sql.type.SqlTypeName; + +import org.apache.impala.analysis.Expr; +import org.apache.impala.calcite.rel.util.ExprConjunctsConverter; +import org.apache.impala.common.AnalysisException; +import org.apache.impala.common.ImpalaException; +import org.apache.impala.planner.PlanNode; +import org.apache.impala.planner.PlanNodeId; + +import java.util.ArrayList; +import java.util.List; + +/** + * ImpalaValuesRel handles the Values RelNode in Calcite. + * + * The Values node will be turned into one of the following: + * - If the parent node is a Union RelNode, no node will be + * created. Instead, the list of Expr will be sent up, and + * the union node will incorporate the Expr list into its node. + * - If the parent node is a filter node, a SelectNode will be + * created. + * - else, a Union node will be created with the values. + */ +public class ImpalaValuesRel extends Values + implements ImpalaPlanRel { + + public ImpalaValuesRel(Values values) { + super(values.getCluster(), values.getRowType(), values.getTuples(), + values.getTraitSet()); + } + + @Override + public NodeWithExprs getPlanNode(ParentPlanRelContext context) throws ImpalaException { + // Value RelNode will generate a Union PlanNode to hold the values. There is + // no need to create another Union node if the parent is already a Union node. + if (context.parentType_ == RelNodeType.UNION && (getTuples().size() == 1)) { + return getValuesExprs(context, getTuples().get(0)); + } + + PlanNodeId nodeId = context.ctx_.getNextNodeId(); + + RelDataType rowType = getRowType(); + + List<NodeWithExprs> nodeWithExprsList = getValuesExprs(context); + + NodeWithExprs retNode = NodeCreationUtils.createUnionPlanNode(nodeId, + context.ctx_.getRootAnalyzer(), rowType, nodeWithExprsList); + + // If there is a filter condition, a SelectNode will get added on top + // of the retNode. + return NodeCreationUtils.wrapInSelectNodeIfNeeded(context, retNode, + getCluster().getRexBuilder()); + } + + private List<NodeWithExprs> getValuesExprs(ParentPlanRelContext context + ) throws ImpalaException { + List<NodeWithExprs> nodeWithExprsList = new ArrayList<>(); + for (List<RexLiteral> literals : getTuples()) { + nodeWithExprsList.add(getValuesExprs(context, literals)); + } + return nodeWithExprsList; + } + + private NodeWithExprs getValuesExprs(ParentPlanRelContext context, + List<RexLiteral> literals) throws ImpalaException { + + PlanNode retNode = null; + + List<Expr> outputExprs = new ArrayList<>(); + for (RexLiteral literal : literals) { + // TODO: IMPALA-13022: Char types are currently disabled. The problem is a bit + // complex. Impala expects string literals to be of type STRING. Calcite does not + // have a type STRING and instead creates literals of type CHAR<x>, where <x> is + // the size of the char literal. This causes a couple of problems: + // 1) If there is a Union on top of a Values (e.g. select 'hello' union select + // 'goodbye') there will be a type mismatch (e.g. char(5) and char(7)) which will + // cause an impalad crash. The server crash is the main reason for disabling this + // 2) The return type of the root expression should be "string". While this really + // only will matter once CTAS support is enabled, it still is something that should + // be flagged as not working right now. + if (literal.getType().getSqlTypeName().equals(SqlTypeName.CHAR)) { + throw new AnalysisException("Char type values are not yet supported."); + } + ExprConjunctsConverter converter = new ExprConjunctsConverter(literal, + new ArrayList<>(), getCluster().getRexBuilder(), + context.ctx_.getRootAnalyzer()); + outputExprs.addAll(converter.getImpalaConjuncts()); + } + + return new NodeWithExprs(retNode, outputExprs); + } + + @Override + public RelNodeType relNodeType() { + return RelNodeType.VALUES; + } +} diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/NodeCreationUtils.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/NodeCreationUtils.java new file mode 100644 index 000000000..ee5c5ee45 --- /dev/null +++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/NodeCreationUtils.java @@ -0,0 +1,115 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.apache.impala.calcite.rel.node; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import org.apache.impala.analysis.Analyzer; +import org.apache.impala.analysis.Expr; +import org.apache.impala.analysis.SlotDescriptor; +import org.apache.impala.analysis.SlotRef; +import org.apache.impala.analysis.TupleDescriptor; +import org.apache.impala.planner.EmptySetNode; +import org.apache.impala.planner.PlanNodeId; +import org.apache.impala.planner.SelectNode; +import org.apache.impala.planner.UnionNode; +import org.apache.impala.calcite.rel.phys.ImpalaUnionNode; +import org.apache.impala.calcite.rel.util.ExprConjunctsConverter; +import org.apache.impala.calcite.rel.util.TupleDescriptorFactory; +import org.apache.impala.common.ImpalaException; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexNode; + +import java.util.List; + +public class NodeCreationUtils { + + /** + * Create an Impala SelectNode. Usually created when a normal Impala node cannot + * handle a filter expression, so we need a standalone node to do the filter. + */ + public static NodeWithExprs createSelectNode(RexNode filterCondition, Analyzer analyzer, + NodeWithExprs nodeWithExprs, PlanNodeId nodeId, RexBuilder rexBuilder + ) throws ImpalaException { + Preconditions.checkNotNull(filterCondition); + ExprConjunctsConverter converter = new ExprConjunctsConverter(filterCondition, + nodeWithExprs.outputExprs_, rexBuilder, analyzer); + List<Expr> filterConjuncts = converter.getImpalaConjuncts(); + SelectNode selectNode = + SelectNode.createFromCalcite(nodeId, nodeWithExprs.planNode_, filterConjuncts); + selectNode.init(analyzer); + return new NodeWithExprs(selectNode, nodeWithExprs.outputExprs_); + } + + public static NodeWithExprs createEmptySetPlanNode(PlanNodeId nodeId, + Analyzer analyzer, RelDataType rowType) throws ImpalaException { + TupleDescriptorFactory tupleDescFactory = + new TupleDescriptorFactory("empty set", rowType); + TupleDescriptor tupleDesc = tupleDescFactory.create(analyzer); + + EmptySetNode emptySetNode = + new EmptySetNode(nodeId, ImmutableList.of(tupleDesc.getId())); + + emptySetNode.init(analyzer); + + List<Expr> outputExprs = createOutputExprs(tupleDesc.getSlots()); + + return new NodeWithExprs(emptySetNode, outputExprs); + } + + public static NodeWithExprs createUnionPlanNode(PlanNodeId nodeId, + Analyzer analyzer, RelDataType rowType, List<NodeWithExprs> childrenPlanNodes + ) throws ImpalaException { + TupleDescriptorFactory tupleDescFactory = + new TupleDescriptorFactory("union", rowType); + TupleDescriptor tupleDesc = tupleDescFactory.create(analyzer); + // The outputexprs are the SlotRef exprs passed to the parent node. + List<Expr> outputExprs = createOutputExprs(tupleDesc.getSlots()); + + UnionNode unionNode = new ImpalaUnionNode(nodeId, tupleDesc.getId(), outputExprs, + childrenPlanNodes); + + unionNode.init(analyzer); + + return new NodeWithExprs(unionNode, outputExprs); + } + + public static List<Expr> createOutputExprs(List<SlotDescriptor> slotDescs) { + ImmutableList.Builder<Expr> builder = new ImmutableList.Builder(); + for (SlotDescriptor slotDesc : slotDescs) { + slotDesc.setIsMaterialized(true); + builder.add(new SlotRef(slotDesc)); + } + return builder.build(); + } + + /** + * wrapInSelectNodeIfNeeded is used for PlanNodes that cannot handle + * filter conditions directly. A SelectNode with the filter is created + * in this case. + */ + public static NodeWithExprs wrapInSelectNodeIfNeeded(ParentPlanRelContext context, + NodeWithExprs planNode, RexBuilder rexBuilder) throws ImpalaException { + return context.filterCondition_ != null + ? NodeCreationUtils.createSelectNode(context.filterCondition_, + context.ctx_.getRootAnalyzer(), planNode, + context.ctx_.getNextNodeId(), rexBuilder) + : planNode; + } +} diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ParentPlanRelContext.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ParentPlanRelContext.java index 1da2bd2b5..50953665f 100644 --- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ParentPlanRelContext.java +++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/node/ParentPlanRelContext.java @@ -19,7 +19,6 @@ package org.apache.impala.calcite.rel.node; import org.apache.calcite.rex.RexNode; import org.apache.calcite.util.ImmutableBitSet; -import org.apache.impala.analysis.Analyzer; import org.apache.impala.planner.PlannerContext; /** @@ -38,24 +37,44 @@ public class ParentPlanRelContext { // The input refs used by the parent PlanRel Node public final ImmutableBitSet inputRefs_; + // type of parent RelNode + public final ImpalaPlanRel.RelNodeType parentType_; + + /** + * Constructor meant for root node. + */ + private ParentPlanRelContext(PlannerContext plannerContext) { + this.ctx_ = plannerContext; + this.filterCondition_ = null; + this.inputRefs_ = null; + this.parentType_ = null; + } + private ParentPlanRelContext(Builder builder) { this.ctx_ = builder.context_; this.filterCondition_ = builder.filterCondition_; this.inputRefs_ = builder.inputRefs_; + this.parentType_ = builder.parentType_; } public static class Builder { private PlannerContext context_; private RexNode filterCondition_; private ImmutableBitSet inputRefs_; + private ImpalaPlanRel.RelNodeType parentType_; + /** + * Should only be called from root level. + */ public Builder(PlannerContext plannerContext) { this.context_ = plannerContext; + this.parentType_ = null; } public Builder(ParentPlanRelContext planRelContext, ImpalaPlanRel planRel) { this.context_ = planRelContext.ctx_; this.filterCondition_ = planRelContext.filterCondition_; + this.parentType_ = planRel.relNodeType(); } public void setFilterCondition(RexNode filterCondition) { @@ -70,4 +89,8 @@ public class ParentPlanRelContext { return new ParentPlanRelContext(this); } } + + public static ParentPlanRelContext createRootContext(PlannerContext context) { + return new ParentPlanRelContext(context); + } } diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/phys/ImpalaUnionNode.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/phys/ImpalaUnionNode.java new file mode 100644 index 000000000..3315d315f --- /dev/null +++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/phys/ImpalaUnionNode.java @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.apache.impala.calcite.rel.phys; + +import org.apache.impala.analysis.Expr; +import org.apache.impala.analysis.TupleId; +import org.apache.impala.calcite.rel.node.NodeWithExprs; +import org.apache.impala.planner.PlanNodeId; +import org.apache.impala.planner.UnionNode; + +import java.util.List; + +/** + * ImpalaUnionNode can be created either from a Union RelNode or a Values RelNode. + * If it comes from a Values RelNode, there is no child node. + */ +public class ImpalaUnionNode extends UnionNode { + + public ImpalaUnionNode(PlanNodeId id, TupleId tupleId, List<Expr> resultExprs, + List<NodeWithExprs> planNodeAndExprsList) { + super(id, tupleId, resultExprs, false /* subPlanId */); + for (NodeWithExprs planNodeAndExprs : planNodeAndExprsList) { + // planNode is null when produced by a LogicalValues + if (planNodeAndExprs.planNode_ == null) { + addConstExprList(planNodeAndExprs.outputExprs_); + } else { + addChild(planNodeAndExprs.planNode_, planNodeAndExprs.outputExprs_); + } + } + } +} diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/util/TupleDescriptorFactory.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/util/TupleDescriptorFactory.java new file mode 100644 index 000000000..d029c7ef3 --- /dev/null +++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/rel/util/TupleDescriptorFactory.java @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package org.apache.impala.calcite.rel.util; + +import com.google.common.base.Preconditions; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeField; + +import org.apache.impala.analysis.Analyzer; +import org.apache.impala.analysis.Expr; +import org.apache.impala.analysis.SlotDescriptor; +import org.apache.impala.analysis.TupleDescriptor; +import org.apache.impala.catalog.Type; +import org.apache.impala.calcite.type.ImpalaTypeConverter; +import org.apache.impala.common.ImpalaException; + +import java.util.ArrayList; +import java.util.List; + +/** + * TupleDescriptorFactory creates an ImpalaTupleDescriptor needed for a PlanNode. + */ +public class TupleDescriptorFactory { + + private final String tupleLabel; + private final List<RelDataTypeField> relDataTypeFields; + private final List<String> fieldLabels; + + public TupleDescriptorFactory(String tupleLabel, List<Expr> exprList, + RelDataType rowType) { + this(tupleLabel, getLabelsFromExprs(exprList), rowType.getFieldList()); + } + + public TupleDescriptorFactory(String tupleLabel, RelDataType rowType) { + this(tupleLabel, getLabelsFromRelDataType(rowType), rowType.getFieldList()); + } + + private TupleDescriptorFactory(String tupleLabel, List<String> fieldLabels, + List<RelDataTypeField> relDataTypeFields) { + this.tupleLabel = tupleLabel; + this.fieldLabels = fieldLabels; + this.relDataTypeFields = relDataTypeFields; + Preconditions.checkArgument(fieldLabels.size() == relDataTypeFields.size()); + } + + /** + * Create the TupleDescriptor. This method will mutate the analyzer by adding its + * TupleDescriptor and associated SlotDescriptors. + */ + public TupleDescriptor create(Analyzer analyzer) throws ImpalaException { + TupleDescriptor tupleDesc = analyzer.getDescTbl().createTupleDescriptor(tupleLabel); + tupleDesc.setIsMaterialized(true); + + for (int i = 0; i < relDataTypeFields.size(); i++) { + RelDataTypeField relDataTypeField = relDataTypeFields.get(i); + String fieldLabel = fieldLabels.get(i); + SlotDescriptor slotDesc = analyzer.addSlotDescriptor(tupleDesc); + Type impalaType = ImpalaTypeConverter.createImpalaType(relDataTypeField.getType()); + slotDesc.setType(impalaType); + slotDesc.setLabel(fieldLabel); + slotDesc.setIsMaterialized(true); + } + tupleDesc.computeMemLayout(); + return tupleDesc; + } + + private static List<String> getLabelsFromExprs(List<Expr> exprs) { + List<String> fieldLabels = new ArrayList<>(); + for (Expr expr : exprs) { + fieldLabels.add(expr.toSql()); + } + return fieldLabels; + } + + private static List<String> getLabelsFromRelDataType(RelDataType rowType) { + List<String> fieldLabels = new ArrayList<>(); + for (RelDataTypeField field : rowType.getFieldList()) { + fieldLabels.add(field.getName()); + } + return fieldLabels; + } +} diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteOptimizer.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteOptimizer.java index 2f747d661..b194ecae9 100644 --- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteOptimizer.java +++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalciteOptimizer.java @@ -31,8 +31,6 @@ import org.apache.impala.calcite.rel.node.ImpalaPlanRel; import org.apache.impala.common.ImpalaException; import org.apache.impala.common.InternalException; -import java.util.List; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -59,6 +57,8 @@ public class CalciteOptimizer implements CompilerStep { ImmutableList.of( new ConvertToImpalaRelRules.ImpalaScanRule(), new ConvertToImpalaRelRules.ImpalaFilterRule(), + new ConvertToImpalaRelRules.ImpalaValuesRule(), + new ConvertToImpalaRelRules.ImpalaUnionRule(), new ConvertToImpalaRelRules.ImpalaProjectRule())); HepPlanner planner = new HepPlanner(builder.build(), diff --git a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalcitePhysPlanCreator.java b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalcitePhysPlanCreator.java index 0c3d459ab..55811219a 100644 --- a/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalcitePhysPlanCreator.java +++ b/java/calcite-planner/src/main/java/org/apache/impala/calcite/service/CalcitePhysPlanCreator.java @@ -21,7 +21,6 @@ import org.apache.impala.calcite.rel.node.NodeWithExprs; import org.apache.impala.calcite.rel.node.ImpalaPlanRel; import org.apache.impala.authorization.AuthorizationFactory; import org.apache.impala.analysis.Analyzer; -import org.apache.impala.analysis.Expr; import org.apache.impala.calcite.rel.node.ParentPlanRelContext; import org.apache.impala.common.ImpalaException; import org.apache.impala.planner.PlannerContext; @@ -63,9 +62,9 @@ public class CalcitePhysPlanCreator implements CompilerStep { * returns the root plan node along with its output expressions. */ public NodeWithExprs create(ImpalaPlanRel optimizedPlan) throws ImpalaException { - ParentPlanRelContext.Builder builder = - new ParentPlanRelContext.Builder(plannerContext_); - NodeWithExprs rootNodeWithExprs = optimizedPlan.getPlanNode(builder.build()); + ParentPlanRelContext rootContext = + ParentPlanRelContext.createRootContext(plannerContext_); + NodeWithExprs rootNodeWithExprs = optimizedPlan.getPlanNode(rootContext); if (LOG.isDebugEnabled()) { LOG.debug("Printing PlanNode tree..."); printPlanNodeTree(rootNodeWithExprs.planNode_, ""); diff --git a/testdata/workloads/functional-query/queries/QueryTest/calcite.test b/testdata/workloads/functional-query/queries/QueryTest/calcite.test index ca5508bb1..240ac532d 100644 --- a/testdata/workloads/functional-query/queries/QueryTest/calcite.test +++ b/testdata/workloads/functional-query/queries/QueryTest/calcite.test @@ -165,3 +165,41 @@ select tinyint_col from calcite_alltypes where bigint_col = 20; ---- TYPES tinyint ==== +---- QUERY +# Values test +select abs(cast(-8 as bigint)); +---- RESULTS +8 +---- TYPES +bigint +==== +---- QUERY +# TODO: Char types are not yet supported, see comment in ImpalaValuesRel for details. +select 'hello' +---- CATCH +Char type values are not yet supported. +==== +---- QUERY +# Union test +select 3 union select 4; +---- RESULTS +3 +4 +---- TYPES +# IMPALA-13022: Calcite always casts literal numbers as INT. To avoid backward compatibility +# issues, the return type should be TINYINT here. There are many tests in the Impala +# test suite which will have this problem. +int +==== +---- QUERY +select * from (values (1)) union (values (2), (3)); +---- RESULTS +1 +2 +3 +---- TYPES +# IMPALA-13022: Calcite always casts literal numbers as INT. To avoid backward compatibility +# issues, the return type should be TINYINT here. There are many tests in the Impala +# test suite which will have this problem. +int +====
