This is an automated email from the ASF dual-hosted git repository. kxiao pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push: new 6644eb40a2 [Improvement](multi catalog) Support Iceberg, Paimon and MaxCompute table in nereids. (#22338) (#22376) 6644eb40a2 is described below commit 6644eb40a27cb00dfd5ae7e040c48a6cddd1714e Author: Jibing-Li <64681310+jibing...@users.noreply.github.com> AuthorDate: Sun Jul 30 21:13:54 2023 +0800 [Improvement](multi catalog) Support Iceberg, Paimon and MaxCompute table in nereids. (#22338) (#22376) --- .../glue/translator/PhysicalPlanTranslator.java | 4 ++ .../doris/nereids/rules/analysis/BindRelation.java | 8 ++- .../doris/planner/external/MaxComputeScanNode.java | 4 ++ .../test_external_catalog_icebergv2_nereids.out | 74 +++++++++++++++++++ .../test_external_catalog_icebergv2_nereids.groovy | 84 ++++++++++++++++++++++ 5 files changed, 172 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 8f9b336585..4b460e944d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -48,6 +48,7 @@ import org.apache.doris.catalog.external.ExternalTable; import org.apache.doris.catalog.external.HMSExternalTable; import org.apache.doris.catalog.external.IcebergExternalTable; import org.apache.doris.catalog.external.JdbcExternalTable; +import org.apache.doris.catalog.external.MaxComputeExternalTable; import org.apache.doris.catalog.external.PaimonExternalTable; import org.apache.doris.common.UserException; import org.apache.doris.common.util.Util; @@ -157,6 +158,7 @@ import org.apache.doris.planner.SortNode; import org.apache.doris.planner.TableFunctionNode; import org.apache.doris.planner.UnionNode; import org.apache.doris.planner.external.HiveScanNode; +import org.apache.doris.planner.external.MaxComputeScanNode; import org.apache.doris.planner.external.hudi.HudiScanNode; import org.apache.doris.planner.external.iceberg.IcebergScanNode; import org.apache.doris.planner.external.jdbc.JdbcScanNode; @@ -403,6 +405,8 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla scanNode = new IcebergScanNode(context.nextPlanNodeId(), tupleDescriptor, false); } else if (table instanceof PaimonExternalTable) { scanNode = new PaimonScanNode(context.nextPlanNodeId(), tupleDescriptor, false); + } else if (table instanceof MaxComputeExternalTable) { + scanNode = new MaxComputeScanNode(context.nextPlanNodeId(), tupleDescriptor, false); } else { throw new RuntimeException("do not support table type " + table.getType()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java index e23078ce54..0775fc7acb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java @@ -23,6 +23,7 @@ import org.apache.doris.catalog.Partition; import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.View; import org.apache.doris.catalog.external.EsExternalTable; +import org.apache.doris.catalog.external.ExternalTable; import org.apache.doris.catalog.external.HMSExternalTable; import org.apache.doris.common.Config; import org.apache.doris.common.util.Util; @@ -212,8 +213,11 @@ public class BindRelation extends OneAnalysisRuleFactory { return new LogicalSubQueryAlias<>(tableQualifier, hiveViewPlan); } } - return new LogicalFileScan(RelationUtil.newRelationId(), - (HMSExternalTable) table, ImmutableList.of(dbName)); + return new LogicalFileScan(RelationUtil.newRelationId(), (HMSExternalTable) table, tableQualifier); + case ICEBERG_EXTERNAL_TABLE: + case PAIMON_EXTERNAL_TABLE: + case MAX_COMPUTE_EXTERNAL_TABLE: + return new LogicalFileScan(RelationUtil.newRelationId(), (ExternalTable) table, tableQualifier); case SCHEMA: return new LogicalSchemaScan(RelationUtil.newRelationId(), table, ImmutableList.of(dbName)); case JDBC_EXTERNAL_TABLE: diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/MaxComputeScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/MaxComputeScanNode.java index 11e9bafd86..1030a67a30 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/MaxComputeScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/MaxComputeScanNode.java @@ -44,6 +44,10 @@ public class MaxComputeScanNode extends FileQueryScanNode { private final MaxComputeExternalCatalog catalog; public static final int MIN_SPLIT_SIZE = 4096; + public MaxComputeScanNode(PlanNodeId id, TupleDescriptor desc, boolean needCheckColumnPriv) { + this(id, desc, "MCScanNode", StatisticalType.MAX_COMPUTE_SCAN_NODE, needCheckColumnPriv); + } + public MaxComputeScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeName, StatisticalType statisticalType, boolean needCheckColumnPriv) { super(id, desc, planNodeName, statisticalType, needCheckColumnPriv); diff --git a/regression-test/data/external_table_emr_p2/iceberg/test_external_catalog_icebergv2_nereids.out b/regression-test/data/external_table_emr_p2/iceberg/test_external_catalog_icebergv2_nereids.out new file mode 100644 index 0000000000..a6e68d2f62 --- /dev/null +++ b/regression-test/data/external_table_emr_p2/iceberg/test_external_catalog_icebergv2_nereids.out @@ -0,0 +1,74 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !q01 -- +149988 + +-- !q02 -- +1 +3 +4 +7 + +-- !q03 -- +8242263 + +-- !q04 -- +0 + +-- !q05 -- +1 Customer#000000001 IVhzIApeRb ot,c,E 15 25-989-741-2988 711.56 BUILDING to the even, regular platelets. regular, ironic epitaphs nag e +3 Customer#000000003 MG9kdTD2WBHm 1 11-719-748-3364 7498.12 AUTOMOBILE deposits eat slyly ironic, even instructions. express foxes detect slyly. blithely even accounts abov +4 Customer#000000004 XxVSJsLAGtn 4 14-128-190-5944 2866.83 MACHINERY requests. final, regular ideas sleep final accou + +-- !q06 -- +604519555 +604519557 +604519558 + +-- !q07 -- +12979.65 +219204.52 +5908.20 + +-- !q08 -- +120001848 + +-- !q09 -- +1 +2 +3 + +-- !q10 -- +150000000 +149999999 +149999996 + +-- !q11 -- +1 +2 +3 + +-- !q12 -- +150000000 +149999999 +149999996 + +-- !q13 -- +1 +4 +7 + +-- !q14 -- +Customer#000000004 +Customer#000000007 + +-- !q15 -- +150000 + +-- !q16 -- +150000 + +-- !q17 -- +150000 + +-- !q18 -- +150000 diff --git a/regression-test/suites/external_table_emr_p2/iceberg/test_external_catalog_icebergv2_nereids.groovy b/regression-test/suites/external_table_emr_p2/iceberg/test_external_catalog_icebergv2_nereids.groovy new file mode 100644 index 0000000000..ab2d45d547 --- /dev/null +++ b/regression-test/suites/external_table_emr_p2/iceberg/test_external_catalog_icebergv2_nereids.groovy @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_external_catalog_icebergv2_nereids", "p2") { + String enabled = context.config.otherConfigs.get("enableExternalHiveTest") + if (enabled != null && enabled.equalsIgnoreCase("true")) { + String extHiveHmsHost = context.config.otherConfigs.get("extHiveHmsHost") + String extHiveHmsPort = context.config.otherConfigs.get("extHiveHmsPort") + String hms_catalog_name = "test_external_hms_catalog_iceberg" + String iceberg_catalog_name = "test_external_iceberg_catalog_nereids" + + sql """drop catalog if exists ${hms_catalog_name};""" + sql """ + create catalog if not exists ${hms_catalog_name} properties ( + 'type'='hms', + 'hive.metastore.uris' = 'thrift://${extHiveHmsHost}:${extHiveHmsPort}' + ); + """ + + sql """drop catalog if exists ${iceberg_catalog_name};""" + sql """ + create catalog if not exists ${iceberg_catalog_name} properties ( + 'type'='iceberg', + 'iceberg.catalog.type'='hms', + 'hive.metastore.uris' = 'thrift://${extHiveHmsHost}:${extHiveHmsPort}' + ); + """ + sql """set enable_nereids_planner=true;""" + sql """set enable_fallback_to_original_planner=false;""" + + sql """switch ${hms_catalog_name};""" + // test parquet format format + def q01 = { + qt_q01 """ select count(1) as c from customer_small """ + qt_q02 """ select c_custkey from customer_small group by c_custkey order by c_custkey limit 4 """ + qt_q03 """ select count(1) from orders_small """ + qt_q04 """ select count(1) from customer_small where c_name = 'Customer#000000005' or c_name = 'Customer#000000006' """ + qt_q05 """ select * from customer_small order by c_custkey limit 3 """ + qt_q06 """ select o_orderkey from orders_small where o_orderkey > 652566 order by o_orderkey limit 3 """ + qt_q07 """ select o_totalprice from orders_small where o_custkey < 3357 order by o_custkey limit 3 """ + qt_q08 """ select count(1) as c from customer """ + } + // test time travel stmt + def q02 = { + qt_q09 """ select c_custkey from customer for time as of '2022-12-27 10:21:36' order by c_custkey limit 3 """ + qt_q10 """ select c_custkey from customer for time as of '2022-12-28 10:21:36' order by c_custkey desc limit 3 """ + qt_q11 """ select c_custkey from customer for version as of 906874575350293177 order by c_custkey limit 3 """ + qt_q12 """ select c_custkey from customer for version as of 6352416983354893547 order by c_custkey desc limit 3 """ + } + // in predicate + def q03 = { + qt_q13 """ select c_custkey from customer_small where c_custkey in (1, 2, 4, 7) order by c_custkey """ + qt_q14 """ select c_name from customer_small where c_name in ('Customer#000000004', 'Customer#000000007') order by c_custkey """ + } + + // test for 'FOR TIME AS OF' and 'FOR VERSION AS OF' + def q04 = { + qt_q15 """ select count(*) from ${hms_catalog_name}.tpch_1000_icebergv2.customer_small FOR TIME AS OF '2022-12-22 02:29:30' """ + qt_q16 """ select count(*) from ${hms_catalog_name}.tpch_1000_icebergv2.customer_small FOR VERSION AS OF 6113938156088124425 """ + qt_q17 """ select count(*) from ${iceberg_catalog_name}.tpch_1000_icebergv2.customer_small FOR TIME AS OF '2022-12-22 02:29:30' """ + qt_q18 """ select count(*) from ${iceberg_catalog_name}.tpch_1000_icebergv2.customer_small FOR VERSION AS OF 6113938156088124425 """ + } + + sql """ use `tpch_1000_icebergv2`; """ + q01() + q02() + q03() + q04() + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org