This is an automated email from the ASF dual-hosted git repository.

kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/branch-2.0 by this push:
     new 6644eb40a2 [Improvement](multi catalog) Support Iceberg, Paimon and 
MaxCompute table in nereids. (#22338) (#22376)
6644eb40a2 is described below

commit 6644eb40a27cb00dfd5ae7e040c48a6cddd1714e
Author: Jibing-Li <64681310+jibing...@users.noreply.github.com>
AuthorDate: Sun Jul 30 21:13:54 2023 +0800

    [Improvement](multi catalog) Support Iceberg, Paimon and MaxCompute table 
in nereids. (#22338) (#22376)
---
 .../glue/translator/PhysicalPlanTranslator.java    |  4 ++
 .../doris/nereids/rules/analysis/BindRelation.java |  8 ++-
 .../doris/planner/external/MaxComputeScanNode.java |  4 ++
 .../test_external_catalog_icebergv2_nereids.out    | 74 +++++++++++++++++++
 .../test_external_catalog_icebergv2_nereids.groovy | 84 ++++++++++++++++++++++
 5 files changed, 172 insertions(+), 2 deletions(-)

diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
 
b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index 8f9b336585..4b460e944d 100644
--- 
a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++ 
b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -48,6 +48,7 @@ import org.apache.doris.catalog.external.ExternalTable;
 import org.apache.doris.catalog.external.HMSExternalTable;
 import org.apache.doris.catalog.external.IcebergExternalTable;
 import org.apache.doris.catalog.external.JdbcExternalTable;
+import org.apache.doris.catalog.external.MaxComputeExternalTable;
 import org.apache.doris.catalog.external.PaimonExternalTable;
 import org.apache.doris.common.UserException;
 import org.apache.doris.common.util.Util;
@@ -157,6 +158,7 @@ import org.apache.doris.planner.SortNode;
 import org.apache.doris.planner.TableFunctionNode;
 import org.apache.doris.planner.UnionNode;
 import org.apache.doris.planner.external.HiveScanNode;
+import org.apache.doris.planner.external.MaxComputeScanNode;
 import org.apache.doris.planner.external.hudi.HudiScanNode;
 import org.apache.doris.planner.external.iceberg.IcebergScanNode;
 import org.apache.doris.planner.external.jdbc.JdbcScanNode;
@@ -403,6 +405,8 @@ public class PhysicalPlanTranslator extends 
DefaultPlanVisitor<PlanFragment, Pla
             scanNode = new IcebergScanNode(context.nextPlanNodeId(), 
tupleDescriptor, false);
         } else if (table instanceof PaimonExternalTable) {
             scanNode = new PaimonScanNode(context.nextPlanNodeId(), 
tupleDescriptor, false);
+        } else if (table instanceof MaxComputeExternalTable) {
+            scanNode = new MaxComputeScanNode(context.nextPlanNodeId(), 
tupleDescriptor, false);
         } else {
             throw new RuntimeException("do not support table type " + 
table.getType());
         }
diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
 
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
index e23078ce54..0775fc7acb 100644
--- 
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
+++ 
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
@@ -23,6 +23,7 @@ import org.apache.doris.catalog.Partition;
 import org.apache.doris.catalog.TableIf;
 import org.apache.doris.catalog.View;
 import org.apache.doris.catalog.external.EsExternalTable;
+import org.apache.doris.catalog.external.ExternalTable;
 import org.apache.doris.catalog.external.HMSExternalTable;
 import org.apache.doris.common.Config;
 import org.apache.doris.common.util.Util;
@@ -212,8 +213,11 @@ public class BindRelation extends OneAnalysisRuleFactory {
                         return new LogicalSubQueryAlias<>(tableQualifier, 
hiveViewPlan);
                     }
                 }
-                return new LogicalFileScan(RelationUtil.newRelationId(),
-                    (HMSExternalTable) table, ImmutableList.of(dbName));
+                return new LogicalFileScan(RelationUtil.newRelationId(), 
(HMSExternalTable) table, tableQualifier);
+            case ICEBERG_EXTERNAL_TABLE:
+            case PAIMON_EXTERNAL_TABLE:
+            case MAX_COMPUTE_EXTERNAL_TABLE:
+                return new LogicalFileScan(RelationUtil.newRelationId(), 
(ExternalTable) table, tableQualifier);
             case SCHEMA:
                 return new LogicalSchemaScan(RelationUtil.newRelationId(), 
table, ImmutableList.of(dbName));
             case JDBC_EXTERNAL_TABLE:
diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/MaxComputeScanNode.java
 
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/MaxComputeScanNode.java
index 11e9bafd86..1030a67a30 100644
--- 
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/MaxComputeScanNode.java
+++ 
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/MaxComputeScanNode.java
@@ -44,6 +44,10 @@ public class MaxComputeScanNode extends FileQueryScanNode {
     private final MaxComputeExternalCatalog catalog;
     public static final int MIN_SPLIT_SIZE = 4096;
 
+    public MaxComputeScanNode(PlanNodeId id, TupleDescriptor desc, boolean 
needCheckColumnPriv) {
+        this(id, desc, "MCScanNode", StatisticalType.MAX_COMPUTE_SCAN_NODE, 
needCheckColumnPriv);
+    }
+
     public MaxComputeScanNode(PlanNodeId id, TupleDescriptor desc, String 
planNodeName,
                               StatisticalType statisticalType, boolean 
needCheckColumnPriv) {
         super(id, desc, planNodeName, statisticalType, needCheckColumnPriv);
diff --git 
a/regression-test/data/external_table_emr_p2/iceberg/test_external_catalog_icebergv2_nereids.out
 
b/regression-test/data/external_table_emr_p2/iceberg/test_external_catalog_icebergv2_nereids.out
new file mode 100644
index 0000000000..a6e68d2f62
--- /dev/null
+++ 
b/regression-test/data/external_table_emr_p2/iceberg/test_external_catalog_icebergv2_nereids.out
@@ -0,0 +1,74 @@
+-- This file is automatically generated. You should know what you did if you 
want to edit this
+-- !q01 --
+149988
+
+-- !q02 --
+1
+3
+4
+7
+
+-- !q03 --
+8242263
+
+-- !q04 --
+0
+
+-- !q05 --
+1      Customer#000000001      IVhzIApeRb ot,c,E       15      25-989-741-2988 
711.56  BUILDING        to the even, regular platelets. regular, ironic 
epitaphs nag e
+3      Customer#000000003      MG9kdTD2WBHm    1       11-719-748-3364 7498.12 
AUTOMOBILE       deposits eat slyly ironic, even instructions. express foxes 
detect slyly. blithely even accounts abov
+4      Customer#000000004      XxVSJsLAGtn     4       14-128-190-5944 2866.83 
MACHINERY        requests. final, regular ideas sleep final accou
+
+-- !q06 --
+604519555
+604519557
+604519558
+
+-- !q07 --
+12979.65
+219204.52
+5908.20
+
+-- !q08 --
+120001848
+
+-- !q09 --
+1
+2
+3
+
+-- !q10 --
+150000000
+149999999
+149999996
+
+-- !q11 --
+1
+2
+3
+
+-- !q12 --
+150000000
+149999999
+149999996
+
+-- !q13 --
+1
+4
+7
+
+-- !q14 --
+Customer#000000004
+Customer#000000007
+
+-- !q15 --
+150000
+
+-- !q16 --
+150000
+
+-- !q17 --
+150000
+
+-- !q18 --
+150000
diff --git 
a/regression-test/suites/external_table_emr_p2/iceberg/test_external_catalog_icebergv2_nereids.groovy
 
b/regression-test/suites/external_table_emr_p2/iceberg/test_external_catalog_icebergv2_nereids.groovy
new file mode 100644
index 0000000000..ab2d45d547
--- /dev/null
+++ 
b/regression-test/suites/external_table_emr_p2/iceberg/test_external_catalog_icebergv2_nereids.groovy
@@ -0,0 +1,84 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_external_catalog_icebergv2_nereids", "p2") {
+    String enabled = context.config.otherConfigs.get("enableExternalHiveTest")
+    if (enabled != null && enabled.equalsIgnoreCase("true")) {
+        String extHiveHmsHost = 
context.config.otherConfigs.get("extHiveHmsHost")
+        String extHiveHmsPort = 
context.config.otherConfigs.get("extHiveHmsPort")
+        String hms_catalog_name = "test_external_hms_catalog_iceberg"
+        String iceberg_catalog_name = "test_external_iceberg_catalog_nereids"
+
+        sql """drop catalog if exists ${hms_catalog_name};"""
+        sql """
+            create catalog if not exists ${hms_catalog_name} properties (
+                'type'='hms',
+                'hive.metastore.uris' = 
'thrift://${extHiveHmsHost}:${extHiveHmsPort}'
+            );
+        """
+
+        sql """drop catalog if exists ${iceberg_catalog_name};"""
+        sql """
+            create catalog if not exists ${iceberg_catalog_name} properties (
+                'type'='iceberg',
+                'iceberg.catalog.type'='hms',
+                'hive.metastore.uris' = 
'thrift://${extHiveHmsHost}:${extHiveHmsPort}'
+            );
+        """
+        sql """set enable_nereids_planner=true;"""
+        sql """set enable_fallback_to_original_planner=false;"""
+
+        sql """switch ${hms_catalog_name};"""
+        // test parquet format format
+        def q01 = {
+            qt_q01 """ select count(1) as c from customer_small """
+            qt_q02 """ select c_custkey from customer_small group by c_custkey 
order by c_custkey limit 4 """
+            qt_q03 """ select count(1) from orders_small """
+            qt_q04 """ select count(1) from customer_small where c_name = 
'Customer#000000005' or c_name = 'Customer#000000006' """
+            qt_q05 """ select * from customer_small order by c_custkey limit 3 
"""
+            qt_q06 """ select o_orderkey from orders_small where o_orderkey > 
652566 order by o_orderkey limit 3 """
+            qt_q07 """ select o_totalprice from orders_small where o_custkey < 
3357 order by o_custkey limit 3 """
+            qt_q08 """ select count(1) as c from customer """
+        }
+        // test time travel stmt
+        def q02 = {
+            qt_q09 """ select c_custkey from customer for time as of 
'2022-12-27 10:21:36' order by c_custkey limit 3 """
+            qt_q10 """ select c_custkey from customer for time as of 
'2022-12-28 10:21:36' order by c_custkey desc limit 3 """
+            qt_q11 """ select c_custkey from customer for version as of 
906874575350293177 order by c_custkey limit 3 """
+            qt_q12 """ select c_custkey from customer for version as of 
6352416983354893547 order by c_custkey desc limit 3 """
+        }
+        // in predicate
+        def q03 = {
+            qt_q13 """ select c_custkey from customer_small where c_custkey in 
(1, 2, 4, 7) order by c_custkey """
+            qt_q14 """ select c_name from customer_small where c_name in 
('Customer#000000004', 'Customer#000000007') order by c_custkey """
+        }
+
+        // test for 'FOR TIME AS OF' and 'FOR VERSION AS OF'
+        def q04 = {
+            qt_q15 """ select count(*) from 
${hms_catalog_name}.tpch_1000_icebergv2.customer_small FOR TIME AS OF 
'2022-12-22 02:29:30' """
+            qt_q16 """ select count(*) from 
${hms_catalog_name}.tpch_1000_icebergv2.customer_small FOR VERSION AS OF 
6113938156088124425 """
+            qt_q17 """ select count(*) from 
${iceberg_catalog_name}.tpch_1000_icebergv2.customer_small FOR TIME AS OF 
'2022-12-22 02:29:30' """
+            qt_q18 """ select count(*) from 
${iceberg_catalog_name}.tpch_1000_icebergv2.customer_small FOR VERSION AS OF 
6113938156088124425 """
+        }
+        
+        sql """ use `tpch_1000_icebergv2`; """
+        q01()
+        q02()
+        q03()
+        q04()
+    }
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org

Reply via email to