This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 57ed781bb66 [fix](regression-test) Add tvf regression tests (#26455)
57ed781bb66 is described below
commit 57ed781bb66b314be7850c6418c6ca7e5b51e2d7
Author: Tiewei Fang <[email protected]>
AuthorDate: Thu Nov 9 12:09:32 2023 +0800
[fix](regression-test) Add tvf regression tests (#26455)
---
.../tvf/queries/test_queries_tvf.out | 4 +
.../data/external_table_p0/tvf/test_s3_tvf.out | 29 +++++
.../external_table_p2/tvf/test_iceberg_meta.out | 22 ++++
.../tvf/queries/test_queries_tvf.groovy | 37 +++++++
.../external_table_p0/tvf/test_numbers.groovy | 30 +++---
.../external_table_p0/tvf/test_s3_tvf.groovy | 120 +++++++++++++++++++++
.../external_table_p2/tvf/test_iceberg_meta.groovy | 51 +++++++++
7 files changed, 278 insertions(+), 15 deletions(-)
diff --git
a/regression-test/data/external_table_p0/tvf/queries/test_queries_tvf.out
b/regression-test/data/external_table_p0/tvf/queries/test_queries_tvf.out
new file mode 100644
index 00000000000..6f7e7cf46ed
--- /dev/null
+++ b/regression-test/data/external_table_p0/tvf/queries/test_queries_tvf.out
@@ -0,0 +1,4 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !select --
+1 doris 10
+
diff --git a/regression-test/data/external_table_p0/tvf/test_s3_tvf.out
b/regression-test/data/external_table_p0/tvf/test_s3_tvf.out
new file mode 100644
index 00000000000..3128c590bc8
--- /dev/null
+++ b/regression-test/data/external_table_p0/tvf/test_s3_tvf.out
@@ -0,0 +1,29 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !select_base --
+1 doris1 18
+2 doris2 19
+3 doris3 99
+4 doris4 \N
+5 doris5 15
+
+-- !select_1 --
+1 doris1 18
+2 doris2 19
+3 doris3 99
+4 doris4 \N
+5 doris5 15
+
+-- !select_2 --
+1 doris1 18
+2 doris2 19
+3 doris3 99
+4 doris4 \N
+5 doris5 15
+
+-- !select_3 --
+1 doris1 18
+2 doris2 19
+3 doris3 99
+4 doris4 \N
+5 doris5 15
+
diff --git a/regression-test/data/external_table_p2/tvf/test_iceberg_meta.out
b/regression-test/data/external_table_p2/tvf/test_iceberg_meta.out
new file mode 100644
index 00000000000..b62e2d7510a
--- /dev/null
+++ b/regression-test/data/external_table_p2/tvf/test_iceberg_meta.out
@@ -0,0 +1,22 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !q01 --
+2879562
+
+-- !q02 --
+1
+11
+3
+5
+6
+7
+8
+
+-- !tvf_1 --
+2023-10-16T21:01:06 4012471924714711043 5784892960796156942 append
+2023-10-16T21:01:06 5784892960796156942 -1 append
+2023-10-16T21:01:06 7235593032487457798 4012471924714711043 append
+2023-10-16T21:01:07 1953697979105284524 7235593032487457798 append
+
+-- !tvf_2 --
+2023-10-16T21:01:06 7235593032487457798 4012471924714711043 append
+
diff --git
a/regression-test/suites/external_table_p0/tvf/queries/test_queries_tvf.groovy
b/regression-test/suites/external_table_p0/tvf/queries/test_queries_tvf.groovy
new file mode 100644
index 00000000000..95cd0d3aaca
--- /dev/null
+++
b/regression-test/suites/external_table_p0/tvf/queries/test_queries_tvf.groovy
@@ -0,0 +1,37 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_queries_tvf","p0,external,tvf,external_docker") {
+ def table_name = "test_queries_tvf"
+ sql """ DROP TABLE IF EXISTS ${table_name} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${table_name} (
+ `user_id` LARGEINT NOT NULL COMMENT "用户id",
+ `name` STRING COMMENT "用户名称",
+ `age` INT COMMENT "用户年龄",
+ )
+ DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1");
+ """
+
+ sql """insert into ${table_name} values (1, 'doris', 10);"""
+
+ sql """select * from ${table_name};"""
+
+ def res = sql """ select QueryId from queries() where `Sql` like
"%${table_name}%"; """
+ logger.info("res = " + res)
+ assertEquals(2, res.size())
+}
\ No newline at end of file
diff --git a/regression-test/suites/external_table_p0/tvf/test_numbers.groovy
b/regression-test/suites/external_table_p0/tvf/test_numbers.groovy
index 0bad88ecc99..6dc09a4f5df 100644
--- a/regression-test/suites/external_table_p0/tvf/test_numbers.groovy
+++ b/regression-test/suites/external_table_p0/tvf/test_numbers.groovy
@@ -1,19 +1,19 @@
// Licensed to the Apache Software Foundation (ASF) under one
- // or more contributor license agreements. See the NOTICE file
- // distributed with this work for additional information
- // regarding copyright ownership. The ASF licenses this file
- // to you under the Apache License, Version 2.0 (the
- // "License"); you may not use this file except in compliance
- // with the License. You may obtain a copy of the License at
- //
- // http://www.apache.org/licenses/LICENSE-2.0
- //
- // Unless required by applicable law or agreed to in writing,
- // software distributed under the License is distributed on an
- // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- // KIND, either express or implied. See the License for the
- // specific language governing permissions and limitations
- // under the License.
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
suite("test_numbers","p0,external,external_docker") {
diff --git a/regression-test/suites/external_table_p0/tvf/test_s3_tvf.groovy
b/regression-test/suites/external_table_p0/tvf/test_s3_tvf.groovy
new file mode 100644
index 00000000000..273b61a716d
--- /dev/null
+++ b/regression-test/suites/external_table_p0/tvf/test_s3_tvf.groovy
@@ -0,0 +1,120 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_s3_tvf", "p0") {
+ // open nereids
+ sql """ set enable_nereids_planner=true """
+ sql """ set enable_fallback_to_original_planner=false """
+
+ String ak = getS3AK()
+ String sk = getS3SK()
+ String s3_endpoint = getS3Endpoint()
+ String region = getS3Region()
+ String bucket = context.config.otherConfigs.get("s3BucketName");
+
+
+ def export_table_name = "test_s3_tvf_export_test"
+ def outFilePath = "${bucket}/est_s3_tvf/export_test/exp_"
+
+
+ def create_table = {table_name ->
+ sql """ DROP TABLE IF EXISTS ${table_name} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${table_name} (
+ `user_id` LARGEINT NOT NULL COMMENT "用户id",
+ `name` STRING COMMENT "用户名称",
+ `age` INT COMMENT "用户年龄",
+ )
+ DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1");
+ """
+ }
+
+ def outfile_to_S3 = {
+ // select ... into outfile ...
+ def res = sql """
+ SELECT * FROM ${export_table_name} t ORDER BY user_id
+ INTO OUTFILE "s3://${outFilePath}"
+ FORMAT AS ORC
+ PROPERTIES (
+ "s3.endpoint" = "${s3_endpoint}",
+ "s3.region" = "${region}",
+ "s3.secret_key"="${sk}",
+ "s3.access_key" = "${ak}"
+ );
+ """
+
+ return res[0][3]
+ }
+
+ // create table to export data
+ create_table(export_table_name)
+
+ // insert data
+ sql """ insert into ${export_table_name} values (1, 'doris1', 18); """
+ sql """ insert into ${export_table_name} values (2, 'doris2', 19); """
+ sql """ insert into ${export_table_name} values (3, 'doris3', 99); """
+ sql """ insert into ${export_table_name} values (4, 'doris4', null); """
+ sql """ insert into ${export_table_name} values (5, 'doris5', 15); """
+
+ // test base data
+ qt_select_base """ SELECT * FROM ${export_table_name} t ORDER BY user_id;
"""
+
+ // test outfile to s3
+ def outfile_url = outfile_to_S3()
+
+ // 1. normal
+ try {
+ order_qt_select_1 """ SELECT * FROM S3 (
+ "uri" =
"http://${s3_endpoint}${outfile_url.substring(4)}0.orc",
+ "ACCESS_KEY"= "${ak}",
+ "SECRET_KEY" = "${sk}",
+ "format" = "orc",
+ "region" = "${region}"
+ );
+ """
+ } finally {
+ }
+
+
+ // 2. test endpoint property
+ try {
+ order_qt_select_2 """ SELECT * FROM S3 (
+ "uri" = "http://${outfile_url.substring(5)}0.orc",
+ "s3.access_key"= "${ak}",
+ "s3.secret_key" = "${sk}",
+ "s3.endpoint" = "${s3_endpoint}",
+ "format" = "orc",
+ "region" = "${region}"
+ );
+ """
+ } finally {
+ }
+
+ // 3.test use_path_style
+ try {
+ order_qt_select_3 """ SELECT * FROM S3 (
+ "uri" =
"http://${s3_endpoint}${outfile_url.substring(4)}0.orc",
+ "s3.access_key"= "${ak}",
+ "s3.secret_key" = "${sk}",
+ "format" = "orc",
+ "use_path_style" = "true",
+ "region" = "${region}"
+ );
+ """
+ } finally {
+ }
+}
diff --git
a/regression-test/suites/external_table_p2/tvf/test_iceberg_meta.groovy
b/regression-test/suites/external_table_p2/tvf/test_iceberg_meta.groovy
new file mode 100644
index 00000000000..de13c48727a
--- /dev/null
+++ b/regression-test/suites/external_table_p2/tvf/test_iceberg_meta.groovy
@@ -0,0 +1,51 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_iceberg_meta",
"p2,external,iceberg,external_remote,external_remote_iceberg") {
+ String enabled = context.config.otherConfigs.get("enableExternalHiveTest")
+ if (enabled != null && enabled.equalsIgnoreCase("true")) {
+ String iceberg_catalog_name = "test_iceberg_meta_tvf"
+ String extHiveHmsHost =
context.config.otherConfigs.get("extHiveHmsHost")
+ String extHdfsPort = context.config.otherConfigs.get("extHdfsPort")
+ String db = "multi_catalog"
+ sql """drop catalog if exists ${iceberg_catalog_name};"""
+ sql """
+ create catalog if not exists ${iceberg_catalog_name} properties (
+ 'type'='iceberg',
+ 'iceberg.catalog.type'='hadoop',
+ 'warehouse' =
'hdfs://${extHiveHmsHost}:${extHdfsPort}/usr/hive/warehouse/hadoop_catalog'
+ );
+ """
+
+ sql """switch ${iceberg_catalog_name};"""
+ sql """ use `${db}`; """
+
+ order_qt_q01 """ select count(*) from iceberg_hadoop_catalog """
+ order_qt_q02 """ select c_custkey from iceberg_hadoop_catalog group by
c_custkey order by c_custkey limit 7 """
+
+ order_qt_tvf_1 """ select committed_at, snapshot_id, parent_id,
operation from iceberg_meta(
+ "table" =
"${iceberg_catalog_name}.${db}.multi_partition",
+ "query_type" = "snapshots");
+ """
+
+ order_qt_tvf_2 """ select committed_at, snapshot_id, parent_id,
operation from iceberg_meta(
+ "table" =
"${iceberg_catalog_name}.${db}.multi_partition",
+ "query_type" = "snapshots")
+ where snapshot_id = 7235593032487457798;
+ """
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
