This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 40638f542a1 [regression] add regression-test for s3 tvf (#35406)
40638f542a1 is described below
commit 40638f542a1d7280c8ebace5ab559ec9b9345a57
Author: 133tosakarin <97331129+133tosaka...@users.noreply.github.com>
AuthorDate: Thu Jun 27 14:12:29 2024 +0800
[regression] add regression-test for s3 tvf (#35406)
I had added some regression test for s3 tvf, all requirse of test cases
refer to https://github.com/apache/doris/issues/32576
---
regression-test/data/load_p0/tvf/test_s3_tvf.out | 358 ++++++++++++--
.../suites/load_p0/tvf/ddl/nest_tbl_basic_tvf.sql | 29 ++
.../load_p0/tvf/ddl/nest_tbl_basic_tvf_drop.sql | 1 +
.../suites/load_p0/tvf/test_s3_tvf.groovy | 526 ++++++++++++++++++++-
4 files changed, 859 insertions(+), 55 deletions(-)
diff --git a/regression-test/data/load_p0/tvf/test_s3_tvf.out
b/regression-test/data/load_p0/tvf/test_s3_tvf.out
index 88b962cfd6c..1e1da717ad9 100644
--- a/regression-test/data/load_p0/tvf/test_s3_tvf.out
+++ b/regression-test/data/load_p0/tvf/test_s3_tvf.out
@@ -11,17 +11,44 @@
-- !select --
20
+-- !select --
+40
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+60
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+20
+
-- !select --
18
-- !select --
-16
+18
-- !select --
-16
+79
-- !select --
-40
+20
-- !select --
20
@@ -29,20 +56,38 @@
-- !select --
20
+-- !select --
+39
+
+-- !select --
+18
+
+-- !select --
+18
+
+-- !select --
+98
+
-- !select --
20
-- !select --
-36
+20
-- !select --
-16
+20
-- !select --
-16
+58
-- !select --
-60
+18
+
+-- !select --
+18
+
+-- !select --
+117
-- !select --
20
@@ -54,16 +99,16 @@
20
-- !select --
-54
+77
-- !select --
-16
+18
-- !select --
-16
+18
-- !select --
-60
+137
-- !select --
20
@@ -75,16 +120,16 @@
20
-- !select --
-54
+95
-- !select --
-16
+18
-- !select --
-16
+18
-- !select --
-70
+157
-- !select --
20
@@ -96,16 +141,19 @@
20
-- !select --
-64
+113
+
+-- !select --
+18
-- !select --
-16
+18
-- !select --
-16
+5
-- !select --
-90
+177
-- !select --
20
@@ -117,16 +165,16 @@
20
-- !select --
-82
+131
-- !select --
-16
+18
-- !select --
-16
+18
-- !select --
-110
+197
-- !select --
20
@@ -138,16 +186,16 @@
20
-- !select --
-100
+149
-- !select --
-16
+18
-- !select --
-16
+18
-- !select --
-130
+217
-- !select --
20
@@ -159,16 +207,16 @@
20
-- !select --
-118
+167
-- !select --
-16
+18
-- !select --
-16
+18
-- !select --
-142
+237
-- !select --
20
@@ -180,13 +228,19 @@
20
-- !select --
-123
+185
-- !select --
-16
+18
-- !select --
-16
+18
+
+-- !select --
+257
+
+-- !select --
+20
-- !select --
20
@@ -195,7 +249,16 @@
20
-- !select --
-162
+203
+
+-- !select --
+18
+
+-- !select --
+18
+
+-- !select --
+277
-- !select --
20
@@ -207,16 +270,16 @@
20
-- !select --
-141
+221
-- !select --
-16
+18
-- !select --
-16
+18
-- !select --
-182
+277
-- !select --
20
@@ -228,16 +291,16 @@
20
-- !select --
-159
+221
-- !select --
-16
+18
-- !select --
-16
+18
-- !select --
-202
+287
-- !select --
20
@@ -249,16 +312,100 @@
20
-- !select --
-177
+231
+
+-- !select --
+18
+
+-- !select --
+18
+
+-- !select --
+307
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+249
+
+-- !select --
+18
+
+-- !select --
+18
+
+-- !select --
+327
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+267
+
+-- !select --
+18
+
+-- !select --
+18
+
+-- !select --
+347
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+285
+
+-- !select --
+18
+
+-- !select --
+18
+
+-- !select --
+367
-- !select --
-16
+20
-- !select --
-16
+20
+
+-- !select --
+20
-- !select --
-222
+305
+
+-- !select --
+18
+
+-- !select --
+18
+
+-- !select --
+379
-- !select --
20
@@ -270,11 +417,122 @@
20
-- !select --
-195
+310
-- !select --
-16
+18
+
+-- !select --
+18
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+399
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+328
+
+-- !select --
+18
+
+-- !select --
+18
+
+-- !select --
+419
+
+-- !select --
+20
+
+-- !select --
+20
-- !select --
-16
+20
+
+-- !select --
+346
+
+-- !select --
+18
+
+-- !select --
+18
+
+-- !select --
+439
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+364
+
+-- !select --
+18
+
+-- !select --
+18
+
+-- !select --
+459
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+382
+
+-- !select --
+18
+
+-- !select --
+18
+
+-- !select --
+479
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+402
+
+-- !select --
+18
+
+-- !select --
+18
diff --git a/regression-test/suites/load_p0/tvf/ddl/nest_tbl_basic_tvf.sql
b/regression-test/suites/load_p0/tvf/ddl/nest_tbl_basic_tvf.sql
new file mode 100644
index 00000000000..74748e532c6
--- /dev/null
+++ b/regression-test/suites/load_p0/tvf/ddl/nest_tbl_basic_tvf.sql
@@ -0,0 +1,29 @@
+
+CREATE TABLE nest_tbl_basic_tvf(
+ id INT NOT NULL,
+ city VARCHAR(100) NULL,
+ code INT NULL,
+ kd01 BOOLEAN NOT NULL DEFAULT "TRUE",
+ kd02 TINYINT NOT NULL DEFAULT "1",
+ kd03 SMALLINT NOT NULL DEFAULT "2",
+ kd04 INT NOT NULL DEFAULT "3",
+ kd05 BIGINT NOT NULL DEFAULT "4",
+ kd06 LARGEINT NOT NULL DEFAULT "5",
+ kd07 FLOAT NOT NULL DEFAULT "6.0",
+ kd08 DOUBLE NOT NULL DEFAULT "7.0",
+ kd09 DECIMAL NOT NULL DEFAULT "888888888",
+ kd10 DECIMALV3 NOT NULL DEFAULT "999999999",
+ kd11 DATE NOT NULL DEFAULT "2023-08-24",
+ kd12 DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ kd13 DATEV2 NOT NULL DEFAULT "2023-08-24",
+ kd14 DATETIMEV2 NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ kd15 CHAR(255) NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+ kd16 VARCHAR(300) NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+ kd17 STRING NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+ kd18 JSON NULL,
+)
+UNIQUE KEY(id)
+DISTRIBUTED BY HASH(id) BUCKETS 32
+PROPERTIES(
+ "replication_num" = "1"
+);
\ No newline at end of file
diff --git a/regression-test/suites/load_p0/tvf/ddl/nest_tbl_basic_tvf_drop.sql
b/regression-test/suites/load_p0/tvf/ddl/nest_tbl_basic_tvf_drop.sql
new file mode 100644
index 00000000000..b6f8ce5f336
--- /dev/null
+++ b/regression-test/suites/load_p0/tvf/ddl/nest_tbl_basic_tvf_drop.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS nest_tbl_basic_tvf
\ No newline at end of file
diff --git a/regression-test/suites/load_p0/tvf/test_s3_tvf.groovy
b/regression-test/suites/load_p0/tvf/test_s3_tvf.groovy
index 83a93fd5d82..4e0616c66d3 100644
--- a/regression-test/suites/load_p0/tvf/test_s3_tvf.groovy
+++ b/regression-test/suites/load_p0/tvf/test_s3_tvf.groovy
@@ -24,7 +24,8 @@ suite("test_s3_tvf", "load_p0") {
"mow_tbl_array_tvf",
"mow_tbl_basic_tvf",
"uniq_tbl_array_tvf",
- "uniq_tbl_basic_tvf"
+ "uniq_tbl_basic_tvf",
+ "nest_tbl_basic_tvf"
]
//deal with agg tables in separate
@@ -49,6 +50,405 @@ suite("test_s3_tvf", "load_p0") {
]
+ def partitionTables = [
+ "uniq_tbl_basic_tvf"
+ ]
+ // ,
+ // path partition key
+ // NOTE: maybe path_partition_keys don't support with broker_load, we just
calulate the insert_count equal to file rows
+ // this parameter is similar with columns from path as (c1, c2, ...) in
broker_load method, look at test case test_tvf_based_broker_load.groovy
+ for (String table : basicTables) {
+ attributeList.add(new TvfAttribute(table, "c1 as k00,c2 as k01,c3 as
k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10 as k09,c11
as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as k16,c18 as
k17,c19 as k18, kd16",
"K00,K01,K02,K03,K04,K05,K06,K07,K08,K09,K10,K11,K12,K13,K14,K15,K16,K17,K18,Kd16",
"", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/kd16=abcdefg/basic_data.csv")
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|")
+ .addProperty("use_path_style", "true")
+ .addProperty("force_parsing_by_standard_uri", "true")
+ .addProperty("path_partition_keys", "kd16"))
+ }
+
// SET (k19=to_bitmap(c6), k20=HLL_HASH(c6), k21=TO_QUANTILE_STATE(c5,
1.0), kd19=to_bitmap(c6), kd20=HLL_HASH(c6), kd21=TO_QUANTILE_STATE(c5, 1.0)
+ attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as
k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10
as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as
k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20,
TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20,
TO_QUANTILE_STATE(c5, 1.0) as kd21, kd16",
+
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21,kd16",
"", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/kd16=abcdefg/basic_data.csv")
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|")
+ .addProperty("use_path_style", "true")
+ .addProperty("force_parsing_by_standard_uri", "true")
+ .addProperty("path_partition_keys", "kd16"))
+ // path partition key maybe don't support array type ?
+ // for(String table : arrayTables) {
+ // attributeList.add(new TvfAttribute(table, ["K00", "K01", "K02",
"K03", "K04", "K05", "K06", "K07", "K08", "K09", "K10", "K11", "K12", "K13",
"K14", "K15", "K16", "K17", "kd16"], "", "")
+ // .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/kd16=[1,2,3,4]/basic_array_data.csv")
+ // .addProperty("format", "csv")
+ // .addProperty("column_separator", "|")
+ // .addProperty("force_parsing_by_standard_uri", "true")
+ // .addProperty("path_partition_keys", "kd16"))
+ // }
+
+ for (String table : basicTables) {
+ attributeList.add(new TvfAttribute(table, "c1 as k00,c2 as k01,c3 as
k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10 as k09,c11
as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as k16,c18 as
k17,c19 as k18, kd16, kd17",
"K00,K01,K02,K03,K04,K05,K06,K07,K08,K09,K10,K11,K12,K13,K14,K15,K16,K17,K18,Kd16,
kd17", "", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/kd16=abcdefg/kd17=hello/basic_data.csv")
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|")
+ .addProperty("use_path_style", "true")
+ .addProperty("force_parsing_by_standard_uri", "true")
+ .addProperty("path_partition_keys", "kd16,kd17"))
+ }
+
+ attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as
k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10
as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as
k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20,
TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20,
TO_QUANTILE_STATE(c5, 1.0) as kd21, kd16",
+
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21,kd16",
"", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/kd16=abcdefg/kd17=hello/basic_data.csv")
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|")
+ .addProperty("use_path_style", "true")
+ .addProperty("force_parsing_by_standard_uri", "true")
+ .addProperty("path_partition_keys", "kd16,kd17"))
+
+ // for (String table : basicTables) {
+ // attributeList.add(new TvfAttribute(table, "c1 as k00,c2 as k01,c3
as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10 as
k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as
k16,c18 as k17,c19 as k18, kd16, kd17",
"K00,K01,K02,K03,K04,K05,K06,K07,K08,K09,K10,K11,K12,K13,K14,K15,K16,K17,K18,Kd16,
kd17", "", "")
+ // .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/kd16=abcdefg/*/basic_data.csv")
+ // .addProperty("format", "csv")
+ // .addProperty("column_separator", "|")
+ // .addProperty("use_path_style", "true")
+ // .addProperty("force_parsing_by_standard_uri", "true")
+ // .addProperty("path_partition_keys", "kd16,kd17"))
+ // }
+
+ // attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2
as k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as
k08,c10 as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as
k15,c17 as k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as
k20, TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as
kd20, TO_QUANTILE_STATE(c5, 1.0) as kd21, kd16",
+ //
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21,kd16",
"", "")
+ // .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/kd16=abcdefg/*/basic_data.csv")
+ // .addProperty("format", "csv")
+ // .addProperty("column_separator", "|")
+ // .addProperty("use_path_style", "true")
+ // .addProperty("force_parsing_by_standard_uri", "true")
+ // .addProperty("path_partition_keys", "kd16,kd17"))
+
+
+
+
+ // trim_double_quotes
+ for(String table : basicTables) {
+ attributeList.add(new TvfAttribute(table, ["K00", "K01", "K02", "K03",
"K04", "K05", "K06", "K07", "K08", "K09", "K10", "K11", "K12", "K13", "K14",
"K15", "K16", "K17", "K18"], "", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_trim_double_quotes.csv")
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|")
+ .addProperty("force_parsing_by_standard_uri", "true")
+ .addProperty("trim_double_quotes", "true"))
+ }
+
+ attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as
k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10
as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as
k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20,
TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20,
TO_QUANTILE_STATE(c5, 1.0) as kd21",
+
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_trim_double_quotes.csv")
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|")
+ .addProperty("force_parsing_by_standard_uri", "true")
+ .addProperty("trim_double_quotes", "true"))
+
+ for(String table : arrayTables) {
+ attributeList.add(new TvfAttribute(table, ["K00", "K01", "K02", "K03",
"K04", "K05", "K06", "K07", "K08", "K09", "K10", "K11", "K12", "K13", "K14",
"K15", "K16", "K17"], "", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data_trim_double_quotes.csv")
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|")
+ .addProperty("force_parsing_by_standard_uri", "true")
+ .addProperty("trim_double_quotes", "true"))
+ }
+
+ // trim_double_quotes with skip_lines : 1
+ for(String table : basicTables) {
+ attributeList.add(new TvfAttribute(table, ["K00", "K01", "K02", "K03",
"K04", "K05", "K06", "K07", "K08", "K09", "K10", "K11", "K12", "K13", "K14",
"K15", "K16", "K17", "K18"], "", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_trim_double_quotes.csv")
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|")
+ .addProperty("force_parsing_by_standard_uri", "true")
+ .addProperty("skip_lines", "1")
+ .addProperty("trim_double_quotes", "true"))
+ }
+
+ attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as
k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10
as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as
k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20,
TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20,
TO_QUANTILE_STATE(c5, 1.0) as kd21",
+
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_trim_double_quotes.csv")
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|")
+ .addProperty("force_parsing_by_standard_uri", "true")
+ .addProperty("skip_lines", "1")
+ .addProperty("trim_double_quotes", "true"))
+
+ for(String table : arrayTables) {
+ attributeList.add(new TvfAttribute(table, ["K00", "K01", "K02", "K03",
"K04", "K05", "K06", "K07", "K08", "K09", "K10", "K11", "K12", "K13", "K14",
"K15", "K16", "K17"], "", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data_trim_double_quotes.csv")
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|")
+ .addProperty("force_parsing_by_standard_uri", "true")
+ .addProperty("skip_lines", "1")
+ .addProperty("trim_double_quotes", "true"))
+ }
+
+ // stip_outer_array
+ for(String table : basicTables) {
+ attributeList.add(new TvfAttribute(table, ["K00", "K01", "K02", "K03",
"K04", "K05", "K06", "K07", "K08", "K09", "K10", "K11", "K12", "K13", "K14",
"K15", "K16", "K17", "K18"], "", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_trim_double_quotes.csv")
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|")
+ .addProperty("force_parsing_by_standard_uri", "true")
+ .addProperty("skip_lines", "1")
+ .addProperty("strip_outer_array", "true")
+ .addProperty("trim_double_quotes", "true"))
+ }
+
+ attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as
k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10
as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as
k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20,
TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20,
TO_QUANTILE_STATE(c5, 1.0) as kd21",
+
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_trim_double_quotes.csv")
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|")
+ .addProperty("force_parsing_by_standard_uri", "true")
+ .addProperty("skip_lines", "1")
+ .addProperty("strip_outer_array", "true")
+ .addProperty("trim_double_quotes", "true"))
+
+ for(String table : arrayTables) {
+ attributeList.add(new TvfAttribute(table, ["K00", "K01", "K02", "K03",
"K04", "K05", "K06", "K07", "K08", "K09", "K10", "K11", "K12", "K13", "K14",
"K15", "K16", "K17"], "", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data_trim_double_quotes.csv")
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|")
+ .addProperty("force_parsing_by_standard_uri", "true")
+ .addProperty("skip_lines", "1")
+ .addProperty("strip_outer_array", "true")
+ .addProperty("trim_double_quotes", "true"))
+ }
+
+ // read_json_by_line
+ for(String table : basicTables) {
+ attributeList.add(new TvfAttribute(table, ["K00", "K01", "K02", "K03",
"K04", "K05", "K06", "K07", "K08", "K09", "K10", "K11", "K12", "K13", "K14",
"K15", "K16", "K17", "K18"], "", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_trim_double_quotes.csv")
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|")
+ .addProperty("force_parsing_by_standard_uri", "true")
+ .addProperty("skip_lines", "1")
+ .addProperty("strip_outer_array", "true")
+ .addProperty("read_json_by_line", "false")
+ .addProperty("trim_double_quotes", "true"))
+ }
+
+ attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as
k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10
as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as
k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20,
TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20,
TO_QUANTILE_STATE(c5, 1.0) as kd21",
+
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_trim_double_quotes.csv")
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|")
+ .addProperty("force_parsing_by_standard_uri", "true")
+ .addProperty("skip_lines", "1")
+ .addProperty("strip_outer_array", "true")
+ .addProperty("read_json_by_line", "false")
+ .addProperty("trim_double_quotes", "true"))
+
+ for(String table : arrayTables) {
+ attributeList.add(new TvfAttribute(table, ["K00", "K01", "K02", "K03",
"K04", "K05", "K06", "K07", "K08", "K09", "K10", "K11", "K12", "K13", "K14",
"K15", "K16", "K17"], "", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data_trim_double_quotes.csv")
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|")
+ .addProperty("force_parsing_by_standard_uri", "true")
+ .addProperty("skip_lines", "1")
+ .addProperty("strip_outer_array", "true")
+ .addProperty("read_json_by_line", "false")
+ .addProperty("trim_double_quotes", "true"))
+ }
+
+ // json format
+ //
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18","k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18"
+ // def a3k = getS3AK()
+ // def s3k = getS3SK()
+ // def jsonpaths = '[\\"$.k00\\", \\"$.k01\\"]'
+ // def sql_str = """
+ // select * from s3(
+ // "s3.access_key" = "$a3k",
+ // "s3.secret_key" = "$s3k",
+ // "s3.region" = "ap-beijing",
+ // "uri" =
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.json",
+ // "format" = "json",
+ // "read_json_by_line" = "false",
+ // "strip_outer_array" = "true",
+ // "use_path_style" = "true",
+ // "jsonpaths" = "$jsonpaths",
+ // "force_parsing_by_standard_uri" = "true"
+ // ) limit 10;
+ // """
+ // def sql_ret = sql """ ${sql_str} """
+ // logger.info("sql_ret: $sql_ret")
+
+ // jsonpaths
+ for(String table : basicTables) {
+ attributeList.add(new TvfAttribute(table, "*","k00,k01,k12", "", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.json")
+ .addProperty("format", "json")
+ .addProperty("read_json_by_line", "false")
+ .addProperty("strip_outer_array", "true")
+ .addProperty("column_separator", "|")
+ .addProperty("use_path_style", "true")
+ .addProperty("jsonpaths", '[\\"$.k00\\", \\"$.k01\\",
\\"$.k12\\"]')
+ .addProperty("force_parsing_by_standard_uri", "true"))
+ }
+
+ attributeList.add(new TvfAttribute("agg_tbl_basic_tvf",
"k00,k01,k02,k03,k04,k05, to_bitmap(k05) as k19, HLL_HASH(k05) as k20,
TO_QUANTILE_STATE(k04, 1.0) as k21, to_bitmap(k05) as kd19, HLL_HASH(k05) as
kd20, TO_QUANTILE_STATE(k04, 1.0) as kd21",
+ "k00,k01,k02, k03,k04, k05,k19, k20, k21, kd19, kd20, kd21" ,"",
"")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.json")
+ .addProperty("format", "json")
+ .addProperty("read_json_by_line", "false")
+ .addProperty("strip_outer_array", "true")
+ .addProperty("column_separator", "|")
+ .addProperty("jsonpaths", '[\\"$.k00\\", \\"$.k01\\", \\"$.k02\\",
\\"$.k03\\", \\"$.k04\\", \\"$.k05\\"]')
+ .addProperty("force_parsing_by_standard_uri", "true"))
+
+ for(String table : arrayTables) {
+ attributeList.add(new TvfAttribute(table, "k00,k01,k02,k03,k04",
"k00,k01,k02,k03,k04", "", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.json")
+ .addProperty("format", "json")
+ .addProperty("read_json_by_line", "false")
+ .addProperty("strip_outer_array", "true")
+ .addProperty("column_separator", "|")
+ .addProperty("jsonpaths", '[\\"$.k00\\", \\"$.k01\\",
\\"$.k02\\", \\"$.k03\\", \\"$.k04\\"]')
+ .addProperty("force_parsing_by_standard_uri", "true"))
+ }
+
+ for(String table : basicTables) {
+ attributeList.add(new TvfAttribute(table, "*","k00,k01,k12", "", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_by_line.json")
+ .addProperty("format", "json")
+ .addProperty("read_json_by_line", "true")
+ .addProperty("strip_outer_array", "true")
+ .addProperty("column_separator", "|")
+ .addProperty("use_path_style", "true")
+ .addProperty("strip_outer_array", "false")
+ .addProperty("jsonpaths", '[\\"$.k00\\", \\"$.k01\\",
\\"$.k12\\"]')
+ .addProperty("num_as_string", "true")
+ .addProperty("force_parsing_by_standard_uri", "true"))
+ }
+
+ attributeList.add(new TvfAttribute("agg_tbl_basic_tvf",
"k00,k01,k02,k03,k04,k05, to_bitmap(k05) as k19, HLL_HASH(k05) as k20,
TO_QUANTILE_STATE(k04, 1.0) as k21, to_bitmap(k05) as kd19, HLL_HASH(k05) as
kd20, TO_QUANTILE_STATE(k04, 1.0) as kd21",
+ "k00,k01,k02, k03,k04, k05,k19, k20, k21, kd19, kd20, kd21" ,"",
"")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_by_line.json")
+ .addProperty("format", "json")
+ .addProperty("read_json_by_line", "true")
+ .addProperty("strip_outer_array", "false")
+ .addProperty("column_separator", "|")
+ .addProperty("jsonpaths", '[\\"$.k00\\", \\"$.k01\\", \\"$.k02\\",
\\"$.k03\\", \\"$.k04\\", \\"$.k05\\"]')
+ .addProperty("num_as_string", "true")
+ .addProperty("force_parsing_by_standard_uri", "true"))
+
+ for(String table : arrayTables) {
+ attributeList.add(new TvfAttribute(table, "k00,k01,k02,k03,k04",
"k00,k01,k02,k03,k04", "", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data_by_line.json")
+ .addProperty("format", "json")
+ .addProperty("read_json_by_line", "true")
+ .addProperty("strip_outer_array", "false")
+ .addProperty("column_separator", "|")
+ .addProperty("jsonpaths", '[\\"$.k00\\", \\"$.k01\\",
\\"$.k02\\", \\"$.k03\\", \\"$.k04\\"]')
+ .addProperty("num_as_string", "true")
+ .addProperty("force_parsing_by_standard_uri", "true"))
+ }
+
+ // json_root
+ attributeList.add(new TvfAttribute("nest_tbl_basic_tvf", "id, city, code",
"id, city, code", "", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/nest_json.json")
+ .addProperty("format", "json")
+ .addProperty("read_json_by_line", "true")
+ .addProperty("json_root", '$.item')
+ .addProperty("force_parsing_by_standard_uri", "true"))
+
+ // num_as_string
+ for(String table : basicTables) {
+ attributeList.add(new TvfAttribute(table,
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18","k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18",
"", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_by_line.json")
+ .addProperty("format", "json")
+ .addProperty("read_json_by_line", "true")
+ .addProperty("strip_outer_array", "false")
+ .addProperty("column_separator", "|")
+ .addProperty("num_as_string", "true")
+ .addProperty("force_parsing_by_standard_uri", "true"))
+ }
+
+ attributeList.add(new TvfAttribute("agg_tbl_basic_tvf",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,
to_bitmap(k05) as k19, HLL_HASH(k05) as k20, TO_QUANTILE_STATE(k04, 1.0) as
k21, to_bitmap(k05) as kd19, HLL_HASH(k05) as kd20, TO_QUANTILE_STATE(k04, 1.0)
as kd21",
+
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_by_line.json")
+ .addProperty("format", "json")
+ .addProperty("read_json_by_line", "true")
+ .addProperty("strip_outer_array", "false")
+ .addProperty("num_as_string", "true")
+ .addProperty("column_separator", "|")
+ .addProperty("force_parsing_by_standard_uri", "true"))
+
+ for(String table : arrayTables) {
+ attributeList.add(new TvfAttribute(table,
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "",
"")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data_by_line.json")
+ .addProperty("format", "json")
+ .addProperty("read_json_by_line", "true")
+ .addProperty("strip_outer_array", "false")
+ .addProperty("column_separator", "|")
+ .addProperty("num_as_string", "true")
+ .addProperty("force_parsing_by_standard_uri", "true"))
+ }
+
+ for(String table : basicTables) {
+ attributeList.add(new TvfAttribute(table,
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18","k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18",
"", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.json")
+ .addProperty("format", "json")
+ .addProperty("read_json_by_line", "false")
+ .addProperty("strip_outer_array", "true")
+ .addProperty("num_as_string", "true")
+ .addProperty("force_parsing_by_standard_uri", "true"))
+ }
+
+ attributeList.add(new TvfAttribute("agg_tbl_basic_tvf",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,
to_bitmap(k05) as k19, HLL_HASH(k05) as k20, TO_QUANTILE_STATE(k04, 1.0) as
k21, to_bitmap(k05) as kd19, HLL_HASH(k05) as kd20, TO_QUANTILE_STATE(k04, 1.0)
as kd21",
+
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.json")
+ .addProperty("format", "json")
+ .addProperty("read_json_by_line", "false")
+ .addProperty("strip_outer_array", "true")
+ .addProperty("column_separator", "|")
+ .addProperty("num_as_string", "true")
+ .addProperty("force_parsing_by_standard_uri", "true"))
+
+ for(String table : arrayTables) {
+ attributeList.add(new TvfAttribute(table,
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "",
"")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.json")
+ .addProperty("format", "json")
+ .addProperty("read_json_by_line", "false")
+ .addProperty("strip_outer_array", "true")
+ .addProperty("column_separator", "|")
+ .addProperty("num_as_string", "true")
+ .addProperty("force_parsing_by_standard_uri", "true"))
+ }
+
+ // fuzzy_parse
+ for(String table : basicTables) {
+ attributeList.add(new TvfAttribute(table,
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18","k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18",
"", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.json")
+ .addProperty("format", "json")
+ .addProperty("read_json_by_line", "false")
+ .addProperty("strip_outer_array", "true")
+ .addProperty("fuzzy_parse", "true")
+ .addProperty("force_parsing_by_standard_uri", "true"))
+ }
+
+ attributeList.add(new TvfAttribute("agg_tbl_basic_tvf",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,
to_bitmap(k05) as k19, HLL_HASH(k05) as k20, TO_QUANTILE_STATE(k04, 1.0) as
k21, to_bitmap(k05) as kd19, HLL_HASH(k05) as kd20, TO_QUANTILE_STATE(k04, 1.0)
as kd21",
+
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.json")
+ .addProperty("format", "json")
+ .addProperty("read_json_by_line", "false")
+ .addProperty("strip_outer_array", "true")
+ .addProperty("fuzzy_parse", "true")
+ .addProperty("force_parsing_by_standard_uri", "true"))
+
+ for(String table : arrayTables) {
+ attributeList.add(new TvfAttribute(table,
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "",
"")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.json")
+ .addProperty("format", "json")
+ .addProperty("read_json_by_line", "false")
+ .addProperty("strip_outer_array", "true")
+ .addProperty("fuzzy_parse", "true")
+ .addProperty("force_parsing_by_standard_uri", "true"))
+ }
+
+
+
// errors: no capital prop, no column mappings, array type loading stuck
/* normal */
@@ -68,7 +468,7 @@ suite("test_s3_tvf", "load_p0") {
.addProperty("force_parsing_by_standard_uri", "true"))
for(String table : arrayTables) {
- attributeList.add(new TvfAttribute(table, ["K00", "K01", "K02", "K03",
"K04", "K05", "K06", "K07", "K08", "K09", "K10", "K11", "K12", "K13", "K14",
"K15", "K16", "K17"], "", "").addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv")
+ attributeList.add(new TvfAttribute(table, ["K00", "K01", "K02", "K03",
"K04", "K05", "K06", "K07", "K08", "K09", "K10", "K11", "K12", "K13", "K14",
"K15", "K16", "K17"], "", "")
.addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv")
.addProperty("format", "csv")
.addProperty("column_separator", "|")
@@ -258,6 +658,57 @@ suite("test_s3_tvf", "load_p0") {
.addProperty("column_separator", "|")
.addProperty("compress_type", "LZ4FRAME"))
}
+
+ // compress_type: lzo
+ for(String table : basicTables) {
+ attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17", "k18"], "", "")
+ .addProperty("uri",
"https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv.lzo";)
+ .addProperty("format", "csv")
+ .addProperty("column_separator",
"|").addProperty("compress_type", "LZO"))
+ }
+
+ attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as
k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10
as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as
k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20,
TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20,
TO_QUANTILE_STATE(c5, 1.0) as kd21",
+
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
+ .addProperty("uri",
"https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv.lzo";)
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|")
+ .addProperty("compress_type", "LZO"))
+
+ // we can upload basic_array_data.csv.lzo to s3
+ for(String table : arrayTables) {
+ attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17"], "", "")
+ .addProperty("uri",
"https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv.lzo";)
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|")
+ .addProperty("compress_type", "LZO"))
+ }
+
+ // compress_type: deflate
+ // for(String table : basicTables) {
+ // attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02",
"k03", "k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13",
"k14", "k15", "k16", "k17", "k18"], "", "")
+ // .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_test.csv.deflate")
+ // .addProperty("format", "csv")
+ // .addProperty("column_separator", "|")
+ // .addProperty("compress_type", "DEFLATE")
+ // .addProperty("force_parsing_by_standard_uri", "true"))
+ // }
+
+ // attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2
as k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as
k08,c10 as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as
k15,c17 as k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as
k20, TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as
kd20, TO_QUANTILE_STATE(c5, 1.0) as kd21",
+ //
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
+ // .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_test.csv.deflate")
+ // .addProperty("format", "csv")
+ // .addProperty("column_separator", "|")
+ // .addProperty("compress_type", "DEFLATE")
+ // .addProperty("force_parsing_by_standard_uri", "true"))
+
+ // for(String table : arrayTables) {
+ // attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02",
"k03", "k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13",
"k14", "k15", "k16", "k17"], "", "")
+ // .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv.deflate")
+ // .addProperty("format", "csv")
+ // .addProperty("column_separator", "|")
+ // .addProperty("compress_type", "DEFLATE")
+ // .addProperty("force_parsing_by_standard_uri", "true"))
+ // }
for(String table : basicTables) {
attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17", "k18"], "WHERE c1 > 50", "")
@@ -394,6 +845,63 @@ suite("test_s3_tvf", "load_p0") {
.addProperty("force_parsing_by_standard_uri", "true"))
}
+
+
+ // line_delimiter: \t
+ for(String table : basicTables) {
+ attributeList.add(new TvfAttribute(table, ["K00", "K01", "K02", "K03",
"K04", "K05", "K06", "K07", "K08", "K09", "K10", "K11", "K12", "K13", "K14",
"K15", "K16", "K17", "K18"], "", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_by_line_delimiter.csv")
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|")
+ .addProperty("line_delimiter", "\t")
+ .addProperty("force_parsing_by_standard_uri", "true"))
+ }
+
+ attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as
k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10
as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as
k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20,
TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20,
TO_QUANTILE_STATE(c5, 1.0) as kd21",
+
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_by_line_delimiter.csv")
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|")
+ .addProperty("line_delimiter", "\t")
+ .addProperty("force_parsing_by_standard_uri", "true"))
+
+ for(String table : arrayTables) {
+ attributeList.add(new TvfAttribute(table, ["K00", "K01", "K02", "K03",
"K04", "K05", "K06", "K07", "K08", "K09", "K10", "K11", "K12", "K13", "K14",
"K15", "K16", "K17"], "", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data_by_tab_line_delimiter.csv")
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|")
+ .addProperty("line_delimiter", "\t")
+ .addProperty("force_parsing_by_standard_uri", "true"))
+ }
+
+ // invalid line delimiter, this will case error
+ // for(String table : basicTables) {
+ // attributeList.add(new TvfAttribute(table, ["K00", "K01", "K02",
"K03", "K04", "K05", "K06", "K07", "K08", "K09", "K10", "K11", "K12", "K13",
"K14", "K15", "K16", "K17", "K18"], "", "")
+ // .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv")
+ // .addProperty("format", "csv")
+ // .addProperty("column_separator", "|")
+ // .addProperty("line_delimiter", ",")
+ // .addProperty("force_parsing_by_standard_uri", "true"))
+ // }
+
+ // attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2
as k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as
k08,c10 as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as
k15,c17 as k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as
k20, TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as
kd20, TO_QUANTILE_STATE(c5, 1.0) as kd21",
+ //
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
+ // .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv")
+ // .addProperty("format", "csv")
+ // .addProperty("column_separator", "|")
+ // .addProperty("line_delimiter", ",")
+ // .addProperty("force_parsing_by_standard_uri", "true"))
+
+ // for(String table : arrayTables) {
+ // attributeList.add(new TvfAttribute(table, ["K00", "K01", "K02",
"K03", "K04", "K05", "K06", "K07", "K08", "K09", "K10", "K11", "K12", "K13",
"K14", "K15", "K16", "K17"], "", "")
+ // .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv")
+ // .addProperty("format", "csv")
+ // .addProperty("column_separator", "|")
+ // .addProperty("line_delimiter", ",")
+ // .addProperty("force_parsing_by_standard_uri", "true"))
+ // }
+
+
def ak = getS3AK()
def sk = getS3SK()
@@ -403,12 +911,12 @@ suite("test_s3_tvf", "load_p0") {
}
def i = 0
+
for (TvfAttribute attribute : attributeList) {
def prop = attribute.getPropertiesStr()
def insertList = attribute.getInsertList()
def selectList = attribute.getSelectList()
-
def sqlStr = """
INSERT INTO ${attribute.tableName} (${insertList})
SELECT ${selectList}
@@ -428,10 +936,10 @@ suite("test_s3_tvf", "load_p0") {
assertTrue(attribute.expectFiled)
logger.info("error: ", ex)
}
-
qt_select """ select count(*) from $attribute.tableName """
++i
- }
+ }
+
}
@@ -483,6 +991,8 @@ class TvfAttribute {
return getInsertList()
}
+
+
String getInsertList() {
if (insertList.length() != 0) {
return insertList
@@ -507,8 +1017,14 @@ class TvfAttribute {
return prop
}
+ String get(String k) {
+ return properties.get(k)
+ }
+
TvfAttribute addProperty(String k, String v) {
properties.put(k, v)
return this
}
+
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
