This is an automated email from the ASF dual-hosted git repository. dataroaring pushed a commit to branch branch-3.0 in repository https://gitbox.apache.org/repos/asf/doris.git
commit aed9b75b20da98f4d7e3d96ee8456ea3f3450b97 Author: Xin Liao <[email protected]> AuthorDate: Wed Jul 10 17:13:58 2024 +0800 [opt](test) Optimize execution time of test_s3_load case (#37562) --- .../data/load_p0/broker_load/test_s3_load.out | 336 -------------- .../broker_load/test_s3_load_properties.out} | 0 .../load_p0/broker_load/test_compress_type.groovy | 38 +- .../suites/load_p0/broker_load/test_s3_load.groovy | 513 +++++---------------- .../broker_load/ddl/agg_tbl_basic.sql | 0 .../broker_load/ddl/agg_tbl_basic_drop.sql | 0 .../broker_load/ddl/basic_data.sql | 0 .../broker_load/ddl/basic_data_drop.sql | 0 .../broker_load/ddl/dup_tbl_array.sql | 0 .../broker_load/ddl/dup_tbl_array_drop.sql | 0 .../broker_load/ddl/dup_tbl_basic.sql | 0 .../broker_load/ddl/dup_tbl_basic_drop.sql | 0 .../broker_load/ddl/mow_tbl_array.sql | 0 .../broker_load/ddl/mow_tbl_array_drop.sql | 0 .../broker_load/ddl/mow_tbl_basic.sql | 0 .../broker_load/ddl/mow_tbl_basic_drop.sql | 0 .../broker_load/ddl/uniq_tbl_array.sql | 0 .../broker_load/ddl/uniq_tbl_array_drop.sql | 0 .../broker_load/ddl/uniq_tbl_basic.sql | 0 .../broker_load/ddl/uniq_tbl_basic_drop.sql | 0 .../broker_load/test_s3_load_properties.groovy} | 2 +- 21 files changed, 143 insertions(+), 746 deletions(-) diff --git a/regression-test/data/load_p0/broker_load/test_s3_load.out b/regression-test/data/load_p0/broker_load/test_s3_load.out index 022c08cf8f3..e99cfc0d79a 100644 --- a/regression-test/data/load_p0/broker_load/test_s3_load.out +++ b/regression-test/data/load_p0/broker_load/test_s3_load.out @@ -2,393 +2,57 @@ -- !select -- 20 --- !select -- -20 - --- !select -- -20 - --- !select -- -20 - --- !select -- -18 - --- !select -- -16 - --- !select -- -16 - -- !select -- 40 --- !select -- -20 - --- !select -- -20 - --- !select -- -20 - --- !select -- -36 - --- !select -- -16 - --- !select -- -16 - -- !select -- 40 --- !select -- -20 - --- !select -- -20 - --- !select -- -20 - --- !select -- -36 - --- !select -- -16 - --- !select -- -16 - -- !select -- 40 --- !select -- -20 - --- !select -- -20 - --- !select -- -20 - --- !select -- -36 - --- !select -- -16 - --- !select -- -16 - -- !select -- 40 --- !select -- -20 - --- !select -- -20 - --- !select -- -20 - --- !select -- -36 - --- !select -- -16 - --- !select -- -16 - -- !select -- 40 --- !select -- -20 - --- !select -- -20 - --- !select -- -20 - --- !select -- -36 - --- !select -- -16 - --- !select -- -16 - -- !select -- 60 --- !select -- -34 - --- !select -- -34 - --- !select -- -34 - --- !select -- -54 - --- !select -- -17 - --- !select -- -17 - -- !select -- 80 --- !select -- -48 - --- !select -- -48 - --- !select -- -48 - --- !select -- -72 - --- !select -- -18 - --- !select -- -18 - -- !select -- 100 --- !select -- -48 - --- !select -- -48 - --- !select -- -48 - --- !select -- -90 - --- !select -- -18 - --- !select -- -18 - -- !select -- 120 --- !select -- -48 - --- !select -- -48 - --- !select -- -48 - --- !select -- -108 - --- !select -- -18 - --- !select -- -18 - -- !select -- 140 --- !select -- -48 - --- !select -- -48 - --- !select -- -48 - --- !select -- -126 - --- !select -- -18 - --- !select -- -18 - -- !select -- 160 --- !select -- -48 - --- !select -- -48 - --- !select -- -48 - --- !select -- -144 - --- !select -- -18 - --- !select -- -18 - -- !select -- 180 --- !select -- -48 - --- !select -- -48 - --- !select -- -48 - --- !select -- -162 - --- !select -- -18 - --- !select -- -18 - -- !select -- 200 --- !select -- -48 - --- !select -- -48 - --- !select -- -48 - --- !select -- -180 - --- !select -- -18 - --- !select -- -18 - --- !select -- -48 - --- !select -- -48 - -- !select -- 220 --- !select -- -48 - --- !select -- -48 - --- !select -- -48 - --- !select -- -198 - --- !select -- -18 - --- !select -- -18 - -- !select -- 240 --- !select -- -48 - --- !select -- -48 - --- !select -- -48 - --- !select -- -216 - --- !select -- -18 - --- !select -- -18 - -- !select -- 260 --- !select -- -48 - --- !select -- -48 - --- !select -- -48 - -- !select -- 280 --- !select -- -48 - --- !select -- -48 - --- !select -- -48 - --- !select -- -47 - --- !select -- -47 - --- !select -- -48 - --- !select -- -216 - -- !select -- 280 --- !select -- -18 - --- !select -- -47 - --- !select -- -18 - --- !select -- -47 - diff --git a/regression-test/data/load_p0/broker_load/test_s3_load.out b/regression-test/data/load_p2/broker_load/test_s3_load_properties.out similarity index 100% copy from regression-test/data/load_p0/broker_load/test_s3_load.out copy to regression-test/data/load_p2/broker_load/test_s3_load_properties.out diff --git a/regression-test/suites/load_p0/broker_load/test_compress_type.groovy b/regression-test/suites/load_p0/broker_load/test_compress_type.groovy index 10370da710b..693e533fa50 100644 --- a/regression-test/suites/load_p0/broker_load/test_compress_type.groovy +++ b/regression-test/suites/load_p0/broker_load/test_compress_type.groovy @@ -88,8 +88,39 @@ suite("test_compress_type", "load_p0") { def i = 0 - sql new File("""${ context.file.parent }/ddl/basic_data_drop.sql""").text - sql new File("""${ context.file.parent }/ddl/basic_data.sql""").text + + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ + CREATE TABLE ${tableName} + ( + k00 INT NOT NULL, + k01 DATE NOT NULL, + k02 BOOLEAN NULL, + k03 TINYINT NULL, + k04 SMALLINT NULL, + k05 INT NULL, + k06 BIGINT NULL, + k07 LARGEINT NULL, + k08 FLOAT NULL, + k09 DOUBLE NULL, + k10 DECIMAL(9,1) NULL, + k11 DECIMALV3(9,1) NULL, + k12 DATETIME NULL, + k13 DATEV2 NULL, + k14 DATETIMEV2 NULL, + k15 CHAR NULL, + k16 VARCHAR NULL, + k17 STRING NULL, + k18 JSON NULL + + ) + DUPLICATE KEY(k00) + DISTRIBUTED BY HASH(k00) BUCKETS 32 + PROPERTIES ( + "bloom_filter_columns"="k05", + "replication_num" = "1" + ) + """ for (String compressType : compressTypes) { def label = "test_s3_load_compress" + UUID.randomUUID().toString().replace("-", "0") + i labels.add(label) @@ -110,9 +141,6 @@ suite("test_compress_type", "load_p0") { "AWS_REGION" = "ap-beijing", "provider" = "${getS3Provider()}" ) - properties( - "use_new_load_scan_node" = "true" - ) """ logger.info("submit sql: ${sql_str}"); sql """${sql_str}""" diff --git a/regression-test/suites/load_p0/broker_load/test_s3_load.groovy b/regression-test/suites/load_p0/broker_load/test_s3_load.groovy index 48158ee8637..f84a44959aa 100644 --- a/regression-test/suites/load_p0/broker_load/test_s3_load.groovy +++ b/regression-test/suites/load_p0/broker_load/test_s3_load.groovy @@ -21,443 +21,178 @@ suite("test_s3_load", "load_p0") { sql "set workload_group=broker_load_test;" - def tables = [ - "agg_tbl_basic", - "dup_tbl_array", - "dup_tbl_basic", - "mow_tbl_array", - "mow_tbl_basic", - "uniq_tbl_array", - "uniq_tbl_basic" - ] - - //deal with agg tables in separate - def basicTables = [ - "dup_tbl_basic", - "mow_tbl_basic", - "uniq_tbl_basic", - ] - - def arrayTables = [ - "dup_tbl_array", - "uniq_tbl_array", - "mow_tbl_array" - ] - - def uniqTables = [ - "mow_tbl_basic", - "uniq_tbl_basic" - ] + def table = "dup_tbl_basic" + + sql """ DROP TABLE IF EXISTS ${table} """ + + sql """ + CREATE TABLE ${table} + ( + k00 INT NOT NULL, + k01 DATE NOT NULL, + k02 BOOLEAN NULL, + k03 TINYINT NULL, + k04 SMALLINT NULL, + k05 INT NULL, + k06 BIGINT NULL, + k07 LARGEINT NULL, + k08 FLOAT NULL, + k09 DOUBLE NULL, + k10 DECIMAL(9,1) NULL, + k11 DECIMALV3(9,1) NULL, + k12 DATETIME NULL, + k13 DATEV2 NULL, + k14 DATETIMEV2 NULL, + k15 CHAR NULL, + k16 VARCHAR NULL, + k17 STRING NULL, + k18 JSON NULL, + kd01 BOOLEAN NOT NULL DEFAULT "TRUE", + kd02 TINYINT NOT NULL DEFAULT "1", + kd03 SMALLINT NOT NULL DEFAULT "2", + kd04 INT NOT NULL DEFAULT "3", + kd05 BIGINT NOT NULL DEFAULT "4", + kd06 LARGEINT NOT NULL DEFAULT "5", + kd07 FLOAT NOT NULL DEFAULT "6.0", + kd08 DOUBLE NOT NULL DEFAULT "7.0", + kd09 DECIMAL NOT NULL DEFAULT "888888888", + kd10 DECIMALV3 NOT NULL DEFAULT "999999999", + kd11 DATE NOT NULL DEFAULT "2023-08-24", + kd12 DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + kd13 DATEV2 NOT NULL DEFAULT "2023-08-24", + kd14 DATETIMEV2 NOT NULL DEFAULT CURRENT_TIMESTAMP, + kd15 CHAR(255) NOT NULL DEFAULT "我能吞下玻璃而不伤身体", + kd16 VARCHAR(300) NOT NULL DEFAULT "我能吞下玻璃而不伤身体", + kd17 STRING NOT NULL DEFAULT "我能吞下玻璃而不伤身体", + kd18 JSON NULL, + + INDEX idx_inverted_k104 (`k05`) USING INVERTED, + INDEX idx_inverted_k110 (`k11`) USING INVERTED, + INDEX idx_inverted_k113 (`k13`) USING INVERTED, + INDEX idx_inverted_k114 (`k14`) USING INVERTED, + INDEX idx_inverted_k117 (`k17`) USING INVERTED PROPERTIES("parser" = "english"), + INDEX idx_ngrambf_k115 (`k15`) USING NGRAM_BF PROPERTIES("gram_size"="3", "bf_size"="256"), + INDEX idx_ngrambf_k116 (`k16`) USING NGRAM_BF PROPERTIES("gram_size"="3", "bf_size"="256"), + INDEX idx_ngrambf_k117 (`k17`) USING NGRAM_BF PROPERTIES("gram_size"="3", "bf_size"="256"), + + INDEX idx_bitmap_k104 (`k02`) USING BITMAP, + INDEX idx_bitmap_k110 (`kd01`) USING BITMAP + + ) + DUPLICATE KEY(k00) + PARTITION BY RANGE(k01) + ( + PARTITION p1 VALUES [('2023-08-01'), ('2023-08-11')), + PARTITION p2 VALUES [('2023-08-11'), ('2023-08-21')), + PARTITION p3 VALUES [('2023-08-21'), ('2023-09-01')) + ) + DISTRIBUTED BY HASH(k00) BUCKETS 32 + PROPERTIES ( + "bloom_filter_columns"="k05", + "replication_num" = "1" + ); + """ def attributesList = [ ] /* ========================================================== normal ========================================================== */ - for (String table : basicTables) { - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv", - "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", - "", "", "", "", "")) - } - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv", - "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\" ", "FORMAT AS \"csv\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", - "", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", "")) - - for (String table : arrayTables) { - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv", - "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)", - "", "", "", "", "")) - } - - for (String table : basicTables) { - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv", - "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(K00,K01,K02,K03,K04,K05,K06,K07,K08,K09,K10,K11,K12,K13,K14,K15,K16,K17,K18)", - "", "", "", "", "")) - } + "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", + "", "", "", "", "")) attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv", - "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\" ", "FORMAT AS \"csv\"", "(K00,K01,K02,K03,K04,K05,K06,k07,K08,K09,K10,K11,K12,K13,K14,K15,K16,K17,K18)", - "", "", "SET (K19=to_bitmap(k04),K20=HLL_HASH(k04),K21=TO_QUANTILE_STATE(K04,1.0),Kd19=to_bitmap(K05),kd20=HLL_HASH(K05),KD21=TO_QUANTILE_STATE(K05,1.0))", "", "")) - - for (String table : arrayTables) { - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv", - "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(K00,K01,K02,K03,K04,K05,K06,K07,K08,K09,K10,K11,K12,K13,K14,K15,K16,K17)", - "", "", "", "", "")) - } - // TODO: should be success ? -// for (String table : basicTables) { -// attributesList.add(new LoadAttributes("s3://cos.ap-beijing.myqcloud.com/doris-build-1308700295/regression/load/data/basic_data.csv", -// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", -// "", "", "", "", "").withPathStyle()) -// } -// -// attributesList.add(new LoadAttributes("s3://cos.ap-beijing.myqcloud.com/doris-build-1308700295/regression/load/data/basic_data.csv", -// "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\" ", "FORMAT AS \"csv\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", -// "", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", "").withPathStyle()) -// -// for (String table : arrayTables) { -// attributesList.add(new LoadAttributes("s3://cos.ap-beijing.myqcloud.com/doris-build-1308700295/regression/load/data/basic_array_data.csv", -// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)", -// "", "", "", "", "").withPathStyle()) -// } -// -// for (String table : basicTables) { -// attributesList.add(new LoadAttributes("s3://cos.ap-beijing.myqcloud.com/doris-build-1308700295/regression/load/data/basic_data.csv", -// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", -// "", "", "", "", "").withPathStyle()) -// } -// -// attributesList.add(new LoadAttributes("s3://cos.ap-beijing.myqcloud.com/doris-build-1308700295/regression/load/data/basic_data.csv", -// "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\" ", "FORMAT AS \"csv\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", -// "", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", "").withPathStyle()) -// -// for (String table : arrayTables) { -// attributesList.add(new LoadAttributes("s3://cos.ap-beijing.myqcloud.com/doris-build-1308700295/regression/load/data/basic_array_data.csv", -// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)", -// "", "", "", "", "").withPathStyle()) -// } + "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(K00,K01,K02,K03,K04,K05,K06,K07,K08,K09,K10,K11,K12,K13,K14,K15,K16,K17,K18)", + "", "", "", "", "")) /* ========================================================== error ========================================================== */ - for (String table : basicTables) { - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_with_errors.csv", - "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", - "", "", "", "", "", true)) - } - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_with_errors.csv", - "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\" ", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", - "", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", "", true)) - - for (String table : arrayTables) { - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data_with_errors.csv", - "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"csv\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)", - "", "", "", "", "", true)) - } - -// has problem, should be success -// for(String table: basicTables) { -// attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_with_errors.csv", -// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", -// "", "", "", "","").addProperties("max_filter_ratio", "0.5")) -// } -// -// attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_with_errors.csv", -// "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\" ", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", -// "", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", "").addProperties("max_filter_ratio", "0.5")) -// -// for(String table : arrayTables) { -// attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data_with_errors.csv", -// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)", -// "", "", "", "","").addProperties("max_filter_ratio", "0.5")) -// } - -// for(String table: basicTables) { -// attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_with_errors.csv", -// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", -// "", "", "", "","", true).addProperties("max_filter_ratio", "0.4")) -// } -// -// attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_with_errors.csv", -// "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\" ", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", -// "", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", "", true).addProperties("max_filter_ratio", "0.4")) -// -// for(String table : arrayTables) { -// attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data_with_errors.csv", -// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)", -// "", "", "", "","", true).addProperties("max_filter_ratio", "0.4")) -// } - - // skip lines -// for(String table: basicTables) { -// attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_with_errors.csv", -// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", -// "", "", "", "","").addProperties("skip_lines", "10")) -// } -// -// attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_with_errors.csv", -// "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\" ", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", -// "", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", "").addProperties("skip_lines", "10")) -// -// for(String table : arrayTables) { -// attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data_with_errors.csv", -// "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)", -// "", "", "", "","").addProperties("skip_lines", "10")) -// } + "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", + "", "", "", "", "", true)) /* ========================================================== wrong column sep ========================================================== */ - for (String table : basicTables) { - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv", - "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \",\"", "FORMAT AS \"csv\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", - "", "", "", "", "", true)) - } - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv", - "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \",\" ", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", - "", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", "", true)) + "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \",\"", "FORMAT AS \"csv\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", + "", "", "", "", "", true)) - for (String table : arrayTables) { - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv", - "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \",\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)", - "", "", "", "", "", true)) - } /* ========================================================== wrong line delim ========================================================== */ - for (String table : basicTables) { - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv", - "${table}", "LINES TERMINATED BY \"\t\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", - "", "", "", "", "", true)) - } - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv", - "agg_tbl_basic", "LINES TERMINATED BY \"\t\"", "COLUMNS TERMINATED BY \"|\" ", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", - "", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", "", true)) + "${table}", "LINES TERMINATED BY \"\t\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", + "", "", "", "", "", true)) - for (String table : arrayTables) { - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv", - "${table}", "LINES TERMINATED BY \"\t\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)", - "", "", "", "", "", true)) - } /* ========================================================== strict mode ========================================================== */ - for(String table: basicTables) { - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_with_errors.csv", - "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", - "", "", "", "","", true).addProperties("strict_mode", "true")) - } - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_with_errors.csv", - "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\" ", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", - "", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", "", true).addProperties("strict_mode","true")) - - for(String table : arrayTables) { - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data_with_errors.csv", - "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)", - "", "", "", "","", true).addProperties("strict_mode", "true")) - } + "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", + "", "", "", "","", true).addProperties("strict_mode", "true")) /* ========================================================== timezone ========================================================== */ - - for(String table: basicTables) { - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv", - "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", - "", "", "SET (k00=unix_timestamp('2023-09-01 12:00:00'))", "","").addProperties("timezone", "Asia/Shanghai")) - } - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv", - "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\" ", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", - "", "", "SET (k00=unix_timestamp('2023-09-01 12:00:00'),k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", "").addProperties("timezone", "Asia/Shanghai")) - - for(String table : arrayTables) { - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv", - "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)", - "", "", "SET (k00=unix_timestamp('2023-09-01 12:00:00'))", "","").addProperties("timezone", "Asia/Shanghai")) - } + "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", + "", "", "SET (k00=unix_timestamp('2023-09-01 12:00:00'))", "","").addProperties("timezone", "Asia/Shanghai")) - for(String table: basicTables) { - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv", - "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", - "", "", "SET (k00=unix_timestamp('2023-09-01 12:00:00'))", "","").addProperties("timezone", "America/Chicago")) - } attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv", - "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\" ", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", - "", "", "SET (k00=unix_timestamp('2023-09-01 12:00:00'),k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", "").addProperties("timezone", "America/Chicago")) - - for(String table : arrayTables) { - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv", - "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)", - "", "", "SET (k00=unix_timestamp('2023-09-01 12:00:00'))", "","").addProperties("timezone", "America/Chicago")) - } + "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", + "", "", "SET (k00=unix_timestamp('2023-09-01 12:00:00'))", "","").addProperties("timezone", "America/Chicago")) /* ========================================================== compress type ========================================================== */ - for (String table : basicTables) { - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.gz", - "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", - "", "", "", "", "")) - } - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.gz", - "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\" ", "FORMAT AS \"csv\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", - "", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", "")) + "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", + "", "", "", "", "")) - for (String table : arrayTables) { - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv.gz", - "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)", - "", "", "", "", "")) - } - - for (String table : basicTables) { - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.bz2", - "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", - "", "", "", "", "")) - } attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.bz2", - "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\" ", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", - "", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", "")) + "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", + "", "", "", "", "")) - for (String table : arrayTables) { - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv.bz2", - "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)", - "", "", "", "", "")) - } - - for (String table : basicTables) { - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.lz4", - "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", - "", "", "", "", "")) - } attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.lz4", - "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\" ", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", - "", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", "")) - - for (String table : arrayTables) { - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv.lz4", - "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)", - "", "", "", "", "")) - } + "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "FORMAT AS \"CSV\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", + "", "", "", "", "")) - for (String table : basicTables) { - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.gz", - "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", - "", "", "", "", "")) - } - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.gz", - "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\" ", "", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", - "", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", "")) - - for (String table : arrayTables) { - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv.gz", - "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)", - "", "", "", "", "")) - } - - for (String table : basicTables) { - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.bz2", - "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", - "", "", "", "", "")) - } + "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", + "", "", "", "", "")) attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.bz2", - "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\" ", "", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", - "", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", "")) - - for (String table : arrayTables) { - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv.bz2", - "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)", - "", "", "", "", "")) - } - - for (String table : basicTables) { - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.lz4", - "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", - "", "", "", "", "")) - } + "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", + "", "", "", "", "")) attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv.lz4", - "agg_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\" ", "", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", - "", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", "")) + "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", + "", "", "", "", "")) - for (String table : arrayTables) { - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.csv.lz4", - "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)", - "", "", "", "", "")) - } /*========================================================== order by ==========================================================*/ - for (String table : uniqTables) { - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv", - "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", - "", "", "", "", "ORDER BY k01")) - } /*========================================================== json ==========================================================*/ - for (String table : basicTables) { - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.json", - "${table}", "", "", "FORMAT AS \"json\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", - "", "", "", "", "PROPERTIES(\"strip_outer_array\" = \"true\", \"fuzzy_parse\" = \"true\")")) - } - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.json", - "agg_tbl_basic", "", "", "FORMAT AS \"json\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", - "", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", "PROPERTIES(\"strip_outer_array\" = \"true\", \"fuzzy_parse\" = \"true\")")) + "${table}", "", "", "FORMAT AS \"json\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", + "", "", "", "", "PROPERTIES(\"strip_outer_array\" = \"true\", \"fuzzy_parse\" = \"true\")")) - for (String table : arrayTables) { - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.json", - "${table}", "", "", "FORMAT AS \"json\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)", - "", "", "", "", "PROPERTIES(\"strip_outer_array\" = \"true\", \"fuzzy_parse\" = \"true\")")) - } - - for (String table : basicTables) { - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_by_line.json", - "${table}", "", "", "FORMAT AS \"JSON\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", - "", "", "", "", "PROPERTIES(\"read_json_by_line\" = \"true\")")) - } attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_by_line.json", - "agg_tbl_basic", "", "", "FORMAT AS \"JSON\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", - "", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", "PROPERTIES(\"read_json_by_line\" = \"true\")")) - - for (String table : arrayTables) { - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data_by_line.json", - "${table}", "", "", "FORMAT AS \"JSON\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)", - "", "", "", "", "PROPERTIES(\"read_json_by_line\" = \"true\")")) - } - - for (String table : basicTables) { - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.parq", - "${table}", "", "", "FORMAT AS \"parquet\"", "(K00,K01,K02,K03,K04,K05,K06,K07,K08,K09,K10,K11,K12,K13,K14,K15,K16,K17,K18)", - "", "", "", "", "")) - } + "${table}", "", "", "FORMAT AS \"JSON\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", + "", "", "", "", "PROPERTIES(\"read_json_by_line\" = \"true\")")) attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.parq", - "agg_tbl_basic", "", "", "FORMAT AS \"PARQUET\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", - "", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", "")) - -// for (String table : arrayTables) { -// attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.parq", -// "${table}", "", "", "FORMAT AS \"parquet\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)", -// "", "", "", "", "")) -// } - - for (String table : basicTables) { - attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.orc", - "${table}", "", "", "FORMAT AS \"orc\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", - "", "", "", "", "")) - } + "${table}", "", "", "FORMAT AS \"parquet\"", "(K00,K01,K02,K03,K04,K05,K06,K07,K08,K09,K10,K11,K12,K13,K14,K15,K16,K17,K18)", + "", "", "", "", "")) attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.orc", - "agg_tbl_basic", "", "", "FORMAT AS \"ORC\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", - "", "", "SET (k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))", "", "")) - -// for (String table : arrayTables) { -// attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.parq", -// "${table}", "", "", "FORMAT AS \"parquet\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)", -// "", "", "", "", "")) -// } - - for(String table : uniqTables) { - def attributes = new LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_delete.csv", - "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED BY \"|\"", "", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,__DEL__)", - "", "", "", "", "DELETE ON __DEL__=true") - attributes.dataDesc.mergeType = "MERGE" - attributesList.add(attributes) - } + "${table}", "", "", "FORMAT AS \"orc\"", "(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)", + "", "", "", "", "")) + def ak = getS3AK() def sk = getS3SK() - for(String tbl : tables) { - sql new File("""${ context.file.parent }/ddl/${tbl}_drop.sql""").text - sql new File("""${ context.file.parent }/ddl/${tbl}.sql""").text - } - def i = 0 for (LoadAttributes attributes : attributesList) { def label = "test_s3_load_" + UUID.randomUUID().toString().replace("-", "_") + "_" + i @@ -521,36 +256,7 @@ suite("test_s3_load", "load_p0") { ++i } -// for (LoadAttributes attributes : attributesList) { -// def max_try_milli_secs = 600000 -// while (max_try_milli_secs > 0) { -// String[][] result = sql """ show load where label="$attributes.label" order by createtime desc limit 1; """ -// if (result[0][2].equals("FINISHED")) { -// if (attributes.isExceptFailed) { -// assertTrue(false, "load should be failed but was success: $result") -// } -// logger.info("Load FINISHED " + attributes.label + ": $result") -// break -// } -// if (result[0][2].equals("CANCELLED")) { -// if (attributes.isExceptFailed) { -// logger.info("Load FINISHED " + attributes.label) -// break -// } -// assertTrue(false, "load failed: $result") -// break -// } -// Thread.sleep(1000) -// max_try_milli_secs -= 1000 -// if (max_try_milli_secs <= 0) { -// assertTrue(false, "load Timeout: $attributes.label") -// } -// } -// } - - for(String tbl : tables) { - qt_select """ select count(*) from ${tbl} """ - } + qt_select """ select count(*) from ${table} """ } class DataDesc { @@ -587,7 +293,6 @@ class LoadAttributes { this.isExceptFailed = isExceptFailed properties = new HashMap<>() - properties.put("use_new_load_scan_node", "true") } LoadAttributes addProperties(String k, String v) { diff --git a/regression-test/suites/load_p0/broker_load/ddl/agg_tbl_basic.sql b/regression-test/suites/load_p2/broker_load/ddl/agg_tbl_basic.sql similarity index 100% rename from regression-test/suites/load_p0/broker_load/ddl/agg_tbl_basic.sql rename to regression-test/suites/load_p2/broker_load/ddl/agg_tbl_basic.sql diff --git a/regression-test/suites/load_p0/broker_load/ddl/agg_tbl_basic_drop.sql b/regression-test/suites/load_p2/broker_load/ddl/agg_tbl_basic_drop.sql similarity index 100% rename from regression-test/suites/load_p0/broker_load/ddl/agg_tbl_basic_drop.sql rename to regression-test/suites/load_p2/broker_load/ddl/agg_tbl_basic_drop.sql diff --git a/regression-test/suites/load_p0/broker_load/ddl/basic_data.sql b/regression-test/suites/load_p2/broker_load/ddl/basic_data.sql similarity index 100% rename from regression-test/suites/load_p0/broker_load/ddl/basic_data.sql rename to regression-test/suites/load_p2/broker_load/ddl/basic_data.sql diff --git a/regression-test/suites/load_p0/broker_load/ddl/basic_data_drop.sql b/regression-test/suites/load_p2/broker_load/ddl/basic_data_drop.sql similarity index 100% rename from regression-test/suites/load_p0/broker_load/ddl/basic_data_drop.sql rename to regression-test/suites/load_p2/broker_load/ddl/basic_data_drop.sql diff --git a/regression-test/suites/load_p0/broker_load/ddl/dup_tbl_array.sql b/regression-test/suites/load_p2/broker_load/ddl/dup_tbl_array.sql similarity index 100% rename from regression-test/suites/load_p0/broker_load/ddl/dup_tbl_array.sql rename to regression-test/suites/load_p2/broker_load/ddl/dup_tbl_array.sql diff --git a/regression-test/suites/load_p0/broker_load/ddl/dup_tbl_array_drop.sql b/regression-test/suites/load_p2/broker_load/ddl/dup_tbl_array_drop.sql similarity index 100% rename from regression-test/suites/load_p0/broker_load/ddl/dup_tbl_array_drop.sql rename to regression-test/suites/load_p2/broker_load/ddl/dup_tbl_array_drop.sql diff --git a/regression-test/suites/load_p0/broker_load/ddl/dup_tbl_basic.sql b/regression-test/suites/load_p2/broker_load/ddl/dup_tbl_basic.sql similarity index 100% rename from regression-test/suites/load_p0/broker_load/ddl/dup_tbl_basic.sql rename to regression-test/suites/load_p2/broker_load/ddl/dup_tbl_basic.sql diff --git a/regression-test/suites/load_p0/broker_load/ddl/dup_tbl_basic_drop.sql b/regression-test/suites/load_p2/broker_load/ddl/dup_tbl_basic_drop.sql similarity index 100% rename from regression-test/suites/load_p0/broker_load/ddl/dup_tbl_basic_drop.sql rename to regression-test/suites/load_p2/broker_load/ddl/dup_tbl_basic_drop.sql diff --git a/regression-test/suites/load_p0/broker_load/ddl/mow_tbl_array.sql b/regression-test/suites/load_p2/broker_load/ddl/mow_tbl_array.sql similarity index 100% rename from regression-test/suites/load_p0/broker_load/ddl/mow_tbl_array.sql rename to regression-test/suites/load_p2/broker_load/ddl/mow_tbl_array.sql diff --git a/regression-test/suites/load_p0/broker_load/ddl/mow_tbl_array_drop.sql b/regression-test/suites/load_p2/broker_load/ddl/mow_tbl_array_drop.sql similarity index 100% rename from regression-test/suites/load_p0/broker_load/ddl/mow_tbl_array_drop.sql rename to regression-test/suites/load_p2/broker_load/ddl/mow_tbl_array_drop.sql diff --git a/regression-test/suites/load_p0/broker_load/ddl/mow_tbl_basic.sql b/regression-test/suites/load_p2/broker_load/ddl/mow_tbl_basic.sql similarity index 100% rename from regression-test/suites/load_p0/broker_load/ddl/mow_tbl_basic.sql rename to regression-test/suites/load_p2/broker_load/ddl/mow_tbl_basic.sql diff --git a/regression-test/suites/load_p0/broker_load/ddl/mow_tbl_basic_drop.sql b/regression-test/suites/load_p2/broker_load/ddl/mow_tbl_basic_drop.sql similarity index 100% rename from regression-test/suites/load_p0/broker_load/ddl/mow_tbl_basic_drop.sql rename to regression-test/suites/load_p2/broker_load/ddl/mow_tbl_basic_drop.sql diff --git a/regression-test/suites/load_p0/broker_load/ddl/uniq_tbl_array.sql b/regression-test/suites/load_p2/broker_load/ddl/uniq_tbl_array.sql similarity index 100% rename from regression-test/suites/load_p0/broker_load/ddl/uniq_tbl_array.sql rename to regression-test/suites/load_p2/broker_load/ddl/uniq_tbl_array.sql diff --git a/regression-test/suites/load_p0/broker_load/ddl/uniq_tbl_array_drop.sql b/regression-test/suites/load_p2/broker_load/ddl/uniq_tbl_array_drop.sql similarity index 100% rename from regression-test/suites/load_p0/broker_load/ddl/uniq_tbl_array_drop.sql rename to regression-test/suites/load_p2/broker_load/ddl/uniq_tbl_array_drop.sql diff --git a/regression-test/suites/load_p0/broker_load/ddl/uniq_tbl_basic.sql b/regression-test/suites/load_p2/broker_load/ddl/uniq_tbl_basic.sql similarity index 100% rename from regression-test/suites/load_p0/broker_load/ddl/uniq_tbl_basic.sql rename to regression-test/suites/load_p2/broker_load/ddl/uniq_tbl_basic.sql diff --git a/regression-test/suites/load_p0/broker_load/ddl/uniq_tbl_basic_drop.sql b/regression-test/suites/load_p2/broker_load/ddl/uniq_tbl_basic_drop.sql similarity index 100% rename from regression-test/suites/load_p0/broker_load/ddl/uniq_tbl_basic_drop.sql rename to regression-test/suites/load_p2/broker_load/ddl/uniq_tbl_basic_drop.sql diff --git a/regression-test/suites/load_p0/broker_load/test_s3_load.groovy b/regression-test/suites/load_p2/broker_load/test_s3_load_properties.groovy similarity index 99% copy from regression-test/suites/load_p0/broker_load/test_s3_load.groovy copy to regression-test/suites/load_p2/broker_load/test_s3_load_properties.groovy index 48158ee8637..b837f28f171 100644 --- a/regression-test/suites/load_p0/broker_load/test_s3_load.groovy +++ b/regression-test/suites/load_p2/broker_load/test_s3_load_properties.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_s3_load", "load_p0") { +suite("test_s3_load_properties", "p2") { sql "create workload group if not exists broker_load_test properties ( 'cpu_share'='1024'); " --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
