This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 0ca0d60d5b7 [regression-test](profile) Refine test for profile #35213
0ca0d60d5b7 is described below
commit 0ca0d60d5b767a0f0c8410fceba35aff6f506c1b
Author: zhiqiang <[email protected]>
AuthorDate: Wed May 22 20:50:25 2024 +0800
[regression-test](profile) Refine test for profile #35213
---
.../apache/doris/common/util/ProfileManager.java | 8 +-
.../data/query_profile/s3_load_profile_test.out | 4 +
.../query_profile/s3_load_profile_test.groovy | 250 +++++++++++++++++++++
.../suites/query_profile/test_profile.groovy | 144 +++---------
4 files changed, 285 insertions(+), 121 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java
index eaeb3635c58..6d5724665af 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/ProfileManager.java
@@ -355,11 +355,16 @@ public class ProfileManager {
try {
loadJobId = Long.parseLong(id);
} catch (Exception e) {
- return futures;
+ throw new IllegalArgumentException("Invalid profile id: " +
id);
}
LoadJob loadJob =
Env.getCurrentEnv().getLoadManager().getLoadJob(loadJobId);
+ if (loadJob == null) {
+ throw new RuntimeException("Profile " + id + " not found");
+ }
+
if (loadJob.getLoadTaskIds() == null) {
+ LOG.warn("Load job {} has no task ids", loadJobId);
return futures;
}
@@ -393,7 +398,6 @@ public class ProfileManager {
public String getProfile(String id) {
List<Future<TGetRealtimeExecStatusResponse>> futures =
createFetchRealTimeProfileTasks(id);
-
// beAddr of reportExecStatus of QeProcessorImpl is meaningless, so
assign a dummy address
// to avoid compile failing.
TNetworkAddress dummyAddr = new TNetworkAddress();
diff --git a/regression-test/data/query_profile/s3_load_profile_test.out
b/regression-test/data/query_profile/s3_load_profile_test.out
new file mode 100644
index 00000000000..2029fd38720
--- /dev/null
+++ b/regression-test/data/query_profile/s3_load_profile_test.out
@@ -0,0 +1,4 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !select --
+20
+
diff --git a/regression-test/suites/query_profile/s3_load_profile_test.groovy
b/regression-test/suites/query_profile/s3_load_profile_test.groovy
new file mode 100644
index 00000000000..f525c74fba4
--- /dev/null
+++ b/regression-test/suites/query_profile/s3_load_profile_test.groovy
@@ -0,0 +1,250 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import groovy.json.JsonSlurper
+
+def getProfile = { id ->
+ def dst = 'http://' + context.config.feHttpAddress
+ def conn = new URL(dst + "/rest/v1/query_profile/$id").openConnection()
+ conn.setRequestMethod("GET")
+ def encoding =
Base64.getEncoder().encodeToString((context.config.feHttpUser + ":" +
+ (context.config.feHttpPassword == null ? "" :
context.config.feHttpPassword)).getBytes("UTF-8"))
+ conn.setRequestProperty("Authorization", "Basic ${encoding}")
+ return conn.getInputStream().getText()
+ }
+
+// ref
https://github.com/apache/doris/blob/3525a03815814f66ec78aa2ad6bbd9225b0e7a6b/regression-test/suites/load_p0/broker_load/test_s3_load.groovy
+suite('s3_load_profile_test') {
+ sql "drop table if exists dup_tbl_basic;"
+ sql """
+ CREATE TABLE dup_tbl_basic
+(
+ k00 INT NOT NULL,
+ k01 DATE NOT NULL,
+ k02 BOOLEAN NULL,
+ k03 TINYINT NULL,
+ k04 SMALLINT NULL,
+ k05 INT NULL,
+ k06 BIGINT NULL,
+ k07 LARGEINT NULL,
+ k08 FLOAT NULL,
+ k09 DOUBLE NULL,
+ k10 DECIMAL(9,1) NULL,
+ k11 DECIMALV3(9,1) NULL,
+ k12 DATETIME NULL,
+ k13 DATEV2 NULL,
+ k14 DATETIMEV2 NULL,
+ k15 CHAR NULL,
+ k16 VARCHAR NULL,
+ k17 STRING NULL,
+ k18 JSON NULL,
+ kd01 BOOLEAN NOT NULL DEFAULT "TRUE",
+ kd02 TINYINT NOT NULL DEFAULT "1",
+ kd03 SMALLINT NOT NULL DEFAULT "2",
+ kd04 INT NOT NULL DEFAULT "3",
+ kd05 BIGINT NOT NULL DEFAULT "4",
+ kd06 LARGEINT NOT NULL DEFAULT "5",
+ kd07 FLOAT NOT NULL DEFAULT "6.0",
+ kd08 DOUBLE NOT NULL DEFAULT "7.0",
+ kd09 DECIMAL NOT NULL DEFAULT "888888888",
+ kd10 DECIMALV3 NOT NULL DEFAULT "999999999",
+ kd11 DATE NOT NULL DEFAULT "2023-08-24",
+ kd12 DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ kd13 DATEV2 NOT NULL DEFAULT "2023-08-24",
+ kd14 DATETIMEV2 NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ kd15 CHAR(255) NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+ kd16 VARCHAR(300) NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+ kd17 STRING NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+ kd18 JSON NULL,
+
+ INDEX idx_inverted_k104 (`k05`) USING INVERTED,
+ INDEX idx_inverted_k110 (`k11`) USING INVERTED,
+ INDEX idx_inverted_k113 (`k13`) USING INVERTED,
+ INDEX idx_inverted_k114 (`k14`) USING INVERTED,
+ INDEX idx_inverted_k117 (`k17`) USING INVERTED PROPERTIES("parser" =
"english"),
+ INDEX idx_ngrambf_k115 (`k15`) USING NGRAM_BF PROPERTIES("gram_size"="3",
"bf_size"="256"),
+ INDEX idx_ngrambf_k116 (`k16`) USING NGRAM_BF PROPERTIES("gram_size"="3",
"bf_size"="256"),
+ INDEX idx_ngrambf_k117 (`k17`) USING NGRAM_BF PROPERTIES("gram_size"="3",
"bf_size"="256"),
+
+ INDEX idx_bitmap_k104 (`k02`) USING BITMAP,
+ INDEX idx_bitmap_k110 (`kd01`) USING BITMAP
+
+)
+ DUPLICATE KEY(k00)
+PARTITION BY RANGE(k01)
+(
+ PARTITION p1 VALUES [('2023-08-01'), ('2023-08-11')),
+ PARTITION p2 VALUES [('2023-08-11'), ('2023-08-21')),
+ PARTITION p3 VALUES [('2023-08-21'), ('2023-09-01'))
+)
+DISTRIBUTED BY HASH(k00) BUCKETS 32
+PROPERTIES (
+ "bloom_filter_columns"="k05",
+ "replication_num" = "1"
+);
+"""
+ def loadAttribute =new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.csv",
+ "dup_tbl_basic", "LINES TERMINATED BY \"\n\"", "COLUMNS
TERMINATED BY \"|\"", "FORMAT AS \"CSV\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+ "", "", "", "", "")
+
+ def ak = getS3AK()
+ def sk = getS3SK()
+
+ sql "set enable_profile=true;"
+
+ def label = "test_s3_load_" + UUID.randomUUID().toString().replace("-",
"_")
+ loadAttribute.label = label
+ def prop = loadAttribute.getPropertiesStr()
+
+ def sql_str = """
+ LOAD LABEL $label (
+ $loadAttribute.dataDesc.mergeType
+ DATA INFILE("$loadAttribute.dataDesc.path")
+ INTO TABLE $loadAttribute.dataDesc.tableName
+ $loadAttribute.dataDesc.columnTermClause
+ $loadAttribute.dataDesc.lineTermClause
+ $loadAttribute.dataDesc.formatClause
+ $loadAttribute.dataDesc.columns
+ $loadAttribute.dataDesc.columnsFromPathClause
+ $loadAttribute.dataDesc.columnMappingClause
+ $loadAttribute.dataDesc.precedingFilterClause
+ $loadAttribute.dataDesc.orderByClause
+ $loadAttribute.dataDesc.whereExpr
+ )
+ WITH S3 (
+ "AWS_ACCESS_KEY" = "$ak",
+ "AWS_SECRET_KEY" = "$sk",
+ "AWS_ENDPOINT" = "cos.ap-beijing.myqcloud.com",
+ "AWS_REGION" = "ap-beijing",
+ "use_path_style" = "$loadAttribute.usePathStyle"
+ )
+ ${prop}
+ """
+ logger.info("submit sql: ${sql_str}");
+ sql """${sql_str}"""
+ logger.info("Submit load with lable: $label, table:
$loadAttribute.dataDesc.tableName, path: $loadAttribute.dataDesc.path")
+
+ def max_try_milli_secs = 600000
+ def jobId = -1
+ while (max_try_milli_secs > 0) {
+ String[][] result = sql """ show load where
label="$loadAttribute.label" order by createtime desc limit 1; """
+ if (result[0][2].equals("FINISHED")) {
+ if (loadAttribute.isExceptFailed) {
+ assertTrue(false, "load should be failed but was success:
$result")
+ }
+ jobId = result[0][0].toInteger()
+ logger.info("Load FINISHED " + loadAttribute.label + ": $result" +
" loadId: $jobId")
+ break
+ }
+ if (result[0][2].equals("CANCELLED")) {
+ if (loadAttribute.isExceptFailed) {
+ logger.info("Load FINISHED " + loadAttribute.label)
+ break
+ }
+ assertTrue(false, "load failed: $result")
+ break
+ }
+ Thread.sleep(1000)
+ max_try_milli_secs -= 1000
+ if (max_try_milli_secs <= 0) {
+ assertTrue(false, "load Timeout: $loadAttribute.label")
+ }
+ }
+
+ qt_select """ select count(*) from $loadAttribute.dataDesc.tableName """
+
+ def profileString = getProfile(jobId)
+ profileJson = new JsonSlurper().parseText(profileString)
+ assertEquals(0, profileJson.code)
+ profileDataString = profileJson.data
+ def taskStateIdx = profileDataString.indexOf("- Task State: FINISHED")
+ assertFalse(taskStateIdx == -1)
+ def fragmentIdx = profileDataString.indexOf(" Fragment 0:")
+ assertFalse(fragmentIdx == -1)
+ def executionProfileIdx = profileDataString.indexOf("Execution Profile")
+ assertFalse(executionProfileIdx == -1)
+ def pattern = ~/Active:\s*([1-9]\d*|0\.\d+|[1-9]\d*\.\d*)ms/
+ def matcher = pattern.matcher(profileDataString)
+ assertTrue(matcher.find())
+}
+
+class DataDesc {
+ public String mergeType = ""
+ public String path
+ public String tableName
+ public String lineTermClause
+ public String columnTermClause
+ public String formatClause
+ public String columns
+ public String columnsFromPathClause
+ public String precedingFilterClause
+ public String columnMappingClause
+ public String whereExpr
+ public String orderByClause
+}
+
+class LoadAttributes {
+ LoadAttributes(String path, String tableName, String lineTermClause,
String columnTermClause, String formatClause,
+ String columns, String columnsFromPathClause, String
precedingFilterClause, String columnMappingClause, String whereExpr, String
orderByClause, boolean isExceptFailed = false) {
+ this.dataDesc = new DataDesc()
+ this.dataDesc.path = path
+ this.dataDesc.tableName = tableName
+ this.dataDesc.lineTermClause = lineTermClause
+ this.dataDesc.columnTermClause = columnTermClause
+ this.dataDesc.formatClause = formatClause
+ this.dataDesc.columns = columns
+ this.dataDesc.columnsFromPathClause = columnsFromPathClause
+ this.dataDesc.precedingFilterClause = precedingFilterClause
+ this.dataDesc.columnMappingClause = columnMappingClause
+ this.dataDesc.whereExpr = whereExpr
+ this.dataDesc.orderByClause = orderByClause
+
+ this.isExceptFailed = isExceptFailed
+
+ properties = new HashMap<>()
+ properties.put("use_new_load_scan_node", "true")
+ }
+
+ LoadAttributes addProperties(String k, String v) {
+ properties.put(k, v)
+ return this
+ }
+
+ String getPropertiesStr() {
+ if (properties.isEmpty()) {
+ return ""
+ }
+ String prop = "PROPERTIES ("
+ properties.forEach (k, v) -> {
+ prop += "\"${k}\" = \"${v}\","
+ }
+ prop = prop.substring(0, prop.size() - 1)
+ prop += ")"
+ return prop
+ }
+
+ LoadAttributes withPathStyle() {
+ usePathStyle = "true"
+ return this
+ }
+
+ public DataDesc dataDesc
+ public Map<String, String> properties
+ public String label
+ public String usePathStyle = "false"
+ public boolean isExceptFailed
+}
diff --git a/regression-test/suites/query_profile/test_profile.groovy
b/regression-test/suites/query_profile/test_profile.groovy
index 37d5993b876..fd442726e1d 100644
--- a/regression-test/suites/query_profile/test_profile.groovy
+++ b/regression-test/suites/query_profile/test_profile.groovy
@@ -18,132 +18,38 @@
import groovy.json.JsonOutput
import groovy.json.JsonSlurper
-/**
-* @Params url is "/xxx"
-* @Return response body
-*/
-def http_get(url) {
+def getProfileList = {
def dst = 'http://' + context.config.feHttpAddress
- def conn = new URL(dst + url).openConnection()
+ def conn = new URL(dst + "/rest/v1/query_profile").openConnection()
conn.setRequestMethod("GET")
- //token for root
- conn.setRequestProperty("Authorization","Basic cm9vdDo=")
+ def encoding =
Base64.getEncoder().encodeToString((context.config.feHttpUser + ":" +
+ (context.config.feHttpPassword == null ? "" :
context.config.feHttpPassword)).getBytes("UTF-8"))
+ conn.setRequestProperty("Authorization", "Basic ${encoding}")
return conn.getInputStream().getText()
}
-def SUCCESS_MSG = 'success'
-def SUCCESS_CODE = 0
-def QUERY_NUM = 5
-
-random = new Random()
-
-def getRandomNumber(int num){
- return random.nextInt(num)
-}
-
-suite('test_profile') {
-
- // nereids not return same profile with legacy planner, fallback to legacy
planner.
- sql """set enable_nereids_planner=false"""
-
- def table = 'test_profile_table'
- def id_data = [1,2,3,4,5,6,7]
- def value_data = [1,2,3,4,5,6,7]
- def len = id_data.size
-
- assertEquals(id_data.size, value_data.size)
-
- sql """ DROP TABLE IF EXISTS ${table} """
-
- sql """
- CREATE TABLE IF NOT EXISTS ${table}(
- `id` INT,
- `cost` INT
- )
- DISTRIBUTED BY HASH(id) BUCKETS 1
- PROPERTIES (
- "replication_num" = "1"
- )
- """
-
- sql """ SET enable_profile = true """
-
- //———————— test for insert stmt ——————————
- for(int i = 0; i < len; i++){
- sql """ INSERT INTO ${table} values (${id_data[i]},
"${value_data[i]}") """
+def getProfile = { id ->
+ def dst = 'http://' + context.config.feHttpAddress
+ def conn = new URL(dst + "/rest/v1/query_profile/$id").openConnection()
+ conn.setRequestMethod("GET")
+ def encoding =
Base64.getEncoder().encodeToString((context.config.feHttpUser + ":" +
+ (context.config.feHttpPassword == null ? "" :
context.config.feHttpPassword)).getBytes("UTF-8"))
+ conn.setRequestProperty("Authorization", "Basic ${encoding}")
+ return conn.getInputStream().getText()
}
- //———————— test for insert stmt (SQL) ——————————
- log.info("test HTTP API interface for insert profile")
- def url = '/rest/v1/query_profile/'
- def query_list_result = http_get(url)
-
- def obj = new JsonSlurper().parseText(query_list_result)
- assertEquals(obj.msg, SUCCESS_MSG)
- assertEquals(obj.code, SUCCESS_CODE)
-
- for(int i = 0 ; i < len ; i++){
- def insert_order = len - i - 1
- def stmt_query_info = obj.data.rows[i]
-
- assertNotNull(stmt_query_info["Profile ID"])
- assertNotEquals(stmt_query_info["Profile ID"], "N/A")
-
- assertEquals(stmt_query_info['Sql Statement'].toString(),
- """ INSERT INTO ${table} values (${id_data[insert_order]},
"${value_data[insert_order]}") """.toString())
- }
-
- //———————— test for select stmt ———————————
- def op_data = ["<", ">", "=", "<=", ">="]
-
- def ops = []
- def nums = []
-
- for(int i = 0 ; i < QUERY_NUM ; i++){
- ops.add(op_data[getRandomNumber(5)])
- nums.add(getRandomNumber(len + 1))
- sql """ SELECT * FROM ${table} WHERE cost ${ops[i]} ${nums[i]} """
- }
-
-
- /* test for `show query profile` stmt
- query profile header
-
JobID|QueryId|User|DefaultDb|SQL|QueryType|StartTime|EndTime|TotalTime|QueryState
*/
- //———————— test for select stmt (SQL) ———————————
- log.info("test for show query profile stmt")
- List<List<Object>> show_query_profile_obj = sql """ show query profile "/"
"""
- log.info("found ${show_query_profile_obj.size} profile data".toString())
- assertTrue(show_query_profile_obj.size >= QUERY_NUM)
-
- for(int i = 0 ; i < QUERY_NUM ; i++){
- def insert_order = QUERY_NUM - i - 1
- def current_obj = show_query_profile_obj[i]
- def stmt_query_info = current_obj[9]
- assertNotEquals(current_obj[1].toString(), "N/A".toString())
- assertEquals(stmt_query_info.toString(), """ SELECT * FROM ${table}
WHERE cost ${ops[insert_order]} ${nums[insert_order]} """.toString())
- }
-
- //———————— test for select stmt (HTTP)————————
- log.info("test HTTP API interface for query profile")
- url = '/rest/v1/query_profile/'
- query_list_result = http_get(url)
-
- obj = new JsonSlurper().parseText(query_list_result)
- assertEquals(obj.msg, SUCCESS_MSG)
- assertEquals(obj.code, SUCCESS_CODE)
+suite('test_profile') {
+ // TODO: more test for normal situation
- for(int i = 0 ; i < QUERY_NUM ; i++){
- def insert_order = QUERY_NUM - i - 1
- def stmt_query_info = obj.data.rows[i]
-
- assertNotNull(stmt_query_info["Profile ID"])
- assertNotEquals(stmt_query_info["Profile ID"].toString(),
"N/A".toString())
-
- assertEquals(stmt_query_info['Sql Statement'].toString(),
- """ SELECT * FROM ${table} WHERE cost ${ops[insert_order]}
${nums[insert_order]} """.toString())
- }
+ // invalidProfileId
+ def invalidProfileString = getProfile("ABCD")
+ logger.info("invalidProfileString:{}", invalidProfileString);
+ def json = new JsonSlurper().parseText(invalidProfileString)
+ assertEquals(500, json.code)
- //———————— clean table and disable profile ————————
- sql """ SET enable_profile = false """
- sql """ DROP TABLE IF EXISTS ${table} """
+ // notExistingProfileId
+ def notExistingProfileString = getProfile("-100")
+ logger.info("notExistingProfileString:{}", notExistingProfileString)
+ def json2 = new JsonSlurper().parseText(notExistingProfileString)
+ assertEquals("ID -100 does not exist", json2.data)
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
