This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 4941b9a94d3 [fix](regression) Check Trino readiness and HDFS load
completion (#68150)
4941b9a94d3 is described below
commit 4941b9a94d36d055c72f9047555afb2b4e6c59e7
Author: Gabriel <[email protected]>
AuthorDate: Fri Sep 18 13:43:34 2026 +0800
[fix](regression) Check Trino readiness and HDFS load completion (#68150)
### What problem does this PR solve?
Two external regression cases can report empty result mismatches without
first confirming that the preceding operation succeeded.
In `test_iceberg_deletion_vector`, the Trino container is restarted
before the comparison query. The test previously retried the data query
while ignoring command failures, so a startup or query error with empty
stdout became a row mismatch. Wait for a successful `SELECT 1` with a
bounded startup deadline, then require the comparison command to succeed
before checking its rows. Report stdout and stderr on failure, and keep
successful empty or incorrect results as assertion failures.
In `test_s3_tvf_number_range`, the HDFS Broker Load waiter previously
returned normally after cancellation or roughly ten seconds of polling.
The caller could then query a freshly truncated table before loading
completed. Wait up to 120 seconds using a monotonic clock, return only
on `FINISHED`, and fail on cancellation or timeout with the job label
and last `SHOW LOAD` result.
### Release note
None. Regression-test changes only.
### Check List (For Author)
- Test: Both modified suites compile with Groovy 4.0.19; `git diff
--check` passes. A local harness evaluates the actual modified
control-flow blocks with simulated command/load responses and
virtualized waiting. All eight scenarios pass: delayed load completion,
cancellation, timeout, missing load job, delayed Trino readiness,
comparison-command failure, readiness timeout, and a successful query
returning no rows. The original implementation fails these checks. Full
external integration execution remains pending CI.
- Behavior changed: Test startup/load failures now report their
execution status before row comparison; expected data results are
unchanged.
- Does this need documentation: No.
---
.../iceberg/test_iceberg_deletion_vector.groovy | 42 ++++++++++++++--------
.../tvf/test_s3_tvf_number_range.groovy | 32 ++++++++---------
2 files changed, 43 insertions(+), 31 deletions(-)
diff --git
a/regression-test/suites/external_table_p0/iceberg/test_iceberg_deletion_vector.groovy
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_deletion_vector.groovy
index 8136df3cef4..a4e23ac7310 100644
---
a/regression-test/suites/external_table_p0/iceberg/test_iceberg_deletion_vector.groovy
+++
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_deletion_vector.groovy
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+import java.util.concurrent.TimeUnit
+
suite("test_iceberg_deletion_vector", "p0,external,nonConcurrent") {
String enabled = context.config.otherConfigs.get("enableIcebergTest")
if (enabled == null || !enabled.equalsIgnoreCase("true")) {
@@ -504,22 +506,34 @@ s3.path-style-access=true
30
)
executeCommand("${dockerCommand} restart ${trinoContainerName}", true, 60)
- String trinoRows = ""
- for (int i = 0; i < 12; i++) {
- Thread.sleep(5000)
- trinoRows = normalizeExternalRows(executeCommand(
- "${dockerCommand} exec ${trinoContainerName} trino
--output-format TSV " +
- "--catalog iceberg --schema format_v3 --execute " +
- "\"SELECT id, batch, data " +
- "FROM dv_delete_matrix_equality_and_dv ORDER BY id\"",
- false,
- 120
- ))
- if (!trinoRows.isEmpty()) {
+ String trinoCommand = "${dockerCommand} exec ${trinoContainerName} trino
--output-format TSV " +
+ "--catalog iceberg --schema format_v3 --execute "
+ // A running container does not imply a ready coordinator. Retry startup
separately so a
+ // failed data query cannot be mistaken for a successful query returning
no rows.
+ long trinoReadyDeadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(300)
+ def readiness = [exitCode: -1, stdout: "", stderr: "Readiness probe has
not run"]
+ while (System.nanoTime() < trinoReadyDeadline) {
+ readiness = executeCommandWithStatus(trinoCommand + '"SELECT 1"', 10,
false, false)
+ if (readiness.exitCode == 0 && readiness.stdout.trim() == "1") {
break
- }
+ }
+ Thread.sleep(1000)
}
- assertEquals(expectedRows, trinoRows)
+ assertTrue(readiness.exitCode == 0 && readiness.stdout.trim() == "1",
+ "Trino did not become ready within 300 seconds. Exit code:
${readiness.exitCode}\n" +
+
"stdout:\n${readiness.stdout}\nstderr:\n${readiness.stderr}")
+
+ def trinoResult = executeCommandWithStatus(
+ trinoCommand + '"SELECT id, batch, data ' +
+ 'FROM dv_delete_matrix_equality_and_dv ORDER BY id"',
+ 120
+ )
+ assertEquals(0, trinoResult.exitCode,
+ "Trino comparison query failed.
stdout:\n${trinoResult.stdout}\nstderr:\n${trinoResult.stderr}")
+ String trinoRows = normalizeExternalRows(trinoResult.stdout)
+ assertEquals(expectedRows, trinoRows,
+ "Trino comparison query returned unexpected rows. " +
+
"stdout:\n${trinoResult.stdout}\nstderr:\n${trinoResult.stderr}")
def profileCounterValues = { String profileText, String counterName ->
def values = []
diff --git
a/regression-test/suites/external_table_p0/tvf/test_s3_tvf_number_range.groovy
b/regression-test/suites/external_table_p0/tvf/test_s3_tvf_number_range.groovy
index 3e84bce35ba..216c869f131 100644
---
a/regression-test/suites/external_table_p0/tvf/test_s3_tvf_number_range.groovy
+++
b/regression-test/suites/external_table_p0/tvf/test_s3_tvf_number_range.groovy
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+import java.util.concurrent.TimeUnit
+
suite("test_s3_tvf_number_range", "p0,external") {
String ak = getS3AK()
@@ -351,29 +353,25 @@ suite("test_s3_tvf_number_range", "p0,external") {
// Helper closure to check load result
def check_hdfs_load_result = {checklabel ->
- def max_try_milli_secs = 10000
- def success = false
- while(max_try_milli_secs) {
- def result = sql """ SHOW LOAD WHERE LABEL =
'${checklabel}' """
+ // Broker Load is asynchronous: cancellation or an unfinished
job must fail here,
+ // before the caller compares rows in the freshly truncated
table.
+ long deadline = System.nanoTime() +
TimeUnit.SECONDS.toNanos(120)
+ def result = []
+ while (System.nanoTime() < deadline) {
+ result = sql """ SHOW LOAD WHERE LABEL = '${checklabel}'
"""
if (result.size() > 0) {
def state = result[0][2] // State column
if (state == "FINISHED") {
sql "sync"
- success = true
- break
- } else if (state == "CANCELLED") {
- logger.error("HDFS load job ${checklabel} was
cancelled: ${result[0]}")
- break
+ return
}
+ assertTrue(state != "CANCELLED",
+ "HDFS load job ${checklabel} was cancelled.
Status: ${result}")
}
- sleep(1000) // wait 1 second every time
- max_try_milli_secs-=1000
- }
-
- if (!success) {
- def result = sql """ SHOW LOAD WHERE LABEL =
'${checklabel}' """
- logger.error("HDFS load job ${checklabel} failed or
timeout. Status: ${result}")
+ sleep(1000)
}
+ assertTrue(false,
+ "HDFS load job ${checklabel} did not finish within 120
seconds. Last status: ${result}")
}
// Test 12: HDFS Broker Load Single range {1..3} - should load
{1,2,3}
@@ -407,4 +405,4 @@ suite("test_s3_tvf_number_range", "p0,external") {
}
}
sql """ DROP TABLE IF EXISTS ${test_table} """
-}
\ No newline at end of file
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
