This is an automated email from the ASF dual-hosted git repository.
zhangdonghao pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new b1bd658c2c [Improve][E2E] Optimize Hudi E2E Cases (#9752)
b1bd658c2c is described below
commit b1bd658c2c074d83e961226dfc09603725423d4e
Author: liucongjy <[email protected]>
AuthorDate: Fri Aug 22 16:41:35 2025 +0800
[Improve][E2E] Optimize Hudi E2E Cases (#9752)
---
.../seatunnel/e2e/connector/hudi/HudiIT.java | 57 ++--------------------
.../e2e/connector/hudi/HudiMultiTableIT.java | 52 +-------------------
.../e2e/connector/hudi/HudiSinkCDCIT.java | 44 +----------------
.../src/test/resources/hudi/fake_to_hudi.conf | 2 +-
.../hudi/fake_to_hudi_with_omit_config_item.conf | 2 +-
.../test/resources/hudi/multi_fake_to_hudi.conf | 2 +-
.../src/test/resources/hudi/mysql_cdc_to_hudi.conf | 2 +-
7 files changed, 12 insertions(+), 149 deletions(-)
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/java/org/apache/seatunnel/e2e/connector/hudi/HudiIT.java
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/java/org/apache/seatunnel/e2e/connector/hudi/HudiIT.java
index 642b94471d..f4b96808ed 100644
---
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/java/org/apache/seatunnel/e2e/connector/hudi/HudiIT.java
+++
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/java/org/apache/seatunnel/e2e/connector/hudi/HudiIT.java
@@ -17,14 +17,11 @@
package org.apache.seatunnel.e2e.connector.hudi;
-import org.apache.seatunnel.common.utils.FileUtils;
import org.apache.seatunnel.e2e.common.TestSuiteBase;
-import org.apache.seatunnel.e2e.common.container.ContainerExtendedFactory;
import org.apache.seatunnel.e2e.common.container.EngineType;
import org.apache.seatunnel.e2e.common.container.TestContainer;
import org.apache.seatunnel.e2e.common.container.TestContainerId;
import org.apache.seatunnel.e2e.common.junit.DisabledOnContainer;
-import org.apache.seatunnel.e2e.common.junit.TestContainerExtension;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.LocalFileSystem;
@@ -36,7 +33,6 @@ import org.apache.parquet.hadoop.example.GroupReadSupport;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.TestTemplate;
import org.testcontainers.containers.Container;
-import org.testcontainers.containers.GenericContainer;
import lombok.extern.slf4j.Slf4j;
@@ -45,6 +41,7 @@ import java.io.IOException;
import java.net.URISyntaxException;
import java.util.concurrent.TimeUnit;
+import static
org.apache.seatunnel.e2e.common.container.AbstractTestContainer.HOST_VOLUME_MOUNT_PATH;
import static org.awaitility.Awaitility.given;
@DisabledOnContainer(
@@ -57,49 +54,7 @@ public class HudiIT extends TestSuiteBase {
private static final String DATABASE = "st";
private static final String DEFAULT_DATABASE = "default";
private static final String TABLE_NAME = "st_test";
- private static final String TABLE_PATH = "/tmp/hudi/";
- private static final String NAMESPACE = "hudi";
- private static final String NAMESPACE_TAR = "hudi.tar.gz";
-
- protected final ContainerExtendedFactory containerExtendedFactory =
- new ContainerExtendedFactory() {
- @Override
- public void extend(GenericContainer<?> container)
- throws IOException, InterruptedException {
- container.execInContainer(
- "sh",
- "-c",
- "cd /tmp" + " && tar -czvf " + NAMESPACE_TAR + " "
+ NAMESPACE);
- container.copyFileFromContainer(
- "/tmp/" + NAMESPACE_TAR, "/tmp/" + NAMESPACE_TAR);
-
- extractFiles();
- }
-
- private void extractFiles() {
- ProcessBuilder processBuilder = new ProcessBuilder();
- processBuilder.command(
- "sh", "-c", "cd /tmp" + " && tar -zxvf " +
NAMESPACE_TAR);
- try {
- Process process = processBuilder.start();
- int exitCode = process.waitFor();
- if (exitCode == 0) {
- log.info("Extract files successful.");
- } else {
- log.error("Extract files failed with exit code " +
exitCode);
- }
- } catch (IOException | InterruptedException e) {
- e.printStackTrace();
- }
- }
- };
-
- @TestContainerExtension
- protected final ContainerExtendedFactory extendedFactory =
- container -> {
- container.execInContainer("sh", "-c", "mkdir -p " +
TABLE_PATH);
- container.execInContainer("sh", "-c", "chmod -R 777 " +
TABLE_PATH);
- };
+ private static final String TABLE_PATH = HOST_VOLUME_MOUNT_PATH + "/hudi/";
@TestTemplate
@DisabledOnContainer(
@@ -120,8 +75,7 @@ public class HudiIT extends TestSuiteBase {
.atMost(60000, TimeUnit.MILLISECONDS)
.untilAsserted(
() -> {
- // copy hudi to local
-
container.executeExtraCommands(containerExtendedFactory);
+ // read hudi data and count rows
ParquetReader<Group> reader =
ParquetReader.builder(new
GroupReadSupport(), inputPath)
.withConf(configuration)
@@ -135,7 +89,6 @@ public class HudiIT extends TestSuiteBase {
}
Assertions.assertEquals(5, rowCount);
});
- FileUtils.deleteFile(TABLE_PATH);
}
@TestTemplate
@@ -163,8 +116,7 @@ public class HudiIT extends TestSuiteBase {
.atMost(60000, TimeUnit.MILLISECONDS)
.untilAsserted(
() -> {
- // copy hudi to local
-
container.executeExtraCommands(containerExtendedFactory);
+ // read hudi data and count rows
ParquetReader<Group> reader =
ParquetReader.builder(new
GroupReadSupport(), inputPath)
.withConf(configuration)
@@ -178,6 +130,5 @@ public class HudiIT extends TestSuiteBase {
}
Assertions.assertEquals(5, rowCount);
});
- FileUtils.deleteFile(TABLE_PATH);
}
}
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/java/org/apache/seatunnel/e2e/connector/hudi/HudiMultiTableIT.java
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/java/org/apache/seatunnel/e2e/connector/hudi/HudiMultiTableIT.java
index c240b85da7..b9f259fc29 100644
---
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/java/org/apache/seatunnel/e2e/connector/hudi/HudiMultiTableIT.java
+++
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/java/org/apache/seatunnel/e2e/connector/hudi/HudiMultiTableIT.java
@@ -17,14 +17,11 @@
package org.apache.seatunnel.e2e.connector.hudi;
-import org.apache.seatunnel.common.utils.FileUtils;
import org.apache.seatunnel.e2e.common.TestSuiteBase;
-import org.apache.seatunnel.e2e.common.container.ContainerExtendedFactory;
import org.apache.seatunnel.e2e.common.container.EngineType;
import org.apache.seatunnel.e2e.common.container.TestContainer;
import org.apache.seatunnel.e2e.common.container.TestContainerId;
import org.apache.seatunnel.e2e.common.junit.DisabledOnContainer;
-import org.apache.seatunnel.e2e.common.junit.TestContainerExtension;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
@@ -37,7 +34,6 @@ import org.apache.parquet.hadoop.example.GroupReadSupport;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.TestTemplate;
import org.testcontainers.containers.Container;
-import org.testcontainers.containers.GenericContainer;
import lombok.extern.slf4j.Slf4j;
@@ -46,6 +42,7 @@ import java.io.IOException;
import java.util.Arrays;
import java.util.concurrent.TimeUnit;
+import static
org.apache.seatunnel.e2e.common.container.AbstractTestContainer.HOST_VOLUME_MOUNT_PATH;
import static org.awaitility.Awaitility.given;
@Slf4j
@@ -55,49 +52,7 @@ public class HudiMultiTableIT extends TestSuiteBase {
private static final String TABLE_NAME_1 = "st_test_1";
private static final String DATABASE_2 = "default";
private static final String TABLE_NAME_2 = "st_test_2";
- private static final String TABLE_PATH = "/tmp/hudi/";
- private static final String NAMESPACE = "hudi";
- private static final String NAMESPACE_TAR = "hudi.tar.gz";
-
- protected final ContainerExtendedFactory containerExtendedFactory =
- new ContainerExtendedFactory() {
- @Override
- public void extend(GenericContainer<?> container)
- throws IOException, InterruptedException {
- container.execInContainer(
- "sh",
- "-c",
- "cd /tmp" + " && tar -czvf " + NAMESPACE_TAR + " "
+ NAMESPACE);
- container.copyFileFromContainer(
- "/tmp/" + NAMESPACE_TAR, "/tmp/" + NAMESPACE_TAR);
-
- extractFiles();
- }
-
- private void extractFiles() {
- ProcessBuilder processBuilder = new ProcessBuilder();
- processBuilder.command(
- "sh", "-c", "cd /tmp" + " && tar -zxvf " +
NAMESPACE_TAR);
- try {
- Process process = processBuilder.start();
- int exitCode = process.waitFor();
- if (exitCode == 0) {
- log.info("Extract files successful.");
- } else {
- log.error("Extract files failed with exit code " +
exitCode);
- }
- } catch (IOException | InterruptedException e) {
- e.printStackTrace();
- }
- }
- };
-
- @TestContainerExtension
- protected final ContainerExtendedFactory extendedFactory =
- container -> {
- container.execInContainer("sh", "-c", "mkdir -p " +
TABLE_PATH);
- container.execInContainer("sh", "-c", "chmod -R 777 " +
TABLE_PATH);
- };
+ private static final String TABLE_PATH = HOST_VOLUME_MOUNT_PATH + "/hudi/";
@TestTemplate
@DisabledOnContainer(
@@ -115,8 +70,6 @@ public class HudiMultiTableIT extends TestSuiteBase {
.atMost(60000, TimeUnit.MILLISECONDS)
.untilAsserted(
() -> {
- // copy hudi to local
-
container.executeExtraCommands(containerExtendedFactory);
Path inputPath1 =
getNewestCommitFilePath(
new File(
@@ -155,7 +108,6 @@ public class HudiMultiTableIT extends TestSuiteBase {
Assertions.assertEquals(100, rowCount1);
Assertions.assertEquals(240, rowCount2);
});
- FileUtils.deleteFile(TABLE_PATH);
}
public static Path getNewestCommitFilePath(File tablePathDir) throws
IOException {
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/java/org/apache/seatunnel/e2e/connector/hudi/HudiSinkCDCIT.java
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/java/org/apache/seatunnel/e2e/connector/hudi/HudiSinkCDCIT.java
index 1610618fef..ab7c91a4b4 100644
---
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/java/org/apache/seatunnel/e2e/connector/hudi/HudiSinkCDCIT.java
+++
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/java/org/apache/seatunnel/e2e/connector/hudi/HudiSinkCDCIT.java
@@ -17,7 +17,6 @@
package org.apache.seatunnel.e2e.connector.hudi;
-import org.apache.seatunnel.common.utils.FileUtils;
import org.apache.seatunnel.common.utils.JsonUtils;
import
org.apache.seatunnel.connectors.seatunnel.cdc.mysql.testutils.MySqlContainer;
import
org.apache.seatunnel.connectors.seatunnel.cdc.mysql.testutils.MySqlVersion;
@@ -43,7 +42,6 @@ import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.TestTemplate;
import org.testcontainers.containers.Container;
-import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.output.Slf4jLogConsumer;
import org.testcontainers.lifecycle.Startables;
import org.testcontainers.utility.DockerLoggerFactory;
@@ -65,6 +63,7 @@ import java.util.concurrent.TimeUnit;
import java.util.stream.Stream;
import static java.lang.Thread.sleep;
+import static
org.apache.seatunnel.e2e.common.container.AbstractTestContainer.HOST_VOLUME_MOUNT_PATH;
import static org.awaitility.Awaitility.given;
@DisabledOnContainer(
@@ -88,7 +87,7 @@ public class HudiSinkCDCIT extends TestSuiteBase implements
TestResource {
private static final String DATABASE = "st";
private static final String TABLE_NAME = "st_test";
- private static final String TABLE_PATH = "/tmp/hudi/";
+ private static final String TABLE_PATH = HOST_VOLUME_MOUNT_PATH + "/hudi/";
private static final String NAMESPACE = "hudi";
private static final String NAMESPACE_TAR = "hudi.tar.gz";
@@ -111,39 +110,6 @@ public class HudiSinkCDCIT extends TestSuiteBase
implements TestResource {
new
Slf4jLogConsumer(DockerLoggerFactory.getLogger("mysql-mysql-image")));
}
- protected final ContainerExtendedFactory containerExtendedFactory =
- new ContainerExtendedFactory() {
- @Override
- public void extend(GenericContainer<?> container)
- throws IOException, InterruptedException {
- container.execInContainer(
- "sh",
- "-c",
- "cd /tmp" + " && tar -czvf " + NAMESPACE_TAR + " "
+ NAMESPACE);
- container.copyFileFromContainer(
- "/tmp/" + NAMESPACE_TAR, "/tmp/" + NAMESPACE_TAR);
-
- extractFiles();
- }
-
- private void extractFiles() {
- ProcessBuilder processBuilder = new ProcessBuilder();
- processBuilder.command(
- "sh", "-c", "cd /tmp" + " && tar -zxvf " +
NAMESPACE_TAR);
- try {
- Process process = processBuilder.start();
- int exitCode = process.waitFor();
- if (exitCode == 0) {
- log.info("Extract files successful.");
- } else {
- log.error("Extract files failed with exit code " +
exitCode);
- }
- } catch (IOException | InterruptedException e) {
- e.printStackTrace();
- }
- }
- };
-
@TestContainerExtension
protected final ContainerExtendedFactory extendedFactory =
container -> {
@@ -220,8 +186,6 @@ public class HudiSinkCDCIT extends TestSuiteBase implements
TestResource {
.atMost(60000, TimeUnit.MILLISECONDS)
.untilAsserted(
() -> {
- // copy hudi to local
-
container.executeExtraCommands(containerExtendedFactory);
Path newestCommitFilePath =
getNewestCommitFilePath(
new File(
@@ -246,7 +210,6 @@ public class HudiSinkCDCIT extends TestSuiteBase implements
TestResource {
}
Assertions.assertEquals(3, rowCount);
});
- FileUtils.deleteFile(TABLE_PATH);
}
private void upsertAndCheckData(TestContainer container)
@@ -262,8 +225,6 @@ public class HudiSinkCDCIT extends TestSuiteBase implements
TestResource {
.atMost(60000, TimeUnit.MILLISECONDS)
.untilAsserted(
() -> {
- // copy hudi to local
-
container.executeExtraCommands(containerExtendedFactory);
Path newestCommitFilePath =
getNewestCommitFilePath(
new File(
@@ -287,7 +248,6 @@ public class HudiSinkCDCIT extends TestSuiteBase implements
TestResource {
}
Assertions.assertEquals(4, rowCount);
});
- FileUtils.deleteFile(TABLE_PATH);
}
public static Path getNewestCommitFilePath(File tablePathDir) throws
IOException {
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/resources/hudi/fake_to_hudi.conf
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/resources/hudi/fake_to_hudi.conf
index 59e2cff56a..7b8cf47c29 100644
---
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/resources/hudi/fake_to_hudi.conf
+++
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/resources/hudi/fake_to_hudi.conf
@@ -47,7 +47,7 @@ source {
sink {
Hudi {
op_type="UPSERT"
- table_dfs_path = "/tmp/hudi"
+ table_dfs_path = "/tmp/seatunnel_mnt/hudi"
database = "st"
table_name = "st_test"
table_type="COPY_ON_WRITE"
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/resources/hudi/fake_to_hudi_with_omit_config_item.conf
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/resources/hudi/fake_to_hudi_with_omit_config_item.conf
index a9445a7823..2ba4554c8c 100644
---
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/resources/hudi/fake_to_hudi_with_omit_config_item.conf
+++
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/resources/hudi/fake_to_hudi_with_omit_config_item.conf
@@ -46,7 +46,7 @@ source {
sink {
Hudi {
- table_dfs_path = "/tmp/hudi"
+ table_dfs_path = "/tmp/seatunnel_mnt/hudi"
table_name = "st_test"
}
}
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/resources/hudi/multi_fake_to_hudi.conf
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/resources/hudi/multi_fake_to_hudi.conf
index 517131331b..aa7f4116e9 100644
---
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/resources/hudi/multi_fake_to_hudi.conf
+++
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/resources/hudi/multi_fake_to_hudi.conf
@@ -74,7 +74,7 @@ source {
sink {
Hudi {
auto_commit = "false"
- table_dfs_path = "/tmp/hudi"
+ table_dfs_path = "/tmp/seatunnel_mnt/hudi"
table_list=[
{
database = "st1"
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/resources/hudi/mysql_cdc_to_hudi.conf
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/resources/hudi/mysql_cdc_to_hudi.conf
index 842d3703a8..c5f122de6e 100644
---
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/resources/hudi/mysql_cdc_to_hudi.conf
+++
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-hudi-e2e/src/test/resources/hudi/mysql_cdc_to_hudi.conf
@@ -45,7 +45,7 @@ transform {
sink {
Hudi {
op_type="UPSERT"
- table_dfs_path = "/tmp/hudi"
+ table_dfs_path = "/tmp/seatunnel_mnt/hudi"
database = "st"
table_name = "st_test"
table_type="COPY_ON_WRITE"
