This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a change to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
from b7ef4936fe [Fix][CI] Pin the k3s version and check out first in the
engine k8s E2E job (#12845)
add be73009224 [Feature][Spark] Support Spark 3.5.8 (#11307)
No new revisions were added by this update.
Summary of changes:
.github/workflows/backend.yml | 4 +
docs/en/engines/command/usage.mdx | 2 +
docs/en/engines/spark.md | 42 ++
docs/en/getting-started/docker/docker.md | 2 +
.../getting-started/locally/quick-start-spark.md | 29 +-
docs/zh/engines/command/usage.mdx | 2 +
docs/zh/engines/spark.md | 35 ++
docs/zh/getting-started/docker/docker.md | 2 +
.../getting-started/locally/quick-start-spark.md | 26 +-
pom.xml | 1 +
seatunnel-core/seatunnel-spark-starter/pom.xml | 12 +-
.../pom.xml | 79 +++-
.../bin/start-seatunnel-spark-3.5-connector-v2.cmd | 80 ++++
.../bin/start-seatunnel-spark-3.5-connector-v2.sh} | 61 ++-
.../core/starter/spark/Spark35LauncherTest.java | 262 +++++++++++
.../spark/TestSpark35StarterJarContentIT.java | 522 +++++++++++++++++++++
.../spark/execution/Spark35BatchWriteTest.java | 269 +++++++++++
.../spark/execution/Spark35RowEncoderTest.java | 18 +-
.../seatunnel/core/starter/spark/SparkStarter.java | 74 ++-
.../starter/spark/execution/SparkRowEncoder.java | 86 ++++
.../spark/execution/TransformExecuteProcessor.java | 5 +-
.../core/starter/spark/SparkStarterTest.java | 175 +++++++
.../spark/execution/SparkRowEncoderTest.java | 60 +++
seatunnel-dist/pom.xml | 12 +
.../main/assembly/assembly-seatunnel-bin-ci.xml | 6 +
.../src/main/assembly/assembly-seatunnel-bin.xml | 6 +
.../src/test/resources/fakesource_to_druid.conf | 2 +-
.../resources/fakesource_to_druid_with_multi.conf | 4 +-
.../connector-hudi-e2e/pom.xml | 39 ++
.../hudi/HudiSeatunnelS3MultiTableIT.java | 6 +-
.../connector/hudi/HudiSparkS3MultiTableIT.java | 43 +-
.../src/test/resources/hudi/s3_fake_to_hudi.conf | 2 +-
.../connector-iceberg-s3-e2e/pom.xml | 39 ++
.../e2e/connector/iceberg/s3/IcebergSourceIT.java | 16 +-
...figs_start_mode_specific_offsets_to_assert.conf | 1 +
seatunnel-e2e/seatunnel-connector-v2-e2e/pom.xml | 6 +
.../e2e/common/container/TestContainerId.java | 1 +
.../spark/AbstractTestSparkContainer.java | 9 +
.../Spark35Container.java} | 30 +-
.../connector-seatunnel-e2e-base/pom.xml | 6 +
seatunnel-e2e/seatunnel-transforms-v2-e2e/pom.xml | 6 +
...ava => PythonTransformContainerPolicyTest.java} | 30 +-
.../e2e/transform/TestPythonTransformIT.java | 3 +-
.../spark/sink/SeaTunnelBatchWrite.java | 9 +
44 files changed, 2009 insertions(+), 115 deletions(-)
copy seatunnel-core/seatunnel-spark-starter/{seatunnel-spark-starter-common =>
seatunnel-spark-3.5-starter}/pom.xml (51%)
create mode 100644
seatunnel-core/seatunnel-spark-starter/seatunnel-spark-3.5-starter/src/main/bin/start-seatunnel-spark-3.5-connector-v2.cmd
copy
seatunnel-core/seatunnel-spark-starter/{seatunnel-spark-2-starter/src/main/bin/start-seatunnel-spark-2-connector-v2.sh
=>
seatunnel-spark-3.5-starter/src/main/bin/start-seatunnel-spark-3.5-connector-v2.sh}
(51%)
create mode 100644
seatunnel-core/seatunnel-spark-starter/seatunnel-spark-3.5-starter/src/test/java/org/apache/seatunnel/core/starter/spark/Spark35LauncherTest.java
create mode 100644
seatunnel-core/seatunnel-spark-starter/seatunnel-spark-3.5-starter/src/test/java/org/apache/seatunnel/core/starter/spark/TestSpark35StarterJarContentIT.java
create mode 100644
seatunnel-core/seatunnel-spark-starter/seatunnel-spark-3.5-starter/src/test/java/org/apache/seatunnel/core/starter/spark/execution/Spark35BatchWriteTest.java
copy
seatunnel-connectors-v2/connector-google-sheets/src/test/java/org/apache/seatunnel/connectors/seatunnel/google/sheets/SheetsFactoryTest.java
=>
seatunnel-core/seatunnel-spark-starter/seatunnel-spark-3.5-starter/src/test/java/org/apache/seatunnel/core/starter/spark/execution/Spark35RowEncoderTest.java
(61%)
create mode 100644
seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/main/java/org/apache/seatunnel/core/starter/spark/execution/SparkRowEncoder.java
create mode 100644
seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/test/java/org/apache/seatunnel/core/starter/spark/SparkStarterTest.java
create mode 100644
seatunnel-core/seatunnel-spark-starter/seatunnel-spark-starter-common/src/test/java/org/apache/seatunnel/core/starter/spark/execution/SparkRowEncoderTest.java
copy
seatunnel-e2e/seatunnel-e2e-common/src/test/java/org/apache/seatunnel/e2e/common/container/{flink/Flink16Container.java
=> spark/Spark35Container.java} (70%)
copy
seatunnel-e2e/seatunnel-transforms-v2-e2e/seatunnel-transforms-v2-e2e-part-2/src/test/java/org/apache/seatunnel/e2e/transform/{TestSparkDateTimeTransformIT.java
=> PythonTransformContainerPolicyTest.java} (54%)