This is an automated email from the ASF dual-hosted git repository.
tyrantlucifer pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 1d57c1a92 [Improve][POM] Improve config file (#3751)
1d57c1a92 is described below
commit 1d57c1a92437161f4bcb6276e522d1875591d51b
Author: Eric <[email protected]>
AuthorDate: Mon Dec 19 22:21:27 2022 +0800
[Improve][POM] Improve config file (#3751)
* improve config file
* remove copy engine yaml config file from engine module
* fix neo4j it error
* fix master init timeout
* add retry policy from hazelcast
---
config/hazelcast-client.yaml | 24 ++++++++++++++
config/hazelcast.yaml | 38 ++++++++++++++++++++++
config/log4j2.properties | 8 ++---
...{log4j2.properties => log4j2_client.properties} | 2 +-
config/seatunnel.yaml | 35 ++++++++++++++++++++
config/v2.batch.config.template | 13 ++------
config/v2.streaming.conf.template | 13 ++------
.../src/main/bin/seatunnel-cluster.sh | 2 +-
.../seatunnel-starter/src/main/bin/seatunnel.sh | 4 +--
.../src/main/assembly/assembly-bin-ci.xml | 9 -----
seatunnel-dist/src/main/assembly/assembly-bin.xml | 9 -----
.../seatunnel/e2e/connector/neo4j/Neo4jIT.java | 2 +-
.../src/test/resources/neo4j/neo4j_to_neo4j.conf | 4 +--
.../src/test/resources/hazelcast-client.yaml | 37 +++++++++++++++++++++
.../src/test/resources/hazelcast.yaml | 33 +++++++++++++++++++
.../src/test/resources/seatunnel.yaml | 35 ++++++++++++++++++++
.../seatunnel/engine/server/SeaTunnelServer.java | 14 ++++++--
17 files changed, 228 insertions(+), 54 deletions(-)
diff --git a/config/hazelcast-client.yaml b/config/hazelcast-client.yaml
new file mode 100644
index 000000000..f4807790a
--- /dev/null
+++ b/config/hazelcast-client.yaml
@@ -0,0 +1,24 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+hazelcast-client:
+ cluster-name: seatunnel
+ properties:
+ hazelcast.logging.type: log4j2
+ network:
+ cluster-members:
+ - localhost:5801
\ No newline at end of file
diff --git a/config/hazelcast.yaml b/config/hazelcast.yaml
new file mode 100644
index 000000000..d0d274358
--- /dev/null
+++ b/config/hazelcast.yaml
@@ -0,0 +1,38 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+hazelcast:
+ cluster-name: seatunnel
+ network:
+ rest-api:
+ enabled: true
+ endpoint-groups:
+ CLUSTER_WRITE:
+ enabled: true
+ join:
+ tcp-ip:
+ enabled: true
+ member-list:
+ - localhost
+ port:
+ auto-increment: false
+ port: 5801
+ properties:
+ hazelcast.invocation.max.retry.count: 20
+ hazelcast.tcp.join.port.try.count: 30
+ hazelcast.logging.type: log4j2
+
diff --git a/config/log4j2.properties b/config/log4j2.properties
index 68ed30982..2462a1c25 100644
--- a/config/log4j2.properties
+++ b/config/log4j2.properties
@@ -25,11 +25,11 @@ property.file_ttl = 7d
rootLogger.level = INFO
############################ log output to console
#############################
-rootLogger.appenderRef.consoleStdout.ref = consoleStdoutAppender
-rootLogger.appenderRef.consoleStderr.ref = consoleStderrAppender
+#rootLogger.appenderRef.consoleStdout.ref = consoleStdoutAppender
+#rootLogger.appenderRef.consoleStderr.ref = consoleStderrAppender
############################ log output to console
#############################
############################ log output to file
#############################
-#rootLogger.appenderRef.file.ref = fileAppender
+rootLogger.appenderRef.file.ref = fileAppender
############################ log output to file
#############################
appender.consoleStdout.name = consoleStdoutAppender
@@ -75,4 +75,4 @@ appender.file.strategy.action.condition.nested_condition.type
= IfAny
appender.file.strategy.action.condition.nested_condition.lastModify.type =
IfLastModified
appender.file.strategy.action.condition.nested_condition.lastModify.age =
${file_ttl}
appender.file.strategy.action.condition.nested_condition.fileCount.type =
IfAccumulatedFileCount
-appender.file.strategy.action.condition.nested_condition.fileCount.exceeds =
${file_count}
+appender.file.strategy.action.condition.nested_condition.fileCount.exceeds =
${file_count}
\ No newline at end of file
diff --git a/config/log4j2.properties b/config/log4j2_client.properties
similarity index 99%
copy from config/log4j2.properties
copy to config/log4j2_client.properties
index 68ed30982..503cebd76 100644
--- a/config/log4j2.properties
+++ b/config/log4j2_client.properties
@@ -75,4 +75,4 @@ appender.file.strategy.action.condition.nested_condition.type
= IfAny
appender.file.strategy.action.condition.nested_condition.lastModify.type =
IfLastModified
appender.file.strategy.action.condition.nested_condition.lastModify.age =
${file_ttl}
appender.file.strategy.action.condition.nested_condition.fileCount.type =
IfAccumulatedFileCount
-appender.file.strategy.action.condition.nested_condition.fileCount.exceeds =
${file_count}
+appender.file.strategy.action.condition.nested_condition.fileCount.exceeds =
${file_count}
\ No newline at end of file
diff --git a/config/seatunnel.yaml b/config/seatunnel.yaml
new file mode 100644
index 000000000..6ac98532f
--- /dev/null
+++ b/config/seatunnel.yaml
@@ -0,0 +1,35 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+seatunnel:
+ engine:
+ backup-count: 1
+ print-execution-info-interval: 10
+ slot-service:
+ dynamic-slot: true
+ checkpoint:
+ interval: 300000
+ timeout: 10000
+ max-concurrent: 5
+ tolerable-failure: 2
+ storage:
+ type: hdfs
+ max-retained: 3
+ namespace: /tmp/seatunnel/checkpoint_snapshot
+ plugin-config:
+ storage-type: hdfs
+ fs.defaultFS: /tmp/ # Ensure that the directory has written
permission
diff --git a/config/v2.batch.config.template b/config/v2.batch.config.template
index 425372b32..57986dc73 100644
--- a/config/v2.batch.config.template
+++ b/config/v2.batch.config.template
@@ -22,6 +22,7 @@ env {
# You can set SeaTunnel environment configuration here
execution.parallelism = 2
job.mode = "BATCH"
+ checkpoint.interval = 10000
#execution.checkpoint.interval = 10000
#execution.checkpoint.data-uri = "hdfs://localhost:9000/checkpoint"
}
@@ -44,20 +45,10 @@ source {
# please go to https://seatunnel.apache.org/docs/category/source-v2
}
-transform {
- sql {
- sql = "select name,age from fake"
- }
-
- # If you would like to get more information about how to configure Seatunnel
and see full list of transform plugins,
- # please go to https://seatunnel.apache.org/docs/category/transform
-}
-
sink {
Console {
- parallelism = 3
}
# If you would like to get more information about how to configure Seatunnel
and see full list of sink plugins,
# please go to https://seatunnel.apache.org/docs/category/sink-v2
-}
+}
\ No newline at end of file
diff --git a/config/v2.streaming.conf.template
b/config/v2.streaming.conf.template
index 80b943ab7..7a1a2e3e5 100644
--- a/config/v2.streaming.conf.template
+++ b/config/v2.streaming.conf.template
@@ -22,6 +22,7 @@ env {
# You can set flink configuration here
execution.parallelism = 2
job.mode = "STREAMING"
+ checkpoint.interval = 2000
#execution.checkpoint.interval = 10000
#execution.checkpoint.data-uri = "hdfs://localhost:9000/checkpoint"
}
@@ -44,20 +45,10 @@ source {
# please go to https://seatunnel.apache.org/docs/category/source-v2
}
-transform {
- sql {
- sql = "select name,age from fake"
- }
-
- # If you would like to get more information about how to configure Seatunnel
and see full list of transform plugins,
- # please go to https://seatunnel.apache.org/docs/category/transform
-}
-
sink {
Console {
- parallelism = 3
}
# If you would like to get more information about how to configure Seatunnel
and see full list of sink plugins,
# please go to https://seatunnel.apache.org/docs/category/sink-v2
-}
+}
\ No newline at end of file
diff --git a/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel-cluster.sh
b/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel-cluster.sh
index 74e946009..eaeb2c03e 100755
--- a/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel-cluster.sh
+++ b/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel-cluster.sh
@@ -81,7 +81,7 @@ JAVA_OPTS="${JAVA_OPTS}
-Dlog4j2.contextSelector=org.apache.logging.log4j.core.a
if [ -e "${CONF_DIR}/log4j2.properties" ]; then
JAVA_OPTS="${JAVA_OPTS}
-Dlog4j2.configurationFile=${CONF_DIR}/log4j2.properties"
JAVA_OPTS="${JAVA_OPTS} -Dseatunnel.logs.path=${APP_DIR}/logs"
- JAVA_OPTS="${JAVA_OPTS} -Dseatunnel.logs.file_name=seatunnel-starter-server"
+ JAVA_OPTS="${JAVA_OPTS} -Dseatunnel.logs.file_name=seatunnel-engine-server"
fi
echo "JAVA_OPTS: ${JAVA_OPTS}"
diff --git a/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel.sh
b/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel.sh
index 05f2b7241..2bd2a6d33 100755
--- a/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel.sh
+++ b/seatunnel-core/seatunnel-starter/src/main/bin/seatunnel.sh
@@ -82,8 +82,8 @@ JAVA_OPTS="${JAVA_OPTS}
-Dseatunnel.config=${SEATUNNEL_CONFIG}"
JAVA_OPTS="${JAVA_OPTS} -Dhazelcast.config=${HAZELCAST_CONFIG}"
# Log4j2 Config
-if [ -e "${CONF_DIR}/log4j2.properties" ]; then
- JAVA_OPTS="${JAVA_OPTS}
-Dlog4j2.configurationFile=${CONF_DIR}/log4j2.properties"
+if [ -e "${CONF_DIR}/log4j2_client.properties" ]; then
+ JAVA_OPTS="${JAVA_OPTS}
-Dlog4j2.configurationFile=${CONF_DIR}/log4j2_client.properties"
JAVA_OPTS="${JAVA_OPTS} -Dseatunnel.logs.path=${APP_DIR}/logs"
if [[ $args == *" -e local"* || $args == *" --deploy-mode local"* ]]; then
JAVA_OPTS="${JAVA_OPTS}
-Dseatunnel.logs.file_name=seatunnel-starter-client-$((`date '+%s'`*1000+`date
'+%N'`/1000000))"
diff --git a/seatunnel-dist/src/main/assembly/assembly-bin-ci.xml
b/seatunnel-dist/src/main/assembly/assembly-bin-ci.xml
index ecda3f384..2f4242d65 100644
--- a/seatunnel-dist/src/main/assembly/assembly-bin-ci.xml
+++ b/seatunnel-dist/src/main/assembly/assembly-bin-ci.xml
@@ -69,15 +69,6 @@
</includes>
<fileMode>0755</fileMode>
</fileSet>
- <!--Engine Config File-->
- <fileSet>
-
<directory>../seatunnel-engine/seatunnel-engine-common/src/main/resources</directory>
- <includes>
- <include>*.yaml</include>
- </includes>
- <outputDirectory>/config</outputDirectory>
- <fileMode>0644</fileMode>
- </fileSet>
<!--Licenses And NOTICE-->
<fileSet>
<directory>release-docs</directory>
diff --git a/seatunnel-dist/src/main/assembly/assembly-bin.xml
b/seatunnel-dist/src/main/assembly/assembly-bin.xml
index b6a5204f6..6d769d90f 100644
--- a/seatunnel-dist/src/main/assembly/assembly-bin.xml
+++ b/seatunnel-dist/src/main/assembly/assembly-bin.xml
@@ -70,15 +70,6 @@
</includes>
<fileMode>0755</fileMode>
</fileSet>
- <!--Engine Config File-->
- <fileSet>
-
<directory>../seatunnel-engine/seatunnel-engine-common/src/main/resources</directory>
- <includes>
- <include>*.yaml</include>
- </includes>
- <outputDirectory>/config</outputDirectory>
- <fileMode>0644</fileMode>
- </fileSet>
<!--Licenses And NOTICE-->
<fileSet>
<directory>release-docs</directory>
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-neo4j-e2e/src/test/java/org/apache/seatunnel/e2e/connector/neo4j/Neo4jIT.java
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-neo4j-e2e/src/test/java/org/apache/seatunnel/e2e/connector/neo4j/Neo4jIT.java
index 2b5122270..7df68c866 100644
---
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-neo4j-e2e/src/test/java/org/apache/seatunnel/e2e/connector/neo4j/Neo4jIT.java
+++
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-neo4j-e2e/src/test/java/org/apache/seatunnel/e2e/connector/neo4j/Neo4jIT.java
@@ -64,7 +64,7 @@ public class Neo4jIT extends TestSuiteBase implements
TestResource {
private static final int HTTP_PORT = 7474;
private static final int BOLT_PORT = 7687;
private static final String CONTAINER_NEO4J_USERNAME = "neo4j";
- private static final String CONTAINER_NEO4J_PASSWORD = "1234";
+ private static final String CONTAINER_NEO4J_PASSWORD = "Test@12343";
private static final URI CONTAINER_URI = URI.create("neo4j://localhost:" +
BOLT_PORT);
private GenericContainer<?> container;
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-neo4j-e2e/src/test/resources/neo4j/neo4j_to_neo4j.conf
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-neo4j-e2e/src/test/resources/neo4j/neo4j_to_neo4j.conf
index d95ab1078..21ee6f69f 100644
---
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-neo4j-e2e/src/test/resources/neo4j/neo4j_to_neo4j.conf
+++
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-neo4j-e2e/src/test/resources/neo4j/neo4j_to_neo4j.conf
@@ -33,7 +33,7 @@ source {
Neo4j {
uri = "neo4j://neo4j-host:7687"
username = "neo4j"
- password = "1234"
+ password = "Test@12343"
database = "neo4j"
max_transaction_retry_time = 1
@@ -67,7 +67,7 @@ sink {
Neo4j {
uri = "neo4j://neo4j-host:7687"
username = "neo4j"
- password = "1234"
+ password = "Test@12343"
database = "neo4j"
max_transaction_retry_time = 1
diff --git
a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/hazelcast-client.yaml
b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/hazelcast-client.yaml
new file mode 100644
index 000000000..1834d9c94
--- /dev/null
+++
b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/hazelcast-client.yaml
@@ -0,0 +1,37 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+hazelcast-client:
+ cluster-name: seatunnel
+
+ network:
+ cluster-members:
+ - localhost:5801
+ - localhost:5802
+ - localhost:5803
+ - localhost:5804
+ - localhost:5805
+ - localhost:5806
+ - localhost:5807
+ - localhost:5808
+ - localhost:5809
+ - localhost:5810
+ - localhost:5811
+ - localhost:5812
+ - localhost:5813
+ - localhost:5814
+ - localhost:5815
\ No newline at end of file
diff --git
a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/hazelcast.yaml
b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/hazelcast.yaml
new file mode 100644
index 000000000..040663327
--- /dev/null
+++
b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/hazelcast.yaml
@@ -0,0 +1,33 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+hazelcast:
+ cluster-name: seatunnel
+ network:
+ join:
+ tcp-ip:
+ enabled: true
+ member-list:
+ - localhost
+ port:
+ auto-increment: true
+ port-count: 100
+ port: 5801
+ properties:
+ hazelcast.invocation.max.retry.count: 20
+ hazelcast.tcp.join.port.try.count: 30
+ hazelcast.logging.type: log4j2
\ No newline at end of file
diff --git
a/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/seatunnel.yaml
b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/seatunnel.yaml
new file mode 100644
index 000000000..51ffc5214
--- /dev/null
+++
b/seatunnel-e2e/seatunnel-engine-e2e/connector-seatunnel-e2e-base/src/test/resources/seatunnel.yaml
@@ -0,0 +1,35 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+seatunnel:
+ engine:
+ backup-count: 2
+ print-execution-info-interval: 10
+ slot-service:
+ dynamic-slot: true
+ checkpoint:
+ interval: 300000
+ timeout: 10000
+ max-concurrent: 5
+ tolerable-failure: 2
+ storage:
+ type: hdfs
+ max-retained: 3
+ plugin-config:
+ namespace: /tmp/seatunnel/checkpoint_snapshot
+ storage-type: hdfs
+ fs.defaultFS: /tmp/ # Ensure that the directory has written
permission
diff --git
a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/SeaTunnelServer.java
b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/SeaTunnelServer.java
index 69f71248b..1213103c4 100644
---
a/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/SeaTunnelServer.java
+++
b/seatunnel-engine/seatunnel-engine-server/src/main/java/org/apache/seatunnel/engine/server/SeaTunnelServer.java
@@ -159,11 +159,19 @@ public class SeaTunnelServer implements ManagedService,
MembershipAwareService,
public CoordinatorService getCoordinatorService() {
int retryCount = 0;
if (isMasterNode()) {
- // TODO the retry count and sleep time need configurable
- while (!coordinatorService.isCoordinatorActive() && retryCount <
120 && isRunning) {
+ // The hazelcast operator request invocation will retry, We must
wait enough time to wait the invocation return.
+ String hazelcastInvocationMaxRetry =
seaTunnelConfig.getHazelcastConfig().getProperty("hazelcast.invocation.max.retry.count");
+ int maxRetry = hazelcastInvocationMaxRetry == null ? 250 * 2 :
Integer.valueOf(hazelcastInvocationMaxRetry) * 2;
+
+ String hazelcastRetryPause =
+
seaTunnelConfig.getHazelcastConfig().getProperty("hazelcast.invocation.retry.pause.millis");
+
+ int retryPause = hazelcastRetryPause == null ? 500 :
Integer.valueOf(hazelcastRetryPause);
+
+ while (!coordinatorService.isCoordinatorActive() && retryCount <
maxRetry && isRunning) {
try {
LOGGER.warning("This is master node, waiting the
coordinator service init finished");
- Thread.sleep(1000);
+ Thread.sleep(retryPause);
retryCount++;
} catch (InterruptedException e) {
throw new RuntimeException(e);
