[GitHub] [inlong] dockerzhang merged pull request #8610: [INLONG-8279][Sort] Fix NPE when run MySqlLoadSqlParseTest
dockerzhang merged PR #8610: URL: https://github.com/apache/inlong/pull/8610 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[inlong] branch master updated: [INLONG-8279][Sort] Fix NPE when run MySqlLoadSqlParseTest (#8610)
This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new adbf930738 [INLONG-8279][Sort] Fix NPE when run MySqlLoadSqlParseTest
(#8610)
adbf930738 is described below
commit adbf930738008bf49a1363b59b6b618aaca66ffb
Author: emhui <111486498+e-m...@users.noreply.github.com>
AuthorDate: Mon Jul 31 15:36:31 2023 +0800
[INLONG-8279][Sort] Fix NPE when run MySqlLoadSqlParseTest (#8610)
---
inlong-sort/sort-core/pom.xml | 6 ++
1 file changed, 6 insertions(+)
diff --git a/inlong-sort/sort-core/pom.xml b/inlong-sort/sort-core/pom.xml
index e376b6c7d6..b33b199c2a 100644
--- a/inlong-sort/sort-core/pom.xml
+++ b/inlong-sort/sort-core/pom.xml
@@ -203,6 +203,12 @@
sort-connector-oracle-cdc
${project.version}
test
+
+
+io.debezium
+debezium-ddl-parser
+
+
org.apache.inlong
[GitHub] [inlong] dockerzhang merged pull request #8588: [INLONG-8360][Manager] Support previewing data of Kafka
dockerzhang merged PR #8588: URL: https://github.com/apache/inlong/pull/8588 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[inlong] branch master updated: [INLONG-8360][Manager] Support previewing data of Kafka (#8588)
This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 90c89f1f9b [INLONG-8360][Manager] Support previewing data of Kafka
(#8588)
90c89f1f9b is described below
commit 90c89f1f9ba21a1a5b65ae9519b70bac490efa16
Author: Hao <1780095+hnrai...@users.noreply.github.com>
AuthorDate: Mon Jul 31 16:10:24 2023 +0800
[INLONG-8360][Manager] Support previewing data of Kafka (#8588)
Co-authored-by: healchow
---
.../manager/plugin/flink/FlinkOperation.java | 6 +-
.../manager/pojo/consume/BriefMQMessage.java | 8 +-
.../message/InlongMsgDeserializeOperator.java | 21 ++-
.../service/message/PbMsgDeserializeOperator.java | 25 ++--
.../service/message/RawMsgDeserializeOperator.java | 7 +-
.../resource/queue/kafka/KafkaOperator.java| 121 -
...rators.java => KafkaQueueResourceOperator.java} | 23 +++-
.../resource/queue/pulsar/PulsarOperator.java | 1 -
...rator.java => PulsarQueueResourceOperator.java} | 2 +-
...rator.java => TubeMQQueueResourceOperator.java} | 2 +-
.../service/source/kafka/KafkaSourceOperator.java | 2 +-
.../source/pulsar/PulsarSourceOperator.java| 2 +-
.../resource/queue/kafka/KafkaOperatorTest.java| 146 +
13 files changed, 327 insertions(+), 39 deletions(-)
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkOperation.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkOperation.java
index 817a9f8197..3ae90e6036 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkOperation.java
+++
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkOperation.java
@@ -313,13 +313,13 @@ public class FlinkOperation {
String jobId = flinkInfo.getJobId();
JobDetailsInfo jobDetailsInfo = flinkService.getJobDetail(jobId);
if (jobDetailsInfo == null) {
-throw new Exception(String.format("delete job failed as the job
not found for %s", jobId));
+throw new Exception(String.format("delete job failed as the job
[%s] not found", jobId));
}
JobStatus jobStatus = jobDetailsInfo.getJobStatus();
if (jobStatus != null && jobStatus.isTerminalState()) {
-String message = String.format("not support delete %s as the task
was terminated", jobId);
-message = jobStatus.isGloballyTerminalState() ? message + "
globally" : " locally";
+String stateName = jobStatus.isGloballyTerminalState() ?
"globally" : "locally";
+String message = String.format("unsupported delete job [%s] as it
was [%s] terminated", jobId, stateName);
throw new Exception(message);
}
diff --git
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/consume/BriefMQMessage.java
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/consume/BriefMQMessage.java
index a09c3c0b0c..51085c5570 100644
---
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/consume/BriefMQMessage.java
+++
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/consume/BriefMQMessage.java
@@ -25,16 +25,16 @@ import lombok.Data;
import lombok.NoArgsConstructor;
/**
- * Inlong display message info
+ * Brief Message info for MQ
*/
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
-@ApiModel("Inlong brief mq message info")
+@ApiModel("Brief Message info for MQ")
public class BriefMQMessage {
-@ApiModelProperty(value = "index id")
+@ApiModelProperty(value = "Message index id")
private Integer id;
@ApiModelProperty(value = "Inlong group id")
@@ -43,7 +43,7 @@ public class BriefMQMessage {
@ApiModelProperty(value = "Inlong stream id")
private String inlongStreamId;
-@ApiModelProperty(value = "Date")
+@ApiModelProperty(value = "Message date")
private Long dt;
@ApiModelProperty(value = "Client ip")
diff --git
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/message/InlongMsgDeserializeOperator.java
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/message/InlongMsgDeserializeOperator.java
index 9ae409425b..b5165eb8bf 100644
---
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/message/InlongMsgDeserializeOperator.java
+++
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/message/InlongMsgDeserializeOperator.java
@@ -44,22 +44,22 @@ public class InlongMsgDeserializeOperator implements
DeserializeOperator {
}
@Override
-public List decodeMsg
[GitHub] [inlong] EMsnap commented on pull request #8475: [INLONG-8445][Sort] Support running tests on both Flink 1.13 and Flink 1.15
EMsnap commented on PR #8475: URL: https://github.com/apache/inlong/pull/8475#issuecomment-1657908840 It seems that the ITs you provide is not running in the pipeline, please check  -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] EMsnap commented on a diff in pull request #8475: [INLONG-8445][Sort] Support running tests on both Flink 1.13 and Flink 1.15
EMsnap commented on code in PR #8475:
URL: https://github.com/apache/inlong/pull/8475#discussion_r1278960736
##
inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.15/src/test/java/org/apache/inlong/sort/tests/PostgresTest.java:
##
@@ -0,0 +1,219 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sort.tests;
+
+import org.apache.inlong.sort.tests.utils.FlinkContainerTestEnv;
+import org.apache.inlong.sort.tests.utils.JdbcProxy;
+import org.apache.inlong.sort.tests.utils.StarRocksContainer;
+import org.apache.inlong.sort.tests.utils.TestUtils;
+
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testcontainers.containers.GenericContainer;
+import org.testcontainers.containers.PostgreSQLContainer;
+import org.testcontainers.containers.output.Slf4jLogConsumer;
+import org.testcontainers.lifecycle.Startables;
+import org.testcontainers.utility.DockerImageName;
+import org.testcontainers.utility.MountableFile;
+
+import java.net.URISyntaxException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Stream;
+
+/**
+ * End-to-end tests for sort-connector-postgres-cdc-v1.15 uber jar.
+ */
+public class PostgresTest extends FlinkContainerTestEnv {
Review Comment:
Maybe use a different name to imply PostGre to Starrocks end to end test
here
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [inlong] vernedeng opened a new pull request, #8616: [INLONG-8598][Sort] Optimize sortstandalone pulsar sink
vernedeng opened a new pull request, #8616: URL: https://github.com/apache/inlong/pull/8616 ### Prepare a Pull Request *(Change the title refer to the following example)* - Title Example: [INLONG-XYZ][Component] Title of the pull request *(The following *XYZ* should be replaced by the actual [GitHub Issue](https://github.com/apache/inlong/issues) number)* - Fixes #8598 ### Motivation *Explain here the context, and why you're making that change. What is the problem you're trying to solve?* ### Modifications *Describe the modifications you've done.* ### Verifying this change *(Please pick either of the following options)* - [ ] This change is a trivial rework/code cleanup without any test coverage. - [ ] This change is already covered by existing tests, such as: *(please describe tests)* - [ ] This change added tests and can be verified as follows: *(example:)* - *Added integration tests for end-to-end deployment with large payloads (10MB)* - *Extended integration test for recovery after broker failure* ### Documentation - Does this pull request introduce a new feature? (yes / no) - If yes, how is the feature documented? (not applicable / docs / JavaDocs / not documented) - If a feature is not applicable for documentation, explain why? - If a feature is not documented yet in this PR, please create a follow-up issue for adding the documentation -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] aloyszhang commented on a diff in pull request #8616: [INLONG-8598][Sort] Optimize sortstandalone pulsar sink
aloyszhang commented on code in PR #8616:
URL: https://github.com/apache/inlong/pull/8616#discussion_r1279176121
##
inlong-sort-standalone/sort-standalone-source/src/main/java/org/apache/inlong/sort/standalone/sink/pulsar/PulsarIdConfig.java:
##
@@ -49,12 +49,12 @@ public PulsarIdConfig() {
/**
* Constructor
- *
+ *
* @param idParam
*/
public PulsarIdConfig(Map idParam) {
this.inlongGroupId = idParam.get(Constants.INLONG_GROUP_ID);
-this.inlongStreamId = idParam.get(Constants.INLONG_STREAM_ID);
+this.inlongStreamId = idParam.getOrDefault(Constants.INLONG_STREAM_ID,
"1");
Review Comment:
It's better to add a DEFAULT_INLONG_STREAM_ID instead of “1”
##
inlong-sort-standalone/sort-standalone-source/src/main/java/org/apache/inlong/sort/standalone/sink/pulsar/PulsarProducerCluster.java:
##
@@ -142,11 +154,11 @@ public void start() {
/**
* getPulsarCompressionType
- *
+ *
* @return CompressionType
*/
private CompressionType getPulsarCompressionType() {
-String type = this.context.getString(KEY_COMPRESSIONTYPE);
+String type = this.context.getString(KEY_COMPRESSIONTYPE, "SNAPPY");
Review Comment:
ditto
##
inlong-sort-standalone/sort-standalone-source/src/main/java/org/apache/inlong/sort/standalone/sink/pulsar/PulsarIdConfig.java:
##
@@ -49,12 +49,12 @@ public PulsarIdConfig() {
/**
* Constructor
- *
+ *
* @param idParam
*/
public PulsarIdConfig(Map idParam) {
this.inlongGroupId = idParam.get(Constants.INLONG_GROUP_ID);
-this.inlongStreamId = idParam.get(Constants.INLONG_STREAM_ID);
+this.inlongStreamId = idParam.getOrDefault(Constants.INLONG_STREAM_ID,
"1");
Review Comment:
It's better to add a DEFAULT_INLONG_STREAM_ID instead of “1”
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [inlong] vernedeng commented on a diff in pull request #8612: [INLONG-8611][Manager] Add full link trace
vernedeng commented on code in PR #8612:
URL: https://github.com/apache/inlong/pull/8612#discussion_r1279185573
##
inlong-agent/bin/agent-env.sh:
##
@@ -61,3 +61,19 @@ if [[ $JMX_ENABLED == 1 ]]; then
else
export AGENT_ARGS="$AGENT_JVM_ARGS -cp $CLASSPATH -Dagent.home=$BASE_DIR"
fi
+
+# Opentelemetry java agent path
+JAVA_AGENT="${BASE_DIR}/bin/opentelemetry-javaagent.jar"
+DOWNLOAD_URL="https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.28.0/opentelemetry-javaagent.jar";
+if [ ! -f "$JAVA_AGENT" ]; then
+wget -O "$JAVA_AGENT" "$DOWNLOAD_URL"
+fi
+
+# Opentelemetry startup parameter configuration
+export OTEL_SERVICE_NAME=inlong_agent
+export OTEL_TRACES_EXPORTER=otlp
+export OTEL_METRICS_EXPORTER=otlp
+export OTEL_LOGS_EXPORTER=otlp
+export OTEL_INSTRUMENTATION_PULSAR_ENABLED=false
+export
OTEL_EXPORTER_OTLP_ENDPOINT="http://otel-collect-proxy.zhiyan.tencent-cloud.net:4317";
Review Comment:
Do not put the internal address to the community
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [inlong] vernedeng commented on a diff in pull request #8612: [INLONG-8611][Manager] Add full link trace
vernedeng commented on code in PR #8612:
URL: https://github.com/apache/inlong/pull/8612#discussion_r1279186456
##
inlong-agent/bin/agent-env.sh:
##
@@ -61,3 +61,19 @@ if [[ $JMX_ENABLED == 1 ]]; then
else
export AGENT_ARGS="$AGENT_JVM_ARGS -cp $CLASSPATH -Dagent.home=$BASE_DIR"
fi
+
+# Opentelemetry java agent path
+JAVA_AGENT="${BASE_DIR}/bin/opentelemetry-javaagent.jar"
+DOWNLOAD_URL="https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v1.28.0/opentelemetry-javaagent.jar";
Review Comment:
Make the version of open-telemetry configurable
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [inlong] vernedeng commented on a diff in pull request #8616: [INLONG-8598][Sort] Optimize sortstandalone pulsar sink
vernedeng commented on code in PR #8616:
URL: https://github.com/apache/inlong/pull/8616#discussion_r1279355005
##
inlong-sort-standalone/sort-standalone-source/src/main/java/org/apache/inlong/sort/standalone/sink/pulsar/PulsarProducerCluster.java:
##
@@ -142,11 +154,11 @@ public void start() {
/**
* getPulsarCompressionType
- *
+ *
* @return CompressionType
*/
private CompressionType getPulsarCompressionType() {
-String type = this.context.getString(KEY_COMPRESSIONTYPE);
+String type = this.context.getString(KEY_COMPRESSIONTYPE, "SNAPPY");
Review Comment:
fixed, thx
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
[GitHub] [inlong] dockerzhang merged pull request #8615: [INLONG-8614][CI] Remove unnecessary packages for UT and build workflow
dockerzhang merged PR #8615: URL: https://github.com/apache/inlong/pull/8615 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[inlong] branch master updated: [INLONG-8614][CI] Remove unnecessary packages for UT and build workflow (#8615)
This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 4d6593985d [INLONG-8614][CI] Remove unnecessary packages for UT and
build workflow (#8615)
4d6593985d is described below
commit 4d6593985d3bf260a329ba424cdfeefce2ab47e7
Author: Charles Zhang
AuthorDate: Tue Aug 1 09:41:41 2023 +0800
[INLONG-8614][CI] Remove unnecessary packages for UT and build workflow
(#8615)
---
.github/workflows/ci_build.yml | 20
.github/workflows/ci_ut.yml| 20
2 files changed, 40 insertions(+)
diff --git a/.github/workflows/ci_build.yml b/.github/workflows/ci_build.yml
index ce5cb0bab7..a6f6dd685e 100644
--- a/.github/workflows/ci_build.yml
+++ b/.github/workflows/ci_build.yml
@@ -76,6 +76,26 @@ jobs:
key: ${{ runner.os }}-inlong-build-${{ hashFiles('**/pom.xml') }}
restore-keys: ${{ runner.os }}-inlong-build
+ - name: Set up swapfile path
+run: |
+ sudo sysctl -w vm.max_map_count=262144
+ sudo sysctl -w fs.file-max=65536
+ sudo fallocate -l 5G /swapfile
+ sudo chmod 600 /swapfile
+ sudo mkswap /swapfile
+ sudo swapon /swapfile
+
+ - name: Remove unnecessary packages
+run: |
+ echo "=== Before pruning ==="
+ df -h
+ sudo rm -rf /usr/share/dotnet
+ sudo rm -rf /usr/local/lib/android
+ sudo rm -rf /opt/ghc
+ echo
+ echo "=== After pruning ==="
+ df -h
+
- name: Build with Maven
run: |
mvn --batch-mode --update-snapshots -e -V clean install -DskipTests
-Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
-Dmaven.wagon.httpconnectionManager.ttlSeconds=120
-Daether.connector.http.reuseConnections=false
-Daether.connector.requestTimeout=6
diff --git a/.github/workflows/ci_ut.yml b/.github/workflows/ci_ut.yml
index aa90250739..180a8e2876 100644
--- a/.github/workflows/ci_ut.yml
+++ b/.github/workflows/ci_ut.yml
@@ -75,6 +75,26 @@ jobs:
key: ${{ runner.os }}-inlong-ut-${{ hashFiles('**/pom.xml') }}
restore-keys: ${{ runner.os }}-inlong-ut
+ - name: Set up swapfile path
+run: |
+ sudo sysctl -w vm.max_map_count=262144
+ sudo sysctl -w fs.file-max=65536
+ sudo fallocate -l 5G /swapfile
+ sudo chmod 600 /swapfile
+ sudo mkswap /swapfile
+ sudo swapon /swapfile
+
+ - name: Remove unnecessary packages
+run: |
+ echo "=== Before pruning ==="
+ df -h
+ sudo rm -rf /usr/share/dotnet
+ sudo rm -rf /usr/local/lib/android
+ sudo rm -rf /opt/ghc
+ echo
+ echo "=== After pruning ==="
+ df -h
+
- name: Build with Maven
run: mvn --batch-mode --update-snapshots -e -V clean install
-DskipTests -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
-Dmaven.wagon.httpconnectionManager.ttlSeconds=120
-Daether.connector.http.reuseConnections=false
-Daether.connector.requestTimeout=6
env:
[GitHub] [inlong] dockerzhang commented on pull request #8612: [INLONG-8611][Manager] Add full link trace
dockerzhang commented on PR #8612: URL: https://github.com/apache/inlong/pull/8612#issuecomment-1659510391 Please rebase from the master to solve the timeout workflow issue. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] EMsnap commented on pull request #8475: [INLONG-8445][Sort] Support running tests on both Flink 1.13 and Flink 1.15
EMsnap commented on PR #8475: URL: https://github.com/apache/inlong/pull/8475#issuecomment-1659566343 LGTM, Great Job ! -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[GitHub] [inlong] EMsnap merged pull request #8475: [INLONG-8445][Sort] Support running tests on both Flink 1.13 and Flink 1.15
EMsnap merged PR #8475: URL: https://github.com/apache/inlong/pull/8475 -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@inlong.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org
[inlong] branch master updated: [INLONG-8445][Sort] Support running tests on both Flink 1.13 and Flink 1.15 (#8475)
This is an automated email from the ASF dual-hosted git repository.
zirui pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new a300d8dac7 [INLONG-8445][Sort] Support running tests on both Flink
1.13 and Flink 1.15 (#8475)
a300d8dac7 is described below
commit a300d8dac7294b4a088e794e1f049444a7a467f7
Author: haibo.duan
AuthorDate: Tue Aug 1 12:55:49 2023 +0800
[INLONG-8445][Sort] Support running tests on both Flink 1.13 and Flink 1.15
(#8475)
---
.github/workflows/ci_ut_flink15.yml| 4 +-
inlong-sort/pom.xml| 4 +-
inlong-sort/sort-dist/pom.xml | 3 +
inlong-sort/sort-end-to-end-tests/pom.xml | 179 +
.../sort-end-to-end-tests-v1.13/pom.xml| 220 +
.../apache/inlong/sort/tests/ClickHouseCase.java | 0
.../org/apache/inlong/sort/tests/KafkaE2ECase.java | 0
.../sort/tests/utils/FlinkContainerTestEnv.java| 0
.../apache/inlong/sort/tests/utils/JdbcProxy.java | 0
.../inlong/sort/tests/utils/MySqlContainer.java| 0
.../sort/tests/utils/PlaceholderResolver.java | 0
.../apache/inlong/sort/tests/utils/TestUtils.java | 0
.../src/test/resources/docker/mysql/my.cnf | 0
.../src/test/resources/docker/mysql/setup.sql | 0
.../test/resources/env/kafka_test_kafka_init.txt | 0
.../test/resources/env/kafka_test_mysql_init.txt | 0
.../test/resources/flinkSql/clickhouse_test.sql| 0
.../src/test/resources/flinkSql/kafka_test.sql | 0
.../src/test/resources/groupFile/kafka_test.json | 0
.../src/test/resources/log4j2-test.properties | 0
.../sort-end-to-end-tests-v1.15/pom.xml| 219
.../inlong/sort/tests/PostgresToStarRocksTest.java | 220 +
.../sort/tests/utils/FlinkContainerTestEnv.java| 44 +
.../apache/inlong/sort/tests/utils/JdbcProxy.java | 0
.../inlong/sort/tests/utils/MySqlContainer.java| 0
.../sort/tests/utils/PlaceholderResolver.java | 0
.../sort/tests/utils/StarRocksContainer.java | 145 ++
.../apache/inlong/sort/tests/utils/TestUtils.java | 0
.../src/test/resources/docker/mysql/my.cnf | 0
.../src/test/resources/docker/mysql/setup.sql | 0
.../test/resources/docker/postgresql}/setup.sql| 9 +-
.../test/resources/docker/starrocks/start_fe_be.sh | 72 +++
.../src/test/resources/flinkSql/postgres_test.sql | 39
.../src/test/resources/log4j2-test.properties | 28 +--
34 files changed, 956 insertions(+), 230 deletions(-)
diff --git a/.github/workflows/ci_ut_flink15.yml
b/.github/workflows/ci_ut_flink15.yml
index abaf605256..343af7dc39 100644
--- a/.github/workflows/ci_ut_flink15.yml
+++ b/.github/workflows/ci_ut_flink15.yml
@@ -55,12 +55,12 @@ jobs:
restore-keys: ${{ runner.os }}-inlong-flink15
- name: Build for Flink 1.15 with Maven
-run: mvn --update-snapshots -e -V package -U -pl inlong-sort/sort-core
-am -Pv1.15 -DskipTests -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
-Dmaven.wagon.httpconnectionManager.ttlSeconds=120
-Daether.connector.http.reuseConnections=false
-Daether.connector.requestTimeout=6
+run: mvn --update-snapshots -e -V clean install -U -pl
:sort-core,:sort-end-to-end-tests-v1.15 -am -Pv1.15 -DskipTests
-Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false
-Dmaven.wagon.httpconnectionManager.ttlSeconds=120
-Daether.connector.http.reuseConnections=false
-Daether.connector.requestTimeout=6
env:
CI: false
- name: Unit test for Flink 1.15 with Maven
-run: mvn --update-snapshots -e -V clean test -am -pl
inlong-sort/sort-core -Pv1.15
+run: mvn --update-snapshots -e -V test -pl
:sort-core,:sort-end-to-end-tests-v1.15 -am -Pv1.15
env:
CI: false
diff --git a/inlong-sort/pom.xml b/inlong-sort/pom.xml
index 17daa583d7..86efbb46bd 100644
--- a/inlong-sort/pom.xml
+++ b/inlong-sort/pom.xml
@@ -37,6 +37,7 @@
sort-core
sort-dist
sort-flink
+sort-end-to-end-tests
@@ -166,9 +167,6 @@
true
-
-sort-end-to-end-tests
-
v1.13
1.13.5
diff --git a/inlong-sort/sort-dist/pom.xml b/inlong-sort/sort-dist/pom.xml
index 0733788826..aa1a0d5278 100644
--- a/inlong-sort/sort-dist/pom.xml
+++ b/inlong-sort/sort-dist/pom.xml
@@ -145,14 +145,17 @@
org.apache.flink
flink-csv
+${flink.version}
org.apache.flink
flink-json
+${flink.version}
