(inlong) branch master updated: [INLONG-10826][SDK] Transform support TRIM(), REPLICATE() function (#10827)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new ee7f3d9732 [INLONG-10826][SDK] Transform support TRIM(), REPLICATE()
function (#10827)
ee7f3d9732 is described below
commit ee7f3d9732fd6706f706b3988d50efb5eae1d090
Author: emptyOVO <118812562+empty...@users.noreply.github.com>
AuthorDate: Wed Aug 21 21:32:53 2024 +0800
[INLONG-10826][SDK] Transform support TRIM(), REPLICATE() function (#10827)
---
.../process/function/ReplicateFunction.java| 71 ++
.../transform/process/function/TrimFunction.java | 44 ++
.../transform/process/operator/OperatorTools.java | 4 ++
.../TestTransformStringFunctionsProcessor.java | 66
4 files changed, 185 insertions(+)
diff --git
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/ReplicateFunction.java
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/ReplicateFunction.java
new file mode 100644
index 00..5270720d44
--- /dev/null
+++
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/ReplicateFunction.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sdk.transform.process.function;
+
+import org.apache.inlong.sdk.transform.decode.SourceData;
+import org.apache.inlong.sdk.transform.process.Context;
+import org.apache.inlong.sdk.transform.process.operator.OperatorTools;
+import org.apache.inlong.sdk.transform.process.parser.ValueParser;
+
+import net.sf.jsqlparser.expression.Expression;
+import net.sf.jsqlparser.expression.Function;
+
+import java.util.List;
+/**
+ * ReplicateFunction
+ * description: replicate(string, numeric)--Repeat the string numeric times
and return a new string
+ */
+public class ReplicateFunction implements ValueParser {
+
+private ValueParser stringParser;
+
+private ValueParser countParser;
+
+public ReplicateFunction(Function expr) {
+List expressions = expr.getParameters().getExpressions();
+stringParser = OperatorTools.buildParser(expressions.get(0));
+countParser = OperatorTools.buildParser(expressions.get(1));
+}
+
+@Override
+public Object parse(SourceData sourceData, int rowIndex, Context context) {
+Object stringObj = stringParser.parse(sourceData, rowIndex, context);
+Object countObj = countParser.parse(sourceData, rowIndex, context);
+String str = OperatorTools.parseString(stringObj);
+double count = OperatorTools.parseBigDecimal(countObj).doubleValue();
+return repeat(str, count);
+}
+private String repeat(String str, double count) {
+if (count == 0) {
+return "";
+}
+if (count == 1) {
+return str;
+}
+StringBuilder repeatedStr = new StringBuilder();
+StringBuilder originStr = new StringBuilder(str);
+while (count > 0) {
+if (count % 2 != 0) {
+repeatedStr.append(originStr);
+}
+count = Math.floor(count / 2);
+originStr.append(originStr);
+}
+return repeatedStr.toString();
+}
+}
diff --git
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/TrimFunction.java
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/TrimFunction.java
new file mode 100644
index 00..b3fbaf26dc
--- /dev/null
+++
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/TrimFunction.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file e
(inlong) branch master updated: [INLONG-10819][SDK] Transform support tan function (#10820)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new d5ca96189d [INLONG-10819][SDK] Transform support tan function (#10820)
d5ca96189d is described below
commit d5ca96189ddd7c6609a91db3f0a881e3bfe5d6ad
Author: emptyOVO <118812562+empty...@users.noreply.github.com>
AuthorDate: Thu Aug 22 16:55:07 2024 +0800
[INLONG-10819][SDK] Transform support tan function (#10820)
---
.../transform/process/function/TanFunction.java| 46 ++
.../transform/process/operator/OperatorTools.java | 2 +
.../TestTransformArithmeticFunctionsProcessor.java | 22 +++
3 files changed, 70 insertions(+)
diff --git
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/TanFunction.java
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/TanFunction.java
new file mode 100644
index 00..ed673f1ef6
--- /dev/null
+++
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/TanFunction.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sdk.transform.process.function;
+
+import org.apache.inlong.sdk.transform.decode.SourceData;
+import org.apache.inlong.sdk.transform.process.Context;
+import org.apache.inlong.sdk.transform.process.operator.OperatorTools;
+import org.apache.inlong.sdk.transform.process.parser.ValueParser;
+
+import net.sf.jsqlparser.expression.Function;
+
+import java.math.BigDecimal;
+/**
+ * TanFunction
+ * description: tan(numeric)--returns the tangent of numeric
+ */
+public class TanFunction implements ValueParser {
+
+private ValueParser numberParser;
+
+public TanFunction(Function expr) {
+numberParser =
OperatorTools.buildParser(expr.getParameters().getExpressions().get(0));
+}
+
+@Override
+public Object parse(SourceData sourceData, int rowIndex, Context context) {
+Object numberObj = numberParser.parse(sourceData, rowIndex, context);
+BigDecimal numberValue = OperatorTools.parseBigDecimal(numberObj);
+return Math.tan(numberValue.doubleValue());
+}
+}
diff --git
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
index 028ef7156f..5ed42095da 100644
---
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
+++
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
@@ -40,6 +40,7 @@ import
org.apache.inlong.sdk.transform.process.function.SinFunction;
import org.apache.inlong.sdk.transform.process.function.SinhFunction;
import org.apache.inlong.sdk.transform.process.function.SqrtFunction;
import org.apache.inlong.sdk.transform.process.function.SubstringFunction;
+import org.apache.inlong.sdk.transform.process.function.TanFunction;
import
org.apache.inlong.sdk.transform.process.function.TimestampExtractFunction;
import org.apache.inlong.sdk.transform.process.function.ToDateFunction;
import org.apache.inlong.sdk.transform.process.function.ToTimestampFunction;
@@ -120,6 +121,7 @@ public class OperatorTools {
functionMap.put("sin", SinFunction::new);
functionMap.put("sinh", SinhFunction::new);
functionMap.put("cos", CosFunction::new);
+functionMap.put("tan", TanFunction::new);
functionMap.put("year", func -> new
DateExtractFunction(DateExtractFunctionType.YEAR, func));
functionMap.put("quarter", func -> new
DateExtractFunction(DateExtractFunctionType.QUARTER, func));
functionMap.put("month", func -> new
DateExtractFunction(DateExtractFunctionType.MONTH, func));
diff --git
a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformAri
(inlong) branch master updated: [INLONG-10841][SDK] Transform SQL supports toBase64 function (#10845)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new fbde3da763 [INLONG-10841][SDK] Transform SQL supports toBase64
function (#10845)
fbde3da763 is described below
commit fbde3da7630f7fb703f37e17370fa0c8ae7f0169
Author: zoy0 <105140381+z...@users.noreply.github.com>
AuthorDate: Thu Aug 22 16:59:37 2024 +0800
[INLONG-10841][SDK] Transform SQL supports toBase64 function (#10845)
---
.../process/function/ToBase64Function.java | 48 ++
.../transform/process/operator/OperatorTools.java | 2 +
.../TestTransformStringFunctionsProcessor.java | 16
3 files changed, 66 insertions(+)
diff --git
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/ToBase64Function.java
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/ToBase64Function.java
new file mode 100644
index 00..a82f36d9c2
--- /dev/null
+++
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/ToBase64Function.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sdk.transform.process.function;
+
+import org.apache.inlong.sdk.transform.decode.SourceData;
+import org.apache.inlong.sdk.transform.process.Context;
+import org.apache.inlong.sdk.transform.process.operator.OperatorTools;
+import org.apache.inlong.sdk.transform.process.parser.ValueParser;
+
+import net.sf.jsqlparser.expression.Function;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Base64;
+
+/**
+ * ToBase64Function
+ * description: to_base64(string1)--returns the base64-encoded result from
string1
+ */
+public class ToBase64Function implements ValueParser {
+
+private final ValueParser stringParser;
+
+public ToBase64Function(Function expr) {
+stringParser =
OperatorTools.buildParser(expr.getParameters().getExpressions().get(0));
+}
+
+@Override
+public Object parse(SourceData sourceData, int rowIndex, Context context) {
+Object stringObj = stringParser.parse(sourceData, rowIndex, context);
+String string = OperatorTools.parseString(stringObj);
+return
Base64.getEncoder().encodeToString(string.getBytes(StandardCharsets.UTF_8));
+}
+}
diff --git
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
index 5ed42095da..c1cffbf88a 100644
---
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
+++
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
@@ -42,6 +42,7 @@ import
org.apache.inlong.sdk.transform.process.function.SqrtFunction;
import org.apache.inlong.sdk.transform.process.function.SubstringFunction;
import org.apache.inlong.sdk.transform.process.function.TanFunction;
import
org.apache.inlong.sdk.transform.process.function.TimestampExtractFunction;
+import org.apache.inlong.sdk.transform.process.function.ToBase64Function;
import org.apache.inlong.sdk.transform.process.function.ToDateFunction;
import org.apache.inlong.sdk.transform.process.function.ToTimestampFunction;
import org.apache.inlong.sdk.transform.process.function.TrimFunction;
@@ -140,6 +141,7 @@ public class OperatorTools {
functionMap.put("from_unixtime", FromUnixTimeFunction::new);
functionMap.put("unix_timestamp", UnixTimestampFunction::new);
functionMap.put("to_timestamp", ToTimestampFunction::new);
+functionMap.put("to_base64", ToBase64Function::new);
}
public static ExpressionOperator buildOperator(Expression expr) {
diff --git
a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformStringFunctionsProcessor.java
b/inlong-sdk/t
(inlong) branch master updated: [INLONG-10884][Manager] Support configuring HTTP type sink (#10890)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 3faf124148 [INLONG-10884][Manager] Support configuring HTTP type sink
(#10890)
3faf124148 is described below
commit 3faf124148a74127e900954a2317faac07b9b66a
Author: fuweng11 <76141879+fuwen...@users.noreply.github.com>
AuthorDate: Mon Aug 26 19:33:59 2024 +0800
[INLONG-10884][Manager] Support configuring HTTP type sink (#10890)
---
.../inlong/manager/common/consts/DataNodeType.java | 1 +
.../inlong/manager/common/consts/SinkType.java | 3 +
.../inlong/manager/common/enums/ClusterType.java | 2 +
.../sort/http/SortHttpClusterInfo.java}| 33 +++---
.../sort/http/SortHttpClusterRequest.java} | 31 +++--
.../manager/pojo/node/http/HttpDataNodeDTO.java| 75
.../manager/pojo/node/http/HttpDataNodeInfo.java | 63 ++
.../http/HttpDataNodeRequest.java} | 37 --
.../inlong/manager/pojo/sink/BaseStreamSink.java | 3 +
.../inlong/manager/pojo/sink/SinkRequest.java | 3 +
.../inlong/manager/pojo/sink/StreamSink.java | 3 +
.../inlong/manager/pojo/sink/http/HttpSink.java| 64 ++
.../inlong/manager/pojo/sink/http/HttpSinkDTO.java | 76
.../HttpSinkRequest.java} | 38 --
.../service/cluster/SortClusterOperator.java | 5 +
.../service/node/http/HttpDataNodeOperator.java| 96 +++
.../service/sink/http/HttpSinkOperator.java| 129 +
17 files changed, 605 insertions(+), 57 deletions(-)
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/DataNodeType.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/DataNodeType.java
index 0f1952c938..47b139f159 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/DataNodeType.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/DataNodeType.java
@@ -37,6 +37,7 @@ public class DataNodeType {
public static final String SQLSERVER = "SQLSERVER";
public static final String MONGODB = "MONGODB";
public static final String DORIS = "DORIS";
+public static final String HTTP = "HTTP";
public static final String OCEANBASE = "OCEANBASE";
/**
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/SinkType.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/SinkType.java
index 5d069e33df..16a1bfd3d8 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/SinkType.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/SinkType.java
@@ -71,6 +71,9 @@ public class SinkType extends StreamType {
@SupportSortType(sortType = SortType.SORT_FLINK)
public static final String TUBEMQ = "TUBEMQ";
+@SupportSortType(sortType = SortType.SORT_STANDALONE)
+public static final String HTTP = "HTTP";
+
@SupportSortType(sortType = SortType.SORT_FLINK)
public static final String OCEANBASE = "OCEANBASE";
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/ClusterType.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/ClusterType.java
index d7e8ba7d4b..d16ab1d46c 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/ClusterType.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/ClusterType.java
@@ -33,6 +33,7 @@ public class ClusterType {
public static final String DATAPROXY = "DATAPROXY";
public static final String KAFKA = "KAFKA";
+public static final String SORT_HTTP = "SORT_HTTP";
public static final String SORT_ES = "SORT_ES";
public static final String SORT_CLS = "SORT_CLS";
public static final String SORT_PULSAR = "SORT_PULSAR";
@@ -48,6 +49,7 @@ public class ClusterType {
add(ClusterType.PULSAR);
add(ClusterType.DATAPROXY);
add(ClusterType.KAFKA);
+add(ClusterType.SORT_HTTP);
add(ClusterType.SORT_ES);
add(ClusterType.SORT_CLS);
add(ClusterType.SORT_PULSAR);
diff --git
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sink/BaseStreamSink.java
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/cluster/sort/http/SortHttpClusterInfo.java
similarity index 58%
copy from
inlong
(inlong) branch master updated: [INLONG-10846][Dashboard] Add cluster name to data source information display (#10848)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new b08cfff127 [INLONG-10846][Dashboard] Add cluster name to data source
information display (#10848)
b08cfff127 is described below
commit b08cfff12739067720b46ed27d0ed5c418fdd537
Author: kamianlaida <165994047+wohainilao...@users.noreply.github.com>
AuthorDate: Mon Aug 26 20:15:49 2024 +0800
[INLONG-10846][Dashboard] Add cluster name to data source information
display (#10848)
---
inlong-dashboard/src/plugins/sources/common/SourceDefaultInfo.ts | 4
inlong-dashboard/src/ui/locales/cn.json | 1 +
inlong-dashboard/src/ui/locales/en.json | 1 +
inlong-dashboard/src/ui/pages/GroupDetail/DataSources/index.tsx | 5 -
4 files changed, 10 insertions(+), 1 deletion(-)
diff --git a/inlong-dashboard/src/plugins/sources/common/SourceDefaultInfo.ts
b/inlong-dashboard/src/plugins/sources/common/SourceDefaultInfo.ts
index d5c768d8a9..970bd181a7 100644
--- a/inlong-dashboard/src/plugins/sources/common/SourceDefaultInfo.ts
+++ b/inlong-dashboard/src/plugins/sources/common/SourceDefaultInfo.ts
@@ -118,6 +118,10 @@ export class SourceDefaultInfo implements DataWithBackend,
RenderRow, RenderList
@I18n('meta.Sources.Name')
sourceName: string;
+ @ColumnDecorator()
+ @I18n('meta.Sources.ClusterName')
+ readonly inlongClusterName: string;
+
@FieldDecorator({
type: 'select',
props: {
diff --git a/inlong-dashboard/src/ui/locales/cn.json
b/inlong-dashboard/src/ui/locales/cn.json
index 379a5a83e1..f6b1e5a493 100644
--- a/inlong-dashboard/src/ui/locales/cn.json
+++ b/inlong-dashboard/src/ui/locales/cn.json
@@ -25,6 +25,7 @@
"basic.SuccessfullyRestart": "重启成功",
"basic.SuccessfullyStop": "停止成功",
"meta.Sources.Name": "数据源名称",
+ "meta.Sources.ClusterName": "集群名",
"meta.Sources.NameRule": "只能包含英文字母、数字、点号(.)、中划线(-)、下划线(_)",
"meta.Sources.Type": "类型",
"meta.Sources.File.SerializationType": "文件类型",
diff --git a/inlong-dashboard/src/ui/locales/en.json
b/inlong-dashboard/src/ui/locales/en.json
index bc1106fdd6..473f53d20d 100644
--- a/inlong-dashboard/src/ui/locales/en.json
+++ b/inlong-dashboard/src/ui/locales/en.json
@@ -25,6 +25,7 @@
"basic.SuccessfullyRestart": "Successfully restart",
"basic.SuccessfullyStop": "Successfully stop",
"meta.Sources.Name": "Source name",
+ "meta.Sources.ClusterName": "Cluster Name",
"meta.Sources.NameRule": "Only English letters, numbers, dots(.), minus(-),
and underscores(_)",
"meta.Sources.Type": "Type",
"meta.Sources.File.SerializationType": "File type",
diff --git a/inlong-dashboard/src/ui/pages/GroupDetail/DataSources/index.tsx
b/inlong-dashboard/src/ui/pages/GroupDetail/DataSources/index.tsx
index e1050f61d6..6af108b3c1 100644
--- a/inlong-dashboard/src/ui/pages/GroupDetail/DataSources/index.tsx
+++ b/inlong-dashboard/src/ui/pages/GroupDetail/DataSources/index.tsx
@@ -209,6 +209,8 @@ const Comp = ({ inlongGroupId, inlongStreamId, readonly }:
Props, ref) => {
{
title: i18n.t('basic.Operating'),
dataIndex: 'action',
+fixed: 'right',
+width: 200,
render: (text, record) =>
readonly ? (
'-'
@@ -235,7 +237,7 @@ const Comp = ({ inlongGroupId, inlongStreamId, readonly }:
Props, ref) => {
},
]);
}, [entityColumns, onDelete, onEdit, readonly]);
-
+ const scroll = { x: 850 };
return (
<>
{
pagination,
loading,
onChange,
+ scroll,
}}
/>
)}
(inlong) branch master updated: [INLONG-10901][SDK] Transform support BIN(integer) function (#10903)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 89ba85ff8b [INLONG-10901][SDK] Transform support BIN(integer) function
(#10903)
89ba85ff8b is described below
commit 89ba85ff8b1acbc8d0c699fc2412e0870e8895fd
Author: emptyOVO <118812562+empty...@users.noreply.github.com>
AuthorDate: Tue Aug 27 12:03:55 2024 +0800
[INLONG-10901][SDK] Transform support BIN(integer) function (#10903)
---
.../transform/process/function/BinFunction.java| 56 ++
.../transform/process/operator/OperatorTools.java | 2 +
.../TestTransformArithmeticFunctionsProcessor.java | 22 +
3 files changed, 80 insertions(+)
diff --git
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/BinFunction.java
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/BinFunction.java
new file mode 100644
index 00..ae4dbc96e7
--- /dev/null
+++
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/BinFunction.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sdk.transform.process.function;
+
+import org.apache.inlong.sdk.transform.decode.SourceData;
+import org.apache.inlong.sdk.transform.process.Context;
+import org.apache.inlong.sdk.transform.process.operator.OperatorTools;
+import org.apache.inlong.sdk.transform.process.parser.ValueParser;
+
+import net.sf.jsqlparser.expression.Expression;
+import net.sf.jsqlparser.expression.Function;
+
+import java.math.BigDecimal;
+import java.util.List;
+/**
+ * BinFunction
+ * description: bin(integer)--Returns a string representation of an integer in
binary format. If the integer is NULL, NULL is returned.
+ */
+public class BinFunction implements ValueParser {
+
+private ValueParser valueParser;
+
+public BinFunction(Function expr) {
+if (expr.getParameters() != null) {
+List expressions =
expr.getParameters().getExpressions();
+if (expressions != null && expressions.size() == 1) {
+valueParser = OperatorTools.buildParser(expressions.get(0));
+}
+}
+}
+
+@Override
+public Object parse(SourceData sourceData, int rowIndex, Context context) {
+if (valueParser != null) {
+Object valueObj = valueParser.parse(sourceData, rowIndex, context);
+BigDecimal value = OperatorTools.parseBigDecimal(valueObj);
+return Integer.toBinaryString(value.intValue());
+}
+return null;
+}
+}
diff --git
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
index c1cffbf88a..72865fa279 100644
---
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
+++
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
@@ -18,6 +18,7 @@
package org.apache.inlong.sdk.transform.process.operator;
import org.apache.inlong.sdk.transform.process.function.AbsFunction;
+import org.apache.inlong.sdk.transform.process.function.BinFunction;
import org.apache.inlong.sdk.transform.process.function.CeilFunction;
import org.apache.inlong.sdk.transform.process.function.ConcatFunction;
import org.apache.inlong.sdk.transform.process.function.CosFunction;
@@ -123,6 +124,7 @@ public class OperatorTools {
functionMap.put("sinh", SinhFunction::new);
functionMap.put("cos", CosFunction::new);
functionMap.put("tan", TanFunction::new);
+functionMap.put("bin", BinFunction::new);
functionMap.put("year", func -> new
DateExtractFunction(DateExtractFunctionType.YEAR, func));
functionMap.put("quarter", func -> new
DateExtractFunction(DateExt
(inlong) branch master updated: [INLONG-10905][SDK] Transform support Length() function (#10908)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 4538f9f1a1 [INLONG-10905][SDK] Transform support Length() function
(#10908)
4538f9f1a1 is described below
commit 4538f9f1a16a6dd95e98fee903693222390c0cdb
Author: Zkplo <87751516+zk...@users.noreply.github.com>
AuthorDate: Tue Aug 27 12:04:18 2024 +0800
[INLONG-10905][SDK] Transform support Length() function (#10908)
---
.../transform/process/function/LengthFunction.java | 49 ++
.../transform/process/operator/OperatorTools.java | 2 +
.../TestTransformStringFunctionsProcessor.java | 23 ++
3 files changed, 74 insertions(+)
diff --git
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/LengthFunction.java
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/LengthFunction.java
new file mode 100644
index 00..5a01652c93
--- /dev/null
+++
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/LengthFunction.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sdk.transform.process.function;
+
+import org.apache.inlong.sdk.transform.decode.SourceData;
+import org.apache.inlong.sdk.transform.process.Context;
+import org.apache.inlong.sdk.transform.process.operator.OperatorTools;
+import org.apache.inlong.sdk.transform.process.parser.ValueParser;
+
+import net.sf.jsqlparser.expression.Function;
+
+/**
+ * LengthFunction
+ * description: length(string)
+ * - return the length of the string
+ * - return NULL if the string is NULL
+ */
+public class LengthFunction implements ValueParser {
+
+private final ValueParser stringParser;
+
+public LengthFunction(Function expr) {
+stringParser =
OperatorTools.buildParser(expr.getParameters().getExpressions().get(0));
+}
+
+@Override
+public Object parse(SourceData sourceData, int rowIndex, Context context) {
+Object stringObject = stringParser.parse(sourceData, rowIndex,
context);
+if (stringObject == null) {
+return null;
+}
+return OperatorTools.parseString(stringObject).length();
+}
+}
diff --git
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
index 72865fa279..b47d39e298 100644
---
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
+++
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
@@ -28,6 +28,7 @@ import
org.apache.inlong.sdk.transform.process.function.DateFormatFunction;
import org.apache.inlong.sdk.transform.process.function.ExpFunction;
import org.apache.inlong.sdk.transform.process.function.FloorFunction;
import org.apache.inlong.sdk.transform.process.function.FromUnixTimeFunction;
+import org.apache.inlong.sdk.transform.process.function.LengthFunction;
import org.apache.inlong.sdk.transform.process.function.LnFunction;
import org.apache.inlong.sdk.transform.process.function.LocateFunction;
import org.apache.inlong.sdk.transform.process.function.Log10Function;
@@ -144,6 +145,7 @@ public class OperatorTools {
functionMap.put("unix_timestamp", UnixTimestampFunction::new);
functionMap.put("to_timestamp", ToTimestampFunction::new);
functionMap.put("to_base64", ToBase64Function::new);
+functionMap.put("length", LengthFunction::new);
}
public static ExpressionOperator buildOperator(Expression expr) {
diff --git
a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformStringFunctionsProcessor.java
b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformStringFunctionsProcessor.java
index 5479b4485a..9
(inlong) branch master updated (4538f9f1a1 -> e30e493d8a)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
from 4538f9f1a1 [INLONG-10905][SDK] Transform support Length() function
(#10908)
add e30e493d8a [INLONG-10833][SDK] Transform SQL supports parsing of Mod
methods and % expressions (#10853)
No new revisions were added by this update.
Summary of changes:
.../{BinFunction.java => ModuloFunction.java} | 32 +++--
.../transform/process/operator/OperatorTools.java | 16 ++-
.../parser/{LongParser.java => DoubleParser.java} | 17 +--
.../{DivisionParser.java => ModuloParser.java} | 18 +--
.../{ParenthesisParser.java => SignParser.java}| 26 ++--
.../TestTransformArithmeticFunctionsProcessor.java | 147 +
6 files changed, 201 insertions(+), 55 deletions(-)
copy
inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/{BinFunction.java
=> ModuloFunction.java} (62%)
copy
inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/{LongParser.java
=> DoubleParser.java} (81%)
copy
inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/{DivisionParser.java
=> ModuloParser.java} (84%)
copy
inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/{ParenthesisParser.java
=> SignParser.java} (66%)
(inlong) branch master updated: [INLONG-10823][SDK] Transform SQL support Reverse function (#10854)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 6ce335e872 [INLONG-10823][SDK] Transform SQL support Reverse function
(#10854)
6ce335e872 is described below
commit 6ce335e872d806847fa42c33f8e3b16e0a3d2b52
Author: Xincheng Huang <60057611+ying-...@users.noreply.github.com>
AuthorDate: Tue Aug 27 12:08:17 2024 +0800
[INLONG-10823][SDK] Transform SQL support Reverse function (#10854)
---
.../process/function/ReverseFunction.java | 56 ++
.../transform/process/operator/OperatorTools.java | 2 +
.../TestTransformStringFunctionsProcessor.java | 32 +
3 files changed, 90 insertions(+)
diff --git
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/ReverseFunction.java
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/ReverseFunction.java
new file mode 100644
index 00..5015e0225a
--- /dev/null
+++
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/ReverseFunction.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sdk.transform.process.function;
+
+import org.apache.inlong.sdk.transform.decode.SourceData;
+import org.apache.inlong.sdk.transform.process.Context;
+import org.apache.inlong.sdk.transform.process.operator.OperatorTools;
+import org.apache.inlong.sdk.transform.process.parser.ValueParser;
+
+import net.sf.jsqlparser.expression.Function;
+/**
+ * ReverseFunction
+ * description: reverse(string)--returns the string with the order of the
characters reversed.
+ * returns NULL if string is a empty string.
+ */
+public class ReverseFunction implements ValueParser {
+
+private ValueParser stringParser;
+public ReverseFunction(Function expr) {
+stringParser =
OperatorTools.buildParser(expr.getParameters().getExpressions().get(0));
+}
+
+/**
+ * parse
+ * Parse and reverse string by reverse() in StringBuilder.
+ * @param sourceData
+ * @param rowIndex
+ * @param context
+ * @return
+ */
+
+@Override
+public Object parse(SourceData sourceData, int rowIndex, Context context) {
+Object stringObj = stringParser.parse(sourceData, rowIndex, context);
+if (stringObj == null) {
+return null;
+}
+String str = OperatorTools.parseString(stringObj);
+return new StringBuilder(str).reverse().toString();
+}
+}
diff --git
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
index 2305d5165c..9c05508db0 100644
---
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
+++
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
@@ -38,6 +38,7 @@ import
org.apache.inlong.sdk.transform.process.function.ModuloFunction;
import org.apache.inlong.sdk.transform.process.function.NowFunction;
import org.apache.inlong.sdk.transform.process.function.PowerFunction;
import org.apache.inlong.sdk.transform.process.function.ReplicateFunction;
+import org.apache.inlong.sdk.transform.process.function.ReverseFunction;
import org.apache.inlong.sdk.transform.process.function.RoundFunction;
import org.apache.inlong.sdk.transform.process.function.SinFunction;
import org.apache.inlong.sdk.transform.process.function.SinhFunction;
@@ -120,6 +121,7 @@ public class OperatorTools {
functionMap.put("log2", Log2Function::new);
functionMap.put("log", LogFunction::new);
functionMap.put("exp", ExpFunction::new);
+functionMap.put("reverse", ReverseFunction::new);
functionMap.put("substring", SubstringFunction::new);
functionMap.put("trim", TrimFuncti
(inlong) branch master updated: [INLONG-10907][Manager] Support reinstalling the installer (#10909)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 3e8afd7ce1 [INLONG-10907][Manager] Support reinstalling the installer
(#10909)
3e8afd7ce1 is described below
commit 3e8afd7ce191cd7ef32c3b689e40e09216f79854
Author: fuweng11 <76141879+fuwen...@users.noreply.github.com>
AuthorDate: Tue Aug 27 12:32:21 2024 +0800
[INLONG-10907][Manager] Support reinstalling the installer (#10909)
---
.../node/AgentClusterNodeInstallOperator.java | 91 +++---
.../node/InlongClusterNodeInstallOperator.java | 8 ++
.../manager/service/cmd/CommandExecutor.java | 3 +
.../manager/service/cmd/CommandExecutorImpl.java | 5 ++
4 files changed, 79 insertions(+), 28 deletions(-)
diff --git
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/cluster/node/AgentClusterNodeInstallOperator.java
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/cluster/node/AgentClusterNodeInstallOperator.java
index d9ace7ab90..77a848dbb6 100644
---
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/cluster/node/AgentClusterNodeInstallOperator.java
+++
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/cluster/node/AgentClusterNodeInstallOperator.java
@@ -21,6 +21,7 @@ import org.apache.inlong.manager.common.enums.ClusterType;
import org.apache.inlong.manager.common.enums.ModuleType;
import org.apache.inlong.manager.common.exceptions.BusinessException;
import org.apache.inlong.manager.common.util.AESUtils;
+import org.apache.inlong.manager.common.util.CommonBeanUtils;
import org.apache.inlong.manager.common.util.Preconditions;
import org.apache.inlong.manager.dao.entity.InlongClusterEntity;
import org.apache.inlong.manager.dao.entity.InlongClusterNodeEntity;
@@ -33,6 +34,7 @@ import
org.apache.inlong.manager.dao.mapper.ModuleConfigEntityMapper;
import org.apache.inlong.manager.dao.mapper.PackageConfigEntityMapper;
import org.apache.inlong.manager.dao.mapper.UserEntityMapper;
import org.apache.inlong.manager.pojo.cluster.ClusterNodeRequest;
+import org.apache.inlong.manager.pojo.cluster.agent.AgentClusterNodeDTO;
import org.apache.inlong.manager.pojo.cluster.agent.AgentClusterNodeRequest;
import org.apache.inlong.manager.service.cmd.CommandExecutor;
@@ -43,6 +45,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
+import java.io.File;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
@@ -50,17 +53,17 @@ import java.util.Objects;
@Service
public class AgentClusterNodeInstallOperator implements
InlongClusterNodeInstallOperator {
-private static final Logger LOGGER =
LoggerFactory.getLogger(AgentClusterNodeInstallOperator.class);
-
public static final String INSTALLER_CONF_PATH =
"/conf/installer.properties";
public static final String INSTALLER_START_CMD = "/bin/installer.sh start";
+public static final String INSTALLER_RESTART_CMD = "/bin/installer.sh
restart";
+public static final String INSTALLER_STOP_CMD = "/bin/installer.sh
restart";
public static final String AGENT_MANAGER_AUTH_SECRET_ID =
"agent.manager.auth.secretId";
public static final String AGENT_MANAGER_AUTH_SECRET_KEY =
"agent.manager.auth.secretKey";
public static final String AGENT_MANAGER_ADDR = "agent.manager.addr";
public static final String AGENT_CLUSTER_NAME = "agent.cluster.name";
public static final String AGENT_CLUSTER_TAG = "agent.cluster.tag";
public static final String AGENT_LOCAL_IP = "agent.local.ip";
-
+private static final Logger LOGGER =
LoggerFactory.getLogger(AgentClusterNodeInstallOperator.class);
@Autowired
private InlongClusterEntityMapper clusterEntityMapper;
@Autowired
@@ -91,45 +94,55 @@ public class AgentClusterNodeInstallOperator implements
InlongClusterNodeInstall
@Override
public boolean install(ClusterNodeRequest clusterNodeRequest, String
operator) {
-LOGGER.info("begin to insert agent inlong cluster node={}",
clusterNodeRequest);
+LOGGER.info("begin to insert agent cluster node={}",
clusterNodeRequest);
try {
-InlongClusterEntity clusterEntity =
clusterEntityMapper.selectById(clusterNodeRequest.getParentId());
AgentClusterNodeRequest request = (AgentClusterNodeRequest)
clusterNodeRequest;
-commandExecutor.mkdir(request, agentInstallPath);
-String downLoadUrl = getInstallerDownLoadUrl(request);
-String fileName =
d
(inlong) branch master updated (3e8afd7ce1 -> 8be8834a78)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
from 3e8afd7ce1 [INLONG-10907][Manager] Support reinstalling the installer
(#10909)
add 8be8834a78 [INLONG-10831][Sort] SortStandalone support HTTP sink
(#10837)
No new revisions were added by this update.
Summary of changes:
.../sink/http/DefaultEvent2HttpRequestHandler.java | 140 +++
.../sort/standalone/sink/http/HttpCallback.java| 80
.../HttpChannelWorker.java}| 72 +---
.../ClsIdConfig.java => http/HttpIdConfig.java}| 62 ++--
.../standalone/sink/http/HttpOutputChannel.java| 146
.../EsIndexRequest.java => http/HttpRequest.java} | 44 +--
.../EsSink.java => http/HttpSink.java} | 49 +--
.../sort/standalone/sink/http/HttpSinkContext.java | 404 +
.../sort/standalone/sink/http/HttpSinkFactory.java | 23 +-
.../IEvent2HttpRequestHandler.java}| 21 +-
10 files changed, 877 insertions(+), 164 deletions(-)
create mode 100644
inlong-sort-standalone/sort-standalone-source/src/main/java/org/apache/inlong/sort/standalone/sink/http/DefaultEvent2HttpRequestHandler.java
create mode 100644
inlong-sort-standalone/sort-standalone-source/src/main/java/org/apache/inlong/sort/standalone/sink/http/HttpCallback.java
copy
inlong-sort-standalone/sort-standalone-source/src/main/java/org/apache/inlong/sort/standalone/sink/{elasticsearch/EsChannelWorker.java
=> http/HttpChannelWorker.java} (56%)
copy
inlong-sort-standalone/sort-standalone-source/src/main/java/org/apache/inlong/sort/standalone/sink/{cls/ClsIdConfig.java
=> http/HttpIdConfig.java} (50%)
create mode 100644
inlong-sort-standalone/sort-standalone-source/src/main/java/org/apache/inlong/sort/standalone/sink/http/HttpOutputChannel.java
copy
inlong-sort-standalone/sort-standalone-source/src/main/java/org/apache/inlong/sort/standalone/sink/{elasticsearch/EsIndexRequest.java
=> http/HttpRequest.java} (63%)
copy
inlong-sort-standalone/sort-standalone-source/src/main/java/org/apache/inlong/sort/standalone/sink/{elasticsearch/EsSink.java
=> http/HttpSink.java} (66%)
create mode 100644
inlong-sort-standalone/sort-standalone-source/src/main/java/org/apache/inlong/sort/standalone/sink/http/HttpSinkContext.java
copy
inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/IntListValuable.java
=>
inlong-sort-standalone/sort-standalone-source/src/main/java/org/apache/inlong/sort/standalone/sink/http/HttpSinkFactory.java
(64%)
copy
inlong-sort-standalone/sort-standalone-source/src/main/java/org/apache/inlong/sort/standalone/sink/{hive/IEventFormatHandler.java
=> http/IEvent2HttpRequestHandler.java} (73%)
(inlong) branch master updated: [INLONG-10898][Dashboard] Data ranking increases according to dt sorting (#10899)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new b93aa368a4 [INLONG-10898][Dashboard] Data ranking increases according
to dt sorting (#10899)
b93aa368a4 is described below
commit b93aa368a41e33838aa7e5b4c29b454d0fabec86
Author: kamianlaida <165994047+wohainilao...@users.noreply.github.com>
AuthorDate: Tue Aug 27 14:08:17 2024 +0800
[INLONG-10898][Dashboard] Data ranking increases according to dt sorting
(#10899)
---
.../src/ui/pages/GroupDetail/DataStream/PreviewModal.tsx | 11 ++-
1 file changed, 10 insertions(+), 1 deletion(-)
diff --git
a/inlong-dashboard/src/ui/pages/GroupDetail/DataStream/PreviewModal.tsx
b/inlong-dashboard/src/ui/pages/GroupDetail/DataStream/PreviewModal.tsx
index a608dbcb7e..fa9edff445 100644
--- a/inlong-dashboard/src/ui/pages/GroupDetail/DataStream/PreviewModal.tsx
+++ b/inlong-dashboard/src/ui/pages/GroupDetail/DataStream/PreviewModal.tsx
@@ -24,6 +24,7 @@ import { useRequest, useUpdateEffect } from '@/ui/hooks';
import i18n from '@/i18n';
import { ColumnsType } from 'antd/es/table';
import dayjs from 'dayjs';
+import { SortOrder } from 'antd/es/table/interface';
export interface Props extends ModalProps {
inlongGroupId: string;
@@ -87,6 +88,12 @@ const Comp: React.FC = ({ inlongGroupId,
inlongStreamId, ...modalProps })
}
return;
};
+ const timestampSorter = (a, b) => {
+const dateA = dayjs(a.dt, '-MM-DD HH:mm:ss');
+const dateB = dayjs(b.dt, '-MM-DD HH:mm:ss');
+return dateA.isBefore(dateB) ? -1 : dateA.isAfter(dateB) ? 1 : 0;
+ };
+ const sortOrder: SortOrder = 'descend';
const detailColumns: ColumnsType = [
{
@@ -94,11 +101,13 @@ const Comp: React.FC = ({ inlongGroupId,
inlongStreamId, ...modalProps })
key: 'dt',
width: 200,
dataIndex: 'dt',
+ sorter: timestampSorter,
+ defaultSortOrder: sortOrder,
},
].concat(
(getColumn() ? getColumn() : []).concat([
{
-title: 'operation',
+title: i18n.t('basic.Operating'),
key: 'operation',
fixed: 'right',
width: 100,
(inlong) branch master updated: [INLONG-10816][SDK] Transform support Replace function.(#10816) (#10818)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 04e3a58ea1 [INLONG-10816][SDK] Transform support Replace
function.(#10816) (#10818)
04e3a58ea1 is described below
commit 04e3a58ea17c166d6d6394a68a9707c3649bef73
Author: rachely <124853723+ybszzz...@users.noreply.github.com>
AuthorDate: Tue Aug 27 14:16:50 2024 +0800
[INLONG-10816][SDK] Transform support Replace function.(#10816) (#10818)
---
.../process/function/ReplaceFunction.java | 57 ++
.../transform/process/operator/OperatorTools.java | 2 +
.../TestTransformStringFunctionsProcessor.java | 36 ++
3 files changed, 95 insertions(+)
diff --git
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/ReplaceFunction.java
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/ReplaceFunction.java
new file mode 100644
index 00..d9d1d26a21
--- /dev/null
+++
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/ReplaceFunction.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sdk.transform.process.function;
+
+import org.apache.inlong.sdk.transform.decode.SourceData;
+import org.apache.inlong.sdk.transform.process.Context;
+import org.apache.inlong.sdk.transform.process.operator.OperatorTools;
+import org.apache.inlong.sdk.transform.process.parser.ValueParser;
+
+import net.sf.jsqlparser.expression.Expression;
+import net.sf.jsqlparser.expression.Function;
+
+import java.util.List;
+
+/**
+ * ReplaceFunction
+ * description: replace(s, s1, s2)--replace string s1 in string s with string
s2.
+ */
+public class ReplaceFunction implements ValueParser {
+
+private ValueParser stringParser;
+private ValueParser targetParser;
+private ValueParser replacementParser;
+
+public ReplaceFunction(Function expr) {
+List expressions = expr.getParameters().getExpressions();
+stringParser = OperatorTools.buildParser(expressions.get(0));
+targetParser = OperatorTools.buildParser(expressions.get(1));
+replacementParser = OperatorTools.buildParser(expressions.get(2));
+}
+
+@Override
+public Object parse(SourceData sourceData, int rowIndex, Context context) {
+Object strObj = stringParser.parse(sourceData, rowIndex, context);
+Object targetObj = targetParser.parse(sourceData, rowIndex, context);
+Object replacementObj = replacementParser.parse(sourceData, rowIndex,
context);
+String str = OperatorTools.parseString(strObj);
+String target = OperatorTools.parseString(targetObj);
+String replacement = OperatorTools.parseString(replacementObj);
+return str.replace(target, replacement);
+}
+}
\ No newline at end of file
diff --git
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
index 9c05508db0..f7f023e748 100644
---
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
+++
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
@@ -37,6 +37,7 @@ import
org.apache.inlong.sdk.transform.process.function.LogFunction;
import org.apache.inlong.sdk.transform.process.function.ModuloFunction;
import org.apache.inlong.sdk.transform.process.function.NowFunction;
import org.apache.inlong.sdk.transform.process.function.PowerFunction;
+import org.apache.inlong.sdk.transform.process.function.ReplaceFunction;
import org.apache.inlong.sdk.transform.process.function.ReplicateFunction;
import org.apache.inlong.sdk.transform.process.function.ReverseFunction;
import org.apache.inlong.sdk.transform.process.function.RoundFunction;
@@ -156,6 +157,7 @@ public class OperatorTools {
functionMap.put(
(inlong) branch release-1.11.0 updated: [INLONG-9706][Manager] Supports jdbc verification under multiple hosts, backport from pull request 9707 (#10852)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch release-1.11.0
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/release-1.11.0 by this push:
new a71e5e81c1 [INLONG-9706][Manager] Supports jdbc verification under
multiple hosts, backport from pull request 9707 (#10852)
a71e5e81c1 is described below
commit a71e5e81c1c5a32cf920963ac926f9c8d2fc9b9f
Author: Yiheng Cao <65160922+crispy-fried-chic...@users.noreply.github.com>
AuthorDate: Tue Aug 27 19:51:43 2024 +0800
[INLONG-9706][Manager] Supports jdbc verification under multiple hosts,
backport from pull request 9707 (#10852)
---
.../inlong/manager/common/consts/InlongConstants.java | 2 ++
.../inlong/manager/pojo/util/MySQLSensitiveUrlUtils.java | 14 --
2 files changed, 14 insertions(+), 2 deletions(-)
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
index 0f6c86056a..c3085972fd 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
@@ -48,6 +48,8 @@ public class InlongConstants {
public static final String EMPTY = "";
+public static final String SHARP = "#";
+
public static final String SLASH = "/";
public static final String COLON = ":";
diff --git
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/util/MySQLSensitiveUrlUtils.java
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/util/MySQLSensitiveUrlUtils.java
index e155887f1e..bb10e33a09 100644
---
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/util/MySQLSensitiveUrlUtils.java
+++
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/util/MySQLSensitiveUrlUtils.java
@@ -71,6 +71,13 @@ public class MySQLSensitiveUrlUtils {
}
resultUrl = resultUrl.replaceAll(InlongConstants.REGEX_WHITESPACE,
InlongConstants.EMPTY);
+for (String key : SENSITIVE_REPLACE_PARAM_MAP.keySet()) {
+resultUrl = StringUtils.replaceIgnoreCase(resultUrl, key +
InlongConstants.EQUAL + "true",
+InlongConstants.EMPTY);
+resultUrl = StringUtils.replaceIgnoreCase(resultUrl, key +
InlongConstants.EQUAL + "yes",
+InlongConstants.EMPTY);
+}
+
if (resultUrl.contains(InlongConstants.QUESTION_MARK)) {
StringBuilder builder = new StringBuilder();
builder.append(StringUtils.substringBefore(resultUrl,
InlongConstants.QUESTION_MARK));
@@ -78,10 +85,13 @@ public class MySQLSensitiveUrlUtils {
List paramList = new ArrayList<>();
String queryString = StringUtils.substringAfter(resultUrl,
InlongConstants.QUESTION_MARK);
-if (queryString.contains("#")) {
-queryString = StringUtils.substringBefore(queryString,
"#");
+if (queryString.contains(InlongConstants.SHARP)) {
+queryString = StringUtils.substringBefore(queryString,
InlongConstants.SHARP);
}
for (String param :
queryString.split(InlongConstants.AMPERSAND)) {
+if (StringUtils.isBlank(param)) {
+continue;
+}
String key = StringUtils.substringBefore(param,
InlongConstants.EQUAL);
String value = StringUtils.substringAfter(param,
InlongConstants.EQUAL);
(inlong) branch master updated: [INLONG-10906][SDK] Transform supports the truncation of left and right strings (#10923)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new fab6756556 [INLONG-10906][SDK] Transform supports the truncation of
left and right strings (#10923)
fab6756556 is described below
commit fab6756556abf56bef71c7b164463dbb08af33d2
Author: Zkplo <87751516+zk...@users.noreply.github.com>
AuthorDate: Wed Aug 28 11:04:34 2024 +0800
[INLONG-10906][SDK] Transform supports the truncation of left and right
strings (#10923)
---
.../transform/process/function/LeftFunction.java | 62 ++
.../transform/process/function/RightFunction.java | 62 ++
.../transform/process/operator/OperatorTools.java | 6 +-
.../TestTransformStringFunctionsProcessor.java | 97 +-
4 files changed, 225 insertions(+), 2 deletions(-)
diff --git
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/LeftFunction.java
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/LeftFunction.java
new file mode 100644
index 00..e86d770f92
--- /dev/null
+++
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/LeftFunction.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sdk.transform.process.function;
+
+import org.apache.inlong.sdk.transform.decode.SourceData;
+import org.apache.inlong.sdk.transform.process.Context;
+import org.apache.inlong.sdk.transform.process.operator.OperatorTools;
+import org.apache.inlong.sdk.transform.process.parser.ValueParser;
+
+import net.sf.jsqlparser.expression.Expression;
+import net.sf.jsqlparser.expression.Function;
+
+import java.util.List;
+
+/**
+ * LeftFunction
+ * description: left(string,length)
+ * - return null if either string or length is null
+ * - return "" if it is less than or equal to zero
+ * - return a substring of length starting from the right side of the string.
+ */
+public class LeftFunction implements ValueParser {
+
+private final ValueParser stringParser;
+private final ValueParser lengthParser;
+
+public LeftFunction(Function expr) {
+List expressions = expr.getParameters().getExpressions();
+stringParser = OperatorTools.buildParser(expressions.get(0));
+lengthParser = OperatorTools.buildParser(expressions.get(1));
+}
+
+@Override
+public Object parse(SourceData sourceData, int rowIndex, Context context) {
+Object stringObj = stringParser.parse(sourceData, rowIndex, context);
+Object lengthObj = lengthParser.parse(sourceData, rowIndex, context);
+if (stringObj == null || lengthObj == null) {
+return null;
+}
+String str = OperatorTools.parseString(stringObj);
+int len = Integer.parseInt(OperatorTools.parseString(lengthObj));
+if (len <= 0) {
+return "";
+}
+return str.substring(0, Math.min(str.length(), len));
+}
+}
diff --git
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/RightFunction.java
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/RightFunction.java
new file mode 100644
index 00..8260d3f299
--- /dev/null
+++
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/RightFunction.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is
(inlong) branch master updated (fab6756556 -> 268ae4d734)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
from fab6756556 [INLONG-10906][SDK] Transform supports the truncation of
left and right strings (#10923)
add 268ae4d734 [INLONG-10866][SDK] Transform SQL support sign function
(#10924)
No new revisions were added by this update.
Summary of changes:
.../{CosFunction.java => SignFunction.java}| 26 +-
.../transform/process/operator/OperatorTools.java | 2 ++
.../TestTransformArithmeticFunctionsProcessor.java | 21 +
3 files changed, 34 insertions(+), 15 deletions(-)
copy
inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/{CosFunction.java
=> SignFunction.java} (79%)
(inlong) branch master updated: [INLONG-10872][SDK] Add a More Complex Json Test Case (#10885)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new ffd5f1b43b [INLONG-10872][SDK] Add a More Complex Json Test Case
(#10885)
ffd5f1b43b is described below
commit ffd5f1b43bc68ffa61b85e5f36166963fa47c070
Author: Xincheng Huang <60057611+ying-...@users.noreply.github.com>
AuthorDate: Wed Aug 28 14:12:21 2024 +0800
[INLONG-10872][SDK] Add a More Complex Json Test Case (#10885)
---
.../transform/process/TestTransformProcessor.java | 74 ++
1 file changed, 61 insertions(+), 13 deletions(-)
diff --git
a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformProcessor.java
b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformProcessor.java
index 52838ab06c..20af097de6 100644
---
a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformProcessor.java
+++
b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformProcessor.java
@@ -148,16 +148,16 @@ public class TestTransformProcessor {
@Test
public void testJson2Csv() throws Exception {
-List fields = this.getTestFieldList();
-JsonSourceInfo jsonSource = new JsonSourceInfo("UTF-8", "msgs");
-CsvSinkInfo csvSink = new CsvSinkInfo("UTF-8", '|', '\\', fields);
-String transformSql = "select
$root.sid,$root.packageID,$child.msgTime,$child.msg from source";
-TransformConfig config = new TransformConfig(transformSql);
+List fields1 = this.getTestFieldList();
+JsonSourceInfo jsonSource1 = new JsonSourceInfo("UTF-8", "msgs");
+CsvSinkInfo csvSink1 = new CsvSinkInfo("UTF-8", '|', '\\', fields1);
+String transformSql1 = "select
$root.sid,$root.packageID,$child.msgTime,$child.msg from source";
+TransformConfig config1 = new TransformConfig(transformSql1);
// case1
-TransformProcessor processor = TransformProcessor
-.create(config,
SourceDecoderFactory.createJsonDecoder(jsonSource),
-SinkEncoderFactory.createCsvEncoder(csvSink));
-String srcString = "{\n"
+TransformProcessor processor1 = TransformProcessor
+.create(config1,
SourceDecoderFactory.createJsonDecoder(jsonSource1),
+SinkEncoderFactory.createCsvEncoder(csvSink1));
+String srcString1 = "{\n"
+ " \"sid\":\"value1\",\n"
+ " \"packageID\":\"value2\",\n"
+ " \"msgs\":[\n"
@@ -165,10 +165,42 @@ public class TestTransformProcessor {
+ " {\"msg\":\"v4\",\"msgTime\":1713243918000}\n"
+ " ]\n"
+ "}";
-List output = processor.transform(srcString, new HashMap<>());
-Assert.assertEquals(2, output.size());
-Assert.assertEquals(output.get(0),
"value1|value2|1713243918000|value4");
-Assert.assertEquals(output.get(1), "value1|value2|1713243918000|v4");
+List output1 = processor1.transform(srcString1, new
HashMap<>());
+Assert.assertEquals(2, output1.size());
+Assert.assertEquals(output1.get(0),
"value1|value2|1713243918000|value4");
+Assert.assertEquals(output1.get(1), "value1|value2|1713243918000|v4");
+// case2
+List fields2 = this.getTestFieldList2();
+JsonSourceInfo jsonSource2 = new JsonSourceInfo("UTF-8", "items");
+CsvSinkInfo csvSink2 = new CsvSinkInfo("UTF-8", '|', '\\', fields2);
+String transformSql2 =
+"select
$root.id,$child.itemId,$child.subItems(0).subItemId,$child.subItems(1).msg from
source";
+TransformConfig config2 = new TransformConfig(transformSql2);
+TransformProcessor processor2 = TransformProcessor
+.create(config2,
SourceDecoderFactory.createJsonDecoder(jsonSource2),
+SinkEncoderFactory.createCsvEncoder(csvSink2));
+String srcString2 = "{\n"
++ " \"id\":\"value1\",\n"
++ " \"name\":\"value2\",\n"
++ " \"items\":[\n"
++ "{\"itemId\":\"item1\",\n"
++ " \"subItems\":[\n"
+
(inlong) branch master updated: [INLONG-10879][SDK] Transform support TIMESTAMPADD() function (#10880)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 389b41d81b [INLONG-10879][SDK] Transform support TIMESTAMPADD()
function (#10880)
389b41d81b is described below
commit 389b41d81b644ef3899117714ac09f144f2b1b1c
Author: Zkplo <87751516+zk...@users.noreply.github.com>
AuthorDate: Wed Aug 28 14:12:35 2024 +0800
[INLONG-10879][SDK] Transform support TIMESTAMPADD() function (#10880)
---
.../process/function/TimestampAddFunction.java | 117 +
.../transform/process/operator/OperatorTools.java | 2 +
.../TestTransformTemporalFunctionsProcessor.java | 40 +++
3 files changed, 159 insertions(+)
diff --git
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/TimestampAddFunction.java
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/TimestampAddFunction.java
new file mode 100644
index 00..e09512cdfd
--- /dev/null
+++
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/TimestampAddFunction.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sdk.transform.process.function;
+
+import org.apache.inlong.sdk.transform.decode.SourceData;
+import org.apache.inlong.sdk.transform.process.Context;
+import org.apache.inlong.sdk.transform.process.operator.OperatorTools;
+import org.apache.inlong.sdk.transform.process.parser.ValueParser;
+
+import net.sf.jsqlparser.expression.Expression;
+import net.sf.jsqlparser.expression.Function;
+
+import java.time.LocalDate;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.List;
+
+/**
+ * TimestampAddFunction
+ * Description: Add integer expression intervals to the date or date time
expression expr.
+ * The unit of the time interval is specified by the unit parameter, which
should be one of the following values:
+ * FRAC_SECOND, SECOND, MINUTE, HOUR, DAY, WEEK, MONTH, QUARTER, or YEAR.
+ */
+public class TimestampAddFunction implements ValueParser {
+
+private ValueParser intervalParser;
+private ValueParser amountParser;
+private ValueParser datetimeParser;
+private static final DateTimeFormatter DEFAULT_FORMAT_DATE_TIME =
+DateTimeFormatter.ofPattern("-MM-dd HH:mm:ss");
+private static final DateTimeFormatter DEFAULT_FORMAT_DATE =
DateTimeFormatter.ofPattern("-MM-dd");
+
+public TimestampAddFunction(Function expr) {
+List expressions = expr.getParameters().getExpressions();
+intervalParser = OperatorTools.buildParser(expressions.get(0));
+amountParser = OperatorTools.buildParser(expressions.get(1));
+datetimeParser = OperatorTools.buildParser(expressions.get(2));
+}
+
+@Override
+public Object parse(SourceData sourceData, int rowIndex, Context context) {
+String interval = intervalParser.parse(sourceData, rowIndex,
context).toString();
+Long amount = Long.parseLong(amountParser.parse(sourceData, rowIndex,
context).toString());
+String dateString = datetimeParser.parse(sourceData, rowIndex,
context).toString();
+return evalDate(dateString, interval, amount);
+}
+
+private String evalDate(String dateString, String interval, Long amount) {
+DateTimeFormatter formatter = null;
+LocalDateTime dateTime = null;
+boolean hasTime = true;
+if (dateString.indexOf(' ') != -1) {
+formatter = DEFAULT_FORMAT_DATE_TIME;
+dateTime = LocalDateTime.parse(dateString, formatter);
+} else {
+formatter = DEFAULT_FORMAT_DATE;
+dateTime = LocalDate.parse(dateString, formatter).atStartOfDay();
+hasTime = false;
+}
+
+switch (interval.toUpperCase()) {
+case "FRAC_SECOND":
+hasTime = true;
+dateTime = dateTime.plusNanos(amount * 1000_000);
+
(inlong) branch master updated: [INLONG-10864][SDK] Transform SQL support md5 function (#10865)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 1394dcdbea [INLONG-10864][SDK] Transform SQL support md5 function
(#10865)
1394dcdbea is described below
commit 1394dcdbea4a18b858d02b4f5319dabcd8dab4a3
Author: Zkplo <87751516+zk...@users.noreply.github.com>
AuthorDate: Wed Aug 28 15:09:23 2024 +0800
[INLONG-10864][SDK] Transform SQL support md5 function (#10865)
---
.../transform/process/function/Md5Function.java| 51 ++
.../transform/process/operator/OperatorTools.java | 2 +
.../TestTransformArithmeticFunctionsProcessor.java | 33 ++
3 files changed, 86 insertions(+)
diff --git
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/Md5Function.java
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/Md5Function.java
new file mode 100644
index 00..027511864a
--- /dev/null
+++
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/Md5Function.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sdk.transform.process.function;
+
+import org.apache.inlong.sdk.transform.decode.SourceData;
+import org.apache.inlong.sdk.transform.process.Context;
+import org.apache.inlong.sdk.transform.process.operator.OperatorTools;
+import org.apache.inlong.sdk.transform.process.parser.ValueParser;
+
+import net.sf.jsqlparser.expression.Function;
+import org.apache.commons.codec.digest.DigestUtils;
+
+import java.nio.charset.StandardCharsets;
+
+/**
+ * Md5Function
+ * description: MD5(string): Return the MD5 hash value of a string in the form
of a 32-bit hexadecimal digit string; If the string is NULL, return NULL.
+ */
+public class Md5Function implements ValueParser {
+
+private ValueParser msgParser;
+
+public Md5Function(Function expr) {
+msgParser =
OperatorTools.buildParser(expr.getParameters().getExpressions().get(0));
+}
+
+@Override
+public Object parse(SourceData sourceData, int rowIndex, Context context) {
+Object msgObj = msgParser.parse(sourceData, rowIndex, context);
+if (msgObj == null) {
+return null;
+}
+String msg = msgObj.toString();
+return DigestUtils.md5Hex(msg.getBytes(StandardCharsets.UTF_8));
+}
+}
diff --git
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
index 66c59a888b..fbd52185e0 100644
---
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
+++
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
@@ -38,6 +38,7 @@ import
org.apache.inlong.sdk.transform.process.function.Log10Function;
import org.apache.inlong.sdk.transform.process.function.Log2Function;
import org.apache.inlong.sdk.transform.process.function.LogFunction;
import org.apache.inlong.sdk.transform.process.function.LowerFunction;
+import org.apache.inlong.sdk.transform.process.function.Md5Function;
import org.apache.inlong.sdk.transform.process.function.ModuloFunction;
import org.apache.inlong.sdk.transform.process.function.NowFunction;
import org.apache.inlong.sdk.transform.process.function.PowerFunction;
@@ -177,6 +178,7 @@ public class OperatorTools {
functionMap.put("left", LeftFunction::new);
functionMap.put("right", RightFunction::new);
functionMap.put("timestampadd", TimestampAddFunction::new);
+functionMap.put("md5", Md5Function::new);
}
public static ExpressionOperator buildOperator(Expression expr) {
diff --git
a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java
b/inlong-sdk/t
(inlong) branch master updated: [INLONG-10832][SDK] Transform SQL support Translate function (#10863)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 04971c37e9 [INLONG-10832][SDK] Transform SQL support Translate
function (#10863)
04971c37e9 is described below
commit 04971c37e908771526b2ac7c888cc55badfeff27
Author: Huan Liang
AuthorDate: Wed Aug 28 16:03:07 2024 +0800
[INLONG-10832][SDK] Transform SQL support Translate function (#10863)
Co-authored-by: AloysZhang
---
.../process/function/TranslateFunction.java| 109 +
.../transform/process/operator/OperatorTools.java | 4 +-
.../TestTransformStringFunctionsProcessor.java | 27 +
3 files changed, 139 insertions(+), 1 deletion(-)
diff --git
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/TranslateFunction.java
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/TranslateFunction.java
new file mode 100644
index 00..ce05eb0fc3
--- /dev/null
+++
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/TranslateFunction.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sdk.transform.process.function;
+
+import org.apache.inlong.sdk.transform.decode.SourceData;
+import org.apache.inlong.sdk.transform.process.Context;
+import org.apache.inlong.sdk.transform.process.operator.OperatorTools;
+import org.apache.inlong.sdk.transform.process.parser.ValueParser;
+
+import net.sf.jsqlparser.expression.Expression;
+import net.sf.jsqlparser.expression.Function;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * TranslateFunction
+ * translate(expression, find_chars, replace_chars)
+ * Description:
+ * For a given expression, replaces all occurrences of specified characters
with specified substitutes.
+ * Existing characters are mapped to replacement characters by their positions
in the find_chars and replace_chars arguments.
+ * If more characters are specified in the find_chars argument than in the
replace_chars argument, the extra characters from the find_chars argument are
omitted in the return value.
+ *
+ * Translate function is similar to the replace function and the
regexp_replace function,
+ * except that replace substitutes one entire string with another string and
regexp_replace lets you search a string for a regular expression pattern,
+ * while translate makes multiple single-character substitutions.
+ *
+ * Arguments:
+ * expression: The expression to be translated.
+ * find_chars: A string containing the characters to be replaced.
+ * replace_chars: A string containing the characters to substitute.
+ * examples:
+ * case1: translate(email, '@', '.') -> original_expression:
ha...@inlong.com target_expression: harry.inlong.com
+ * case2: translate(hello WorD, 'WD', 'wd') -> original_expression: hello
WorD target_expression: hello word
+ */
+public class TranslateFunction implements ValueParser {
+
+private ValueParser originalStrParser;
+
+private ValueParser findCharsParser;
+
+private ValueParser replaceCharsParser;
+
+public TranslateFunction(Function expr) {
+List expressions = expr.getParameters().getExpressions();
+originalStrParser = OperatorTools.buildParser(expressions.get(0));
+findCharsParser = OperatorTools.buildParser(expressions.get(1));
+replaceCharsParser = OperatorTools.buildParser(expressions.get(2));
+}
+
+@Override
+public Object parse(SourceData sourceData, int rowIndex, Context context) {
+Object originalStrObject = originalStrParser.parse(sourceData,
rowIndex, context);
+Object findCharsObject = findCharsParser.parse(sourceData, rowIndex,
context);
+Object replaceCharsObject = replaceCharsParser.parse(sourceData,
rowIndex, context);
+String originalStr = OperatorTools.parseString(originalStrObject);
+String findC
(inlong) branch master updated: [INLONG-10930][SDK] Transform support DAYOFWEEK function (#10933)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 2409da5c66 [INLONG-10930][SDK] Transform support DAYOFWEEK function
(#10933)
2409da5c66 is described below
commit 2409da5c66897210f38cf1b7e88579501528f7bf
Author: Xincheng Huang <60057611+ying-...@users.noreply.github.com>
AuthorDate: Wed Aug 28 17:08:14 2024 +0800
[INLONG-10930][SDK] Transform support DAYOFWEEK function (#10933)
---
.../sdk/transform/process/function/DateExtractFunction.java| 6 +-
.../inlong/sdk/transform/process/operator/OperatorTools.java | 1 +
.../process/TestTransformTemporalFunctionsProcessor.java | 10 ++
3 files changed, 16 insertions(+), 1 deletion(-)
diff --git
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/DateExtractFunction.java
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/DateExtractFunction.java
index d763b657e2..896b67c31a 100644
---
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/DateExtractFunction.java
+++
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/DateExtractFunction.java
@@ -41,6 +41,7 @@ import java.util.Locale;
* - week(date)--returns the week of a year (an integer between 1 and 53) from
SQL date
* - dayofyear(date)--returns the day of a year (an integer between 1 and 366)
from SQL date
* - dayofmonth(date)--returns the day of a month (an integer between 1 and
31) from SQL date
+ * - dayofweek(date)--returns the day of a week (an integer between 1(Sunday)
and 7(Saturday)) from SQL date
*/
public class DateExtractFunction implements ValueParser {
@@ -49,7 +50,7 @@ public class DateExtractFunction implements ValueParser {
private static final TemporalField weekOfYearField =
WeekFields.of(Locale.getDefault()).weekOfWeekBasedYear();
public enum DateExtractFunctionType {
-YEAR, QUARTER, MONTH, WEEK, DAY_OF_YEAR, DAY_OF_MONTH
+YEAR, QUARTER, MONTH, WEEK, DAY_OF_YEAR, DAY_OF_MONTH, DAY_OF_WEEK
}
public DateExtractFunction(DateExtractFunctionType type, Function expr) {
@@ -82,6 +83,9 @@ public class DateExtractFunction implements ValueParser {
// dayofmonth(between 1 and 31)
case DAY_OF_MONTH:
return localDate.getDayOfMonth();
+// dayofweek(between 1 and 7)
+case DAY_OF_WEEK:
+return localDate.getDayOfWeek().getValue() % 7 + 1;
default:
return null;
}
diff --git
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
index e4bfb2cf62..eb39599b47 100644
---
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
+++
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
@@ -158,6 +158,7 @@ public class OperatorTools {
functionMap.put("week", func -> new
DateExtractFunction(DateExtractFunctionType.WEEK, func));
functionMap.put("dayofyear", func -> new
DateExtractFunction(DateExtractFunctionType.DAY_OF_YEAR, func));
functionMap.put("dayofmonth", func -> new
DateExtractFunction(DateExtractFunctionType.DAY_OF_MONTH, func));
+functionMap.put("dayofweek", func -> new
DateExtractFunction(DateExtractFunctionType.DAY_OF_WEEK, func));
functionMap.put("hour",
func -> new
TimestampExtractFunction(TimestampExtractFunction.TimestampExtractFunctionType.HOUR,
func));
functionMap.put("minute",
diff --git
a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformTemporalFunctionsProcessor.java
b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformTemporalFunctionsProcessor.java
index 6ccfc357b3..8c99b2a6bd 100644
---
a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformTemporalFunctionsProcessor.java
+++
b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformTemporalFunctionsProcessor.java
@@ -205,6 +205,16 @@ public class TestTransformTemporalFunctionsProcessor {
List output6 = processor6.transform("2024-02-29", new
HashMap<>());
Assert.assertEquals(1, output6.size());
Assert.assertEquals(output6.get(0), "result=29");
+
+String transformSql7 = "select dayofweek(string1) from source";
+Tran
(inlong) branch master updated: [INLONG-10847][Dashboard] Agent type cluster node management adds restart、reinstall, install log 、heartbeat detection and unLoad operations (#10926)
This is an automated email from the ASF dual-hosted git repository. aloyszhang pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/inlong.git The following commit(s) were added to refs/heads/master by this push: new 4cf7ccba60 [INLONG-10847][Dashboard] Agent type cluster node management adds restart、reinstall, install log 、heartbeat detection and unLoad operations (#10926) 4cf7ccba60 is described below commit 4cf7ccba604251a34a89981b5176b5988675d0d9 Author: kamianlaida <165994047+wohainilao...@users.noreply.github.com> AuthorDate: Thu Aug 29 14:21:42 2024 +0800 [INLONG-10847][Dashboard] Agent type cluster node management adds restart、reinstall, install log 、heartbeat detection and unLoad operations (#10926) --- inlong-dashboard/src/ui/locales/cn.json| 16 ++ inlong-dashboard/src/ui/locales/en.json| 17 +- .../src/ui/pages/Clusters/HeartBeatModal.tsx | 113 .../src/ui/pages/Clusters/LogModal.tsx | 76 .../src/ui/pages/Clusters/NodeEditModal.tsx| 19 +- .../src/ui/pages/Clusters/NodeManage.tsx | 191 - 6 files changed, 421 insertions(+), 11 deletions(-) diff --git a/inlong-dashboard/src/ui/locales/cn.json b/inlong-dashboard/src/ui/locales/cn.json index f6b1e5a493..b97e66643b 100644 --- a/inlong-dashboard/src/ui/locales/cn.json +++ b/inlong-dashboard/src/ui/locales/cn.json @@ -810,6 +810,21 @@ "pages.Clusters.Node.ProtocolType": "协议类型", "pages.Clusters.Node.Agent": "Agent", "pages.Clusters.Node.Agent.Version": "版本", + "pages.Cluster.Node.More": "更多", + "pages.Cluster.Node.Install": "重新安装", + "pages.Nodes.Restart": "重启", + "pages.Cluster.Node.InstallTitle": "确认重新安装吗？", + "pages.Cluster.Node.RestartTitle": "确认重启吗？", + "pages.Cluster.Node.Unload": "卸载", + "pages.Cluster.Node.UnloadTitle": "确认卸载吗？", + "pages.Cluster.Node.InstallLog": "安装日志", + "pages.Cluster.Node.InstallLog.None": "暂无日志", + "pages.Clusters.Node.Agent.HeartbeatDetection": "心跳检测", + "pages.Clusters.Node.Agent.HeartbeatInfo": "心跳信息", + "pages.Clusters.Node.Agent.HeartbeatInfo.ModifyTime": "修改时间", + "pages.Clusters.Node.Agent.HeartbeatInfo.ReportTime": "报告时间", + "pages.Clusters.Node.Agent.HeartbeatInfo.Component": "类型", + "pages.Clusters.Node.Agent.HeartbeatInfo.Instance": "Ip", "pages.Clusters.Node.AgentInstaller": "安装包", "pages.Clusters.Node.IsInstall": "安装方式", "pages.Clusters.Node.ManualInstall": "手动安装", @@ -824,6 +839,7 @@ "pages.Clusters.Node.Status.Normal": "正常", "pages.Clusters.Node.Status.Timeout": "心跳超时", "pages.Clusters.Node.LastModifier": "最后操作", + "pages.Clusters.Node.Creator": "创建人", "pages.Clusters.Node.Create": "新建节点", "pages.Clusters.Node.IpRule": "请输入正确的IP地址", "pages.Clusters.Node.PortRule": "请输入正确的端口", diff --git a/inlong-dashboard/src/ui/locales/en.json b/inlong-dashboard/src/ui/locales/en.json index 473f53d20d..c281478532 100644 --- a/inlong-dashboard/src/ui/locales/en.json +++ b/inlong-dashboard/src/ui/locales/en.json @@ -810,6 +810,21 @@ "pages.Clusters.Node.ProtocolType": "Protocol type", "pages.Clusters.Node.Agent": "Agent", "pages.Clusters.Node.Agent.Version": "Version", + "pages.Cluster.Node.More": "More", + "pages.Cluster.Node.Install": "Reinstall", + "pages.Nodes.Restart": "Restart", + "pages.Cluster.Node.InstallTitle": "Are you sure to reinstall?", + "pages.Cluster.Node.RestartTitle": "Are you sure to restart?", + "pages.Cluster.Node.Unload": "Uninstall", + "pages.Cluster.Node.UnloadTitle": "Are you sure to uninstall?", + "pages.Cluster.Node.InstallLog": "Install log", + "pages.Cluster.Node.InstallLog.None": "No logs yet", + "pages.Clusters.Node.Agent.HeartbeatDetection": "Heartbeat Detection", + "pages.Clusters.Node.Agent.HeartbeatInfo": "Heartbeat Info", + "pages.Clusters.Node.Agent.HeartbeatInfo.ModifyTime": "Modify Time", + "pages.Clusters.Node.Agent.HeartbeatInfo.ReportTime": "Report Time", + "pages.Clusters.Node.Agent.HeartbeatInfo.Component": "Type", + "pages.Clusters.Node.Agent.HeartbeatIn
(inlong) branch master updated (4cf7ccba60 -> 6ba319369e)
This is an automated email from the ASF dual-hosted git repository. aloyszhang pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/inlong.git from 4cf7ccba60 [INLONG-10847][Dashboard] Agent type cluster node management adds restart、reinstall, install log 、heartbeat detection and unLoad operations (#10926) add 6ba319369e [INLONG-10931][Manager] Data preview supports data containing escape characters (#10932) No new revisions were added by this update. Summary of changes: inlong-manager/manager-service/pom.xml | 5 + .../manager/service/datatype/CsvDataTypeOperator.java| 16 2 files changed, 17 insertions(+), 4 deletions(-)
(inlong) branch master updated: [INLONG-10947][Dashboard] Modify field description missing for English locale (#10948)
This is an automated email from the ASF dual-hosted git repository. aloyszhang pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/inlong.git The following commit(s) were added to refs/heads/master by this push: new 1b62fdd49e [INLONG-10947][Dashboard] Modify field description missing for English locale (#10948) 1b62fdd49e is described below commit 1b62fdd49edd3577732a8f8b7fec0bdbedb09b2e Author: kamianlaida <165994047+wohainilao...@users.noreply.github.com> AuthorDate: Thu Aug 29 16:29:45 2024 +0800 [INLONG-10947][Dashboard] Modify field description missing for English locale (#10948) --- inlong-dashboard/src/ui/locales/en.json | 1 + 1 file changed, 1 insertion(+) diff --git a/inlong-dashboard/src/ui/locales/en.json b/inlong-dashboard/src/ui/locales/en.json index c281478532..ebf2bbfcb1 100644 --- a/inlong-dashboard/src/ui/locales/en.json +++ b/inlong-dashboard/src/ui/locales/en.json @@ -838,6 +838,7 @@ "pages.Clusters.Node.Status": "Status", "pages.Clusters.Node.Status.Normal": "Normal", "pages.Clusters.Node.Status.Timeout": "Timeout", + "pages.Clusters.Node.LastModifier": "Last modifier", "pages.Clusters.Node.Creator": "Creator", "pages.Clusters.Node.Create": "Create", "pages.Clusters.Node.IpRule": "Please enter the IP address correctly",
(inlong-website) branch master updated: [INLONG-968][Doc] Add excalidraw raw file directories (#969)
This is an automated email from the ASF dual-hosted git repository. aloyszhang pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/inlong-website.git The following commit(s) were added to refs/heads/master by this push: new 07fdec66a2 [INLONG-968][Doc] Add excalidraw raw file directories (#969) 07fdec66a2 is described below commit 07fdec66a2cbb5db9bf4887491db142a3b58b138 Author: AloysZhang AuthorDate: Thu Aug 29 18:41:23 2024 +0800 [INLONG-968][Doc] Add excalidraw raw file directories (#969) --- image_raw/administration/.gitkeep| 0 image_raw/components/.gitkeep| 0 image_raw/data_nodes/.gitkeep| 0 image_raw/deployment/.gitkeep| 0 image_raw/design_and_concept/.gitkeep| 0 image_raw/development/.gitkeep | 0 image_raw/introduction/architecture.pptx | Bin 0 -> 815539 bytes image_raw/quick_start/.gitkeep | 0 image_raw/sdk/.gitkeep | 0 image_raw/user_guide/.gitkeep| 0 10 files changed, 0 insertions(+), 0 deletions(-) diff --git a/image_raw/administration/.gitkeep b/image_raw/administration/.gitkeep new file mode 100644 index 00..e69de29bb2 diff --git a/image_raw/components/.gitkeep b/image_raw/components/.gitkeep new file mode 100644 index 00..e69de29bb2 diff --git a/image_raw/data_nodes/.gitkeep b/image_raw/data_nodes/.gitkeep new file mode 100644 index 00..e69de29bb2 diff --git a/image_raw/deployment/.gitkeep b/image_raw/deployment/.gitkeep new file mode 100644 index 00..e69de29bb2 diff --git a/image_raw/design_and_concept/.gitkeep b/image_raw/design_and_concept/.gitkeep new file mode 100644 index 00..e69de29bb2 diff --git a/image_raw/development/.gitkeep b/image_raw/development/.gitkeep new file mode 100644 index 00..e69de29bb2 diff --git a/image_raw/introduction/architecture.pptx b/image_raw/introduction/architecture.pptx new file mode 100644 index 00..f8c6e54ef6 Binary files /dev/null and b/image_raw/introduction/architecture.pptx differ diff --git a/image_raw/quick_start/.gitkeep b/image_raw/quick_start/.gitkeep new file mode 100644 index 00..e69de29bb2 diff --git a/image_raw/sdk/.gitkeep b/image_raw/sdk/.gitkeep new file mode 100644 index 00..e69de29bb2 diff --git a/image_raw/user_guide/.gitkeep b/image_raw/user_guide/.gitkeep new file mode 100644 index 00..e69de29bb2
(inlong) branch master updated: [INLONG-10934][SDK] Transform support LocalDate function (#10951)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new f35ab6f619 [INLONG-10934][SDK] Transform support LocalDate function
(#10951)
f35ab6f619 is described below
commit f35ab6f619004fac58a06f79b98e681e778622f0
Author: Xincheng Huang <60057611+ying-...@users.noreply.github.com>
AuthorDate: Thu Aug 29 20:19:44 2024 +0800
[INLONG-10934][SDK] Transform support LocalDate function (#10951)
---
.../process/function/LocalDateFunction.java| 56 ++
.../TestTransformTemporalFunctionsProcessor.java | 42
2 files changed, 98 insertions(+)
diff --git
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/LocalDateFunction.java
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/LocalDateFunction.java
new file mode 100644
index 00..ba2d909d95
--- /dev/null
+++
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/LocalDateFunction.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sdk.transform.process.function;
+
+import org.apache.inlong.sdk.transform.decode.SourceData;
+import org.apache.inlong.sdk.transform.process.Context;
+import org.apache.inlong.sdk.transform.process.operator.OperatorTools;
+import org.apache.inlong.sdk.transform.process.parser.ValueParser;
+
+import net.sf.jsqlparser.expression.Function;
+
+import java.time.LocalDate;
+import java.time.ZoneId;
+
+/**
+ * LocalDateFunction
+ * description:
+ * localDate([string1]) returns the current date in the specified time zone.
+ * (by default: the current date in the system time zone)
+ */
+@TransformFunction(names = {"localdate", "currentdate", "current_date",
"curdate"})
+public class LocalDateFunction implements ValueParser {
+
+private ValueParser stringParser;
+
+public LocalDateFunction(Function expr) {
+if (expr.getParameters() != null) {
+stringParser =
OperatorTools.buildParser(expr.getParameters().getExpressions().get(0));
+}
+}
+
+@Override
+public Object parse(SourceData sourceData, int rowIndex, Context context) {
+if (stringParser != null) {
+String zoneString =
OperatorTools.parseString(stringParser.parse(sourceData, rowIndex, context));
+return LocalDate.now(ZoneId.of(zoneString));
+} else {
+return LocalDate.now(ZoneId.systemDefault());
+}
+}
+}
\ No newline at end of file
diff --git
a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformTemporalFunctionsProcessor.java
b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformTemporalFunctionsProcessor.java
index e0830dd72e..354b8b7f18 100644
---
a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformTemporalFunctionsProcessor.java
+++
b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformTemporalFunctionsProcessor.java
@@ -29,6 +29,7 @@ import org.junit.Before;
import org.junit.Test;
import java.time.Duration;
+import java.time.LocalDate;
import java.time.LocalTime;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
@@ -421,6 +422,47 @@ public class TestTransformTemporalFunctionsProcessor {
Assert.assertTrue(duration3.getSeconds() < 1);
}
+@Test
+public void testLocalDateFunction() throws Exception {
+DateTimeFormatter formatter =
DateTimeFormatter.ofPattern("-MM-dd");
+
+// case1: localDate() - default system time zone
+String transformSql1 = "select localdate() from source";
+TransformConfig config1 = new TransformConfig(transformSql1);
+TransformProcessor processor1 = TransformProcessor
+.create(config1,
SourceDecoderFactory.createCsvDecoder(csvSou
(inlong) branch master updated (1b62fdd49e -> a9719038f1)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
from 1b62fdd49e [INLONG-10947][Dashboard] Modify field description missing
for English locale (#10948)
add a9719038f1 [INLONG-10946][SDK] Transform SQL supports SPACE function
(#10949)
No new revisions were added by this update.
Summary of changes:
.../{Md5Function.java => SpaceFunction.java} | 35 +++
.../TestTransformStringFunctionsProcessor.java | 39 ++
2 files changed, 60 insertions(+), 14 deletions(-)
copy
inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/{Md5Function.java
=> SpaceFunction.java} (62%)
(inlong) branch master updated: [INLONG-10954][Manager] Support fields of timestamptz type (#10955)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 88f0d4e1fb [INLONG-10954][Manager] Support fields of timestamptz type
(#10955)
88f0d4e1fb is described below
commit 88f0d4e1fbe9364157ed4b6642d51f77d141b5d7
Author: fuweng11 <76141879+fuwen...@users.noreply.github.com>
AuthorDate: Fri Aug 30 17:42:41 2024 +0800
[INLONG-10954][Manager] Support fields of timestamptz type (#10955)
---
.../java/org/apache/inlong/manager/pojo/sort/util/FieldInfoUtils.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/util/FieldInfoUtils.java
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/util/FieldInfoUtils.java
index 05a541770e..0893caf853 100644
---
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/util/FieldInfoUtils.java
+++
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/util/FieldInfoUtils.java
@@ -298,7 +298,6 @@ public class FieldInfoUtils {
formatInfo = new TimeFormatInfo();
}
break;
-case TIMESTAMPTZ:
case TIMESTAMP:
case DATETIME:
if (StringUtils.isNotBlank(format)) {
@@ -307,6 +306,7 @@ public class FieldInfoUtils {
formatInfo = new TimestampFormatInfo();
}
break;
+case TIMESTAMPTZ:
case LOCAL_ZONE_TIMESTAMP:
if (StringUtils.isNotBlank(format)) {
formatInfo = new
LocalZonedTimestampFormatInfo(convertTimestampOrDataFormat(format), 2);
(inlong) branch master updated (5fd7cc0923 -> dfe9ab590b)
This is an automated email from the ASF dual-hosted git repository. aloyszhang pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/inlong.git from 5fd7cc0923 [INLONG-10725][CI] Add UT test workflow for flink 1.18 connectors. (#10726) add dfe9ab590b [INLONG-10977][Manager] Data preview supports escaping for KV data type (#10978) No new revisions were added by this update. Summary of changes: .../service/datatype/KvDataTypeOperator.java | 26 +- 1 file changed, 15 insertions(+), 11 deletions(-)
(inlong) branch master updated: [INLONG-10975][Manager] Fix the problem of when saving the group, only the existence of the groupid under the current tenant was verified (#10976)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new a3dd2a9a8a [INLONG-10975][Manager] Fix the problem of when saving the
group, only the existence of the groupid under the current tenant was verified
(#10976)
a3dd2a9a8a is described below
commit a3dd2a9a8ae2dcbbda476ffb3a1154da9b45ee1d
Author: fuweng11 <76141879+fuwen...@users.noreply.github.com>
AuthorDate: Mon Sep 2 14:15:54 2024 +0800
[INLONG-10975][Manager] Fix the problem of when saving the group, only the
existence of the groupid under the current tenant was verified (#10976)
---
.../apache/inlong/manager/service/group/InlongGroupServiceImpl.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/group/InlongGroupServiceImpl.java
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/group/InlongGroupServiceImpl.java
index 5440768191..3ee94a23ce 100644
---
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/group/InlongGroupServiceImpl.java
+++
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/group/InlongGroupServiceImpl.java
@@ -205,7 +205,7 @@ public class InlongGroupServiceImpl implements
InlongGroupService {
Preconditions.expectNotNull(request, "inlong group request cannot be
empty");
String groupId = request.getInlongGroupId();
-InlongGroupEntity entity = groupMapper.selectByGroupId(groupId);
+InlongGroupEntity entity =
groupMapper.selectByGroupIdWithoutTenant(groupId);
if (entity != null) {
LOGGER.error("groupId={} has already exists", groupId);
throw new BusinessException(ErrorCodeEnum.GROUP_DUPLICATE);
@@ -278,7 +278,7 @@ public class InlongGroupServiceImpl implements
InlongGroupService {
@Override
public Boolean exist(String groupId) {
Preconditions.expectNotNull(groupId,
ErrorCodeEnum.GROUP_ID_IS_EMPTY.getMessage());
-InlongGroupEntity entity = groupMapper.selectByGroupId(groupId);
+InlongGroupEntity entity =
groupMapper.selectByGroupIdWithoutTenant(groupId);
LOGGER.debug("success to check inlong group {}, exist? {}", groupId,
entity != null);
return entity != null;
}
(inlong) branch master updated (a3dd2a9a8a -> 7ecff9cd25)
This is an automated email from the ASF dual-hosted git repository. aloyszhang pushed a change to branch master in repository https://gitbox.apache.org/repos/asf/inlong.git from a3dd2a9a8a [INLONG-10975][Manager] Fix the problem of when saving the group, only the existence of the groupid under the current tenant was verified (#10976) add 7ecff9cd25 [INLONG-10947][Dashboard] A wrong error occurs in data preview (#10974) No new revisions were added by this update. Summary of changes: .../src/ui/pages/GroupDetail/DataStream/PreviewModal.tsx | 6 +++--- inlong-dashboard/src/ui/pages/ModuleAudit/AuditModule/config.tsx | 7 +++ 2 files changed, 6 insertions(+), 7 deletions(-)
(inlong) branch master updated: [INLONG-10966][SDK] Fix HexFunction set the Java keyword 'public' (#10967)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 5c44cde517 [INLONG-10966][SDK] Fix HexFunction set the Java keyword
'public' (#10967)
5c44cde517 is described below
commit 5c44cde51727cd3fa918358e624b751050530d01
Author: emptyOVO <118812562+empty...@users.noreply.github.com>
AuthorDate: Mon Sep 2 14:26:22 2024 +0800
[INLONG-10966][SDK] Fix HexFunction set the Java keyword 'public' (#10967)
---
.../org/apache/inlong/sdk/transform/process/function/HexFunction.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/HexFunction.java
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/HexFunction.java
index c641209162..0d33056c72 100644
---
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/HexFunction.java
+++
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/HexFunction.java
@@ -36,7 +36,7 @@ import java.util.regex.Pattern;
* - If the input argument is a string, the HEX function converts each
character in the string to its corresponding hexadecimal ASCII encoding and
returns the hexadecimal representation of the entire string.
*/
@TransformFunction(names = {"hex"})
-class HexFunction implements ValueParser {
+public class HexFunction implements ValueParser {
private static final Pattern BIG_DECIMAL_PATTERN =
Pattern.compile("^[-+]?\\d+(\\.\\d+)?([eE][-+]?\\d+)?$");
(inlong) branch master updated: [INLONG-10942][SDK] Add official function names for all Transform functions (#10943)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 4de972e932 [INLONG-10942][SDK] Add official function names for all
Transform functions (#10943)
4de972e932 is described below
commit 4de972e9324f443ab3c71f88b5c43f93c9825ee6
Author: Zkplo <87751516+zk...@users.noreply.github.com>
AuthorDate: Mon Sep 2 14:28:32 2024 +0800
[INLONG-10942][SDK] Add official function names for all Transform functions
(#10943)
---
.../inlong/sdk/transform/process/function/DateExtractFunction.java | 6 +++---
.../inlong/sdk/transform/process/function/FromUnixTimeFunction.java | 2 +-
.../inlong/sdk/transform/process/function/SubstringFunction.java| 4 +++-
.../inlong/sdk/transform/process/function/TimestampAddFunction.java | 2 +-
4 files changed, 8 insertions(+), 6 deletions(-)
diff --git
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/DateExtractFunction.java
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/DateExtractFunction.java
index cac2273619..a1ab7e2dfa 100644
---
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/DateExtractFunction.java
+++
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/DateExtractFunction.java
@@ -86,7 +86,7 @@ public abstract class DateExtractFunction implements
ValueParser {
}
}
-@TransformFunction(names = {"day_of_year"})
+@TransformFunction(names = {"day_of_year", "dayofyear"})
public static class DayOfYearExtractFunction extends DateExtractFunction {
public DayOfYearExtractFunction(Function expr) {
@@ -94,7 +94,7 @@ public abstract class DateExtractFunction implements
ValueParser {
}
}
-@TransformFunction(names = {"day_of_month"})
+@TransformFunction(names = {"day_of_month", "dayofmonth"})
public static class DayOfMonthExtractFunction extends DateExtractFunction {
public DayOfMonthExtractFunction(Function expr) {
@@ -102,7 +102,7 @@ public abstract class DateExtractFunction implements
ValueParser {
}
}
-@TransformFunction(names = {"day_of_week"})
+@TransformFunction(names = {"day_of_week", "dayofweek"})
public static class DayOfWeekExtractFunction extends DateExtractFunction {
public DayOfWeekExtractFunction(Function expr) {
diff --git
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FromUnixTimeFunction.java
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FromUnixTimeFunction.java
index b7b68a0e73..6a897ee3e0 100644
---
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FromUnixTimeFunction.java
+++
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FromUnixTimeFunction.java
@@ -41,7 +41,7 @@ import java.util.concurrent.ConcurrentHashMap;
* format(default is ‘-MM-dd HH:mm:ss’). numeric is an internal timestamp
value representing seconds
* since ‘1970-01-01 00:00:00’ UTC, such as produced by the UNIX_TIMESTAMP()
function.
*/
-@TransformFunction(names = {"from_unix_time"})
+@TransformFunction(names = {"from_unix_time", "form_unixtime"})
public class FromUnixTimeFunction implements ValueParser {
private ValueParser numericParser;
diff --git
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/SubstringFunction.java
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/SubstringFunction.java
index 772dc2950c..727c0484b4 100644
---
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/SubstringFunction.java
+++
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/SubstringFunction.java
@@ -32,7 +32,7 @@ import java.util.List;
* description: substring(string FROM INT1 [ FOR INT2 ])--returns a substring
of STRING starting from position INT1 with
* length INT2 (to the end by default)
*/
-@TransformFunction(names = {"substring"})
+@TransformFunction(names = {"substring", "substr"})
public class SubstringFunction implements ValueParser {
private ValueParser stringParser;
@@ -41,6 +41,7 @@ public class SubstringFunction implements ValueParser {
/**
* Constructor
+ *
* @param expr
*/
public SubstringFunction(Function expr) {
@@ -55,6 +56,7 @@ public class SubstringFunction implements ValueParser {
/**
* parse
+ *
* @param sourceData
* @param
(inlong) branch master updated: [INLONG-10988][Manager] Data preview filters data in tubes based on streamId (#10989)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new c4bf54bbb1 [INLONG-10988][Manager] Data preview filters data in tubes
based on streamId (#10989)
c4bf54bbb1 is described below
commit c4bf54bbb1bb0d83429c1aa9b93ef721bf376e90
Author: fuweng11 <76141879+fuwen...@users.noreply.github.com>
AuthorDate: Mon Sep 2 20:24:25 2024 +0800
[INLONG-10988][Manager] Data preview filters data in tubes based on
streamId (#10989)
---
.../inlong/manager/service/resource/queue/tubemq/TubeMQOperator.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/queue/tubemq/TubeMQOperator.java
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/queue/tubemq/TubeMQOperator.java
index 050de07805..66d1fcf59a 100644
---
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/queue/tubemq/TubeMQOperator.java
+++
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/queue/tubemq/TubeMQOperator.java
@@ -71,6 +71,7 @@ public class TubeMQOperator {
private static final String CREATE_USER = "&createUser=";
private static final String CONF_MOD_AUTH_TOKEN = "&confModAuthToken=";
private static final String MSG_COUNT = "&msgCount=";
+private static final String FILTER_CONDS = "&filterConds=";
private static final String QUERY_TOPIC_PATH =
"/webapi.htm?method=admin_query_cluster_topic_view";
private static final String QUERY_BROKER_PATH =
"/webapi.htm?method=admin_query_broker_run_status";
@@ -288,7 +289,7 @@ public class TubeMQOperator {
}
String url = "http://"; + brokerUrl + QUERY_MESSAGE_PATH +
TOPIC_NAME + topicName + MSG_COUNT
-+ request.getMessageCount();
++ request.getMessageCount() + FILTER_CONDS +
streamInfo.getInlongStreamId();
TubeMessageResponse response = HttpUtils.request(restTemplate,
url, HttpMethod.GET,
null, new HttpHeaders(), TubeMessageResponse.class);
if (response.getErrCode() != SUCCESS_CODE && response.getErrCode()
!= 200) {
(inlong) branch master updated: [INLONG-10968][SDK] Support Inlong Transform operator annotation (#10969)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new f1285ed4b1 [INLONG-10968][SDK] Support Inlong Transform operator
annotation (#10969)
f1285ed4b1 is described below
commit f1285ed4b1c9ec1ff16942c6324f6fad46c6898c
Author: emptyOVO <118812562+empty...@users.noreply.github.com>
AuthorDate: Tue Sep 3 10:33:10 2024 +0800
[INLONG-10968][SDK] Support Inlong Transform operator annotation (#10969)
---
.../transform/process/operator/AndOperator.java| 1 +
.../process/operator/EqualsToOperator.java | 1 +
.../operator/GreaterThanEqualsOperator.java| 1 +
.../process/operator/GreaterThanOperator.java | 1 +
.../process/operator/MinorThanEqualsOperator.java | 1 +
.../process/operator/MinorThanOperator.java| 1 +
.../process/operator/NotEqualsToOperator.java | 1 +
.../transform/process/operator/NotOperator.java| 1 +
.../transform/process/operator/OperatorTools.java | 89 ++
.../sdk/transform/process/operator/OrOperator.java | 1 +
.../process/operator/ParenthesisOperator.java | 1 +
.../{NotOperator.java => TransformOperator.java} | 33 ++--
12 files changed, 77 insertions(+), 55 deletions(-)
diff --git
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/AndOperator.java
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/AndOperator.java
index a9dcd42606..f438d4295c 100644
---
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/AndOperator.java
+++
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/AndOperator.java
@@ -26,6 +26,7 @@ import
net.sf.jsqlparser.expression.operators.conditional.AndExpression;
* AndOperator
*
*/
+@TransformOperator(values = AndExpression.class)
public class AndOperator implements ExpressionOperator {
private final ExpressionOperator left;
diff --git
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/EqualsToOperator.java
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/EqualsToOperator.java
index 709537e8a0..13010c854b 100644
---
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/EqualsToOperator.java
+++
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/EqualsToOperator.java
@@ -27,6 +27,7 @@ import
net.sf.jsqlparser.expression.operators.relational.EqualsTo;
* EqualsToOperator
*
*/
+@TransformOperator(values = EqualsTo.class)
public class EqualsToOperator implements ExpressionOperator {
private final ValueParser left;
diff --git
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/GreaterThanEqualsOperator.java
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/GreaterThanEqualsOperator.java
index 3a53968e10..e703afdbda 100644
---
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/GreaterThanEqualsOperator.java
+++
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/GreaterThanEqualsOperator.java
@@ -27,6 +27,7 @@ import
net.sf.jsqlparser.expression.operators.relational.GreaterThanEquals;
* GreaterThanEqualsOperator
*
*/
+@TransformOperator(values = GreaterThanEquals.class)
public class GreaterThanEqualsOperator implements ExpressionOperator {
private final ValueParser left;
diff --git
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/GreaterThanOperator.java
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/GreaterThanOperator.java
index a1cd8c2ea2..ba73ef4c4d 100644
---
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/GreaterThanOperator.java
+++
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/GreaterThanOperator.java
@@ -27,6 +27,7 @@ import
net.sf.jsqlparser.expression.operators.relational.GreaterThan;
* GreaterThanOperator
*
*/
+@TransformOperator(values = GreaterThan.class)
public class GreaterThanOperator implements ExpressionOperator {
private final ValueParser left;
diff --git
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/MinorThanEqualsOperator.java
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/MinorThanEqualsOperator.java
index 4248cf1d36..e8104e2cc3 100644
---
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/MinorThanEqualsOperator.java
+++
(inlong) branch master updated: [INLONG-10997][Manager] Incorrect setting of transformSQL in dataflowconfig (#10998)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new e188410668 [INLONG-10997][Manager] Incorrect setting of transformSQL
in dataflowconfig (#10998)
e188410668 is described below
commit e1884106681a4062555d413f6f2cdc24ef741229
Author: fuweng11 <76141879+fuwen...@users.noreply.github.com>
AuthorDate: Tue Sep 3 14:07:56 2024 +0800
[INLONG-10997][Manager] Incorrect setting of transformSQL in dataflowconfig
(#10998)
---
.../inlong/manager/service/resource/sort/DefaultSortConfigOperator.java | 1 +
1 file changed, 1 insertion(+)
diff --git
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/DefaultSortConfigOperator.java
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/DefaultSortConfigOperator.java
index 116a8f0ea2..17ca4b9bad 100644
---
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/DefaultSortConfigOperator.java
+++
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/resource/sort/DefaultSortConfigOperator.java
@@ -175,6 +175,7 @@ public class DefaultSortConfigOperator implements
SortConfigOperator {
.dataflowId(String.valueOf(sink.getId()))
.sourceConfig(getSourceConfig(groupInfo, streamInfo, sink))
.auditTag(String.valueOf(sink.getId()))
+.transformSql(sink.getTransformSql())
.sinkConfig(getSinkConfig(groupInfo, streamInfo, sink))
.inlongGroupId(groupInfo.getInlongGroupId())
.inlongStreamId(streamInfo.getInlongStreamId())
(inlong) branch master updated: [INLONG-10938][SDK] Transform SQL supports fuzzy matching of LIKE and NOT LIKE (#11009)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 6104e21984 [INLONG-10938][SDK] Transform SQL supports fuzzy matching
of LIKE and NOT LIKE (#11009)
6104e21984 is described below
commit 6104e219843df3e733bfcde95771bb1188446599
Author: Zkplo <87751516+zk...@users.noreply.github.com>
AuthorDate: Tue Sep 10 10:30:59 2024 +0800
[INLONG-10938][SDK] Transform SQL supports fuzzy matching of LIKE and NOT
LIKE (#11009)
---
.../org/apache/inlong/common/util/StringUtil.java | 4 +
.../transform/process/function/FunctionTools.java | 3 +
.../sdk/transform/process/parser/LikeParser.java | 110 ++
.../sdk/transform/process/parser/ParserTools.java | 3 +
.../transform/process/TestLikeParserProcessor.java | 169 +
5 files changed, 289 insertions(+)
diff --git
a/inlong-common/src/main/java/org/apache/inlong/common/util/StringUtil.java
b/inlong-common/src/main/java/org/apache/inlong/common/util/StringUtil.java
index 7df1eb6975..c10a921acb 100644
--- a/inlong-common/src/main/java/org/apache/inlong/common/util/StringUtil.java
+++ b/inlong-common/src/main/java/org/apache/inlong/common/util/StringUtil.java
@@ -283,4 +283,8 @@ public class StringUtil {
}
return charStr;
}
+
+public static boolean isEmpty(Object str) {
+return str == null || str.toString().isEmpty();
+}
}
diff --git
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FunctionTools.java
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FunctionTools.java
index b351aee06b..1c6bd1501a 100644
---
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FunctionTools.java
+++
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FunctionTools.java
@@ -70,6 +70,9 @@ public class FunctionTools {
}
public static ValueParser getTransformFunction(Function func) {
+if (func == null) {
+return null;
+}
String functionName = func.getName();
Class clazz = functionMap.get(functionName);
if (clazz == null) {
diff --git
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/LikeParser.java
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/LikeParser.java
new file mode 100644
index 00..ee9c828d8e
--- /dev/null
+++
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/LikeParser.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sdk.transform.process.parser;
+
+import org.apache.inlong.common.util.StringUtil;
+import org.apache.inlong.sdk.transform.decode.SourceData;
+import org.apache.inlong.sdk.transform.process.Context;
+import org.apache.inlong.sdk.transform.process.operator.OperatorTools;
+
+import lombok.extern.slf4j.Slf4j;
+import net.sf.jsqlparser.expression.operators.relational.LikeExpression;
+
+import java.util.regex.Pattern;
+
+/**
+ * LikeParser
+ */
+@Slf4j
+@TransformParser(values = LikeExpression.class)
+public class LikeParser implements ValueParser {
+
+private final ValueParser destParser;
+private final ValueParser patternParser;
+private final ValueParser escapeParser;
+private final boolean isNot;
+private static final String REGEX_SPECIAL_CHAR = "[]()|^-+*?{}$\\.";
+
+public LikeParser(LikeExpression expr) {
+destParser = OperatorTools.buildParser(expr.getLeftExpression());
+patternParser = OperatorTools.buildParser(expr.getRightExpression());
+escapeParser = OperatorTools.buildParser(expr.getEscape());
+isNot = expr.isNot();
+}
+
+@Override
+public Object parse(SourceData sourceData, int rowIndex, Context context) {
+Object destObj = destParser.parse(sourceData, rowIndex, context);
+Object patternObj = patternParser
(inlong) branch master updated (6104e21984 -> 8588d6ceaa)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
from 6104e21984 [INLONG-10938][SDK] Transform SQL supports fuzzy matching
of LIKE and NOT LIKE (#11009)
add 8588d6ceaa [INLONG-11036][SDK] Transform update ReplicateFunction
(#11039)
No new revisions were added by this update.
Summary of changes:
...{ReplicateFunction.java => RepeatFunction.java} | 17 ++---
...licateFunction.java => TestRepeatFunction.java} | 40 +++---
2 files changed, 40 insertions(+), 17 deletions(-)
rename
inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/{ReplicateFunction.java
=> RepeatFunction.java} (83%)
rename
inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/function/string/{TestReplicateFunction.java
=> TestRepeatFunction.java} (66%)
(inlong) branch master updated: [INLONG-10900][SDK] Transform SQL support soundex function (#11061)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 9ec6785c63 [INLONG-10900][SDK] Transform SQL support soundex function
(#11061)
9ec6785c63 is described below
commit 9ec6785c631ef7d6160aa7ee8f54e603ae279361
Author: Huan Liang
AuthorDate: Tue Sep 10 10:33:17 2024 +0800
[INLONG-10900][SDK] Transform SQL support soundex function (#11061)
---
.../process/function/SoundexFunction.java | 79 ++
.../function/string/TestSoundexFunction.java | 49 ++
2 files changed, 128 insertions(+)
diff --git
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/SoundexFunction.java
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/SoundexFunction.java
new file mode 100644
index 00..2a46c2a8a6
--- /dev/null
+++
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/SoundexFunction.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sdk.transform.process.function;
+
+import org.apache.inlong.sdk.transform.decode.SourceData;
+import org.apache.inlong.sdk.transform.process.Context;
+import org.apache.inlong.sdk.transform.process.operator.OperatorTools;
+import org.apache.inlong.sdk.transform.process.parser.ValueParser;
+
+import net.sf.jsqlparser.expression.Function;
+
+import java.nio.charset.StandardCharsets;
+
+/**
+ * SoundexFunction
+ * description: soundex(string)---Returns a four character code representing
the sound of a string. This method returns a string, or null if parameter is
null.
+ * See https://en.wikipedia.org/wiki/Soundex for more information.
+ */
+@TransformFunction(names = {"soundex"})
+public class SoundexFunction implements ValueParser {
+
+private ValueParser stringParser;
+
+private static final byte[] SOUNDEX_INDEX =
+
"71237128722455712623718272\000\000\000\000\000\00071237128722455712623718272"
+.getBytes(StandardCharsets.ISO_8859_1);
+
+public SoundexFunction(Function expr) {
+stringParser =
OperatorTools.buildParser(expr.getParameters().getExpressions().get(0));
+}
+@Override
+public Object parse(SourceData sourceData, int rowIndex, Context context) {
+Object stringObject = stringParser.parse(sourceData, rowIndex,
context);
+String str = OperatorTools.parseString(stringObject);
+if (str == null) {
+return null;
+}
+return new String(getSoundex(str), StandardCharsets.ISO_8859_1);
+}
+
+private static byte[] getSoundex(String str) {
+byte[] chars = {'0', '0', '0', '0'};
+byte lastDigit = '0';
+for (int i = 0, j = 0, l = str.length(); i < l && j < 4; i++) {
+char c = str.charAt(i);
+if (c >= 'A' && c <= 'z') {
+byte newDigit = SOUNDEX_INDEX[c - 'A'];
+if (newDigit != 0) {
+if (j == 0) {
+chars[j++] = (byte) (c & 0xdf); // Converts a-z to A-Z
+lastDigit = newDigit;
+} else if (newDigit <= '6') {
+if (newDigit != lastDigit) {
+chars[j++] = lastDigit = newDigit;
+}
+} else if (newDigit == '7') {
+lastDigit = newDigit;
+}
+}
+}
+}
+return chars;
+}
+}
diff --git
a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/function/string/TestSoundexFunction.java
b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/function/string/TestSoundexFunction.java
new file mode 100644
index 00..8b673ab06f
--- /dev/null
+++
b/inlong-sdk/t
(inlong) branch master updated: [INLONG-11056][SDK] Transform support RAND_INTEGER() function (#11063)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 5b0eb6df9c [INLONG-11056][SDK] Transform support RAND_INTEGER()
function (#11063)
5b0eb6df9c is described below
commit 5b0eb6df9ca74e68c63499206baeb40d2810bf68
Author: emptyOVO <118812562+empty...@users.noreply.github.com>
AuthorDate: Tue Sep 10 20:12:55 2024 +0800
[INLONG-11056][SDK] Transform support RAND_INTEGER() function (#11063)
---
.../process/function/RandIntegerFunction.java | 76 ++
.../arithmetic/TestRandIntegerFunction.java| 61 +
2 files changed, 137 insertions(+)
diff --git
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/RandIntegerFunction.java
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/RandIntegerFunction.java
new file mode 100644
index 00..446d4aca58
--- /dev/null
+++
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/RandIntegerFunction.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sdk.transform.process.function;
+
+import org.apache.inlong.sdk.transform.decode.SourceData;
+import org.apache.inlong.sdk.transform.process.Context;
+import org.apache.inlong.sdk.transform.process.operator.OperatorTools;
+import org.apache.inlong.sdk.transform.process.parser.ValueParser;
+
+import net.sf.jsqlparser.expression.Expression;
+import net.sf.jsqlparser.expression.Function;
+
+import java.util.List;
+import java.util.Random;
+
+/**
+ * RandIntegerFunction
+ * description: RAND_INTEGER(INT1)--Returns a pseudorandom integer value in
the range [0, INT)
+ * RAND_INTEGER(INT1, INT2)--Returns a pseudorandom integer value
in the range [0, INT1) with an initial seed INT2.
+ * Two RAND_INTEGER functions will return idential sequences of
numbers if they have the same initial seed and bound.
+ */
+@TransformFunction(names = {"rand_integer"})
+public class RandIntegerFunction implements ValueParser {
+
+private ValueParser firstIntParser;
+
+private ValueParser secondIntParser;
+
+private Random random;
+
+public RandIntegerFunction(Function expr) {
+random = new Random();
+List expressions = expr.getParameters().getExpressions();
+if (expressions != null) {
+firstIntParser = OperatorTools.buildParser(expressions.get(0));
+if (expressions.size() >= 2) {
+secondIntParser =
OperatorTools.buildParser(expressions.get(1));
+}
+}
+}
+@Override
+public Object parse(SourceData sourceData, int rowIndex, Context context) {
+Object firstIntObj = firstIntParser.parse(sourceData, rowIndex,
context);
+int firstInt = OperatorTools.parseBigDecimal(firstIntObj).intValue();
+if (secondIntParser != null) {
+Object secondIntObj = secondIntParser.parse(sourceData, rowIndex,
context);
+int secondInt =
OperatorTools.parseBigDecimal(secondIntObj).intValue();
+return randInteger(firstInt, secondInt);
+}
+return randInteger(firstInt);
+}
+
+private int randInteger(int range) {
+return random.nextInt(range);
+}
+
+private int randInteger(int range, int seed) {
+random.setSeed(seed);
+return random.nextInt(range);
+}
+}
diff --git
a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/function/arithmetic/TestRandIntegerFunction.java
b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/function/arithmetic/TestRandIntegerFunction.java
new file mode 100644
index 00..f388e962d3
--- /dev/null
+++
b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/function/arithmetic/TestRandIntegerFunction.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor
(inlong) branch master updated: [INLONG-11089][Manager] Optimize Sort filter function (#11090)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 5056e49b41 [INLONG-11089][Manager] Optimize Sort filter function
(#11090)
5056e49b41 is described below
commit 5056e49b416fcd5e19522069d30022bd1e9686f8
Author: vernedeng
AuthorDate: Fri Sep 13 10:01:16 2024 +0800
[INLONG-11089][Manager] Optimize Sort filter function (#11090)
---
.../apache/inlong/manager/pojo/sort/util/FilterFunctionUtils.java | 6 +++---
.../apache/inlong/sort/protocol/transformation/FilterFunction.java | 7 ++-
.../sort/protocol/transformation/function/BetweenFunction.java | 6 ++
.../protocol/transformation/function/MultiValueFilterFunction.java | 5 ++---
.../transformation/function/SingleValueFilterFunction.java | 4 +---
5 files changed, 14 insertions(+), 14 deletions(-)
diff --git
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/util/FilterFunctionUtils.java
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/util/FilterFunctionUtils.java
index fbc93f4e91..1b3eadc7f2 100644
---
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/util/FilterFunctionUtils.java
+++
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/util/FilterFunctionUtils.java
@@ -98,11 +98,11 @@ public class FilterFunctionUtils {
.map(filterRule -> createFilterFunction(filterRule,
transformName)).collect(Collectors.toList());
// Move logicOperator to preFunction
for (int index = filterFunctions.size() - 1; index > 0; index--) {
-SingleValueFilterFunction function = (SingleValueFilterFunction)
filterFunctions.get(index);
-SingleValueFilterFunction preFunction =
(SingleValueFilterFunction) filterFunctions.get(index - 1);
+FilterFunction function = filterFunctions.get(index);
+FilterFunction preFunction = filterFunctions.get(index - 1);
function.setLogicOperator(preFunction.getLogicOperator());
}
-((SingleValueFilterFunction)
filterFunctions.get(0)).setLogicOperator(EmptyOperator.getInstance());
+(filterFunctions.get(0)).setLogicOperator(EmptyOperator.getInstance());
return filterFunctions;
}
diff --git
a/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/transformation/FilterFunction.java
b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/transformation/FilterFunction.java
index bc07a89d80..77efea72b9 100644
---
a/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/transformation/FilterFunction.java
+++
b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/transformation/FilterFunction.java
@@ -21,6 +21,8 @@ import
org.apache.inlong.sort.protocol.transformation.function.BetweenFunction;
import
org.apache.inlong.sort.protocol.transformation.function.MultiValueFilterFunction;
import
org.apache.inlong.sort.protocol.transformation.function.SingleValueFilterFunction;
+import lombok.Data;
+import
org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonProperty;
import
org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonSubTypes;
import
org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonTypeInfo;
@@ -33,6 +35,9 @@ import
org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonTyp
@JsonSubTypes.Type(value = MultiValueFilterFunction.class, name =
"multiValueFilter"),
@JsonSubTypes.Type(value = BetweenFunction.class, name =
"betweenFunction")
})
-public interface FilterFunction extends Function {
+@Data
+public abstract class FilterFunction implements Function {
+@JsonProperty("logicOperator")
+protected LogicOperator logicOperator;
}
diff --git
a/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/transformation/function/BetweenFunction.java
b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/transformation/function/BetweenFunction.java
index 3a4d639074..28d0a0c371 100644
---
a/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/transformation/function/BetweenFunction.java
+++
b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/transformation/function/BetweenFunction.java
@@ -30,6 +30,7 @@ import
org.apache.flink.shaded.jackson2.com.fasterxml.jackson.annotation.JsonTyp
import javax.annotation.Nonnull;
+import java.io.Serializable;
import java.util.Arrays;
import java.util.List;
@@ -38,7 +39,7 @@ import java.util.List;
*/
@JsonTypeName("betweenFunction")
@Data
-public class BetweenFunction implements FilterFunction {
+public class BetweenFunction
(inlong) branch master updated: [INLONG-11092][SDK] Fix the issue of inconsistent capitalization of Function names in FunctionalTools (#11093)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 0aae16f45d [INLONG-11092][SDK] Fix the issue of inconsistent
capitalization of Function names in FunctionalTools (#11093)
0aae16f45d is described below
commit 0aae16f45dd7cf314c8af34fdce4474591490afe
Author: Zkplo <87751516+zk...@users.noreply.github.com>
AuthorDate: Sat Sep 14 10:23:27 2024 +0800
[INLONG-11092][SDK] Fix the issue of inconsistent capitalization of
Function names in FunctionalTools (#11093)
---
.../org/apache/inlong/sdk/transform/process/function/FunctionTools.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FunctionTools.java
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FunctionTools.java
index 1c6bd1501a..8da0bf622e 100644
---
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FunctionTools.java
+++
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/FunctionTools.java
@@ -73,7 +73,7 @@ public class FunctionTools {
if (func == null) {
return null;
}
-String functionName = func.getName();
+String functionName = func.getName().toLowerCase();
Class clazz = functionMap.get(functionName);
if (clazz == null) {
return new ColumnParser(func);
(inlong) branch master updated (0aae16f45d -> af9acab33e)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
from 0aae16f45d [INLONG-11092][SDK] Fix the issue of inconsistent
capitalization of Function names in FunctionalTools (#11093)
add af9acab33e [INLONG-11018][SDK] Transform SQL supports COMPRESS and
UNCOMPRESS Functions (#11048)
No new revisions were added by this update.
Summary of changes:
.../process/function/CompressFunction.java | 117 +
.../process/function/UnCompressFunction.java | 100 ++
.../factory/CompressionAlgorithmFactory.java} | 34 +++---
.../factory/UnCompressionAlgorithmFactory.java}| 34 +++---
.../process/function/handler/CompressHandler.java | 10 +-
.../process/function/handler/DeflaterCompress.java | 39 +++
.../function/handler/DeflaterUncompress.java | 37 +++
.../process/function/handler/GzipCompress.java | 24 ++---
.../process/function/handler/GzipUncompress.java | 38 +++
.../function/handler/UncompressHandler.java| 7 +-
.../process/function/handler/ZipCompress.java | 38 +++
.../process/function/handler/ZipUncompress.java| 43
...ngthFunction.java => TestCompressFunction.java} | 42 +---
.../function/string/TestUnCompressFunction.java| 110 +++
14 files changed, 500 insertions(+), 173 deletions(-)
create mode 100644
inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/CompressFunction.java
create mode 100644
inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/UnCompressFunction.java
copy
inlong-sdk/{dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/utils/ConcurrentHashSet.java
=>
transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/factory/CompressionAlgorithmFactory.java}
(51%)
copy
inlong-sdk/{dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/utils/ConcurrentHashSet.java
=>
transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/factory/UnCompressionAlgorithmFactory.java}
(50%)
copy
inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/AuthenticationType.java
=>
inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/handler/CompressHandler.java
(84%)
copy
inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/ByteTypeInfo.java
=>
inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/handler/DeflaterCompress.java
(54%)
copy
inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/ByteTypeInfo.java
=>
inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/handler/DeflaterUncompress.java
(55%)
copy
inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/FlinkSortPollerPlugin.java
=>
inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/handler/GzipCompress.java
(62%)
copy
inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/IntTypeInfo.java
=>
inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/handler/GzipUncompress.java
(53%)
copy
inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/task/MockInstanceManager.java
=>
inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/handler/UncompressHandler.java
(83%)
copy
inlong-common/src/main/java/org/apache/inlong/common/pojo/sort/dataflow/field/format/IntTypeInfo.java
=>
inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/handler/ZipCompress.java
(56%)
create mode 100644
inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/handler/ZipUncompress.java
copy
inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/function/string/{TestBitLengthFunction.java
=> TestCompressFunction.java} (62%)
create mode 100644
inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/function/string/TestUnCompressFunction.java
(inlong) branch master updated: [INLONG-11030][SDK] Add AVRO formatted data source for Transform (#11082)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 9df3d56a03 [INLONG-11030][SDK] Add AVRO formatted data source for
Transform (#11082)
9df3d56a03 is described below
commit 9df3d56a0387f07375f805793e7e08ee38a681f4
Author: Xincheng Huang <60057611+ying-...@users.noreply.github.com>
AuthorDate: Sat Sep 14 10:32:43 2024 +0800
[INLONG-11030][SDK] Add AVRO formatted data source for Transform (#11082)
---
.../inlong/sdk/transform/decode/AvroNode.java | 58 +++
.../sdk/transform/decode/AvroSourceData.java | 170 +
.../sdk/transform/decode/AvroSourceDecoder.java| 139 +
.../sdk/transform/decode/SourceDecoderFactory.java | 5 +
.../inlong/sdk/transform/pojo/AvroSourceInfo.java | 56 +++
.../processor/AbstractProcessorTestBase.java | 13 ++
.../process/processor/TestAvro2CsvProcessor.java | 52 +++
7 files changed, 493 insertions(+)
diff --git
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/AvroNode.java
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/AvroNode.java
new file mode 100644
index 00..856d164fb7
--- /dev/null
+++
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/AvroNode.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sdk.transform.decode;
+
+import lombok.Data;
+import org.apache.commons.lang.math.NumberUtils;
+import org.apache.commons.lang3.StringUtils;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * AvroNode
+ */
+@Data
+public class AvroNode {
+
+private String name;
+private boolean isArray = false;
+private List arrayIndices = new ArrayList<>();
+
+public AvroNode(String nodeString) {
+int beginIndex = nodeString.indexOf('(');
+if (beginIndex < 0) {
+this.name = nodeString;
+} else {
+this.name = StringUtils.trim(nodeString.substring(0, beginIndex));
+int endIndex = nodeString.lastIndexOf(')');
+if (endIndex >= 0) {
+this.isArray = true;
+String indicesString = nodeString.substring(beginIndex + 1,
endIndex);
+String[] indices = indicesString.split(",");
+for (String index : indices) {
+int arrayIndex =
NumberUtils.toInt(StringUtils.trim(index), -1);
+if (arrayIndex < 0) {
+arrayIndex = 0;
+}
+this.arrayIndices.add(arrayIndex);
+}
+}
+}
+}
+}
\ No newline at end of file
diff --git
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/AvroSourceData.java
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/AvroSourceData.java
new file mode 100644
index 00..c060c89af4
--- /dev/null
+++
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/AvroSourceData.java
@@ -0,0 +1,170 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sdk.trans
(inlong) 03/03: [INLONG-9822][Manager] Support flink job runtime execution mode configuration (#9823)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch dev-offline-sync
in repository https://gitbox.apache.org/repos/asf/inlong.git
commit 1323736a0b54ad402d12104dcfba61398ff6cbda
Author: AloysZhang
AuthorDate: Fri Mar 15 15:12:53 2024 +0800
[INLONG-9822][Manager] Support flink job runtime execution mode
configuration (#9823)
---
.../inlong/manager/common/consts/InlongConstants.java | 3 +++
.../apache/inlong/manager/plugin/flink/FlinkService.java | 2 ++
.../apache/inlong/manager/plugin/flink/dto/FlinkInfo.java | 2 ++
.../inlong/manager/plugin/listener/DeleteSortListener.java | 1 +
.../manager/plugin/listener/RestartSortListener.java | 8 +++-
.../manager/plugin/listener/StartupSortListener.java | 14 ++
.../org/apache/inlong/sort/configuration/Constants.java| 8
.../src/main/java/org/apache/inlong/sort/Entrance.java | 11 ++-
8 files changed, 43 insertions(+), 6 deletions(-)
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
index 581ebb3098..e84dcc6602 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
@@ -90,6 +90,9 @@ public class InlongConstants {
public static final Integer DATASYNC_REALTIME_MODE = 1;
public static final Integer DATASYNC_OFFLINE_MODE = 2;
+public static final String RUNTIME_EXECUTION_MODE_STREAMING = "streaming";
+public static final String RUNTIME_EXECUTION_MODE_BATCH = "batch";
+
public static final Integer DISABLE_ZK = 0;
public static final Integer ENABLE_ZK = 1;
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkService.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkService.java
index 69a32b73fa..33a7ca59cd 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkService.java
+++
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkService.java
@@ -258,6 +258,8 @@ public class FlinkService {
list.add(flinkInfo.getLocalConfPath());
list.add("-checkpoint.interval");
list.add("6");
+list.add("-runtime.execution.mode");
+list.add(flinkInfo.getRuntimeExecutionMode());
list.add("-metrics.audit.proxy.hosts");
list.add(flinkConfig.getAuditProxyHosts());
return list.toArray(new String[0]);
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/dto/FlinkInfo.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/dto/FlinkInfo.java
index f7071ceb4b..4c3c75f855 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/dto/FlinkInfo.java
+++
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/dto/FlinkInfo.java
@@ -52,4 +52,6 @@ public class FlinkInfo {
private boolean isException = false;
private String exceptionMsg;
+
+private String runtimeExecutionMode;
}
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/DeleteSortListener.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/DeleteSortListener.java
index 009374e2a6..5e30ad8cb5 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/DeleteSortListener.java
+++
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/DeleteSortListener.java
@@ -119,6 +119,7 @@ public class DeleteSortListener implements
SortOperateListener {
FlinkOperation flinkOperation = FlinkOperation.getInstance();
try {
flinkOperation.delete(flinkInfo);
+// TODO if the job is OFFLINE, should delete the scheduler
information
log.info("job delete success for jobId={}", jobId);
} catch (Exception e) {
flinkInfo.setException(true);
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/RestartSortListener.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/RestartSortListener.java
index 0c6828c984..242138c1e4 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/RestartSortList
(inlong) 02/03: [INLONG-9813][Manager] Support offline data sync management (#9814)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch dev-offline-sync
in repository https://gitbox.apache.org/repos/asf/inlong.git
commit 5cdd94d4c9d927342b45570c86523948f115d3b2
Author: AloysZhang
AuthorDate: Wed Mar 13 16:18:04 2024 +0800
[INLONG-9813][Manager] Support offline data sync management (#9814)
---
.../inlong/manager/plugin/listener/StartupSortListener.java | 13 +
.../inlong/manager/service/core/impl/AuditServiceImpl.java | 6 --
.../service/listener/group/InitGroupCompleteListener.java | 3 ++-
.../service/listener/group/UpdateGroupCompleteListener.java | 3 ++-
.../listener/group/apply/ApproveApplyProcessListener.java | 1 -
.../service/listener/queue/QueueResourceListener.java | 6 --
.../service/listener/stream/InitStreamCompleteListener.java | 3 ++-
.../manager/service/source/AbstractSourceOperator.java | 1 +
.../manager/service/source/StreamSourceServiceImpl.java | 3 ++-
9 files changed, 26 insertions(+), 13 deletions(-)
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
index ab9e9b55d8..fae5faa278 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
+++
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
@@ -73,7 +73,8 @@ public class StartupSortListener implements
SortOperateListener {
}
log.info("add startup group listener for groupId [{}]", groupId);
-return
InlongConstants.DATASYNC_REALTIME_MODE.equals(groupProcessForm.getGroupInfo().getInlongGroupMode());
+return
(InlongConstants.DATASYNC_REALTIME_MODE.equals(groupProcessForm.getGroupInfo().getInlongGroupMode())
+||
InlongConstants.DATASYNC_OFFLINE_MODE.equals(groupProcessForm.getGroupInfo().getInlongGroupMode()));
}
@Override
@@ -141,9 +142,13 @@ public class StartupSortListener implements
SortOperateListener {
FlinkOperation flinkOperation = FlinkOperation.getInstance();
try {
flinkOperation.genPath(flinkInfo, dataflow);
-flinkOperation.start(flinkInfo);
-log.info("job submit success for groupId = {}, streamId = {},
jobId = {}", groupId,
-streamInfo.getInlongStreamId(), flinkInfo.getJobId());
+// only start job for real-time mode
+if (InlongConstants.DATASYNC_REALTIME_MODE
+
.equals(groupResourceForm.getGroupInfo().getInlongGroupMode())) {
+flinkOperation.start(flinkInfo);
+log.info("job submit success for groupId = {}, streamId =
{}, jobId = {}", groupId,
+streamInfo.getInlongStreamId(),
flinkInfo.getJobId());
+}
} catch (Exception e) {
flinkInfo.setException(true);
flinkInfo.setExceptionMsg(getExceptionStackMsg(e));
diff --git
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/AuditServiceImpl.java
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/AuditServiceImpl.java
index c0086ada23..217f3b5d25 100644
---
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/AuditServiceImpl.java
+++
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/AuditServiceImpl.java
@@ -297,7 +297,8 @@ public class AuditServiceImpl implements AuditService {
if (CollectionUtils.isEmpty(request.getAuditIds())) {
// properly overwrite audit ids by role and stream config
-if
(InlongConstants.DATASYNC_REALTIME_MODE.equals(groupEntity.getInlongGroupMode()))
{
+if
(InlongConstants.DATASYNC_REALTIME_MODE.equals(groupEntity.getInlongGroupMode())
+||
InlongConstants.DATASYNC_OFFLINE_MODE.equals(groupEntity.getInlongGroupMode()))
{
auditIdMap.put(getAuditId(sourceNodeType, false),
sourceNodeType);
request.setAuditIds(getAuditIds(groupId, streamId,
sourceNodeType, sinkNodeType));
} else {
@@ -462,7 +463,8 @@ public class AuditServiceImpl implements AuditService {
} else {
auditSet.add(getAuditId(sinkNodeType, true));
InlongGroupEntity inlongGroup =
inlongGroupMapper.selectByGroupId(groupId);
-if
(InlongConstants.DATASYNC_REALTIME_MODE.equals(inlongGroup.getInlongGroupMode()))
{
+if
(InlongConstants.DATASYNC_REALTIME_MODE.equals(
(inlong) 01/03: [INLONG-9781][Manager] Add offline sync task type definition (#9787)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch dev-offline-sync
in repository https://gitbox.apache.org/repos/asf/inlong.git
commit 875e07cd74e83ee2e0011eb84f6b81c53578601e
Author: AloysZhang
AuthorDate: Thu Mar 7 11:44:20 2024 +0800
[INLONG-9781][Manager] Add offline sync task type definition (#9787)
---
.../java/org/apache/inlong/manager/client/ut/BaseTest.java | 2 +-
.../inlong/manager/common/consts/InlongConstants.java | 3 ++-
.../org/apache/inlong/manager/common/enums/GroupMode.java | 14 +++---
.../manager/plugin/listener/StartupSortListener.java | 2 +-
.../apache/inlong/manager/pojo/group/InlongGroupInfo.java | 3 ++-
.../inlong/manager/pojo/group/InlongGroupPageRequest.java | 3 ++-
.../inlong/manager/pojo/group/InlongGroupRequest.java | 5 +++--
.../inlong/manager/service/core/impl/AuditServiceImpl.java | 4 ++--
.../manager/service/group/InlongGroupServiceImpl.java | 2 +-
.../service/listener/group/InitGroupCompleteListener.java | 2 +-
.../listener/group/UpdateGroupCompleteListener.java| 2 +-
.../service/listener/queue/QueueResourceListener.java | 2 +-
.../listener/stream/InitStreamCompleteListener.java| 2 +-
.../manager/service/source/AbstractSourceOperator.java | 2 +-
.../manager/service/source/StreamSourceServiceImpl.java| 2 +-
15 files changed, 31 insertions(+), 19 deletions(-)
diff --git
a/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/ut/BaseTest.java
b/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/ut/BaseTest.java
index 4d218918f5..9961c16ef8 100644
---
a/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/ut/BaseTest.java
+++
b/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/ut/BaseTest.java
@@ -110,7 +110,7 @@ public class BaseTest {
// set enable zk, create resource, group mode, and cluster tag
pulsarInfo.setEnableZookeeper(InlongConstants.DISABLE_ZK);
pulsarInfo.setEnableCreateResource(InlongConstants.ENABLE_CREATE_RESOURCE);
-pulsarInfo.setInlongGroupMode(InlongConstants.DATASYNC_MODE);
+pulsarInfo.setInlongGroupMode(InlongConstants.DATASYNC_REALTIME_MODE);
pulsarInfo.setInlongClusterTag("default_cluster");
pulsarInfo.setDailyRecords(1000);
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
index c3085972fd..581ebb3098 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
@@ -87,7 +87,8 @@ public class InlongConstants {
public static final Integer DELETED_STATUS = 10;
public static final Integer STANDARD_MODE = 0;
-public static final Integer DATASYNC_MODE = 1;
+public static final Integer DATASYNC_REALTIME_MODE = 1;
+public static final Integer DATASYNC_OFFLINE_MODE = 2;
public static final Integer DISABLE_ZK = 0;
public static final Integer ENABLE_ZK = 1;
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java
index f0db2353b6..d4b417f39a 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java
@@ -31,10 +31,18 @@ public enum GroupMode {
STANDARD("standard"),
/**
- * DataSync mode(only Data Synchronization): group init only with sort in
InLong Cluster
- * StreamSource -> Sort -> StreamSink
+ * DataSync mode(only Data Synchronization): real-time data sync in stream
way, group init only with
+ * sort in InLong Cluster.
+ * StreamSource -> Sort -> Sink
*/
-DATASYNC("datasync");
+DATASYNC("datasync"),
+
+/**
+ * DataSync mode(only Data Synchronization): offline data sync in batch
way, group init only with sort
+ * in InLong Cluster.
+ * BatchSource -> Sort -> Sink
+ */
+DATASYNC_BATCH("datasync_offline");
@Getter
private final String mode;
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
index 0b0e55e369..ab9e9b55d8 100644
---
a/inlong-manager/manager-
(inlong) branch dev-offline-sync updated (0ce3d755da -> 1323736a0b)
This is an automated email from the ASF dual-hosted git repository. aloyszhang pushed a change to branch dev-offline-sync in repository https://gitbox.apache.org/repos/asf/inlong.git discard 0ce3d755da [INLONG-9822][Manager] Support flink job runtime execution mode configuration (#9823) omit d5af6a94e2 [INLONG-9813][Manager] Support offline data sync management (#9814) omit 9e96c4e37b [INLONG-9781][Manager] Add offline sync task type definition (#9787) add 293ab1ab3e [INLONG-9766][Audit] Support user-defined SocketAddress loader getting AuditProxy (#9767) add 55f639537d [INLONG-9784][Audit] Optimize sending memory management when the audit-proxy config is null (#9786) add 7c4adbf564 [INLONG-9773][Manager] SortSDK configuration support acquire tenant from InlongGroup (#9776) add 03933e4072 [INLONG-9783][Sort] Add compatibility processing of tid to streamId changes in the message deserialization base class (#9785) add 29574c04cf [INLONG-9774][Sort] Support rowdata way of sort InLong message tlog-kv format (#9780) add 2f0bab5e8d [INLONG-9735][Manager] Bump the Spring version 5.3.27 to 5.3.32 (#9790) add 45e7afc108 [INLONG-9788][Sort] Supports data parse that contains delimiters in kv and csv data format (#9789) add 44f2b04677 [INLONG-9793][Manager] Fix the problem of Manager client workflowApi.listprocess failed to pass parameters correctly (#9794) add 5a58040909 [INLONG-9791][Sort] Return null instead of throwing an exception when deserialization by type fails (#9792) add 829a09bb9f [INLONG-9795][Sort] Optimize the definition of enumeration variables and remove the semicolon (#9764) add 0585278b81 [INLONG-9797][Audit] Audit-SDK reporting supports version number dimension (#9800) add 70e36029ea [INLONG-9807][Audit] Add debug log for audit-proxy (#9809) add 039e681f11 [INLONG-9008][Manager] Set the ignoreParseError field to null (#9810) add 3ae27abb3f [INLONG-9802][Agent] Add an agent installer module for agent installation (#9803) add 2ea39a2765 [INLONG-9809][Audit] SDK supports both singleton and non-singleton usage (#9812) add 5bd1010743 [INLONG-9804][Agent] Add Pulsar source for Agent (#9805) add b6e1064dcf [INLONG-9816][Agent] Add config class for installer (#9817) add b08d1612b0 [INLONG-9806][Agent] Add installer configuration file (#9815) add 58b2fc963c [INLONG-9818][Manager] Decode Msg based on the manager's configuration (#9819) add f84ad372a4 [INLONG-8676][Manager] Elasticsearch - Modify the calling method from SDK to HTTP (#9057) add 7189f34d0b [INLONG-9755][Sort] NPE exception when using default configuration to start inlong-sort-standalone (#9824) add 4649b8e546 [INLONG-9825][Manager] Reduce the creation of RestClusterClient (#9826) add 43bcb9d859 [INLONG-9827][Manager] Fix the problem of failed to check if the consumption group exists (#9828) add fd95e72ba5 [INLONG-9833][Agent] Add module state to distinguish whether the module has been downloaded or installed (#9834) add 23bbce2de1 [INLONG-9831][Agent] Increase configuration acquisition capability (#9832) add 8e5dd5003e [INLONG-9829][Agent] Add guardian scripts (#9830) new 875e07cd74 [INLONG-9781][Manager] Add offline sync task type definition (#9787) new 5cdd94d4c9 [INLONG-9813][Manager] Support offline data sync management (#9814) new 1323736a0b [INLONG-9822][Manager] Support flink job runtime execution mode configuration (#9823) This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (0ce3d755da) \ N -- N -- N refs/heads/dev-offline-sync (1323736a0b) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. The 3 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../inlong/agent/conf/AbstractConfiguration.java | 2 +- .../inlong/agent/constant/FetcherConstants.java| 21 +- .../inlong/agent/constant/TaskConstants.java | 10 + .../org/apache/inlong/agent/pojo/PulsarTask.java | 36 ++- .../apache/inlong/agent/pojo/TaskProfileDto.java | 29 +++ .../org/apache/inlong/agent/utils/HttpManager.java | 26 +- .../apache/inlong/agent/core/HeartbeatManager.java | 4 +- inlong-agen
(inlong) branch master updated: [INLONG-9873][Manager] Support adding data add tasks for file collection (#9874)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new e42f5ec276 [INLONG-9873][Manager] Support adding data add tasks for
file collection (#9874)
e42f5ec276 is described below
commit e42f5ec27688089eb50ae48695c8fe39684eefb7
Author: fuweng11 <76141879+fuwen...@users.noreply.github.com>
AuthorDate: Thu Mar 28 21:42:59 2024 +0800
[INLONG-9873][Manager] Support adding data add tasks for file collection
(#9874)
* [INLONG-9873][Manager] Support adding supplementary recording tasks for
file collection
* [INLONG-9873][Manager] Fix error
* [INLONG-9873][Manager] Fix error
* [INLONG-9873][Manager] Fix error
* [INLONG-9873][Manager] Fix error
* [INLONG-9873][Manager] Fix error
* [INLONG-9873][Manager] Fix code style
* [INLONG-9873][Manager] Fix error
* [INLONG-9873][Manager] Fix error
* [INLONG-9873][Manager] Fix error
* [INLONG-9873][Manager] Fix error
---
.../manager/client/api/InlongGroupContext.java | 2 +-
.../manager/dao/entity/StreamSourceEntity.java | 4 +-
.../dao/mapper/StreamSourceEntityMapper.java | 15 +---
.../resources/mappers/StreamSourceEntityMapper.xml | 37 +++---
.../{SubSourceDTO.java => DataAddTaskDTO.java} | 16
.../manager/pojo/source/DataAddTaskRequest.java| 44 ++
.../inlong/manager/pojo/source/SourceRequest.java | 4 +-
.../inlong/manager/pojo/source/StreamSource.java | 8 ++--
.../pojo/source/file/FileDataAddTaskRequest.java} | 27 ++---
.../manager/pojo/source/pulsar/PulsarSource.java | 3 ++
.../pojo/source/pulsar/PulsarSourceDTO.java| 3 ++
.../service/core/impl/AgentServiceImpl.java| 33 +---
.../source/AbstractSourceOperateListener.java | 2 +-
.../listener/source/SourceRestartListener.java | 2 +-
.../listener/source/SourceStopListener.java| 2 +-
.../service/source/AbstractSourceOperator.java | 7
.../service/source/StreamSourceOperator.java | 10 +
.../service/source/StreamSourceService.java| 10 +
.../service/source/StreamSourceServiceImpl.java| 15 +++-
.../service/source/file/FileSourceOperator.java| 43 +++--
.../manager/service/task/DataCleansingTask.java| 8 ++--
.../service/task/DeleteStreamSourceTask.java | 2 +-
.../service/core/impl/AgentServiceTest.java| 8 ++--
.../main/resources/h2/apache_inlong_manager.sql| 4 +-
.../manager-web/sql/apache_inlong_manager.sql | 4 +-
inlong-manager/manager-web/sql/changes-1.12.0.sql | 3 ++
.../web/controller/StreamSourceController.java | 7
.../src/main/resources/application-dev.properties | 21 +++
.../src/main/resources/application-prod.properties | 21 +++
.../src/main/resources/application-test.properties | 21 +++
30 files changed, 294 insertions(+), 92 deletions(-)
diff --git
a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/InlongGroupContext.java
b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/InlongGroupContext.java
index 8600ecdafa..fe2c5ed04c 100644
---
a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/InlongGroupContext.java
+++
b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/InlongGroupContext.java
@@ -102,7 +102,7 @@ public class InlongGroupContext implements Serializable {
StreamSource source = entry.getValue();
// when template id is null it is considered as normal
source other than template source
// sub sources are filtered because they are already
collected in template source's sub source list
-if (source != null && source.getTemplateId() == null) {
+if (source != null && source.getTaskMapId() == null) {
groupSources.add(source);
}
}
diff --git
a/inlong-manager/manager-dao/src/main/java/org/apache/inlong/manager/dao/entity/StreamSourceEntity.java
b/inlong-manager/manager-dao/src/main/java/org/apache/inlong/manager/dao/entity/StreamSourceEntity.java
index 4b28525333..b577fd48da 100644
---
a/inlong-manager/manager-dao/src/main/java/org/apache/inlong/manager/dao/entity/StreamSourceEntity.java
+++
b/inlong-manager/manager-dao/src/main/java/org/apache/inlong/manager/dao/entity/StreamSourceEntity.java
@@ -37,7 +37,7 @@ public class StreamSourceEntity implements Serializable {
private String inlongStreamId;
private String sourceType;
private String sourceName;
-private Integer templateId;
(inlong) branch master updated: [INLONG-9889][Agent] Delete test code, it will be added back in unit testing (#9890)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 79528ac5aa [INLONG-9889][Agent] Delete test code, it will be added
back in unit testing (#9890)
79528ac5aa is described below
commit 79528ac5aacc584f0e9bde2bb56c66db04ed51a7
Author: justinwwhuang
AuthorDate: Fri Mar 29 09:53:59 2024 +0800
[INLONG-9889][Agent] Delete test code, it will be added back in unit
testing (#9890)
* [INLONG-9889][Agent] Delete test code
* Update
inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ManagerFetcher.java
Co-authored-by: AloysZhang
* [INLONG-9889][Agent] Delete test code
-
Co-authored-by: AloysZhang
---
.../inlong/agent/installer/ManagerFetcher.java | 56 ++
1 file changed, 4 insertions(+), 52 deletions(-)
diff --git
a/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ManagerFetcher.java
b/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ManagerFetcher.java
index 3ec62cad61..ab17b6a5a9 100644
---
a/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ManagerFetcher.java
+++
b/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ManagerFetcher.java
@@ -25,8 +25,6 @@ import org.apache.inlong.agent.utils.HttpManager;
import org.apache.inlong.agent.utils.ThreadUtils;
import org.apache.inlong.common.pojo.agent.installer.ConfigRequest;
import org.apache.inlong.common.pojo.agent.installer.ConfigResult;
-import org.apache.inlong.common.pojo.agent.installer.ModuleConfig;
-import org.apache.inlong.common.pojo.agent.installer.PackageConfig;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
@@ -35,10 +33,6 @@ import com.google.gson.JsonObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
import static
org.apache.inlong.agent.constant.FetcherConstants.AGENT_FETCHER_INTERVAL;
import static
org.apache.inlong.agent.constant.FetcherConstants.AGENT_MANAGER_RETURN_PARAM_DATA;
import static
org.apache.inlong.agent.constant.FetcherConstants.DEFAULT_AGENT_FETCHER_INTERVAL;
@@ -128,63 +122,21 @@ public class ManagerFetcher extends AbstractDaemon
implements ProfileFetcher {
Thread.currentThread().setName("ManagerFetcher");
while (isRunnable()) {
try {
-int configSleepTime = conf.getInt(AGENT_FETCHER_INTERVAL,
DEFAULT_AGENT_FETCHER_INTERVAL);
-ConfigResult config = getTestConfig();
+ConfigResult config = getConfig();
if (config != null) {
manager.getModuleManager().submitConfig(config);
}
-
TimeUnit.SECONDS.sleep(AgentUtils.getRandomBySeed(configSleepTime));
} catch (Throwable ex) {
LOGGER.warn("exception caught", ex);
ThreadUtils.threadThrowableHandler(Thread.currentThread(),
ex);
+} finally {
+AgentUtils
+
.silenceSleepInSeconds(conf.getInt(AGENT_FETCHER_INTERVAL,
DEFAULT_AGENT_FETCHER_INTERVAL));
}
}
};
}
-private ConfigResult getTestConfig() {
-List configs = new ArrayList<>();
-configs.add(getTestModuleConfig(1, "inlong-agent",
"inlong-agent-md5-185454", "1.0", 1,
-"cd ~/inlong-agent/bin;sh agent.sh start", "cd
~/inlong-agent/bin;sh agent.sh stop",
-"ps aux | grep core.AgentMain | grep java | grep -v grep | awk
'{print $2}'",
-"cd ~/inlong-agent/bin;sh agent.sh stop;rm -rf
~/inlong-agent/;mkdir ~/inlong-agent;cd /tmp;tar -xzvf
agent-release-1.12.0-SNAPSHOT-bin.tar.gz -C ~/inlong-agent;cd
~/inlong-agent/bin;sh agent.sh start",
-"echo empty uninstall cmd",
"agent-release-1.12.0-SNAPSHOT-bin.tar.gz",
-
"https://inlong-anager.data.qq.com:8099/api/download?file=agent-release-1.12.0-SNAPSHOT-bin.tar.gz";,
-"package-md5-190300"));
-configs.add(getTestModuleConfig(2, "inlong-agent-installer",
"inlong-agent-installer-md5-190353", "1.0", 1,
-"ps aux | grep installer | grep -v grep | awk '{print $2}' |
xargs kill -9 ;sleep 3;cd ~/inlong-agent-installer/;sh ./bin/monitor.sh",
-"",
-"ps aux | grep installer.Main | grep java | grep -v grep | awk
(inlong) branch master updated: [INLONG-9906][Agent] Add configuration comparison logic and processing of comparison results (#9908)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 09de40 [INLONG-9906][Agent] Add configuration comparison logic and
processing of comparison results (#9908)
09de40 is described below
commit 09de4084f2345e56d8c8646e72508e9a8503
Author: justinwwhuang
AuthorDate: Mon Apr 1 15:36:02 2024 +0800
[INLONG-9906][Agent] Add configuration comparison logic and processing of
comparison results (#9908)
* [INLONG-9906][Agent] Add configuration comparison logic and processing of
comparison results
* [INLONG-9906][Agent] Modify based on comments
* Update
inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ModuleManager.java
Co-authored-by: AloysZhang
-
Co-authored-by: AloysZhang
---
.../inlong/agent/installer/ModuleManager.java | 151 +
1 file changed, 151 insertions(+)
diff --git
a/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ModuleManager.java
b/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ModuleManager.java
index bc7bf89e8c..00dfc9668d 100755
---
a/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ModuleManager.java
+++
b/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ModuleManager.java
@@ -22,10 +22,12 @@ import org.apache.inlong.agent.constant.AgentConstants;
import org.apache.inlong.agent.installer.conf.InstallerConfiguration;
import org.apache.inlong.agent.metrics.audit.AuditUtils;
import org.apache.inlong.agent.utils.AgentUtils;
+import org.apache.inlong.agent.utils.ExcuteLinux;
import org.apache.inlong.agent.utils.HttpManager;
import org.apache.inlong.agent.utils.ThreadUtils;
import org.apache.inlong.common.pojo.agent.installer.ConfigResult;
import org.apache.inlong.common.pojo.agent.installer.ModuleConfig;
+import org.apache.inlong.common.pojo.agent.installer.ModuleStateEnum;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
@@ -216,9 +218,158 @@ public class ModuleManager extends AbstractDaemon {
managerModuleList.forEach((moduleConfig) -> {
modulesFromManager.put(moduleConfig.getId(), moduleConfig);
});
+traverseManagerModulesToLocal(modulesFromManager);
+traverseLocalModulesToManager(modulesFromManager);
return true;
}
+private void traverseManagerModulesToLocal(Map
modulesFromManager) {
+modulesFromManager.values().forEach((managerModule) -> {
+ModuleConfig localModule =
currentModules.get(managerModule.getId());
+if (localModule == null) {
+LOGGER.info("traverseManagerModulesToLocal module {} {} {} not
found in local, add it",
+managerModule.getId(), managerModule.getName(),
managerModule.getVersion());
+addModule(managerModule);
+} else {
+if (managerModule.getMd5().equals(localModule.getMd5())) {
+LOGGER.info("traverseManagerModulesToLocal module {} {} {}
md5 no change, do nothing",
+localModule.getId(), localModule.getName(),
localModule.getVersion());
+} else {
+LOGGER.info("traverseManagerModulesToLocal module {} {} {}
md5 changed, update it",
+localModule.getId(), localModule.getName(),
localModule.getVersion());
+updateModule(localModule, managerModule);
+}
+}
+});
+}
+
+private void traverseLocalModulesToManager(Map
modulesFromManager) {
+currentModules.values().forEach((localModule) -> {
+ModuleConfig managerModule =
modulesFromManager.get(localModule.getId());
+if (managerModule == null) {
+LOGGER.info("traverseLocalModulesToManager module {} {} {} not
found in local, delete it",
+localModule.getId(), localModule.getName(),
localModule.getVersion());
+deleteModule(localModule);
+}
+});
+}
+
+private void addModule(ModuleConfig module) {
+LOGGER.info("add module {} start", module.getName());
+addAndSaveModuleConfig(module);
+if (!downloadModule(module)) {
+LOGGER.error("add module {} but download failed",
module.getName());
+return;
+}
+saveModuleState(module.getId(), ModuleStateEnum.DOWNLOADED);
+installModule(module);
+saveModuleState(module.getId(), ModuleStateEnum.INSTALLED);
+startModule(module);
+LOGGER.info("add module {} end", module.getId());
+
(inlong) branch master updated: [INLONG-9934][doc] update copyright date to 2024 (#9935)
This is an automated email from the ASF dual-hosted git repository. aloyszhang pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/inlong.git The following commit(s) were added to refs/heads/master by this push: new 1df84dca9f [INLONG-9934][doc] update copyright date to 2024 (#9935) 1df84dca9f is described below commit 1df84dca9f3e56a84d1370a3fa0fab61b729b110 Author: AloysZhang AuthorDate: Mon Apr 8 10:22:04 2024 +0800 [INLONG-9934][doc] update copyright date to 2024 (#9935) --- NOTICE | 2 +- licenses/NOTICE| 2 +- licenses/inlong-agent/NOTICE | 2 +- licenses/inlong-audit/NOTICE | 2 +- licenses/inlong-dashboard/NOTICE | 2 +- licenses/inlong-dataproxy/NOTICE | 2 +- licenses/inlong-manager/NOTICE | 2 +- licenses/inlong-sort-connectors/NOTICE | 2 +- licenses/inlong-sort-standalone/NOTICE | 2 +- licenses/inlong-sort/NOTICE| 2 +- licenses/inlong-tubemq-manager/NOTICE | 2 +- licenses/inlong-tubemq-server/NOTICE | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/NOTICE b/NOTICE index e5d29f9d7b..bd764f36a2 100644 --- a/NOTICE +++ b/NOTICE @@ -1,5 +1,5 @@ Apache InLong -Copyright 2019-2023 The Apache Software Foundation. +Copyright 2019-2024 The Apache Software Foundation. This product includes software developed at The Apache Software Foundation (https://www.apache.org/). diff --git a/licenses/NOTICE b/licenses/NOTICE index e5d29f9d7b..bd764f36a2 100644 --- a/licenses/NOTICE +++ b/licenses/NOTICE @@ -1,5 +1,5 @@ Apache InLong -Copyright 2019-2023 The Apache Software Foundation. +Copyright 2019-2024 The Apache Software Foundation. This product includes software developed at The Apache Software Foundation (https://www.apache.org/). diff --git a/licenses/inlong-agent/NOTICE b/licenses/inlong-agent/NOTICE index ad68a5c04b..fd5db7ce46 100644 --- a/licenses/inlong-agent/NOTICE +++ b/licenses/inlong-agent/NOTICE @@ -1,5 +1,5 @@ Apache InLong -Copyright 2019-2023 The Apache Software Foundation. +Copyright 2019-2024 The Apache Software Foundation. This product includes software developed at The Apache Software Foundation (https://www.apache.org/). diff --git a/licenses/inlong-audit/NOTICE b/licenses/inlong-audit/NOTICE index 529e2a862a..c9b80cc535 100644 --- a/licenses/inlong-audit/NOTICE +++ b/licenses/inlong-audit/NOTICE @@ -1,5 +1,5 @@ Apache InLong -Copyright 2019-2023 The Apache Software Foundation. +Copyright 2019-2024 The Apache Software Foundation. This product includes software developed at The Apache Software Foundation (https://www.apache.org/). diff --git a/licenses/inlong-dashboard/NOTICE b/licenses/inlong-dashboard/NOTICE index 66b3df4903..3309b21824 100644 --- a/licenses/inlong-dashboard/NOTICE +++ b/licenses/inlong-dashboard/NOTICE @@ -1,5 +1,5 @@ Apache InLong -Copyright 2019-2023 The Apache Software Foundation. +Copyright 2019-2024 The Apache Software Foundation. This product includes software developed at The Apache Software Foundation (https://www.apache.org/). diff --git a/licenses/inlong-dataproxy/NOTICE b/licenses/inlong-dataproxy/NOTICE index 2c3f00bc90..a448b96c3a 100644 --- a/licenses/inlong-dataproxy/NOTICE +++ b/licenses/inlong-dataproxy/NOTICE @@ -1,5 +1,5 @@ Apache InLong -Copyright 2019-2023 The Apache Software Foundation. +Copyright 2019-2024 The Apache Software Foundation. This product includes software developed at The Apache Software Foundation (https://www.apache.org/). diff --git a/licenses/inlong-manager/NOTICE b/licenses/inlong-manager/NOTICE index 98edfd8442..13fb77230e 100644 --- a/licenses/inlong-manager/NOTICE +++ b/licenses/inlong-manager/NOTICE @@ -1,5 +1,5 @@ Apache InLong -Copyright 2019-2023 The Apache Software Foundation. +Copyright 2019-2024 The Apache Software Foundation. This product includes software developed at The Apache Software Foundation (https://www.apache.org/). diff --git a/licenses/inlong-sort-connectors/NOTICE b/licenses/inlong-sort-connectors/NOTICE index d35911144e..de79d72435 100644 --- a/licenses/inlong-sort-connectors/NOTICE +++ b/licenses/inlong-sort-connectors/NOTICE @@ -1,5 +1,5 @@ Apache InLong -Copyright 2019-2023 The Apache Software Foundation. +Copyright 2019-2024 The Apache Software Foundation. This product includes software developed at The Apache Software Foundation (https://www.apache.org/). diff --git a/licenses/inlong-sort-standalone/NOTICE b/licenses/inlong-sort-standalone/NOTICE index bbfcd63a51..ca64ee7681 100644 --- a/licenses/inlong-sort-standalone/NOTICE +++ b/licenses/inlong-sort-standalone/NOTICE @@ -1,5 +1,5 @@ Apache InLong -Copyright 2019-2023 The Apache Software Foundation. +Copyright 2019-2024 The Apache Software Foundation. This product includes software developed at The Apache Software Foundation (https://www.apache.org/). diff --git a/licenses/inlong-sort/NOTICE b/licenses/inlong-sort/NOTICE
(inlong) branch master updated: [INLONG-9921][Manager] Fix the problem of manager can't stop data sync job (#9942)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new f5950eed5a [INLONG-9921][Manager] Fix the problem of manager can't
stop data sync job (#9942)
f5950eed5a is described below
commit f5950eed5a4f1a0479d2a6e1547b763c83d5c9bf
Author: fuweng11 <76141879+fuwen...@users.noreply.github.com>
AuthorDate: Tue Apr 9 17:15:32 2024 +0800
[INLONG-9921][Manager] Fix the problem of manager can't stop data sync job
(#9942)
* [INLONG-9921][Manager] Fix the problem of manager can't stop data sync job
---
.../inlong/manager/common/enums/GroupStatus.java| 21 +++--
.../manager/common/enums/SimpleGroupStatus.java | 4 ++--
.../inlong/manager/common/enums/StreamStatus.java | 15 +++
.../manager/service/core/impl/AgentServiceImpl.java | 2 +-
.../service/group/InlongGroupProcessService.java| 2 +-
.../listener/group/UpdateGroupCompleteListener.java | 16 +++-
.../stream/UpdateStreamCompleteListener.java| 4 ++--
.../listener/stream/UpdateStreamListener.java | 4 ++--
.../service/stream/InlongStreamProcessService.java | 8
.../manager/service/core/impl/AgentServiceTest.java | 8 +---
.../group/InlongGroupProcessServiceTest.java| 2 +-
.../source/listener/StreamSourceListenerTest.java | 2 +-
12 files changed, 52 insertions(+), 36 deletions(-)
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupStatus.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupStatus.java
index 79e89ad53b..759f5a03b4 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupStatus.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupStatus.java
@@ -41,13 +41,13 @@ public enum GroupStatus {
CONFIG_FAILED(120, "configuration failed"),
CONFIG_SUCCESSFUL(130, "configuration successful"),
-CONFIG_OFFLINE_ING(141, "in configure offline"),
-CONFIGURATION_OFFLINE(140, "configure offline successful"),
+CONFIG_OFFLINE_ING(141, "configuration is going offline"),
+CONFIG_OFFLINE_SUCCESSFUL(140, "configuration offline successful"),
-CONFIG_ONLINE_ING(151, "in configure online"),
+CONFIG_ONLINE_ING(151, "configuration is going online"),
-CONFIG_DELETING(41, "configure deleting"),
-CONFIG_DELETED(40, "configure deleted"),
+CONFIG_DELETING(41, "configuration deleting"),
+CONFIG_DELETED(40, "configuration deleted"),
// FINISH is used for batch task.
FINISH(131, "finish");
@@ -71,9 +71,10 @@ public enum GroupStatus {
Sets.newHashSet(CONFIG_SUCCESSFUL, TO_BE_APPROVAL, CONFIG_ING,
CONFIG_OFFLINE_ING, CONFIG_DELETING));
GROUP_STATE_AUTOMATON.put(
-CONFIG_OFFLINE_ING, Sets.newHashSet(CONFIG_OFFLINE_ING,
CONFIGURATION_OFFLINE, CONFIG_FAILED));
-GROUP_STATE_AUTOMATON.put(CONFIGURATION_OFFLINE,
Sets.newHashSet(CONFIGURATION_OFFLINE, CONFIG_ONLINE_ING,
-CONFIG_DELETING));
+CONFIG_OFFLINE_ING, Sets.newHashSet(CONFIG_OFFLINE_ING,
CONFIG_OFFLINE_SUCCESSFUL, CONFIG_FAILED));
+GROUP_STATE_AUTOMATON.put(
+CONFIG_OFFLINE_SUCCESSFUL,
Sets.newHashSet(CONFIG_OFFLINE_SUCCESSFUL, CONFIG_ONLINE_ING,
+CONFIG_DELETING));
GROUP_STATE_AUTOMATON.put(CONFIG_ONLINE_ING,
Sets.newHashSet(CONFIG_ONLINE_ING, CONFIG_FAILED,
CONFIG_SUCCESSFUL));
@@ -143,7 +144,7 @@ public enum GroupStatus {
return status == GroupStatus.APPROVE_PASSED
|| status == GroupStatus.CONFIG_FAILED
|| status == GroupStatus.CONFIG_SUCCESSFUL
-|| status == GroupStatus.CONFIGURATION_OFFLINE
+|| status == GroupStatus.CONFIG_OFFLINE_SUCCESSFUL
|| status == GroupStatus.FINISH;
}
@@ -163,7 +164,7 @@ public enum GroupStatus {
*/
public static boolean allowedSuspend(GroupStatus status) {
return status == GroupStatus.CONFIG_SUCCESSFUL
-|| status == GroupStatus.CONFIGURATION_OFFLINE
+|| status == GroupStatus.CONFIG_OFFLINE_SUCCESSFUL
|| status == GroupStatus.FINISH;
}
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/SimpleGroupStatus.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/SimpleGroupStatus.java
index d9f6bdc98d..879902f9d4 100644
---
a/inlong-manager/manager-common/src/main/java/
(inlong) branch master updated: [INLONG-9946][Agent] Verify the return code, only proceed with the subsequent process if the return code is successful (#9947)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new bcc1f3fd9f [INLONG-9946][Agent] Verify the return code, only proceed
with the subsequent process if the return code is successful (#9947)
bcc1f3fd9f is described below
commit bcc1f3fd9f2ae50607dda2365bcafa41b2f58db1
Author: justinwwhuang
AuthorDate: Tue Apr 9 17:31:02 2024 +0800
[INLONG-9946][Agent] Verify the return code, only proceed with the
subsequent process if the return code is successful (#9947)
---
.../main/java/org/apache/inlong/agent/installer/ManagerFetcher.java | 3 ++-
.../java/org/apache/inlong/agent/plugin/fetcher/ManagerFetcher.java | 6 +++---
2 files changed, 5 insertions(+), 4 deletions(-)
diff --git
a/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ManagerFetcher.java
b/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ManagerFetcher.java
index 63fbffa76e..68189dc6d1 100644
---
a/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ManagerFetcher.java
+++
b/inlong-agent/agent-installer/src/main/java/org/apache/inlong/agent/installer/ManagerFetcher.java
@@ -25,6 +25,7 @@ import org.apache.inlong.agent.utils.HttpManager;
import org.apache.inlong.agent.utils.ThreadUtils;
import org.apache.inlong.common.pojo.agent.installer.ConfigRequest;
import org.apache.inlong.common.pojo.agent.installer.ConfigResult;
+import org.apache.inlong.common.pojo.agent.installer.InstallerCode;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
@@ -123,7 +124,7 @@ public class ManagerFetcher extends AbstractDaemon
implements ProfileFetcher {
while (isRunnable()) {
try {
ConfigResult config = getConfig();
-if (config != null) {
+if (config != null &&
config.getCode().equals(InstallerCode.SUCCESS)) {
manager.getModuleManager().submitConfig(config);
}
} catch (Throwable ex) {
diff --git
a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/fetcher/ManagerFetcher.java
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/fetcher/ManagerFetcher.java
index 632bb29021..f5fad95d23 100644
---
a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/fetcher/ManagerFetcher.java
+++
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/fetcher/ManagerFetcher.java
@@ -44,7 +44,6 @@ import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
-import java.util.concurrent.TimeUnit;
import static
org.apache.inlong.agent.constant.AgentConstants.AGENT_CLUSTER_NAME;
import static org.apache.inlong.agent.constant.AgentConstants.AGENT_UNIQ_ID;
@@ -177,7 +176,6 @@ public class ManagerFetcher extends AbstractDaemon
implements ProfileFetcher {
Thread.currentThread().setName("ManagerFetcher");
while (isRunnable()) {
try {
-int configSleepTime = conf.getInt(AGENT_FETCHER_INTERVAL,
DEFAULT_AGENT_FETCHER_INTERVAL);
TaskResult taskResult = getStaticConfig();
if (taskResult != null) {
List taskProfiles = new ArrayList<>();
@@ -187,10 +185,12 @@ public class ManagerFetcher extends AbstractDaemon
implements ProfileFetcher {
});
agentManager.getTaskManager().submitTaskProfiles(taskProfiles);
}
-
TimeUnit.SECONDS.sleep(AgentUtils.getRandomBySeed(configSleepTime));
} catch (Throwable ex) {
LOGGER.warn("exception caught", ex);
ThreadUtils.threadThrowableHandler(Thread.currentThread(),
ex);
+} finally {
+
AgentUtils.silenceSleepInSeconds(AgentUtils.getRandomBySeed(
+conf.getInt(AGENT_FETCHER_INTERVAL,
DEFAULT_AGENT_FETCHER_INTERVAL)));
}
}
};
(inlong) 04/04: [INLONG-9862][Manager] Support submit flink job for offline sync (#9865)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch dev-offline-sync
in repository https://gitbox.apache.org/repos/asf/inlong.git
commit dadc2d194eab39e412faac6c0a0ee193afc692b5
Author: AloysZhang
AuthorDate: Fri Mar 22 18:24:26 2024 +0800
[INLONG-9862][Manager] Support submit flink job for offline sync (#9865)
---
.../manager/common/consts/InlongConstants.java | 4 +
.../plugin/listener/StartupSortListener.java | 28 ---
.../manager/pojo/sort/util/StreamParseUtils.java | 21 +
.../listener/StreamTaskListenerFactory.java| 28 +++
.../schedule/StreamScheduleResourceListener.java | 95 ++
.../service/listener/sort/SortConfigListener.java | 8 ++
.../stream/CreateStreamWorkflowDefinition.java | 13 ++-
.../workflow/definition/ServiceTaskType.java | 1 +
.../task/ScheduleOperateListener.java} | 36
.../manager/workflow/plugin/ProcessPlugin.java | 5 ++
10 files changed, 204 insertions(+), 35 deletions(-)
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
index e84dcc6602..f0156cd0e9 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
@@ -141,6 +141,10 @@ public class InlongConstants {
*/
public static final String DATAFLOW = "dataflow";
+public static final String REGISTER_SCHEDULE_STATUS =
"register.schedule.status";
+
+public static final String REGISTERED = "registered";
+
public static final String STREAMS = "streams";
public static final String RELATIONS = "relations";
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
index f6b3b6061a..fec87280d1 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
+++
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
@@ -26,6 +26,7 @@ import org.apache.inlong.manager.plugin.flink.FlinkOperation;
import org.apache.inlong.manager.plugin.flink.dto.FlinkInfo;
import org.apache.inlong.manager.plugin.flink.enums.Constants;
import org.apache.inlong.manager.pojo.sink.StreamSink;
+import org.apache.inlong.manager.pojo.sort.util.StreamParseUtils;
import org.apache.inlong.manager.pojo.stream.InlongStreamExtInfo;
import org.apache.inlong.manager.pojo.stream.InlongStreamInfo;
import
org.apache.inlong.manager.pojo.workflow.form.process.GroupResourceProcessForm;
@@ -98,6 +99,14 @@ public class StartupSortListener implements
SortOperateListener {
}
for (InlongStreamInfo streamInfo : streamInfos) {
+boolean isOfflineSync = InlongConstants.DATASYNC_OFFLINE_MODE
+
.equals(groupResourceForm.getGroupInfo().getInlongGroupMode());
+// do not submit flink job if the group mode is offline and the
stream is not config successfully
+if (isOfflineSync &&
!StreamParseUtils.isRegisterScheduleSuccess(streamInfo)) {
+log.info("no need to submit flink job for groupId={}
streamId={} as the mode is offline "
++ "and the stream is not config successfully yet",
groupId, streamInfo.getInlongStreamId());
+continue;
+}
List sinkList = streamInfo.getSinkList();
List sinkTypes =
sinkList.stream().map(StreamSink::getSinkType).collect(Collectors.toList());
if (CollectionUtils.isEmpty(sinkList) ||
!SinkType.containSortFlinkSink(sinkTypes)) {
@@ -131,9 +140,6 @@ public class StartupSortListener implements
SortOperateListener {
return ListenerResult.fail(message);
}
-boolean isRealTimeSync = InlongConstants.DATASYNC_REALTIME_MODE
-
.equals(groupResourceForm.getGroupInfo().getInlongGroupMode());
-
FlinkInfo flinkInfo = new FlinkInfo();
String jobName =
Constants.SORT_JOB_NAME_GENERATOR.apply(processForm) + InlongConstants.HYPHEN
@@ -141,20 +147,18 @@ public class StartupSortListener implements
SortOperateListener {
flinkInfo.setJobName(jobName);
flinkInfo.setEndpoint(kvConf.get(InlongConstants.SORT_URL));
flinkInfo.setInlongStreamInfoList(Collections.singletonList(streamInfo));
-if (isRealTimeSync) {
-
flin
(inlong) 03/04: [INLONG-9822][Manager] Support flink job runtime execution mode configuration (#9823)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch dev-offline-sync
in repository https://gitbox.apache.org/repos/asf/inlong.git
commit ddc72ed417cbd12f06f463d646bf6ed841ffd67d
Author: AloysZhang
AuthorDate: Fri Mar 15 15:12:53 2024 +0800
[INLONG-9822][Manager] Support flink job runtime execution mode
configuration (#9823)
---
.../inlong/manager/common/consts/InlongConstants.java | 3 +++
.../apache/inlong/manager/plugin/flink/FlinkService.java | 2 ++
.../apache/inlong/manager/plugin/flink/dto/FlinkInfo.java | 2 ++
.../inlong/manager/plugin/listener/DeleteSortListener.java | 1 +
.../manager/plugin/listener/RestartSortListener.java | 8 +++-
.../manager/plugin/listener/StartupSortListener.java | 14 ++
.../org/apache/inlong/sort/configuration/Constants.java| 8
.../src/main/java/org/apache/inlong/sort/Entrance.java | 11 ++-
8 files changed, 43 insertions(+), 6 deletions(-)
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
index 581ebb3098..e84dcc6602 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
@@ -90,6 +90,9 @@ public class InlongConstants {
public static final Integer DATASYNC_REALTIME_MODE = 1;
public static final Integer DATASYNC_OFFLINE_MODE = 2;
+public static final String RUNTIME_EXECUTION_MODE_STREAMING = "streaming";
+public static final String RUNTIME_EXECUTION_MODE_BATCH = "batch";
+
public static final Integer DISABLE_ZK = 0;
public static final Integer ENABLE_ZK = 1;
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkService.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkService.java
index 69a32b73fa..33a7ca59cd 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkService.java
+++
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkService.java
@@ -258,6 +258,8 @@ public class FlinkService {
list.add(flinkInfo.getLocalConfPath());
list.add("-checkpoint.interval");
list.add("6");
+list.add("-runtime.execution.mode");
+list.add(flinkInfo.getRuntimeExecutionMode());
list.add("-metrics.audit.proxy.hosts");
list.add(flinkConfig.getAuditProxyHosts());
return list.toArray(new String[0]);
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/dto/FlinkInfo.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/dto/FlinkInfo.java
index f7071ceb4b..4c3c75f855 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/dto/FlinkInfo.java
+++
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/dto/FlinkInfo.java
@@ -52,4 +52,6 @@ public class FlinkInfo {
private boolean isException = false;
private String exceptionMsg;
+
+private String runtimeExecutionMode;
}
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/DeleteSortListener.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/DeleteSortListener.java
index 009374e2a6..5e30ad8cb5 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/DeleteSortListener.java
+++
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/DeleteSortListener.java
@@ -119,6 +119,7 @@ public class DeleteSortListener implements
SortOperateListener {
FlinkOperation flinkOperation = FlinkOperation.getInstance();
try {
flinkOperation.delete(flinkInfo);
+// TODO if the job is OFFLINE, should delete the scheduler
information
log.info("job delete success for jobId={}", jobId);
} catch (Exception e) {
flinkInfo.setException(true);
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/RestartSortListener.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/RestartSortListener.java
index 0c6828c984..242138c1e4 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/RestartSortList
(inlong) 02/04: [INLONG-9813][Manager] Support offline data sync management (#9814)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch dev-offline-sync
in repository https://gitbox.apache.org/repos/asf/inlong.git
commit e66cf075fe41e45f44c9b13e250242b0aaf4e0a3
Author: AloysZhang
AuthorDate: Wed Mar 13 16:18:04 2024 +0800
[INLONG-9813][Manager] Support offline data sync management (#9814)
---
.../inlong/manager/plugin/listener/StartupSortListener.java | 13 +
.../service/listener/group/InitGroupCompleteListener.java | 3 ++-
.../service/listener/group/UpdateGroupCompleteListener.java | 3 ++-
.../service/listener/queue/QueueResourceListener.java | 6 --
.../service/listener/stream/InitStreamCompleteListener.java | 3 ++-
.../manager/service/source/AbstractSourceOperator.java | 1 +
.../manager/service/source/StreamSourceServiceImpl.java | 3 ++-
7 files changed, 22 insertions(+), 10 deletions(-)
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
index ab9e9b55d8..fae5faa278 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
+++
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
@@ -73,7 +73,8 @@ public class StartupSortListener implements
SortOperateListener {
}
log.info("add startup group listener for groupId [{}]", groupId);
-return
InlongConstants.DATASYNC_REALTIME_MODE.equals(groupProcessForm.getGroupInfo().getInlongGroupMode());
+return
(InlongConstants.DATASYNC_REALTIME_MODE.equals(groupProcessForm.getGroupInfo().getInlongGroupMode())
+||
InlongConstants.DATASYNC_OFFLINE_MODE.equals(groupProcessForm.getGroupInfo().getInlongGroupMode()));
}
@Override
@@ -141,9 +142,13 @@ public class StartupSortListener implements
SortOperateListener {
FlinkOperation flinkOperation = FlinkOperation.getInstance();
try {
flinkOperation.genPath(flinkInfo, dataflow);
-flinkOperation.start(flinkInfo);
-log.info("job submit success for groupId = {}, streamId = {},
jobId = {}", groupId,
-streamInfo.getInlongStreamId(), flinkInfo.getJobId());
+// only start job for real-time mode
+if (InlongConstants.DATASYNC_REALTIME_MODE
+
.equals(groupResourceForm.getGroupInfo().getInlongGroupMode())) {
+flinkOperation.start(flinkInfo);
+log.info("job submit success for groupId = {}, streamId =
{}, jobId = {}", groupId,
+streamInfo.getInlongStreamId(),
flinkInfo.getJobId());
+}
} catch (Exception e) {
flinkInfo.setException(true);
flinkInfo.setExceptionMsg(getExceptionStackMsg(e));
diff --git
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupCompleteListener.java
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupCompleteListener.java
index c96c12c9ff..c8c087320f 100644
---
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupCompleteListener.java
+++
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupCompleteListener.java
@@ -98,7 +98,8 @@ public class InitGroupCompleteListener implements
ProcessEventListener {
// update status of other related configs
if
(InlongConstants.DISABLE_CREATE_RESOURCE.equals(groupInfo.getEnableCreateResource()))
{
-if
(InlongConstants.DATASYNC_REALTIME_MODE.equals(groupInfo.getInlongGroupMode()))
{
+if
(InlongConstants.DATASYNC_REALTIME_MODE.equals(groupInfo.getInlongGroupMode())
+||
InlongConstants.DATASYNC_OFFLINE_MODE.equals(groupInfo.getInlongGroupMode())) {
sourceService.updateStatus(groupId, null,
SourceStatus.SOURCE_NORMAL.getCode(), operator);
} else {
sourceService.updateStatus(groupId, null,
SourceStatus.TO_BE_ISSUED_ADD.getCode(), operator);
diff --git
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/UpdateGroupCompleteListener.java
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/UpdateGroupCompleteListener.java
index a6dd8fc786..1a6e1f7fe1 100644
---
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/UpdateGroupCompleteListener.java
+++
b/inlong-ma
(inlong) 01/04: [INLONG-9781][Manager] Add offline sync task type definition (#9787)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch dev-offline-sync
in repository https://gitbox.apache.org/repos/asf/inlong.git
commit cfd9136286502a4f33d3482461218f814f4f2a6a
Author: AloysZhang
AuthorDate: Thu Mar 7 11:44:20 2024 +0800
[INLONG-9781][Manager] Add offline sync task type definition (#9787)
---
.../java/org/apache/inlong/manager/client/ut/BaseTest.java | 2 +-
.../inlong/manager/common/consts/InlongConstants.java | 3 ++-
.../org/apache/inlong/manager/common/enums/GroupMode.java | 14 +++---
.../manager/plugin/listener/StartupSortListener.java | 2 +-
.../apache/inlong/manager/pojo/group/InlongGroupInfo.java | 3 ++-
.../inlong/manager/pojo/group/InlongGroupPageRequest.java | 3 ++-
.../inlong/manager/pojo/group/InlongGroupRequest.java | 5 +++--
.../inlong/manager/service/core/impl/AuditServiceImpl.java | 6 --
.../manager/service/group/InlongGroupServiceImpl.java | 2 +-
.../service/listener/group/InitGroupCompleteListener.java | 2 +-
.../listener/group/UpdateGroupCompleteListener.java| 2 +-
.../service/listener/queue/QueueResourceListener.java | 2 +-
.../listener/stream/InitStreamCompleteListener.java| 2 +-
.../manager/service/source/AbstractSourceOperator.java | 2 +-
.../manager/service/source/StreamSourceServiceImpl.java| 2 +-
15 files changed, 33 insertions(+), 19 deletions(-)
diff --git
a/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/ut/BaseTest.java
b/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/ut/BaseTest.java
index 4d218918f5..9961c16ef8 100644
---
a/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/ut/BaseTest.java
+++
b/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/ut/BaseTest.java
@@ -110,7 +110,7 @@ public class BaseTest {
// set enable zk, create resource, group mode, and cluster tag
pulsarInfo.setEnableZookeeper(InlongConstants.DISABLE_ZK);
pulsarInfo.setEnableCreateResource(InlongConstants.ENABLE_CREATE_RESOURCE);
-pulsarInfo.setInlongGroupMode(InlongConstants.DATASYNC_MODE);
+pulsarInfo.setInlongGroupMode(InlongConstants.DATASYNC_REALTIME_MODE);
pulsarInfo.setInlongClusterTag("default_cluster");
pulsarInfo.setDailyRecords(1000);
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
index c3085972fd..581ebb3098 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
@@ -87,7 +87,8 @@ public class InlongConstants {
public static final Integer DELETED_STATUS = 10;
public static final Integer STANDARD_MODE = 0;
-public static final Integer DATASYNC_MODE = 1;
+public static final Integer DATASYNC_REALTIME_MODE = 1;
+public static final Integer DATASYNC_OFFLINE_MODE = 2;
public static final Integer DISABLE_ZK = 0;
public static final Integer ENABLE_ZK = 1;
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java
index f0db2353b6..d4b417f39a 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java
@@ -31,10 +31,18 @@ public enum GroupMode {
STANDARD("standard"),
/**
- * DataSync mode(only Data Synchronization): group init only with sort in
InLong Cluster
- * StreamSource -> Sort -> StreamSink
+ * DataSync mode(only Data Synchronization): real-time data sync in stream
way, group init only with
+ * sort in InLong Cluster.
+ * StreamSource -> Sort -> Sink
*/
-DATASYNC("datasync");
+DATASYNC("datasync"),
+
+/**
+ * DataSync mode(only Data Synchronization): offline data sync in batch
way, group init only with sort
+ * in InLong Cluster.
+ * BatchSource -> Sort -> Sink
+ */
+DATASYNC_BATCH("datasync_offline");
@Getter
private final String mode;
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
index 0b0e55e369..ab9e9b55d8 100644
---
a/inlong-manager/manager-
(inlong) 03/04: [INLONG-9822][Manager] Support flink job runtime execution mode configuration (#9823)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch rebase
in repository https://gitbox.apache.org/repos/asf/inlong.git
commit 045e04b7683fc6823e6c8debd59e657538bb9bd8
Author: AloysZhang
AuthorDate: Fri Mar 15 15:12:53 2024 +0800
[INLONG-9822][Manager] Support flink job runtime execution mode
configuration (#9823)
---
.../inlong/manager/common/consts/InlongConstants.java | 3 +++
.../apache/inlong/manager/plugin/flink/FlinkService.java | 4
.../apache/inlong/manager/plugin/flink/dto/FlinkInfo.java | 2 ++
.../inlong/manager/plugin/listener/DeleteSortListener.java | 1 +
.../manager/plugin/listener/RestartSortListener.java | 8 +++-
.../manager/plugin/listener/StartupSortListener.java | 14 ++
.../org/apache/inlong/sort/configuration/Constants.java| 8
.../src/main/java/org/apache/inlong/sort/Entrance.java | 11 ++-
8 files changed, 45 insertions(+), 6 deletions(-)
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
index 581ebb3098..e84dcc6602 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
@@ -90,6 +90,9 @@ public class InlongConstants {
public static final Integer DATASYNC_REALTIME_MODE = 1;
public static final Integer DATASYNC_OFFLINE_MODE = 2;
+public static final String RUNTIME_EXECUTION_MODE_STREAMING = "streaming";
+public static final String RUNTIME_EXECUTION_MODE_BATCH = "batch";
+
public static final Integer DISABLE_ZK = 0;
public static final Integer ENABLE_ZK = 1;
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkService.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkService.java
index 129b7aa444..33a7ca59cd 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkService.java
+++
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkService.java
@@ -258,6 +258,10 @@ public class FlinkService {
list.add(flinkInfo.getLocalConfPath());
list.add("-checkpoint.interval");
list.add("6");
+list.add("-runtime.execution.mode");
+list.add(flinkInfo.getRuntimeExecutionMode());
+list.add("-metrics.audit.proxy.hosts");
+list.add(flinkConfig.getAuditProxyHosts());
return list.toArray(new String[0]);
}
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/dto/FlinkInfo.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/dto/FlinkInfo.java
index f7071ceb4b..4c3c75f855 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/dto/FlinkInfo.java
+++
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/dto/FlinkInfo.java
@@ -52,4 +52,6 @@ public class FlinkInfo {
private boolean isException = false;
private String exceptionMsg;
+
+private String runtimeExecutionMode;
}
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/DeleteSortListener.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/DeleteSortListener.java
index 009374e2a6..5e30ad8cb5 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/DeleteSortListener.java
+++
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/DeleteSortListener.java
@@ -119,6 +119,7 @@ public class DeleteSortListener implements
SortOperateListener {
FlinkOperation flinkOperation = FlinkOperation.getInstance();
try {
flinkOperation.delete(flinkInfo);
+// TODO if the job is OFFLINE, should delete the scheduler
information
log.info("job delete success for jobId={}", jobId);
} catch (Exception e) {
flinkInfo.setException(true);
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/RestartSortListener.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/RestartSortListener.java
index 0c6828c984..242138c1e4 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/RestartSortList
(inlong) 04/04: [INLONG-9862][Manager] Support submit flink job for offline sync (#9865)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch rebase
in repository https://gitbox.apache.org/repos/asf/inlong.git
commit 05c04c80c93e882362c9b08ceea582ce61d4c23d
Author: AloysZhang
AuthorDate: Fri Mar 22 18:24:26 2024 +0800
[INLONG-9862][Manager] Support submit flink job for offline sync (#9865)
---
.../manager/common/consts/InlongConstants.java | 4 +
.../plugin/listener/StartupSortListener.java | 28 ---
.../manager/pojo/sort/util/StreamParseUtils.java | 21 +
.../listener/StreamTaskListenerFactory.java| 28 +++
.../schedule/StreamScheduleResourceListener.java | 95 ++
.../service/listener/sort/SortConfigListener.java | 8 ++
.../stream/CreateStreamWorkflowDefinition.java | 13 ++-
.../workflow/definition/ServiceTaskType.java | 1 +
.../task/ScheduleOperateListener.java} | 36
.../manager/workflow/plugin/ProcessPlugin.java | 5 ++
10 files changed, 204 insertions(+), 35 deletions(-)
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
index e84dcc6602..f0156cd0e9 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
@@ -141,6 +141,10 @@ public class InlongConstants {
*/
public static final String DATAFLOW = "dataflow";
+public static final String REGISTER_SCHEDULE_STATUS =
"register.schedule.status";
+
+public static final String REGISTERED = "registered";
+
public static final String STREAMS = "streams";
public static final String RELATIONS = "relations";
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
index f6b3b6061a..fec87280d1 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
+++
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
@@ -26,6 +26,7 @@ import org.apache.inlong.manager.plugin.flink.FlinkOperation;
import org.apache.inlong.manager.plugin.flink.dto.FlinkInfo;
import org.apache.inlong.manager.plugin.flink.enums.Constants;
import org.apache.inlong.manager.pojo.sink.StreamSink;
+import org.apache.inlong.manager.pojo.sort.util.StreamParseUtils;
import org.apache.inlong.manager.pojo.stream.InlongStreamExtInfo;
import org.apache.inlong.manager.pojo.stream.InlongStreamInfo;
import
org.apache.inlong.manager.pojo.workflow.form.process.GroupResourceProcessForm;
@@ -98,6 +99,14 @@ public class StartupSortListener implements
SortOperateListener {
}
for (InlongStreamInfo streamInfo : streamInfos) {
+boolean isOfflineSync = InlongConstants.DATASYNC_OFFLINE_MODE
+
.equals(groupResourceForm.getGroupInfo().getInlongGroupMode());
+// do not submit flink job if the group mode is offline and the
stream is not config successfully
+if (isOfflineSync &&
!StreamParseUtils.isRegisterScheduleSuccess(streamInfo)) {
+log.info("no need to submit flink job for groupId={}
streamId={} as the mode is offline "
++ "and the stream is not config successfully yet",
groupId, streamInfo.getInlongStreamId());
+continue;
+}
List sinkList = streamInfo.getSinkList();
List sinkTypes =
sinkList.stream().map(StreamSink::getSinkType).collect(Collectors.toList());
if (CollectionUtils.isEmpty(sinkList) ||
!SinkType.containSortFlinkSink(sinkTypes)) {
@@ -131,9 +140,6 @@ public class StartupSortListener implements
SortOperateListener {
return ListenerResult.fail(message);
}
-boolean isRealTimeSync = InlongConstants.DATASYNC_REALTIME_MODE
-
.equals(groupResourceForm.getGroupInfo().getInlongGroupMode());
-
FlinkInfo flinkInfo = new FlinkInfo();
String jobName =
Constants.SORT_JOB_NAME_GENERATOR.apply(processForm) + InlongConstants.HYPHEN
@@ -141,20 +147,18 @@ public class StartupSortListener implements
SortOperateListener {
flinkInfo.setJobName(jobName);
flinkInfo.setEndpoint(kvConf.get(InlongConstants.SORT_URL));
flinkInfo.setInlongStreamInfoList(Collections.singletonList(streamInfo));
-if (isRealTimeSync) {
-
flin
(inlong) branch dev-offline-sync updated (dadc2d194e -> 2932887cb5)
This is an automated email from the ASF dual-hosted git repository. aloyszhang pushed a change to branch dev-offline-sync in repository https://gitbox.apache.org/repos/asf/inlong.git discard dadc2d194e [INLONG-9862][Manager] Support submit flink job for offline sync (#9865) discard ddc72ed417 [INLONG-9822][Manager] Support flink job runtime execution mode configuration (#9823) discard e66cf075fe [INLONG-9813][Manager] Support offline data sync management (#9814) discard cfd9136286 [INLONG-9781][Manager] Add offline sync task type definition (#9787) add 3b70a93d96 [INLONG-10145][Manager] Change spring version 5.3.32 to 5.3.34. (#10149) add 762a026a47 [INLONG-10147][Audit] Less usage costs when the Audit uses the MySQL as storage (#10151) add 2c2e52b568 [INLONG-10150][Manager] Remove metrics. audit. proxy. hosts from flink-sort-plugins. properties (#10153) add 9bd422ffe9 [INLONG-10144][Sort] Redis connectors support audit ID (#10146) add e40550c30a [INLONG-10154][SDK] Support to transform CSV/KV data to CSV/KV data without field list configuration (#10155) add 00595bb156 [INLONG-10157][Audit] Use a custom thread pool instead of the system thread pool (#10158) add 011e96ba95 [INLONG-10152][Sort] Refactor MetricOption code structure. (#10156) add 0fd55f85b5 [INLONG-10162][DataProxy] Upgrade golang.org/x/net to version 0.23.0. (#10163) add d34dcc0bc6 [INLONG-10161][DataProxy] Upgrade google.golang.org/protobuf v1.32.0 to v1.34.1 (#10165) add c060b8a566 [INLONG-10171][Audit] Update the standalone deploy scripts for the audit service (#10172) add 9407b2f757 [INLONG-10167][TubeMQ] Upgrade google.golang.org/protobuf v1.23.0 to v1.34.1 (#10168) add 384ca56494 [INLONG-10169][Manager] Support configure sort cluster for kafka (#10170) add bc7a2d99f2 [INLONG-10176][Manager] Fix the problem of sort_config table creation statement error (#10177) add 4da8c8e392 [INLONG-10125][Script] Fix the agent can not start successfully (#10175) add c37301897a [INLONG-10160][Sort] Upgrade elasticsearch version 7.9.2 to 7.17.13 (#10166) add fb10c2b6e2 [INLONG-10124][Script] Update the scripts for standalone deployment (#10178) add ce1ca12799 [INLONG-10181][Manager] Remove useless exceptions for DataNodeOperator.getNodeConfig (#10182) add 888c6aab52 [INLONG-10184][SDK] The header in ProxyEvent need contain both 'rtms' and 'auditVersion'. (#10185) add 1bfac499b5 [INLONG-10173][Sort] SortStandalone support request unified configuration (#10174) add 632aa88e44 [INLONG-10187][Agent] Handle situations where time offset is empty (#10188) add 808cf799c8 [INLONG-10091][Agent] Delete the useless code related to 8008 listen port (#10192) add bbd29d961e [INLONG-10197][Manager] Support OpenAPI for querying audit data (#10199) add cdf81cd791 [INLONG-10200][Manager] Define module type mapping relationships (#10203) add 50fc3024b6 [INLONG-10205][Script] Add the metrics.audit.proxy.hosts for standalone deployment (#10206) add f91542b75a [INLONG-10179][DashBoard] Remove the All types In cluster management (#10209) add 1fa64267e3 [INLONG-10204][Manager] Kafka sink supports automatic allocation of sort standalone cluster (#10207) add 06d5e370ec [INLONG-10201][Audit] Renamed configuration variables for clarity (#10202) add 61d2b74ce1 [INLONG-10189][Agent] Handling SDK initialization exceptions (#10190) add 065392d3a2 [INLONG-10210][Agent] Add a script for environment initialization (#10211) add 3b6600baf2 [INLONG-10096][Manager] Support installing agents by SSH (#10098) add 974ef70d7f [INLONG-10215][DashBoard] Add default value for file data source in data access module (#10216) add e8dbf282bf [INLONG-10076][Manager] Doris data node supports test connections (#10217) add e68c69f986 [INLONG-10208][Sort] ClsSink support unified configuration (#10220) add c5dbfabf22 [INLONG-10194][Sort] Sqlserver connector support audit ID (#10212) add d77048497a [INLONG-10213][SDK] SortSDK support unified sort configuration (#10219) add 540fd51fb4 [INLONG-10193][Sort] Postgres connector support audit ID (#10198) add 11e52ba618 [INLONG-10221][SDK] DataProxy SDK of cpp supports automatic installation of log4cplus components (#10222) add a81d7cf286 [INLONG-10183][Sort] MongoDB connector support audit ID (#10186) add 54f831aad5 [INLONG-10226][Dashboard] Fix audit item search failure (#10227) add b9c9688fc5 [INLONG-10159][Sort] Kafka connector support audit ID (#10180) new 57d7f733c7 [INLONG-9781][Manager] Add offline sync task type definition (#9787) new 06a84650c7 [INLONG-9813][Manager] Support offline data sync management (#9814) new 9024c1b58d [INLONG-9822][Manager] Support flink job runtime execution mode configuration (#9823) new 6996891c69 [INLONG-9862][Manager] Support submit f
(inlong) 02/05: [INLONG-9813][Manager] Support offline data sync management (#9814)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch dev-offline-sync
in repository https://gitbox.apache.org/repos/asf/inlong.git
commit 06a84650c7c5eceb86183f35b24a9486c9a5ca60
Author: AloysZhang
AuthorDate: Wed Mar 13 16:18:04 2024 +0800
[INLONG-9813][Manager] Support offline data sync management (#9814)
---
.../inlong/manager/plugin/listener/StartupSortListener.java | 13 +
.../service/listener/group/InitGroupCompleteListener.java | 3 ++-
.../service/listener/group/UpdateGroupCompleteListener.java | 3 ++-
.../service/listener/queue/QueueResourceListener.java | 6 --
.../service/listener/stream/InitStreamCompleteListener.java | 3 ++-
.../manager/service/source/AbstractSourceOperator.java | 1 +
.../manager/service/source/StreamSourceServiceImpl.java | 3 ++-
7 files changed, 22 insertions(+), 10 deletions(-)
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
index ab9e9b55d8..fae5faa278 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
+++
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
@@ -73,7 +73,8 @@ public class StartupSortListener implements
SortOperateListener {
}
log.info("add startup group listener for groupId [{}]", groupId);
-return
InlongConstants.DATASYNC_REALTIME_MODE.equals(groupProcessForm.getGroupInfo().getInlongGroupMode());
+return
(InlongConstants.DATASYNC_REALTIME_MODE.equals(groupProcessForm.getGroupInfo().getInlongGroupMode())
+||
InlongConstants.DATASYNC_OFFLINE_MODE.equals(groupProcessForm.getGroupInfo().getInlongGroupMode()));
}
@Override
@@ -141,9 +142,13 @@ public class StartupSortListener implements
SortOperateListener {
FlinkOperation flinkOperation = FlinkOperation.getInstance();
try {
flinkOperation.genPath(flinkInfo, dataflow);
-flinkOperation.start(flinkInfo);
-log.info("job submit success for groupId = {}, streamId = {},
jobId = {}", groupId,
-streamInfo.getInlongStreamId(), flinkInfo.getJobId());
+// only start job for real-time mode
+if (InlongConstants.DATASYNC_REALTIME_MODE
+
.equals(groupResourceForm.getGroupInfo().getInlongGroupMode())) {
+flinkOperation.start(flinkInfo);
+log.info("job submit success for groupId = {}, streamId =
{}, jobId = {}", groupId,
+streamInfo.getInlongStreamId(),
flinkInfo.getJobId());
+}
} catch (Exception e) {
flinkInfo.setException(true);
flinkInfo.setExceptionMsg(getExceptionStackMsg(e));
diff --git
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupCompleteListener.java
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupCompleteListener.java
index c96c12c9ff..c8c087320f 100644
---
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupCompleteListener.java
+++
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupCompleteListener.java
@@ -98,7 +98,8 @@ public class InitGroupCompleteListener implements
ProcessEventListener {
// update status of other related configs
if
(InlongConstants.DISABLE_CREATE_RESOURCE.equals(groupInfo.getEnableCreateResource()))
{
-if
(InlongConstants.DATASYNC_REALTIME_MODE.equals(groupInfo.getInlongGroupMode()))
{
+if
(InlongConstants.DATASYNC_REALTIME_MODE.equals(groupInfo.getInlongGroupMode())
+||
InlongConstants.DATASYNC_OFFLINE_MODE.equals(groupInfo.getInlongGroupMode())) {
sourceService.updateStatus(groupId, null,
SourceStatus.SOURCE_NORMAL.getCode(), operator);
} else {
sourceService.updateStatus(groupId, null,
SourceStatus.TO_BE_ISSUED_ADD.getCode(), operator);
diff --git
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/UpdateGroupCompleteListener.java
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/UpdateGroupCompleteListener.java
index a6dd8fc786..1a6e1f7fe1 100644
---
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/UpdateGroupCompleteListener.java
+++
b/inlong-ma
(inlong) 05/05: Support scheudle infomartion management
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch dev-offline-sync
in repository https://gitbox.apache.org/repos/asf/inlong.git
commit 2932887cb5a5da63b2498b4bb0f97513d05f5aa9
Author: aloyszhang
AuthorDate: Thu May 16 16:37:56 2024 +0800
Support scheudle infomartion management
---
.../inlong/manager/common/enums/ErrorCodeEnum.java | 3 +
.../manager/common/enums/OperationTarget.java | 4 +-
.../manager/common/enums/ScheduleStatus.java | 29 +
.../inlong/manager/dao/entity/ScheduleEntity.java | 41 +++
.../manager/dao/mapper/ScheduleEntityMapper.java | 16 +++
.../resources/mappers/ScheduleEntityMapper.xml | 120 +
.../inlong/manager/pojo/schedule/ScheduleInfo.java | 57 ++
.../manager/pojo/schedule/ScheduleInfoRequest.java | 56 ++
.../manager/service/schedule/ScheduleService.java | 45
.../service/schedule/ScheduleServiceImpl.java | 96 +
.../manager-web/sql/apache_inlong_manager.sql | 38 +++
.../web/controller/InLongSchedulerController.java | 65 +++
12 files changed, 569 insertions(+), 1 deletion(-)
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/ErrorCodeEnum.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/ErrorCodeEnum.java
index 98a7f50bd7..ce3bf9d015 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/ErrorCodeEnum.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/ErrorCodeEnum.java
@@ -125,6 +125,9 @@ public enum ErrorCodeEnum {
MQ_TYPE_IS_NULL(1600, "MQ type is null"),
MQ_TYPE_NOT_SUPPORT(1601, "MQ type '%s' not support"),
+SCHEDULE_NOT_FOUND(1700, "Schedule info not found"),
+SCHEDULE_DUPLICATE(1701, "Schedule info already exist"),
+
WORKFLOW_EXE_FAILED(4000, "Workflow execution exception"),
WORKFLOW_APPROVER_NOT_FOUND(4001, "Workflow approver does not exist/no
operation authority"),
WORKFLOW_DELETE_RECORD_FAILED(4002, "Workflow delete record failure"),
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/OperationTarget.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/OperationTarget.java
index e6c866a0e1..93d4e1f92c 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/OperationTarget.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/OperationTarget.java
@@ -44,6 +44,8 @@ public enum OperationTarget {
INLONG_ROLE,
-TENANT_ROLE
+TENANT_ROLE,
+
+SCHEDULE
}
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/ScheduleStatus.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/ScheduleStatus.java
new file mode 100644
index 00..818c8277a2
--- /dev/null
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/ScheduleStatus.java
@@ -0,0 +1,29 @@
+package org.apache.inlong.manager.common.enums;
+
+import lombok.Getter;
+
+@Getter
+public enum ScheduleStatus {
+
+NEW(100, "new"),
+DELETED(40, "deleted");
+
+private final Integer code;
+private final String description;
+
+ScheduleStatus(Integer code, String description) {
+this.code = code;
+this.description = description;
+}
+
+
+public static ScheduleStatus forCode(int code) {
+for (ScheduleStatus status : values()) {
+if (status.getCode() == code) {
+return status;
+}
+}
+throw new IllegalStateException(String.format("Illegal code=%s for
ScheduleStatus", code));
+}
+
+}
diff --git
a/inlong-manager/manager-dao/src/main/java/org/apache/inlong/manager/dao/entity/ScheduleEntity.java
b/inlong-manager/manager-dao/src/main/java/org/apache/inlong/manager/dao/entity/ScheduleEntity.java
new file mode 100644
index 00..5502e8c57d
--- /dev/null
+++
b/inlong-manager/manager-dao/src/main/java/org/apache/inlong/manager/dao/entity/ScheduleEntity.java
@@ -0,0 +1,41 @@
+package org.apache.inlong.manager.dao.entity;
+
+import java.io.Serializable;
+import java.util.Date;
+import lombok.Data;
+
+@Data
+public class ScheduleEntity implements Serializable {
+private static final long serialVersionUID = 1L;
+
+private Integer id;
+// inLong group id
+private String groupId;
+// schedule type, support [normal, crontab], 0 for normal and 1 for crontab
+private Integer scheduleType;
+// time unit for offline task schedule interval, support [month, week,
day, hour, minute, oneway]
+
(inlong) branch rebase created (now 05c04c80c9)
This is an automated email from the ASF dual-hosted git repository. aloyszhang pushed a change to branch rebase in repository https://gitbox.apache.org/repos/asf/inlong.git at 05c04c80c9 [INLONG-9862][Manager] Support submit flink job for offline sync (#9865) This branch includes the following new commits: new 84511f056f [INLONG-9781][Manager] Add offline sync task type definition (#9787) new ce64702219 [INLONG-9813][Manager] Support offline data sync management (#9814) new 045e04b768 [INLONG-9822][Manager] Support flink job runtime execution mode configuration (#9823) new 05c04c80c9 [INLONG-9862][Manager] Support submit flink job for offline sync (#9865) The 4 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
(inlong) 02/04: [INLONG-9813][Manager] Support offline data sync management (#9814)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch rebase
in repository https://gitbox.apache.org/repos/asf/inlong.git
commit ce64702219a9bb840496da58a2e0ce6fa5492d7e
Author: AloysZhang
AuthorDate: Wed Mar 13 16:18:04 2024 +0800
[INLONG-9813][Manager] Support offline data sync management (#9814)
---
.../inlong/manager/plugin/listener/StartupSortListener.java | 13 +
.../service/listener/group/InitGroupCompleteListener.java | 3 ++-
.../service/listener/group/UpdateGroupCompleteListener.java | 3 ++-
.../service/listener/queue/QueueResourceListener.java | 6 --
.../service/listener/stream/InitStreamCompleteListener.java | 3 ++-
.../manager/service/source/AbstractSourceOperator.java | 1 +
.../manager/service/source/StreamSourceServiceImpl.java | 3 ++-
7 files changed, 22 insertions(+), 10 deletions(-)
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
index ab9e9b55d8..fae5faa278 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
+++
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
@@ -73,7 +73,8 @@ public class StartupSortListener implements
SortOperateListener {
}
log.info("add startup group listener for groupId [{}]", groupId);
-return
InlongConstants.DATASYNC_REALTIME_MODE.equals(groupProcessForm.getGroupInfo().getInlongGroupMode());
+return
(InlongConstants.DATASYNC_REALTIME_MODE.equals(groupProcessForm.getGroupInfo().getInlongGroupMode())
+||
InlongConstants.DATASYNC_OFFLINE_MODE.equals(groupProcessForm.getGroupInfo().getInlongGroupMode()));
}
@Override
@@ -141,9 +142,13 @@ public class StartupSortListener implements
SortOperateListener {
FlinkOperation flinkOperation = FlinkOperation.getInstance();
try {
flinkOperation.genPath(flinkInfo, dataflow);
-flinkOperation.start(flinkInfo);
-log.info("job submit success for groupId = {}, streamId = {},
jobId = {}", groupId,
-streamInfo.getInlongStreamId(), flinkInfo.getJobId());
+// only start job for real-time mode
+if (InlongConstants.DATASYNC_REALTIME_MODE
+
.equals(groupResourceForm.getGroupInfo().getInlongGroupMode())) {
+flinkOperation.start(flinkInfo);
+log.info("job submit success for groupId = {}, streamId =
{}, jobId = {}", groupId,
+streamInfo.getInlongStreamId(),
flinkInfo.getJobId());
+}
} catch (Exception e) {
flinkInfo.setException(true);
flinkInfo.setExceptionMsg(getExceptionStackMsg(e));
diff --git
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupCompleteListener.java
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupCompleteListener.java
index c96c12c9ff..c8c087320f 100644
---
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupCompleteListener.java
+++
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupCompleteListener.java
@@ -98,7 +98,8 @@ public class InitGroupCompleteListener implements
ProcessEventListener {
// update status of other related configs
if
(InlongConstants.DISABLE_CREATE_RESOURCE.equals(groupInfo.getEnableCreateResource()))
{
-if
(InlongConstants.DATASYNC_REALTIME_MODE.equals(groupInfo.getInlongGroupMode()))
{
+if
(InlongConstants.DATASYNC_REALTIME_MODE.equals(groupInfo.getInlongGroupMode())
+||
InlongConstants.DATASYNC_OFFLINE_MODE.equals(groupInfo.getInlongGroupMode())) {
sourceService.updateStatus(groupId, null,
SourceStatus.SOURCE_NORMAL.getCode(), operator);
} else {
sourceService.updateStatus(groupId, null,
SourceStatus.TO_BE_ISSUED_ADD.getCode(), operator);
diff --git
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/UpdateGroupCompleteListener.java
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/UpdateGroupCompleteListener.java
index a6dd8fc786..1a6e1f7fe1 100644
---
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/UpdateGroupCompleteListener.java
+++
b/inlong-manager/man
(inlong) 01/04: [INLONG-9781][Manager] Add offline sync task type definition (#9787)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch rebase
in repository https://gitbox.apache.org/repos/asf/inlong.git
commit 84511f056fa66a0f3ca008d9621587c80231f582
Author: AloysZhang
AuthorDate: Thu Mar 7 11:44:20 2024 +0800
[INLONG-9781][Manager] Add offline sync task type definition (#9787)
---
.../java/org/apache/inlong/manager/client/ut/BaseTest.java | 2 +-
.../inlong/manager/common/consts/InlongConstants.java | 3 ++-
.../org/apache/inlong/manager/common/enums/GroupMode.java | 14 +++---
.../manager/plugin/listener/StartupSortListener.java | 2 +-
.../apache/inlong/manager/pojo/group/InlongGroupInfo.java | 3 ++-
.../inlong/manager/pojo/group/InlongGroupPageRequest.java | 3 ++-
.../inlong/manager/pojo/group/InlongGroupRequest.java | 5 +++--
.../inlong/manager/service/core/impl/AuditServiceImpl.java | 6 --
.../manager/service/group/InlongGroupServiceImpl.java | 2 +-
.../service/listener/group/InitGroupCompleteListener.java | 2 +-
.../listener/group/UpdateGroupCompleteListener.java| 2 +-
.../service/listener/queue/QueueResourceListener.java | 2 +-
.../listener/stream/InitStreamCompleteListener.java| 2 +-
.../manager/service/source/AbstractSourceOperator.java | 2 +-
.../manager/service/source/StreamSourceServiceImpl.java| 2 +-
15 files changed, 33 insertions(+), 19 deletions(-)
diff --git
a/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/ut/BaseTest.java
b/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/ut/BaseTest.java
index 4d218918f5..9961c16ef8 100644
---
a/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/ut/BaseTest.java
+++
b/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/ut/BaseTest.java
@@ -110,7 +110,7 @@ public class BaseTest {
// set enable zk, create resource, group mode, and cluster tag
pulsarInfo.setEnableZookeeper(InlongConstants.DISABLE_ZK);
pulsarInfo.setEnableCreateResource(InlongConstants.ENABLE_CREATE_RESOURCE);
-pulsarInfo.setInlongGroupMode(InlongConstants.DATASYNC_MODE);
+pulsarInfo.setInlongGroupMode(InlongConstants.DATASYNC_REALTIME_MODE);
pulsarInfo.setInlongClusterTag("default_cluster");
pulsarInfo.setDailyRecords(1000);
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
index c3085972fd..581ebb3098 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
@@ -87,7 +87,8 @@ public class InlongConstants {
public static final Integer DELETED_STATUS = 10;
public static final Integer STANDARD_MODE = 0;
-public static final Integer DATASYNC_MODE = 1;
+public static final Integer DATASYNC_REALTIME_MODE = 1;
+public static final Integer DATASYNC_OFFLINE_MODE = 2;
public static final Integer DISABLE_ZK = 0;
public static final Integer ENABLE_ZK = 1;
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java
index f0db2353b6..d4b417f39a 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java
@@ -31,10 +31,18 @@ public enum GroupMode {
STANDARD("standard"),
/**
- * DataSync mode(only Data Synchronization): group init only with sort in
InLong Cluster
- * StreamSource -> Sort -> StreamSink
+ * DataSync mode(only Data Synchronization): real-time data sync in stream
way, group init only with
+ * sort in InLong Cluster.
+ * StreamSource -> Sort -> Sink
*/
-DATASYNC("datasync");
+DATASYNC("datasync"),
+
+/**
+ * DataSync mode(only Data Synchronization): offline data sync in batch
way, group init only with sort
+ * in InLong Cluster.
+ * BatchSource -> Sort -> Sink
+ */
+DATASYNC_BATCH("datasync_offline");
@Getter
private final String mode;
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
index 0b0e55e369..ab9e9b55d8 100644
---
a/inlong-manager/manager-plugins/base/src/main/jav
(inlong) 03/05: [INLONG-9822][Manager] Support flink job runtime execution mode configuration (#9823)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch dev-offline-sync
in repository https://gitbox.apache.org/repos/asf/inlong.git
commit 9024c1b58d521c6aabda9df937d2c15d7f5c6c23
Author: AloysZhang
AuthorDate: Fri Mar 15 15:12:53 2024 +0800
[INLONG-9822][Manager] Support flink job runtime execution mode
configuration (#9823)
---
.../inlong/manager/common/consts/InlongConstants.java | 3 +++
.../apache/inlong/manager/plugin/flink/FlinkService.java | 4
.../apache/inlong/manager/plugin/flink/dto/FlinkInfo.java | 2 ++
.../inlong/manager/plugin/listener/DeleteSortListener.java | 1 +
.../manager/plugin/listener/RestartSortListener.java | 8 +++-
.../manager/plugin/listener/StartupSortListener.java | 14 ++
.../org/apache/inlong/sort/configuration/Constants.java| 8
.../src/main/java/org/apache/inlong/sort/Entrance.java | 11 ++-
8 files changed, 45 insertions(+), 6 deletions(-)
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
index 581ebb3098..e84dcc6602 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
@@ -90,6 +90,9 @@ public class InlongConstants {
public static final Integer DATASYNC_REALTIME_MODE = 1;
public static final Integer DATASYNC_OFFLINE_MODE = 2;
+public static final String RUNTIME_EXECUTION_MODE_STREAMING = "streaming";
+public static final String RUNTIME_EXECUTION_MODE_BATCH = "batch";
+
public static final Integer DISABLE_ZK = 0;
public static final Integer ENABLE_ZK = 1;
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkService.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkService.java
index 129b7aa444..33a7ca59cd 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkService.java
+++
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkService.java
@@ -258,6 +258,10 @@ public class FlinkService {
list.add(flinkInfo.getLocalConfPath());
list.add("-checkpoint.interval");
list.add("6");
+list.add("-runtime.execution.mode");
+list.add(flinkInfo.getRuntimeExecutionMode());
+list.add("-metrics.audit.proxy.hosts");
+list.add(flinkConfig.getAuditProxyHosts());
return list.toArray(new String[0]);
}
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/dto/FlinkInfo.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/dto/FlinkInfo.java
index f7071ceb4b..4c3c75f855 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/dto/FlinkInfo.java
+++
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/dto/FlinkInfo.java
@@ -52,4 +52,6 @@ public class FlinkInfo {
private boolean isException = false;
private String exceptionMsg;
+
+private String runtimeExecutionMode;
}
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/DeleteSortListener.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/DeleteSortListener.java
index 009374e2a6..5e30ad8cb5 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/DeleteSortListener.java
+++
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/DeleteSortListener.java
@@ -119,6 +119,7 @@ public class DeleteSortListener implements
SortOperateListener {
FlinkOperation flinkOperation = FlinkOperation.getInstance();
try {
flinkOperation.delete(flinkInfo);
+// TODO if the job is OFFLINE, should delete the scheduler
information
log.info("job delete success for jobId={}", jobId);
} catch (Exception e) {
flinkInfo.setException(true);
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/RestartSortListener.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/RestartSortListener.java
index 0c6828c984..242138c1e4 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/Restar
(inlong) 01/05: [INLONG-9781][Manager] Add offline sync task type definition (#9787)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch dev-offline-sync
in repository https://gitbox.apache.org/repos/asf/inlong.git
commit 57d7f733c7b113091fd59e9364e4577631d05055
Author: AloysZhang
AuthorDate: Thu Mar 7 11:44:20 2024 +0800
[INLONG-9781][Manager] Add offline sync task type definition (#9787)
---
.../java/org/apache/inlong/manager/client/ut/BaseTest.java | 2 +-
.../inlong/manager/common/consts/InlongConstants.java | 3 ++-
.../org/apache/inlong/manager/common/enums/GroupMode.java | 14 +++---
.../manager/plugin/listener/StartupSortListener.java | 2 +-
.../apache/inlong/manager/pojo/group/InlongGroupInfo.java | 3 ++-
.../inlong/manager/pojo/group/InlongGroupPageRequest.java | 3 ++-
.../inlong/manager/pojo/group/InlongGroupRequest.java | 5 +++--
.../inlong/manager/service/core/impl/AuditServiceImpl.java | 6 --
.../manager/service/group/InlongGroupServiceImpl.java | 2 +-
.../service/listener/group/InitGroupCompleteListener.java | 2 +-
.../listener/group/UpdateGroupCompleteListener.java| 2 +-
.../service/listener/queue/QueueResourceListener.java | 2 +-
.../listener/stream/InitStreamCompleteListener.java| 2 +-
.../manager/service/source/AbstractSourceOperator.java | 2 +-
.../manager/service/source/StreamSourceServiceImpl.java| 2 +-
15 files changed, 33 insertions(+), 19 deletions(-)
diff --git
a/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/ut/BaseTest.java
b/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/ut/BaseTest.java
index 4d218918f5..9961c16ef8 100644
---
a/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/ut/BaseTest.java
+++
b/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/ut/BaseTest.java
@@ -110,7 +110,7 @@ public class BaseTest {
// set enable zk, create resource, group mode, and cluster tag
pulsarInfo.setEnableZookeeper(InlongConstants.DISABLE_ZK);
pulsarInfo.setEnableCreateResource(InlongConstants.ENABLE_CREATE_RESOURCE);
-pulsarInfo.setInlongGroupMode(InlongConstants.DATASYNC_MODE);
+pulsarInfo.setInlongGroupMode(InlongConstants.DATASYNC_REALTIME_MODE);
pulsarInfo.setInlongClusterTag("default_cluster");
pulsarInfo.setDailyRecords(1000);
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
index c3085972fd..581ebb3098 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
@@ -87,7 +87,8 @@ public class InlongConstants {
public static final Integer DELETED_STATUS = 10;
public static final Integer STANDARD_MODE = 0;
-public static final Integer DATASYNC_MODE = 1;
+public static final Integer DATASYNC_REALTIME_MODE = 1;
+public static final Integer DATASYNC_OFFLINE_MODE = 2;
public static final Integer DISABLE_ZK = 0;
public static final Integer ENABLE_ZK = 1;
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java
index f0db2353b6..d4b417f39a 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java
@@ -31,10 +31,18 @@ public enum GroupMode {
STANDARD("standard"),
/**
- * DataSync mode(only Data Synchronization): group init only with sort in
InLong Cluster
- * StreamSource -> Sort -> StreamSink
+ * DataSync mode(only Data Synchronization): real-time data sync in stream
way, group init only with
+ * sort in InLong Cluster.
+ * StreamSource -> Sort -> Sink
*/
-DATASYNC("datasync");
+DATASYNC("datasync"),
+
+/**
+ * DataSync mode(only Data Synchronization): offline data sync in batch
way, group init only with sort
+ * in InLong Cluster.
+ * BatchSource -> Sort -> Sink
+ */
+DATASYNC_BATCH("datasync_offline");
@Getter
private final String mode;
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
index 0b0e55e369..ab9e9b55d8 100644
---
a/inlong-manager/manager-
(inlong) 04/05: [INLONG-9862][Manager] Support submit flink job for offline sync (#9865)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch dev-offline-sync
in repository https://gitbox.apache.org/repos/asf/inlong.git
commit 6996891c69316e07bc75faa545e1f49bb06f8892
Author: AloysZhang
AuthorDate: Fri Mar 22 18:24:26 2024 +0800
[INLONG-9862][Manager] Support submit flink job for offline sync (#9865)
---
.../manager/common/consts/InlongConstants.java | 4 +
.../plugin/listener/StartupSortListener.java | 28 ---
.../manager/pojo/sort/util/StreamParseUtils.java | 21 +
.../listener/StreamTaskListenerFactory.java| 28 +++
.../schedule/StreamScheduleResourceListener.java | 95 ++
.../service/listener/sort/SortConfigListener.java | 8 ++
.../stream/CreateStreamWorkflowDefinition.java | 13 ++-
.../workflow/definition/ServiceTaskType.java | 1 +
.../task/ScheduleOperateListener.java} | 36
.../manager/workflow/plugin/ProcessPlugin.java | 5 ++
10 files changed, 204 insertions(+), 35 deletions(-)
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
index e84dcc6602..f0156cd0e9 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
@@ -141,6 +141,10 @@ public class InlongConstants {
*/
public static final String DATAFLOW = "dataflow";
+public static final String REGISTER_SCHEDULE_STATUS =
"register.schedule.status";
+
+public static final String REGISTERED = "registered";
+
public static final String STREAMS = "streams";
public static final String RELATIONS = "relations";
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
index f6b3b6061a..fec87280d1 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
+++
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
@@ -26,6 +26,7 @@ import org.apache.inlong.manager.plugin.flink.FlinkOperation;
import org.apache.inlong.manager.plugin.flink.dto.FlinkInfo;
import org.apache.inlong.manager.plugin.flink.enums.Constants;
import org.apache.inlong.manager.pojo.sink.StreamSink;
+import org.apache.inlong.manager.pojo.sort.util.StreamParseUtils;
import org.apache.inlong.manager.pojo.stream.InlongStreamExtInfo;
import org.apache.inlong.manager.pojo.stream.InlongStreamInfo;
import
org.apache.inlong.manager.pojo.workflow.form.process.GroupResourceProcessForm;
@@ -98,6 +99,14 @@ public class StartupSortListener implements
SortOperateListener {
}
for (InlongStreamInfo streamInfo : streamInfos) {
+boolean isOfflineSync = InlongConstants.DATASYNC_OFFLINE_MODE
+
.equals(groupResourceForm.getGroupInfo().getInlongGroupMode());
+// do not submit flink job if the group mode is offline and the
stream is not config successfully
+if (isOfflineSync &&
!StreamParseUtils.isRegisterScheduleSuccess(streamInfo)) {
+log.info("no need to submit flink job for groupId={}
streamId={} as the mode is offline "
++ "and the stream is not config successfully yet",
groupId, streamInfo.getInlongStreamId());
+continue;
+}
List sinkList = streamInfo.getSinkList();
List sinkTypes =
sinkList.stream().map(StreamSink::getSinkType).collect(Collectors.toList());
if (CollectionUtils.isEmpty(sinkList) ||
!SinkType.containSortFlinkSink(sinkTypes)) {
@@ -131,9 +140,6 @@ public class StartupSortListener implements
SortOperateListener {
return ListenerResult.fail(message);
}
-boolean isRealTimeSync = InlongConstants.DATASYNC_REALTIME_MODE
-
.equals(groupResourceForm.getGroupInfo().getInlongGroupMode());
-
FlinkInfo flinkInfo = new FlinkInfo();
String jobName =
Constants.SORT_JOB_NAME_GENERATOR.apply(processForm) + InlongConstants.HYPHEN
@@ -141,20 +147,18 @@ public class StartupSortListener implements
SortOperateListener {
flinkInfo.setJobName(jobName);
flinkInfo.setEndpoint(kvConf.get(InlongConstants.SORT_URL));
flinkInfo.setInlongStreamInfoList(Collections.singletonList(streamInfo));
-if (isRealTimeSync) {
-
flin
(inlong) 02/04: [INLONG-9813][Manager] Support offline data sync management (#9814)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch rebase
in repository https://gitbox.apache.org/repos/asf/inlong.git
commit dfb22f4d8e1016de28a4818dd5739827f152b844
Author: AloysZhang
AuthorDate: Wed Mar 13 16:18:04 2024 +0800
[INLONG-9813][Manager] Support offline data sync management (#9814)
---
.../inlong/manager/plugin/listener/StartupSortListener.java | 13 +
.../service/listener/group/InitGroupCompleteListener.java | 3 ++-
.../service/listener/group/UpdateGroupCompleteListener.java | 3 ++-
.../service/listener/queue/QueueResourceListener.java | 6 --
.../service/listener/stream/InitStreamCompleteListener.java | 3 ++-
.../manager/service/source/AbstractSourceOperator.java | 1 +
.../manager/service/source/StreamSourceServiceImpl.java | 3 ++-
7 files changed, 22 insertions(+), 10 deletions(-)
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
index ab9e9b55d8..fae5faa278 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
+++
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
@@ -73,7 +73,8 @@ public class StartupSortListener implements
SortOperateListener {
}
log.info("add startup group listener for groupId [{}]", groupId);
-return
InlongConstants.DATASYNC_REALTIME_MODE.equals(groupProcessForm.getGroupInfo().getInlongGroupMode());
+return
(InlongConstants.DATASYNC_REALTIME_MODE.equals(groupProcessForm.getGroupInfo().getInlongGroupMode())
+||
InlongConstants.DATASYNC_OFFLINE_MODE.equals(groupProcessForm.getGroupInfo().getInlongGroupMode()));
}
@Override
@@ -141,9 +142,13 @@ public class StartupSortListener implements
SortOperateListener {
FlinkOperation flinkOperation = FlinkOperation.getInstance();
try {
flinkOperation.genPath(flinkInfo, dataflow);
-flinkOperation.start(flinkInfo);
-log.info("job submit success for groupId = {}, streamId = {},
jobId = {}", groupId,
-streamInfo.getInlongStreamId(), flinkInfo.getJobId());
+// only start job for real-time mode
+if (InlongConstants.DATASYNC_REALTIME_MODE
+
.equals(groupResourceForm.getGroupInfo().getInlongGroupMode())) {
+flinkOperation.start(flinkInfo);
+log.info("job submit success for groupId = {}, streamId =
{}, jobId = {}", groupId,
+streamInfo.getInlongStreamId(),
flinkInfo.getJobId());
+}
} catch (Exception e) {
flinkInfo.setException(true);
flinkInfo.setExceptionMsg(getExceptionStackMsg(e));
diff --git
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupCompleteListener.java
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupCompleteListener.java
index c96c12c9ff..c8c087320f 100644
---
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupCompleteListener.java
+++
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupCompleteListener.java
@@ -98,7 +98,8 @@ public class InitGroupCompleteListener implements
ProcessEventListener {
// update status of other related configs
if
(InlongConstants.DISABLE_CREATE_RESOURCE.equals(groupInfo.getEnableCreateResource()))
{
-if
(InlongConstants.DATASYNC_REALTIME_MODE.equals(groupInfo.getInlongGroupMode()))
{
+if
(InlongConstants.DATASYNC_REALTIME_MODE.equals(groupInfo.getInlongGroupMode())
+||
InlongConstants.DATASYNC_OFFLINE_MODE.equals(groupInfo.getInlongGroupMode())) {
sourceService.updateStatus(groupId, null,
SourceStatus.SOURCE_NORMAL.getCode(), operator);
} else {
sourceService.updateStatus(groupId, null,
SourceStatus.TO_BE_ISSUED_ADD.getCode(), operator);
diff --git
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/UpdateGroupCompleteListener.java
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/UpdateGroupCompleteListener.java
index a6dd8fc786..1a6e1f7fe1 100644
---
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/UpdateGroupCompleteListener.java
+++
b/inlong-manager/man
(inlong) 03/04: [INLONG-9822][Manager] Support flink job runtime execution mode configuration (#9823)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch rebase
in repository https://gitbox.apache.org/repos/asf/inlong.git
commit ff3c351401b2bf0c349792915e733c18ffd01386
Author: AloysZhang
AuthorDate: Fri Mar 15 15:12:53 2024 +0800
[INLONG-9822][Manager] Support flink job runtime execution mode
configuration (#9823)
---
.../inlong/manager/common/consts/InlongConstants.java | 3 +++
.../apache/inlong/manager/plugin/flink/FlinkService.java | 2 ++
.../apache/inlong/manager/plugin/flink/dto/FlinkInfo.java | 2 ++
.../inlong/manager/plugin/listener/DeleteSortListener.java | 1 +
.../manager/plugin/listener/RestartSortListener.java | 8 +++-
.../manager/plugin/listener/StartupSortListener.java | 14 ++
.../org/apache/inlong/sort/configuration/Constants.java| 8
.../src/main/java/org/apache/inlong/sort/Entrance.java | 11 ++-
8 files changed, 43 insertions(+), 6 deletions(-)
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
index 581ebb3098..e84dcc6602 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
@@ -90,6 +90,9 @@ public class InlongConstants {
public static final Integer DATASYNC_REALTIME_MODE = 1;
public static final Integer DATASYNC_OFFLINE_MODE = 2;
+public static final String RUNTIME_EXECUTION_MODE_STREAMING = "streaming";
+public static final String RUNTIME_EXECUTION_MODE_BATCH = "batch";
+
public static final Integer DISABLE_ZK = 0;
public static final Integer ENABLE_ZK = 1;
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkService.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkService.java
index 129b7aa444..9188c6b73f 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkService.java
+++
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkService.java
@@ -258,6 +258,8 @@ public class FlinkService {
list.add(flinkInfo.getLocalConfPath());
list.add("-checkpoint.interval");
list.add("6");
+list.add("-runtime.execution.mode");
+list.add(flinkInfo.getRuntimeExecutionMode());
return list.toArray(new String[0]);
}
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/dto/FlinkInfo.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/dto/FlinkInfo.java
index f7071ceb4b..4c3c75f855 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/dto/FlinkInfo.java
+++
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/dto/FlinkInfo.java
@@ -52,4 +52,6 @@ public class FlinkInfo {
private boolean isException = false;
private String exceptionMsg;
+
+private String runtimeExecutionMode;
}
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/DeleteSortListener.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/DeleteSortListener.java
index 009374e2a6..5e30ad8cb5 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/DeleteSortListener.java
+++
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/DeleteSortListener.java
@@ -119,6 +119,7 @@ public class DeleteSortListener implements
SortOperateListener {
FlinkOperation flinkOperation = FlinkOperation.getInstance();
try {
flinkOperation.delete(flinkInfo);
+// TODO if the job is OFFLINE, should delete the scheduler
information
log.info("job delete success for jobId={}", jobId);
} catch (Exception e) {
flinkInfo.setException(true);
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/RestartSortListener.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/RestartSortListener.java
index 0c6828c984..242138c1e4 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/RestartSortListener.java
+++
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/liste
(inlong) branch rebase updated (05c04c80c9 -> 65d3a928ea)
This is an automated email from the ASF dual-hosted git repository. aloyszhang pushed a change to branch rebase in repository https://gitbox.apache.org/repos/asf/inlong.git discard 05c04c80c9 [INLONG-9862][Manager] Support submit flink job for offline sync (#9865) discard 045e04b768 [INLONG-9822][Manager] Support flink job runtime execution mode configuration (#9823) discard ce64702219 [INLONG-9813][Manager] Support offline data sync management (#9814) discard 84511f056f [INLONG-9781][Manager] Add offline sync task type definition (#9787) new 1e0fddc0a8 [INLONG-9781][Manager] Add offline sync task type definition (#9787) new dfb22f4d8e [INLONG-9813][Manager] Support offline data sync management (#9814) new ff3c351401 [INLONG-9822][Manager] Support flink job runtime execution mode configuration (#9823) new 65d3a928ea [INLONG-9862][Manager] Support submit flink job for offline sync (#9865) This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (05c04c80c9) \ N -- N -- N refs/heads/rebase (65d3a928ea) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. The 4 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference. Summary of changes: .../main/java/org/apache/inlong/manager/plugin/flink/FlinkService.java | 2 -- 1 file changed, 2 deletions(-)
(inlong) 04/04: [INLONG-9862][Manager] Support submit flink job for offline sync (#9865)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch rebase
in repository https://gitbox.apache.org/repos/asf/inlong.git
commit 65d3a928ea702b41613a60b917bab1d59152f23a
Author: AloysZhang
AuthorDate: Fri Mar 22 18:24:26 2024 +0800
[INLONG-9862][Manager] Support submit flink job for offline sync (#9865)
---
.../manager/common/consts/InlongConstants.java | 4 +
.../plugin/listener/StartupSortListener.java | 28 ---
.../manager/pojo/sort/util/StreamParseUtils.java | 21 +
.../listener/StreamTaskListenerFactory.java| 28 +++
.../schedule/StreamScheduleResourceListener.java | 95 ++
.../service/listener/sort/SortConfigListener.java | 8 ++
.../stream/CreateStreamWorkflowDefinition.java | 13 ++-
.../workflow/definition/ServiceTaskType.java | 1 +
.../task/ScheduleOperateListener.java} | 36
.../manager/workflow/plugin/ProcessPlugin.java | 5 ++
10 files changed, 204 insertions(+), 35 deletions(-)
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
index e84dcc6602..f0156cd0e9 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
@@ -141,6 +141,10 @@ public class InlongConstants {
*/
public static final String DATAFLOW = "dataflow";
+public static final String REGISTER_SCHEDULE_STATUS =
"register.schedule.status";
+
+public static final String REGISTERED = "registered";
+
public static final String STREAMS = "streams";
public static final String RELATIONS = "relations";
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
index f6b3b6061a..fec87280d1 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
+++
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
@@ -26,6 +26,7 @@ import org.apache.inlong.manager.plugin.flink.FlinkOperation;
import org.apache.inlong.manager.plugin.flink.dto.FlinkInfo;
import org.apache.inlong.manager.plugin.flink.enums.Constants;
import org.apache.inlong.manager.pojo.sink.StreamSink;
+import org.apache.inlong.manager.pojo.sort.util.StreamParseUtils;
import org.apache.inlong.manager.pojo.stream.InlongStreamExtInfo;
import org.apache.inlong.manager.pojo.stream.InlongStreamInfo;
import
org.apache.inlong.manager.pojo.workflow.form.process.GroupResourceProcessForm;
@@ -98,6 +99,14 @@ public class StartupSortListener implements
SortOperateListener {
}
for (InlongStreamInfo streamInfo : streamInfos) {
+boolean isOfflineSync = InlongConstants.DATASYNC_OFFLINE_MODE
+
.equals(groupResourceForm.getGroupInfo().getInlongGroupMode());
+// do not submit flink job if the group mode is offline and the
stream is not config successfully
+if (isOfflineSync &&
!StreamParseUtils.isRegisterScheduleSuccess(streamInfo)) {
+log.info("no need to submit flink job for groupId={}
streamId={} as the mode is offline "
++ "and the stream is not config successfully yet",
groupId, streamInfo.getInlongStreamId());
+continue;
+}
List sinkList = streamInfo.getSinkList();
List sinkTypes =
sinkList.stream().map(StreamSink::getSinkType).collect(Collectors.toList());
if (CollectionUtils.isEmpty(sinkList) ||
!SinkType.containSortFlinkSink(sinkTypes)) {
@@ -131,9 +140,6 @@ public class StartupSortListener implements
SortOperateListener {
return ListenerResult.fail(message);
}
-boolean isRealTimeSync = InlongConstants.DATASYNC_REALTIME_MODE
-
.equals(groupResourceForm.getGroupInfo().getInlongGroupMode());
-
FlinkInfo flinkInfo = new FlinkInfo();
String jobName =
Constants.SORT_JOB_NAME_GENERATOR.apply(processForm) + InlongConstants.HYPHEN
@@ -141,20 +147,18 @@ public class StartupSortListener implements
SortOperateListener {
flinkInfo.setJobName(jobName);
flinkInfo.setEndpoint(kvConf.get(InlongConstants.SORT_URL));
flinkInfo.setInlongStreamInfoList(Collections.singletonList(streamInfo));
-if (isRealTimeSync) {
-
flin
(inlong) 01/04: [INLONG-9781][Manager] Add offline sync task type definition (#9787)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch rebase
in repository https://gitbox.apache.org/repos/asf/inlong.git
commit 1e0fddc0a8d6ab9134f7b6dd7dfafb3d08fd3a91
Author: AloysZhang
AuthorDate: Thu Mar 7 11:44:20 2024 +0800
[INLONG-9781][Manager] Add offline sync task type definition (#9787)
---
.../java/org/apache/inlong/manager/client/ut/BaseTest.java | 2 +-
.../inlong/manager/common/consts/InlongConstants.java | 3 ++-
.../org/apache/inlong/manager/common/enums/GroupMode.java | 14 +++---
.../manager/plugin/listener/StartupSortListener.java | 2 +-
.../apache/inlong/manager/pojo/group/InlongGroupInfo.java | 3 ++-
.../inlong/manager/pojo/group/InlongGroupPageRequest.java | 3 ++-
.../inlong/manager/pojo/group/InlongGroupRequest.java | 5 +++--
.../inlong/manager/service/core/impl/AuditServiceImpl.java | 6 --
.../manager/service/group/InlongGroupServiceImpl.java | 2 +-
.../service/listener/group/InitGroupCompleteListener.java | 2 +-
.../listener/group/UpdateGroupCompleteListener.java| 2 +-
.../service/listener/queue/QueueResourceListener.java | 2 +-
.../listener/stream/InitStreamCompleteListener.java| 2 +-
.../manager/service/source/AbstractSourceOperator.java | 2 +-
.../manager/service/source/StreamSourceServiceImpl.java| 2 +-
15 files changed, 33 insertions(+), 19 deletions(-)
diff --git
a/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/ut/BaseTest.java
b/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/ut/BaseTest.java
index 4d218918f5..9961c16ef8 100644
---
a/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/ut/BaseTest.java
+++
b/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/ut/BaseTest.java
@@ -110,7 +110,7 @@ public class BaseTest {
// set enable zk, create resource, group mode, and cluster tag
pulsarInfo.setEnableZookeeper(InlongConstants.DISABLE_ZK);
pulsarInfo.setEnableCreateResource(InlongConstants.ENABLE_CREATE_RESOURCE);
-pulsarInfo.setInlongGroupMode(InlongConstants.DATASYNC_MODE);
+pulsarInfo.setInlongGroupMode(InlongConstants.DATASYNC_REALTIME_MODE);
pulsarInfo.setInlongClusterTag("default_cluster");
pulsarInfo.setDailyRecords(1000);
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
index c3085972fd..581ebb3098 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
@@ -87,7 +87,8 @@ public class InlongConstants {
public static final Integer DELETED_STATUS = 10;
public static final Integer STANDARD_MODE = 0;
-public static final Integer DATASYNC_MODE = 1;
+public static final Integer DATASYNC_REALTIME_MODE = 1;
+public static final Integer DATASYNC_OFFLINE_MODE = 2;
public static final Integer DISABLE_ZK = 0;
public static final Integer ENABLE_ZK = 1;
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java
index f0db2353b6..d4b417f39a 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java
@@ -31,10 +31,18 @@ public enum GroupMode {
STANDARD("standard"),
/**
- * DataSync mode(only Data Synchronization): group init only with sort in
InLong Cluster
- * StreamSource -> Sort -> StreamSink
+ * DataSync mode(only Data Synchronization): real-time data sync in stream
way, group init only with
+ * sort in InLong Cluster.
+ * StreamSource -> Sort -> Sink
*/
-DATASYNC("datasync");
+DATASYNC("datasync"),
+
+/**
+ * DataSync mode(only Data Synchronization): offline data sync in batch
way, group init only with sort
+ * in InLong Cluster.
+ * BatchSource -> Sort -> Sink
+ */
+DATASYNC_BATCH("datasync_offline");
@Getter
private final String mode;
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
index 0b0e55e369..ab9e9b55d8 100644
---
a/inlong-manager/manager-plugins/base/src/main/jav
(inlong) branch rebase deleted (was 65d3a928ea)
This is an automated email from the ASF dual-hosted git repository. aloyszhang pushed a change to branch rebase in repository https://gitbox.apache.org/repos/asf/inlong.git was 65d3a928ea [INLONG-9862][Manager] Support submit flink job for offline sync (#9865) This change permanently discards the following revisions: discard 65d3a928ea [INLONG-9862][Manager] Support submit flink job for offline sync (#9865) discard ff3c351401 [INLONG-9822][Manager] Support flink job runtime execution mode configuration (#9823) discard dfb22f4d8e [INLONG-9813][Manager] Support offline data sync management (#9814) discard 1e0fddc0a8 [INLONG-9781][Manager] Add offline sync task type definition (#9787)
(inlong) branch dev-offline-sync deleted (was 2932887cb5)
This is an automated email from the ASF dual-hosted git repository. aloyszhang pushed a change to branch dev-offline-sync in repository https://gitbox.apache.org/repos/asf/inlong.git was 2932887cb5 Support scheudle infomartion management This change permanently discards the following revisions: discard 2932887cb5 Support scheudle infomartion management discard 6996891c69 [INLONG-9862][Manager] Support submit flink job for offline sync (#9865) discard 9024c1b58d [INLONG-9822][Manager] Support flink job runtime execution mode configuration (#9823) discard 06a84650c7 [INLONG-9813][Manager] Support offline data sync management (#9814) discard 57d7f733c7 [INLONG-9781][Manager] Add offline sync task type definition (#9787)
(inlong) 02/04: [INLONG-9813][Manager] Support offline data sync management (#9814)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch rebase
in repository https://gitbox.apache.org/repos/asf/inlong.git
commit dfb22f4d8e1016de28a4818dd5739827f152b844
Author: AloysZhang
AuthorDate: Wed Mar 13 16:18:04 2024 +0800
[INLONG-9813][Manager] Support offline data sync management (#9814)
---
.../inlong/manager/plugin/listener/StartupSortListener.java | 13 +
.../service/listener/group/InitGroupCompleteListener.java | 3 ++-
.../service/listener/group/UpdateGroupCompleteListener.java | 3 ++-
.../service/listener/queue/QueueResourceListener.java | 6 --
.../service/listener/stream/InitStreamCompleteListener.java | 3 ++-
.../manager/service/source/AbstractSourceOperator.java | 1 +
.../manager/service/source/StreamSourceServiceImpl.java | 3 ++-
7 files changed, 22 insertions(+), 10 deletions(-)
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
index ab9e9b55d8..fae5faa278 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
+++
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
@@ -73,7 +73,8 @@ public class StartupSortListener implements
SortOperateListener {
}
log.info("add startup group listener for groupId [{}]", groupId);
-return
InlongConstants.DATASYNC_REALTIME_MODE.equals(groupProcessForm.getGroupInfo().getInlongGroupMode());
+return
(InlongConstants.DATASYNC_REALTIME_MODE.equals(groupProcessForm.getGroupInfo().getInlongGroupMode())
+||
InlongConstants.DATASYNC_OFFLINE_MODE.equals(groupProcessForm.getGroupInfo().getInlongGroupMode()));
}
@Override
@@ -141,9 +142,13 @@ public class StartupSortListener implements
SortOperateListener {
FlinkOperation flinkOperation = FlinkOperation.getInstance();
try {
flinkOperation.genPath(flinkInfo, dataflow);
-flinkOperation.start(flinkInfo);
-log.info("job submit success for groupId = {}, streamId = {},
jobId = {}", groupId,
-streamInfo.getInlongStreamId(), flinkInfo.getJobId());
+// only start job for real-time mode
+if (InlongConstants.DATASYNC_REALTIME_MODE
+
.equals(groupResourceForm.getGroupInfo().getInlongGroupMode())) {
+flinkOperation.start(flinkInfo);
+log.info("job submit success for groupId = {}, streamId =
{}, jobId = {}", groupId,
+streamInfo.getInlongStreamId(),
flinkInfo.getJobId());
+}
} catch (Exception e) {
flinkInfo.setException(true);
flinkInfo.setExceptionMsg(getExceptionStackMsg(e));
diff --git
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupCompleteListener.java
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupCompleteListener.java
index c96c12c9ff..c8c087320f 100644
---
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupCompleteListener.java
+++
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupCompleteListener.java
@@ -98,7 +98,8 @@ public class InitGroupCompleteListener implements
ProcessEventListener {
// update status of other related configs
if
(InlongConstants.DISABLE_CREATE_RESOURCE.equals(groupInfo.getEnableCreateResource()))
{
-if
(InlongConstants.DATASYNC_REALTIME_MODE.equals(groupInfo.getInlongGroupMode()))
{
+if
(InlongConstants.DATASYNC_REALTIME_MODE.equals(groupInfo.getInlongGroupMode())
+||
InlongConstants.DATASYNC_OFFLINE_MODE.equals(groupInfo.getInlongGroupMode())) {
sourceService.updateStatus(groupId, null,
SourceStatus.SOURCE_NORMAL.getCode(), operator);
} else {
sourceService.updateStatus(groupId, null,
SourceStatus.TO_BE_ISSUED_ADD.getCode(), operator);
diff --git
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/UpdateGroupCompleteListener.java
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/UpdateGroupCompleteListener.java
index a6dd8fc786..1a6e1f7fe1 100644
---
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/UpdateGroupCompleteListener.java
+++
b/inlong-manager/man
(inlong) 01/04: [INLONG-9781][Manager] Add offline sync task type definition (#9787)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch rebase
in repository https://gitbox.apache.org/repos/asf/inlong.git
commit 1e0fddc0a8d6ab9134f7b6dd7dfafb3d08fd3a91
Author: AloysZhang
AuthorDate: Thu Mar 7 11:44:20 2024 +0800
[INLONG-9781][Manager] Add offline sync task type definition (#9787)
---
.../java/org/apache/inlong/manager/client/ut/BaseTest.java | 2 +-
.../inlong/manager/common/consts/InlongConstants.java | 3 ++-
.../org/apache/inlong/manager/common/enums/GroupMode.java | 14 +++---
.../manager/plugin/listener/StartupSortListener.java | 2 +-
.../apache/inlong/manager/pojo/group/InlongGroupInfo.java | 3 ++-
.../inlong/manager/pojo/group/InlongGroupPageRequest.java | 3 ++-
.../inlong/manager/pojo/group/InlongGroupRequest.java | 5 +++--
.../inlong/manager/service/core/impl/AuditServiceImpl.java | 6 --
.../manager/service/group/InlongGroupServiceImpl.java | 2 +-
.../service/listener/group/InitGroupCompleteListener.java | 2 +-
.../listener/group/UpdateGroupCompleteListener.java| 2 +-
.../service/listener/queue/QueueResourceListener.java | 2 +-
.../listener/stream/InitStreamCompleteListener.java| 2 +-
.../manager/service/source/AbstractSourceOperator.java | 2 +-
.../manager/service/source/StreamSourceServiceImpl.java| 2 +-
15 files changed, 33 insertions(+), 19 deletions(-)
diff --git
a/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/ut/BaseTest.java
b/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/ut/BaseTest.java
index 4d218918f5..9961c16ef8 100644
---
a/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/ut/BaseTest.java
+++
b/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/ut/BaseTest.java
@@ -110,7 +110,7 @@ public class BaseTest {
// set enable zk, create resource, group mode, and cluster tag
pulsarInfo.setEnableZookeeper(InlongConstants.DISABLE_ZK);
pulsarInfo.setEnableCreateResource(InlongConstants.ENABLE_CREATE_RESOURCE);
-pulsarInfo.setInlongGroupMode(InlongConstants.DATASYNC_MODE);
+pulsarInfo.setInlongGroupMode(InlongConstants.DATASYNC_REALTIME_MODE);
pulsarInfo.setInlongClusterTag("default_cluster");
pulsarInfo.setDailyRecords(1000);
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
index c3085972fd..581ebb3098 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
@@ -87,7 +87,8 @@ public class InlongConstants {
public static final Integer DELETED_STATUS = 10;
public static final Integer STANDARD_MODE = 0;
-public static final Integer DATASYNC_MODE = 1;
+public static final Integer DATASYNC_REALTIME_MODE = 1;
+public static final Integer DATASYNC_OFFLINE_MODE = 2;
public static final Integer DISABLE_ZK = 0;
public static final Integer ENABLE_ZK = 1;
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java
index f0db2353b6..d4b417f39a 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java
@@ -31,10 +31,18 @@ public enum GroupMode {
STANDARD("standard"),
/**
- * DataSync mode(only Data Synchronization): group init only with sort in
InLong Cluster
- * StreamSource -> Sort -> StreamSink
+ * DataSync mode(only Data Synchronization): real-time data sync in stream
way, group init only with
+ * sort in InLong Cluster.
+ * StreamSource -> Sort -> Sink
*/
-DATASYNC("datasync");
+DATASYNC("datasync"),
+
+/**
+ * DataSync mode(only Data Synchronization): offline data sync in batch
way, group init only with sort
+ * in InLong Cluster.
+ * BatchSource -> Sort -> Sink
+ */
+DATASYNC_BATCH("datasync_offline");
@Getter
private final String mode;
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
index 0b0e55e369..ab9e9b55d8 100644
---
a/inlong-manager/manager-plugins/base/src/main/jav
(inlong) branch rebase created (now 65d3a928ea)
This is an automated email from the ASF dual-hosted git repository. aloyszhang pushed a change to branch rebase in repository https://gitbox.apache.org/repos/asf/inlong.git at 65d3a928ea [INLONG-9862][Manager] Support submit flink job for offline sync (#9865) This branch includes the following new commits: new 1e0fddc0a8 [INLONG-9781][Manager] Add offline sync task type definition (#9787) new dfb22f4d8e [INLONG-9813][Manager] Support offline data sync management (#9814) new ff3c351401 [INLONG-9822][Manager] Support flink job runtime execution mode configuration (#9823) new 65d3a928ea [INLONG-9862][Manager] Support submit flink job for offline sync (#9865) The 4 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
(inlong) 03/04: [INLONG-9822][Manager] Support flink job runtime execution mode configuration (#9823)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch rebase
in repository https://gitbox.apache.org/repos/asf/inlong.git
commit ff3c351401b2bf0c349792915e733c18ffd01386
Author: AloysZhang
AuthorDate: Fri Mar 15 15:12:53 2024 +0800
[INLONG-9822][Manager] Support flink job runtime execution mode
configuration (#9823)
---
.../inlong/manager/common/consts/InlongConstants.java | 3 +++
.../apache/inlong/manager/plugin/flink/FlinkService.java | 2 ++
.../apache/inlong/manager/plugin/flink/dto/FlinkInfo.java | 2 ++
.../inlong/manager/plugin/listener/DeleteSortListener.java | 1 +
.../manager/plugin/listener/RestartSortListener.java | 8 +++-
.../manager/plugin/listener/StartupSortListener.java | 14 ++
.../org/apache/inlong/sort/configuration/Constants.java| 8
.../src/main/java/org/apache/inlong/sort/Entrance.java | 11 ++-
8 files changed, 43 insertions(+), 6 deletions(-)
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
index 581ebb3098..e84dcc6602 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
@@ -90,6 +90,9 @@ public class InlongConstants {
public static final Integer DATASYNC_REALTIME_MODE = 1;
public static final Integer DATASYNC_OFFLINE_MODE = 2;
+public static final String RUNTIME_EXECUTION_MODE_STREAMING = "streaming";
+public static final String RUNTIME_EXECUTION_MODE_BATCH = "batch";
+
public static final Integer DISABLE_ZK = 0;
public static final Integer ENABLE_ZK = 1;
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkService.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkService.java
index 129b7aa444..9188c6b73f 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkService.java
+++
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkService.java
@@ -258,6 +258,8 @@ public class FlinkService {
list.add(flinkInfo.getLocalConfPath());
list.add("-checkpoint.interval");
list.add("6");
+list.add("-runtime.execution.mode");
+list.add(flinkInfo.getRuntimeExecutionMode());
return list.toArray(new String[0]);
}
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/dto/FlinkInfo.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/dto/FlinkInfo.java
index f7071ceb4b..4c3c75f855 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/dto/FlinkInfo.java
+++
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/dto/FlinkInfo.java
@@ -52,4 +52,6 @@ public class FlinkInfo {
private boolean isException = false;
private String exceptionMsg;
+
+private String runtimeExecutionMode;
}
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/DeleteSortListener.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/DeleteSortListener.java
index 009374e2a6..5e30ad8cb5 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/DeleteSortListener.java
+++
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/DeleteSortListener.java
@@ -119,6 +119,7 @@ public class DeleteSortListener implements
SortOperateListener {
FlinkOperation flinkOperation = FlinkOperation.getInstance();
try {
flinkOperation.delete(flinkInfo);
+// TODO if the job is OFFLINE, should delete the scheduler
information
log.info("job delete success for jobId={}", jobId);
} catch (Exception e) {
flinkInfo.setException(true);
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/RestartSortListener.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/RestartSortListener.java
index 0c6828c984..242138c1e4 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/RestartSortListener.java
+++
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/liste
(inlong) 04/04: [INLONG-9862][Manager] Support submit flink job for offline sync (#9865)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch rebase
in repository https://gitbox.apache.org/repos/asf/inlong.git
commit 65d3a928ea702b41613a60b917bab1d59152f23a
Author: AloysZhang
AuthorDate: Fri Mar 22 18:24:26 2024 +0800
[INLONG-9862][Manager] Support submit flink job for offline sync (#9865)
---
.../manager/common/consts/InlongConstants.java | 4 +
.../plugin/listener/StartupSortListener.java | 28 ---
.../manager/pojo/sort/util/StreamParseUtils.java | 21 +
.../listener/StreamTaskListenerFactory.java| 28 +++
.../schedule/StreamScheduleResourceListener.java | 95 ++
.../service/listener/sort/SortConfigListener.java | 8 ++
.../stream/CreateStreamWorkflowDefinition.java | 13 ++-
.../workflow/definition/ServiceTaskType.java | 1 +
.../task/ScheduleOperateListener.java} | 36
.../manager/workflow/plugin/ProcessPlugin.java | 5 ++
10 files changed, 204 insertions(+), 35 deletions(-)
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
index e84dcc6602..f0156cd0e9 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
@@ -141,6 +141,10 @@ public class InlongConstants {
*/
public static final String DATAFLOW = "dataflow";
+public static final String REGISTER_SCHEDULE_STATUS =
"register.schedule.status";
+
+public static final String REGISTERED = "registered";
+
public static final String STREAMS = "streams";
public static final String RELATIONS = "relations";
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
index f6b3b6061a..fec87280d1 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
+++
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
@@ -26,6 +26,7 @@ import org.apache.inlong.manager.plugin.flink.FlinkOperation;
import org.apache.inlong.manager.plugin.flink.dto.FlinkInfo;
import org.apache.inlong.manager.plugin.flink.enums.Constants;
import org.apache.inlong.manager.pojo.sink.StreamSink;
+import org.apache.inlong.manager.pojo.sort.util.StreamParseUtils;
import org.apache.inlong.manager.pojo.stream.InlongStreamExtInfo;
import org.apache.inlong.manager.pojo.stream.InlongStreamInfo;
import
org.apache.inlong.manager.pojo.workflow.form.process.GroupResourceProcessForm;
@@ -98,6 +99,14 @@ public class StartupSortListener implements
SortOperateListener {
}
for (InlongStreamInfo streamInfo : streamInfos) {
+boolean isOfflineSync = InlongConstants.DATASYNC_OFFLINE_MODE
+
.equals(groupResourceForm.getGroupInfo().getInlongGroupMode());
+// do not submit flink job if the group mode is offline and the
stream is not config successfully
+if (isOfflineSync &&
!StreamParseUtils.isRegisterScheduleSuccess(streamInfo)) {
+log.info("no need to submit flink job for groupId={}
streamId={} as the mode is offline "
++ "and the stream is not config successfully yet",
groupId, streamInfo.getInlongStreamId());
+continue;
+}
List sinkList = streamInfo.getSinkList();
List sinkTypes =
sinkList.stream().map(StreamSink::getSinkType).collect(Collectors.toList());
if (CollectionUtils.isEmpty(sinkList) ||
!SinkType.containSortFlinkSink(sinkTypes)) {
@@ -131,9 +140,6 @@ public class StartupSortListener implements
SortOperateListener {
return ListenerResult.fail(message);
}
-boolean isRealTimeSync = InlongConstants.DATASYNC_REALTIME_MODE
-
.equals(groupResourceForm.getGroupInfo().getInlongGroupMode());
-
FlinkInfo flinkInfo = new FlinkInfo();
String jobName =
Constants.SORT_JOB_NAME_GENERATOR.apply(processForm) + InlongConstants.HYPHEN
@@ -141,20 +147,18 @@ public class StartupSortListener implements
SortOperateListener {
flinkInfo.setJobName(jobName);
flinkInfo.setEndpoint(kvConf.get(InlongConstants.SORT_URL));
flinkInfo.setInlongStreamInfoList(Collections.singletonList(streamInfo));
-if (isRealTimeSync) {
-
flin
(inlong) branch dev-offline-sync created (now 2932887cb5)
This is an automated email from the ASF dual-hosted git repository. aloyszhang pushed a change to branch dev-offline-sync in repository https://gitbox.apache.org/repos/asf/inlong.git at 2932887cb5 Support scheudle infomartion management This branch includes the following new commits: new 57d7f733c7 [INLONG-9781][Manager] Add offline sync task type definition (#9787) new 06a84650c7 [INLONG-9813][Manager] Support offline data sync management (#9814) new 9024c1b58d [INLONG-9822][Manager] Support flink job runtime execution mode configuration (#9823) new 6996891c69 [INLONG-9862][Manager] Support submit flink job for offline sync (#9865) new 2932887cb5 Support scheudle infomartion management The 5 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
(inlong) 05/05: Support scheudle infomartion management
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch dev-offline-sync
in repository https://gitbox.apache.org/repos/asf/inlong.git
commit 2932887cb5a5da63b2498b4bb0f97513d05f5aa9
Author: aloyszhang
AuthorDate: Thu May 16 16:37:56 2024 +0800
Support scheudle infomartion management
---
.../inlong/manager/common/enums/ErrorCodeEnum.java | 3 +
.../manager/common/enums/OperationTarget.java | 4 +-
.../manager/common/enums/ScheduleStatus.java | 29 +
.../inlong/manager/dao/entity/ScheduleEntity.java | 41 +++
.../manager/dao/mapper/ScheduleEntityMapper.java | 16 +++
.../resources/mappers/ScheduleEntityMapper.xml | 120 +
.../inlong/manager/pojo/schedule/ScheduleInfo.java | 57 ++
.../manager/pojo/schedule/ScheduleInfoRequest.java | 56 ++
.../manager/service/schedule/ScheduleService.java | 45
.../service/schedule/ScheduleServiceImpl.java | 96 +
.../manager-web/sql/apache_inlong_manager.sql | 38 +++
.../web/controller/InLongSchedulerController.java | 65 +++
12 files changed, 569 insertions(+), 1 deletion(-)
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/ErrorCodeEnum.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/ErrorCodeEnum.java
index 98a7f50bd7..ce3bf9d015 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/ErrorCodeEnum.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/ErrorCodeEnum.java
@@ -125,6 +125,9 @@ public enum ErrorCodeEnum {
MQ_TYPE_IS_NULL(1600, "MQ type is null"),
MQ_TYPE_NOT_SUPPORT(1601, "MQ type '%s' not support"),
+SCHEDULE_NOT_FOUND(1700, "Schedule info not found"),
+SCHEDULE_DUPLICATE(1701, "Schedule info already exist"),
+
WORKFLOW_EXE_FAILED(4000, "Workflow execution exception"),
WORKFLOW_APPROVER_NOT_FOUND(4001, "Workflow approver does not exist/no
operation authority"),
WORKFLOW_DELETE_RECORD_FAILED(4002, "Workflow delete record failure"),
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/OperationTarget.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/OperationTarget.java
index e6c866a0e1..93d4e1f92c 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/OperationTarget.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/OperationTarget.java
@@ -44,6 +44,8 @@ public enum OperationTarget {
INLONG_ROLE,
-TENANT_ROLE
+TENANT_ROLE,
+
+SCHEDULE
}
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/ScheduleStatus.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/ScheduleStatus.java
new file mode 100644
index 00..818c8277a2
--- /dev/null
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/ScheduleStatus.java
@@ -0,0 +1,29 @@
+package org.apache.inlong.manager.common.enums;
+
+import lombok.Getter;
+
+@Getter
+public enum ScheduleStatus {
+
+NEW(100, "new"),
+DELETED(40, "deleted");
+
+private final Integer code;
+private final String description;
+
+ScheduleStatus(Integer code, String description) {
+this.code = code;
+this.description = description;
+}
+
+
+public static ScheduleStatus forCode(int code) {
+for (ScheduleStatus status : values()) {
+if (status.getCode() == code) {
+return status;
+}
+}
+throw new IllegalStateException(String.format("Illegal code=%s for
ScheduleStatus", code));
+}
+
+}
diff --git
a/inlong-manager/manager-dao/src/main/java/org/apache/inlong/manager/dao/entity/ScheduleEntity.java
b/inlong-manager/manager-dao/src/main/java/org/apache/inlong/manager/dao/entity/ScheduleEntity.java
new file mode 100644
index 00..5502e8c57d
--- /dev/null
+++
b/inlong-manager/manager-dao/src/main/java/org/apache/inlong/manager/dao/entity/ScheduleEntity.java
@@ -0,0 +1,41 @@
+package org.apache.inlong.manager.dao.entity;
+
+import java.io.Serializable;
+import java.util.Date;
+import lombok.Data;
+
+@Data
+public class ScheduleEntity implements Serializable {
+private static final long serialVersionUID = 1L;
+
+private Integer id;
+// inLong group id
+private String groupId;
+// schedule type, support [normal, crontab], 0 for normal and 1 for crontab
+private Integer scheduleType;
+// time unit for offline task schedule interval, support [month, week,
day, hour, minute, oneway]
+
(inlong) 03/05: [INLONG-9822][Manager] Support flink job runtime execution mode configuration (#9823)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch dev-offline-sync
in repository https://gitbox.apache.org/repos/asf/inlong.git
commit 9024c1b58d521c6aabda9df937d2c15d7f5c6c23
Author: AloysZhang
AuthorDate: Fri Mar 15 15:12:53 2024 +0800
[INLONG-9822][Manager] Support flink job runtime execution mode
configuration (#9823)
---
.../inlong/manager/common/consts/InlongConstants.java | 3 +++
.../apache/inlong/manager/plugin/flink/FlinkService.java | 4
.../apache/inlong/manager/plugin/flink/dto/FlinkInfo.java | 2 ++
.../inlong/manager/plugin/listener/DeleteSortListener.java | 1 +
.../manager/plugin/listener/RestartSortListener.java | 8 +++-
.../manager/plugin/listener/StartupSortListener.java | 14 ++
.../org/apache/inlong/sort/configuration/Constants.java| 8
.../src/main/java/org/apache/inlong/sort/Entrance.java | 11 ++-
8 files changed, 45 insertions(+), 6 deletions(-)
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
index 581ebb3098..e84dcc6602 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
@@ -90,6 +90,9 @@ public class InlongConstants {
public static final Integer DATASYNC_REALTIME_MODE = 1;
public static final Integer DATASYNC_OFFLINE_MODE = 2;
+public static final String RUNTIME_EXECUTION_MODE_STREAMING = "streaming";
+public static final String RUNTIME_EXECUTION_MODE_BATCH = "batch";
+
public static final Integer DISABLE_ZK = 0;
public static final Integer ENABLE_ZK = 1;
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkService.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkService.java
index 129b7aa444..33a7ca59cd 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkService.java
+++
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkService.java
@@ -258,6 +258,10 @@ public class FlinkService {
list.add(flinkInfo.getLocalConfPath());
list.add("-checkpoint.interval");
list.add("6");
+list.add("-runtime.execution.mode");
+list.add(flinkInfo.getRuntimeExecutionMode());
+list.add("-metrics.audit.proxy.hosts");
+list.add(flinkConfig.getAuditProxyHosts());
return list.toArray(new String[0]);
}
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/dto/FlinkInfo.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/dto/FlinkInfo.java
index f7071ceb4b..4c3c75f855 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/dto/FlinkInfo.java
+++
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/dto/FlinkInfo.java
@@ -52,4 +52,6 @@ public class FlinkInfo {
private boolean isException = false;
private String exceptionMsg;
+
+private String runtimeExecutionMode;
}
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/DeleteSortListener.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/DeleteSortListener.java
index 009374e2a6..5e30ad8cb5 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/DeleteSortListener.java
+++
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/DeleteSortListener.java
@@ -119,6 +119,7 @@ public class DeleteSortListener implements
SortOperateListener {
FlinkOperation flinkOperation = FlinkOperation.getInstance();
try {
flinkOperation.delete(flinkInfo);
+// TODO if the job is OFFLINE, should delete the scheduler
information
log.info("job delete success for jobId={}", jobId);
} catch (Exception e) {
flinkInfo.setException(true);
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/RestartSortListener.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/RestartSortListener.java
index 0c6828c984..242138c1e4 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/Restar
(inlong) 01/05: [INLONG-9781][Manager] Add offline sync task type definition (#9787)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch dev-offline-sync
in repository https://gitbox.apache.org/repos/asf/inlong.git
commit 57d7f733c7b113091fd59e9364e4577631d05055
Author: AloysZhang
AuthorDate: Thu Mar 7 11:44:20 2024 +0800
[INLONG-9781][Manager] Add offline sync task type definition (#9787)
---
.../java/org/apache/inlong/manager/client/ut/BaseTest.java | 2 +-
.../inlong/manager/common/consts/InlongConstants.java | 3 ++-
.../org/apache/inlong/manager/common/enums/GroupMode.java | 14 +++---
.../manager/plugin/listener/StartupSortListener.java | 2 +-
.../apache/inlong/manager/pojo/group/InlongGroupInfo.java | 3 ++-
.../inlong/manager/pojo/group/InlongGroupPageRequest.java | 3 ++-
.../inlong/manager/pojo/group/InlongGroupRequest.java | 5 +++--
.../inlong/manager/service/core/impl/AuditServiceImpl.java | 6 --
.../manager/service/group/InlongGroupServiceImpl.java | 2 +-
.../service/listener/group/InitGroupCompleteListener.java | 2 +-
.../listener/group/UpdateGroupCompleteListener.java| 2 +-
.../service/listener/queue/QueueResourceListener.java | 2 +-
.../listener/stream/InitStreamCompleteListener.java| 2 +-
.../manager/service/source/AbstractSourceOperator.java | 2 +-
.../manager/service/source/StreamSourceServiceImpl.java| 2 +-
15 files changed, 33 insertions(+), 19 deletions(-)
diff --git
a/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/ut/BaseTest.java
b/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/ut/BaseTest.java
index 4d218918f5..9961c16ef8 100644
---
a/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/ut/BaseTest.java
+++
b/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/ut/BaseTest.java
@@ -110,7 +110,7 @@ public class BaseTest {
// set enable zk, create resource, group mode, and cluster tag
pulsarInfo.setEnableZookeeper(InlongConstants.DISABLE_ZK);
pulsarInfo.setEnableCreateResource(InlongConstants.ENABLE_CREATE_RESOURCE);
-pulsarInfo.setInlongGroupMode(InlongConstants.DATASYNC_MODE);
+pulsarInfo.setInlongGroupMode(InlongConstants.DATASYNC_REALTIME_MODE);
pulsarInfo.setInlongClusterTag("default_cluster");
pulsarInfo.setDailyRecords(1000);
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
index c3085972fd..581ebb3098 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
@@ -87,7 +87,8 @@ public class InlongConstants {
public static final Integer DELETED_STATUS = 10;
public static final Integer STANDARD_MODE = 0;
-public static final Integer DATASYNC_MODE = 1;
+public static final Integer DATASYNC_REALTIME_MODE = 1;
+public static final Integer DATASYNC_OFFLINE_MODE = 2;
public static final Integer DISABLE_ZK = 0;
public static final Integer ENABLE_ZK = 1;
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java
index f0db2353b6..d4b417f39a 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java
@@ -31,10 +31,18 @@ public enum GroupMode {
STANDARD("standard"),
/**
- * DataSync mode(only Data Synchronization): group init only with sort in
InLong Cluster
- * StreamSource -> Sort -> StreamSink
+ * DataSync mode(only Data Synchronization): real-time data sync in stream
way, group init only with
+ * sort in InLong Cluster.
+ * StreamSource -> Sort -> Sink
*/
-DATASYNC("datasync");
+DATASYNC("datasync"),
+
+/**
+ * DataSync mode(only Data Synchronization): offline data sync in batch
way, group init only with sort
+ * in InLong Cluster.
+ * BatchSource -> Sort -> Sink
+ */
+DATASYNC_BATCH("datasync_offline");
@Getter
private final String mode;
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
index 0b0e55e369..ab9e9b55d8 100644
---
a/inlong-manager/manager-
(inlong) 04/05: [INLONG-9862][Manager] Support submit flink job for offline sync (#9865)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch dev-offline-sync
in repository https://gitbox.apache.org/repos/asf/inlong.git
commit 6996891c69316e07bc75faa545e1f49bb06f8892
Author: AloysZhang
AuthorDate: Fri Mar 22 18:24:26 2024 +0800
[INLONG-9862][Manager] Support submit flink job for offline sync (#9865)
---
.../manager/common/consts/InlongConstants.java | 4 +
.../plugin/listener/StartupSortListener.java | 28 ---
.../manager/pojo/sort/util/StreamParseUtils.java | 21 +
.../listener/StreamTaskListenerFactory.java| 28 +++
.../schedule/StreamScheduleResourceListener.java | 95 ++
.../service/listener/sort/SortConfigListener.java | 8 ++
.../stream/CreateStreamWorkflowDefinition.java | 13 ++-
.../workflow/definition/ServiceTaskType.java | 1 +
.../task/ScheduleOperateListener.java} | 36
.../manager/workflow/plugin/ProcessPlugin.java | 5 ++
10 files changed, 204 insertions(+), 35 deletions(-)
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
index e84dcc6602..f0156cd0e9 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
@@ -141,6 +141,10 @@ public class InlongConstants {
*/
public static final String DATAFLOW = "dataflow";
+public static final String REGISTER_SCHEDULE_STATUS =
"register.schedule.status";
+
+public static final String REGISTERED = "registered";
+
public static final String STREAMS = "streams";
public static final String RELATIONS = "relations";
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
index f6b3b6061a..fec87280d1 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
+++
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
@@ -26,6 +26,7 @@ import org.apache.inlong.manager.plugin.flink.FlinkOperation;
import org.apache.inlong.manager.plugin.flink.dto.FlinkInfo;
import org.apache.inlong.manager.plugin.flink.enums.Constants;
import org.apache.inlong.manager.pojo.sink.StreamSink;
+import org.apache.inlong.manager.pojo.sort.util.StreamParseUtils;
import org.apache.inlong.manager.pojo.stream.InlongStreamExtInfo;
import org.apache.inlong.manager.pojo.stream.InlongStreamInfo;
import
org.apache.inlong.manager.pojo.workflow.form.process.GroupResourceProcessForm;
@@ -98,6 +99,14 @@ public class StartupSortListener implements
SortOperateListener {
}
for (InlongStreamInfo streamInfo : streamInfos) {
+boolean isOfflineSync = InlongConstants.DATASYNC_OFFLINE_MODE
+
.equals(groupResourceForm.getGroupInfo().getInlongGroupMode());
+// do not submit flink job if the group mode is offline and the
stream is not config successfully
+if (isOfflineSync &&
!StreamParseUtils.isRegisterScheduleSuccess(streamInfo)) {
+log.info("no need to submit flink job for groupId={}
streamId={} as the mode is offline "
++ "and the stream is not config successfully yet",
groupId, streamInfo.getInlongStreamId());
+continue;
+}
List sinkList = streamInfo.getSinkList();
List sinkTypes =
sinkList.stream().map(StreamSink::getSinkType).collect(Collectors.toList());
if (CollectionUtils.isEmpty(sinkList) ||
!SinkType.containSortFlinkSink(sinkTypes)) {
@@ -131,9 +140,6 @@ public class StartupSortListener implements
SortOperateListener {
return ListenerResult.fail(message);
}
-boolean isRealTimeSync = InlongConstants.DATASYNC_REALTIME_MODE
-
.equals(groupResourceForm.getGroupInfo().getInlongGroupMode());
-
FlinkInfo flinkInfo = new FlinkInfo();
String jobName =
Constants.SORT_JOB_NAME_GENERATOR.apply(processForm) + InlongConstants.HYPHEN
@@ -141,20 +147,18 @@ public class StartupSortListener implements
SortOperateListener {
flinkInfo.setJobName(jobName);
flinkInfo.setEndpoint(kvConf.get(InlongConstants.SORT_URL));
flinkInfo.setInlongStreamInfoList(Collections.singletonList(streamInfo));
-if (isRealTimeSync) {
-
flin
(inlong) 02/05: [INLONG-9813][Manager] Support offline data sync management (#9814)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch dev-offline-sync
in repository https://gitbox.apache.org/repos/asf/inlong.git
commit 06a84650c7c5eceb86183f35b24a9486c9a5ca60
Author: AloysZhang
AuthorDate: Wed Mar 13 16:18:04 2024 +0800
[INLONG-9813][Manager] Support offline data sync management (#9814)
---
.../inlong/manager/plugin/listener/StartupSortListener.java | 13 +
.../service/listener/group/InitGroupCompleteListener.java | 3 ++-
.../service/listener/group/UpdateGroupCompleteListener.java | 3 ++-
.../service/listener/queue/QueueResourceListener.java | 6 --
.../service/listener/stream/InitStreamCompleteListener.java | 3 ++-
.../manager/service/source/AbstractSourceOperator.java | 1 +
.../manager/service/source/StreamSourceServiceImpl.java | 3 ++-
7 files changed, 22 insertions(+), 10 deletions(-)
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
index ab9e9b55d8..fae5faa278 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
+++
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
@@ -73,7 +73,8 @@ public class StartupSortListener implements
SortOperateListener {
}
log.info("add startup group listener for groupId [{}]", groupId);
-return
InlongConstants.DATASYNC_REALTIME_MODE.equals(groupProcessForm.getGroupInfo().getInlongGroupMode());
+return
(InlongConstants.DATASYNC_REALTIME_MODE.equals(groupProcessForm.getGroupInfo().getInlongGroupMode())
+||
InlongConstants.DATASYNC_OFFLINE_MODE.equals(groupProcessForm.getGroupInfo().getInlongGroupMode()));
}
@Override
@@ -141,9 +142,13 @@ public class StartupSortListener implements
SortOperateListener {
FlinkOperation flinkOperation = FlinkOperation.getInstance();
try {
flinkOperation.genPath(flinkInfo, dataflow);
-flinkOperation.start(flinkInfo);
-log.info("job submit success for groupId = {}, streamId = {},
jobId = {}", groupId,
-streamInfo.getInlongStreamId(), flinkInfo.getJobId());
+// only start job for real-time mode
+if (InlongConstants.DATASYNC_REALTIME_MODE
+
.equals(groupResourceForm.getGroupInfo().getInlongGroupMode())) {
+flinkOperation.start(flinkInfo);
+log.info("job submit success for groupId = {}, streamId =
{}, jobId = {}", groupId,
+streamInfo.getInlongStreamId(),
flinkInfo.getJobId());
+}
} catch (Exception e) {
flinkInfo.setException(true);
flinkInfo.setExceptionMsg(getExceptionStackMsg(e));
diff --git
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupCompleteListener.java
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupCompleteListener.java
index c96c12c9ff..c8c087320f 100644
---
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupCompleteListener.java
+++
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupCompleteListener.java
@@ -98,7 +98,8 @@ public class InitGroupCompleteListener implements
ProcessEventListener {
// update status of other related configs
if
(InlongConstants.DISABLE_CREATE_RESOURCE.equals(groupInfo.getEnableCreateResource()))
{
-if
(InlongConstants.DATASYNC_REALTIME_MODE.equals(groupInfo.getInlongGroupMode()))
{
+if
(InlongConstants.DATASYNC_REALTIME_MODE.equals(groupInfo.getInlongGroupMode())
+||
InlongConstants.DATASYNC_OFFLINE_MODE.equals(groupInfo.getInlongGroupMode())) {
sourceService.updateStatus(groupId, null,
SourceStatus.SOURCE_NORMAL.getCode(), operator);
} else {
sourceService.updateStatus(groupId, null,
SourceStatus.TO_BE_ISSUED_ADD.getCode(), operator);
diff --git
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/UpdateGroupCompleteListener.java
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/UpdateGroupCompleteListener.java
index a6dd8fc786..1a6e1f7fe1 100644
---
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/UpdateGroupCompleteListener.java
+++
b/inlong-ma
(inlong) branch rebase deleted (was 65d3a928ea)
This is an automated email from the ASF dual-hosted git repository. aloyszhang pushed a change to branch rebase in repository https://gitbox.apache.org/repos/asf/inlong.git was 65d3a928ea [INLONG-9862][Manager] Support submit flink job for offline sync (#9865) The revisions that were on this branch are still contained in other references; therefore, this change does not discard any commits from the repository.
(inlong) branch dev-offline-sync updated (2932887cb5 -> 65d3a928ea)
This is an automated email from the ASF dual-hosted git repository. aloyszhang pushed a change to branch dev-offline-sync in repository https://gitbox.apache.org/repos/asf/inlong.git discard 2932887cb5 Support scheudle infomartion management discard 6996891c69 [INLONG-9862][Manager] Support submit flink job for offline sync (#9865) discard 9024c1b58d [INLONG-9822][Manager] Support flink job runtime execution mode configuration (#9823) discard 06a84650c7 [INLONG-9813][Manager] Support offline data sync management (#9814) discard 57d7f733c7 [INLONG-9781][Manager] Add offline sync task type definition (#9787) add 1e0fddc0a8 [INLONG-9781][Manager] Add offline sync task type definition (#9787) add dfb22f4d8e [INLONG-9813][Manager] Support offline data sync management (#9814) add ff3c351401 [INLONG-9822][Manager] Support flink job runtime execution mode configuration (#9823) add 65d3a928ea [INLONG-9862][Manager] Support submit flink job for offline sync (#9865) This update added new revisions after undoing existing revisions. That is to say, some revisions that were in the old version of the branch are not in the new version. This situation occurs when a user --force pushes a change and generates a repository containing something like this: * -- * -- B -- O -- O -- O (2932887cb5) \ N -- N -- N refs/heads/dev-offline-sync (65d3a928ea) You should already have received notification emails for all of the O revisions, and so the following emails describe only the N revisions from the common base, B. Any revisions marked "omit" are not gone; other references still refer to them. Any revisions marked "discard" are gone forever. No new revisions were added by this update. Summary of changes: .../inlong/manager/common/enums/ErrorCodeEnum.java | 3 - .../manager/common/enums/OperationTarget.java | 4 +- .../manager/common/enums/ScheduleStatus.java | 29 - .../inlong/manager/dao/entity/ScheduleEntity.java | 41 --- .../manager/dao/mapper/ScheduleEntityMapper.java | 16 --- .../resources/mappers/ScheduleEntityMapper.xml | 120 - .../inlong/manager/plugin/flink/FlinkService.java | 2 - .../inlong/manager/pojo/schedule/ScheduleInfo.java | 57 -- .../manager/pojo/schedule/ScheduleInfoRequest.java | 56 -- .../manager/service/schedule/ScheduleService.java | 45 .../service/schedule/ScheduleServiceImpl.java | 96 - .../manager-web/sql/apache_inlong_manager.sql | 38 --- .../web/controller/InLongSchedulerController.java | 65 --- 13 files changed, 1 insertion(+), 571 deletions(-) delete mode 100644 inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/ScheduleStatus.java delete mode 100644 inlong-manager/manager-dao/src/main/java/org/apache/inlong/manager/dao/entity/ScheduleEntity.java delete mode 100644 inlong-manager/manager-dao/src/main/java/org/apache/inlong/manager/dao/mapper/ScheduleEntityMapper.java delete mode 100644 inlong-manager/manager-dao/src/main/resources/mappers/ScheduleEntityMapper.xml delete mode 100644 inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/schedule/ScheduleInfo.java delete mode 100644 inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/schedule/ScheduleInfoRequest.java delete mode 100644 inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/schedule/ScheduleService.java delete mode 100644 inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/schedule/ScheduleServiceImpl.java delete mode 100644 inlong-manager/manager-web/src/main/java/org/apache/inlong/manager/web/controller/InLongSchedulerController.java
(inlong) 03/05: [INLONG-9822][Manager] Support flink job runtime execution mode configuration (#9823)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch dev-offline-sync
in repository https://gitbox.apache.org/repos/asf/inlong.git
commit ed21473a97181420df50719c64d96200b0bec136
Author: AloysZhang
AuthorDate: Fri Mar 15 15:12:53 2024 +0800
[INLONG-9822][Manager] Support flink job runtime execution mode
configuration (#9823)
---
.../apache/inlong/manager/common/consts/InlongConstants.java | 3 +++
.../org/apache/inlong/manager/plugin/flink/FlinkService.java | 2 ++
.../org/apache/inlong/manager/plugin/flink/dto/FlinkInfo.java | 2 ++
.../inlong/manager/plugin/listener/DeleteSortListener.java| 1 +
.../inlong/manager/plugin/listener/RestartSortListener.java | 8 +++-
.../java/org/apache/inlong/sort/configuration/Constants.java | 8
.../src/main/java/org/apache/inlong/sort/Entrance.java| 11 ++-
7 files changed, 33 insertions(+), 2 deletions(-)
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
index 581ebb3098..e84dcc6602 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
@@ -90,6 +90,9 @@ public class InlongConstants {
public static final Integer DATASYNC_REALTIME_MODE = 1;
public static final Integer DATASYNC_OFFLINE_MODE = 2;
+public static final String RUNTIME_EXECUTION_MODE_STREAMING = "streaming";
+public static final String RUNTIME_EXECUTION_MODE_BATCH = "batch";
+
public static final Integer DISABLE_ZK = 0;
public static final Integer ENABLE_ZK = 1;
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkService.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkService.java
index 129b7aa444..9188c6b73f 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkService.java
+++
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/FlinkService.java
@@ -258,6 +258,8 @@ public class FlinkService {
list.add(flinkInfo.getLocalConfPath());
list.add("-checkpoint.interval");
list.add("6");
+list.add("-runtime.execution.mode");
+list.add(flinkInfo.getRuntimeExecutionMode());
return list.toArray(new String[0]);
}
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/dto/FlinkInfo.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/dto/FlinkInfo.java
index f7071ceb4b..4c3c75f855 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/dto/FlinkInfo.java
+++
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/flink/dto/FlinkInfo.java
@@ -52,4 +52,6 @@ public class FlinkInfo {
private boolean isException = false;
private String exceptionMsg;
+
+private String runtimeExecutionMode;
}
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/DeleteSortListener.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/DeleteSortListener.java
index 009374e2a6..5e30ad8cb5 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/DeleteSortListener.java
+++
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/DeleteSortListener.java
@@ -119,6 +119,7 @@ public class DeleteSortListener implements
SortOperateListener {
FlinkOperation flinkOperation = FlinkOperation.getInstance();
try {
flinkOperation.delete(flinkInfo);
+// TODO if the job is OFFLINE, should delete the scheduler
information
log.info("job delete success for jobId={}", jobId);
} catch (Exception e) {
flinkInfo.setException(true);
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/RestartSortListener.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/RestartSortListener.java
index 0c6828c984..242138c1e4 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/RestartSortListener.java
+++
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/RestartSortListener.java
@@ -86,7 +86,13 @@ publ
(inlong) 05/05: [INLONG-10247][Manager] Support schedule information management for offline sync (#10254)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch dev-offline-sync
in repository https://gitbox.apache.org/repos/asf/inlong.git
commit b68bf5c5e597bb6870e6b9addb37aa7c3b1d9aab
Author: AloysZhang
AuthorDate: Thu May 23 18:18:31 2024 +0800
[INLONG-10247][Manager] Support schedule information management for
offline sync (#10254)
---
.../client/api/inner/client/ClientFactory.java | 2 +
.../api/inner/client/InLongScheduleClient.java | 67 +
.../client/api/service/InLongScheduleApi.java | 49 +++
.../inlong/manager/common/enums/ErrorCodeEnum.java | 3 +
.../manager/common/enums/OperationTarget.java | 4 +-
.../{OperationTarget.java => ScheduleStatus.java} | 52 ---
.../inlong/manager/dao/entity/ScheduleEntity.java | 61
.../manager/dao/mapper/ScheduleEntityMapper.java} | 34 ++---
.../resources/mappers/ScheduleEntityMapper.xml | 144 +++
.../manager/dao/mapper/ScheduleEntityTest.java | 118
.../inlong/manager/pojo/schedule/ScheduleInfo.java | 82 +++
.../manager/pojo/schedule/ScheduleInfoRequest.java | 76 ++
.../manager/service/schedule/ScheduleService.java | 71 ++
.../service/schedule/ScheduleServiceImpl.java | 155 +
.../main/resources/h2/apache_inlong_manager.sql| 31 +
.../manager-web/sql/apache_inlong_manager.sql | 30
inlong-manager/manager-web/sql/changes-1.13.0.sql | 30
.../web/controller/InLongSchedulerController.java | 88
18 files changed, 1045 insertions(+), 52 deletions(-)
diff --git
a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/inner/client/ClientFactory.java
b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/inner/client/ClientFactory.java
index 3c67f1fa71..01c9fb6473 100644
---
a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/inner/client/ClientFactory.java
+++
b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/inner/client/ClientFactory.java
@@ -55,6 +55,7 @@ public class ClientFactory {
private final AuditClient auditClient;
private final InlongTenantClient inlongTenantClient;
private final InlongTenantRoleClient inlongTenantRoleClient;
+private final InLongScheduleClient inLongScheduleClient;
public ClientFactory(ClientConfiguration configuration) {
groupClient = new InlongGroupClient(configuration);
@@ -74,5 +75,6 @@ public class ClientFactory {
auditClient = new AuditClient(configuration);
inlongTenantClient = new InlongTenantClient(configuration);
inlongTenantRoleClient = new InlongTenantRoleClient(configuration);
+inLongScheduleClient = new InLongScheduleClient(configuration);
}
}
diff --git
a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/inner/client/InLongScheduleClient.java
b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/inner/client/InLongScheduleClient.java
new file mode 100644
index 00..86638dbae8
--- /dev/null
+++
b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/inner/client/InLongScheduleClient.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.manager.client.api.inner.client;
+
+import org.apache.inlong.manager.client.api.ClientConfiguration;
+import org.apache.inlong.manager.client.api.service.InLongScheduleApi;
+import org.apache.inlong.manager.client.api.util.ClientUtils;
+import org.apache.inlong.manager.common.enums.ErrorCodeEnum;
+import org.apache.inlong.manager.common.util.Preconditions;
+import org.apache.inlong.manager.pojo.common.Response;
+import org.apache.inlong.manager.pojo.schedule.ScheduleInfo;
+import org.apache.inlong.manager.pojo.schedule.ScheduleInfoRequest;
+
+public class InLongScheduleClient {
+
+private InLongScheduleApi scheduleApi;
+
+public InLongScheduleClient(ClientConfiguration clientConfiguration) {
+scheduleApi =
Cli
(inlong) 04/05: [INLONG-9862][Manager] Support submit flink job for offline sync (#9865)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch dev-offline-sync
in repository https://gitbox.apache.org/repos/asf/inlong.git
commit b5279c097b53e8bd1ad35cace748fa0646063bf3
Author: AloysZhang
AuthorDate: Fri Mar 22 18:24:26 2024 +0800
[INLONG-9862][Manager] Support submit flink job for offline sync (#9865)
---
.../manager/common/consts/InlongConstants.java | 4 +
.../manager/pojo/sort/util/StreamParseUtils.java | 21 +
.../listener/StreamTaskListenerFactory.java| 28 +++
.../schedule/StreamScheduleResourceListener.java | 95 ++
.../service/listener/sort/SortConfigListener.java | 8 ++
.../stream/CreateStreamWorkflowDefinition.java | 13 ++-
.../workflow/definition/ServiceTaskType.java | 1 +
.../task/ScheduleOperateListener.java} | 36
.../manager/workflow/plugin/ProcessPlugin.java | 5 ++
9 files changed, 188 insertions(+), 23 deletions(-)
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
index e84dcc6602..f0156cd0e9 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
@@ -141,6 +141,10 @@ public class InlongConstants {
*/
public static final String DATAFLOW = "dataflow";
+public static final String REGISTER_SCHEDULE_STATUS =
"register.schedule.status";
+
+public static final String REGISTERED = "registered";
+
public static final String STREAMS = "streams";
public static final String RELATIONS = "relations";
diff --git
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/util/StreamParseUtils.java
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/util/StreamParseUtils.java
index 92470968ac..203bcc6778 100644
---
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/util/StreamParseUtils.java
+++
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/util/StreamParseUtils.java
@@ -17,11 +17,14 @@
package org.apache.inlong.manager.pojo.sort.util;
+import org.apache.inlong.manager.common.consts.InlongConstants;
import org.apache.inlong.manager.common.enums.ErrorCodeEnum;
import org.apache.inlong.manager.common.enums.TransformType;
import org.apache.inlong.manager.common.util.Preconditions;
import org.apache.inlong.manager.pojo.sink.StreamSink;
import org.apache.inlong.manager.pojo.source.StreamSource;
+import org.apache.inlong.manager.pojo.stream.InlongStreamExtInfo;
+import org.apache.inlong.manager.pojo.stream.InlongStreamInfo;
import org.apache.inlong.manager.pojo.stream.StreamNode;
import org.apache.inlong.manager.pojo.stream.StreamPipeline;
import org.apache.inlong.manager.pojo.stream.StreamTransform;
@@ -38,6 +41,8 @@ import
org.apache.inlong.manager.pojo.transform.splitter.SplitterDefinition;
import com.google.gson.Gson;
import com.google.gson.JsonObject;
+import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
/**
* Utils of stream parse.
@@ -154,4 +159,20 @@ public class StreamParseUtils {
return GSON.fromJson(tempView, StreamPipeline.class);
}
+public static String getStreamExtProperty(String key, InlongStreamInfo
streamInfo) {
+if (StringUtils.isNotBlank(key) && streamInfo != null &&
CollectionUtils.isNotEmpty(streamInfo.getExtList())) {
+for (InlongStreamExtInfo ext : streamInfo.getExtList()) {
+if (key.equalsIgnoreCase(ext.getKeyName())) {
+return ext.getKeyValue();
+}
+}
+}
+return null;
+}
+
+public static boolean isRegisterScheduleSuccess(InlongStreamInfo
streamInfo) {
+return InlongConstants.REGISTERED
+
.equalsIgnoreCase(getStreamExtProperty(InlongConstants.REGISTER_SCHEDULE_STATUS,
streamInfo));
+}
+
}
diff --git
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/StreamTaskListenerFactory.java
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/StreamTaskListenerFactory.java
index 43fdfe4666..87c3a867c2 100644
---
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/StreamTaskListenerFactory.java
+++
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/StreamTaskListenerFactory.java
@@ -20,12 +20,14 @@ package org.apache.inlong.manager.service.listene
(inlong) 01/05: [INLONG-9781][Manager] Add offline sync task type definition (#9787)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch dev-offline-sync
in repository https://gitbox.apache.org/repos/asf/inlong.git
commit a23ad3234b9b3cbb91d10e8965f75664a3a5e308
Author: AloysZhang
AuthorDate: Thu Mar 7 11:44:20 2024 +0800
[INLONG-9781][Manager] Add offline sync task type definition (#9787)
---
.../java/org/apache/inlong/manager/client/ut/BaseTest.java | 2 +-
.../inlong/manager/common/consts/InlongConstants.java | 3 ++-
.../org/apache/inlong/manager/common/enums/GroupMode.java | 14 +++---
.../manager/plugin/listener/StartupSortListener.java | 2 +-
.../apache/inlong/manager/pojo/group/InlongGroupInfo.java | 3 ++-
.../inlong/manager/pojo/group/InlongGroupPageRequest.java | 3 ++-
.../inlong/manager/pojo/group/InlongGroupRequest.java | 5 +++--
.../inlong/manager/service/core/impl/AuditServiceImpl.java | 6 --
.../manager/service/group/InlongGroupServiceImpl.java | 2 +-
.../service/listener/group/InitGroupCompleteListener.java | 2 +-
.../listener/group/UpdateGroupCompleteListener.java| 2 +-
.../service/listener/queue/QueueResourceListener.java | 2 +-
.../listener/stream/InitStreamCompleteListener.java| 2 +-
.../manager/service/source/AbstractSourceOperator.java | 2 +-
.../manager/service/source/StreamSourceServiceImpl.java| 2 +-
15 files changed, 33 insertions(+), 19 deletions(-)
diff --git
a/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/ut/BaseTest.java
b/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/ut/BaseTest.java
index 4d218918f5..9961c16ef8 100644
---
a/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/ut/BaseTest.java
+++
b/inlong-manager/manager-client-examples/src/test/java/org/apache/inlong/manager/client/ut/BaseTest.java
@@ -110,7 +110,7 @@ public class BaseTest {
// set enable zk, create resource, group mode, and cluster tag
pulsarInfo.setEnableZookeeper(InlongConstants.DISABLE_ZK);
pulsarInfo.setEnableCreateResource(InlongConstants.ENABLE_CREATE_RESOURCE);
-pulsarInfo.setInlongGroupMode(InlongConstants.DATASYNC_MODE);
+pulsarInfo.setInlongGroupMode(InlongConstants.DATASYNC_REALTIME_MODE);
pulsarInfo.setInlongClusterTag("default_cluster");
pulsarInfo.setDailyRecords(1000);
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
index c3085972fd..581ebb3098 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/consts/InlongConstants.java
@@ -87,7 +87,8 @@ public class InlongConstants {
public static final Integer DELETED_STATUS = 10;
public static final Integer STANDARD_MODE = 0;
-public static final Integer DATASYNC_MODE = 1;
+public static final Integer DATASYNC_REALTIME_MODE = 1;
+public static final Integer DATASYNC_OFFLINE_MODE = 2;
public static final Integer DISABLE_ZK = 0;
public static final Integer ENABLE_ZK = 1;
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java
index f0db2353b6..d4b417f39a 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/GroupMode.java
@@ -31,10 +31,18 @@ public enum GroupMode {
STANDARD("standard"),
/**
- * DataSync mode(only Data Synchronization): group init only with sort in
InLong Cluster
- * StreamSource -> Sort -> StreamSink
+ * DataSync mode(only Data Synchronization): real-time data sync in stream
way, group init only with
+ * sort in InLong Cluster.
+ * StreamSource -> Sort -> Sink
*/
-DATASYNC("datasync");
+DATASYNC("datasync"),
+
+/**
+ * DataSync mode(only Data Synchronization): offline data sync in batch
way, group init only with sort
+ * in InLong Cluster.
+ * BatchSource -> Sort -> Sink
+ */
+DATASYNC_BATCH("datasync_offline");
@Getter
private final String mode;
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
index 038f35543f..9997c2abd2 100644
---
a/inlong-manager/manager-
(inlong) 02/05: [INLONG-9813][Manager] Support offline data sync management (#9814)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch dev-offline-sync
in repository https://gitbox.apache.org/repos/asf/inlong.git
commit 7b0ba77d0c4545491a045031b96e4924704a4794
Author: AloysZhang
AuthorDate: Wed Mar 13 16:18:04 2024 +0800
[INLONG-9813][Manager] Support offline data sync management (#9814)
---
.../apache/inlong/manager/plugin/listener/StartupSortListener.java | 3 ++-
.../manager/service/listener/group/InitGroupCompleteListener.java | 3 ++-
.../manager/service/listener/group/UpdateGroupCompleteListener.java | 3 ++-
.../manager/service/listener/queue/QueueResourceListener.java | 6 --
.../manager/service/listener/stream/InitStreamCompleteListener.java | 3 ++-
.../inlong/manager/service/source/AbstractSourceOperator.java | 1 +
.../inlong/manager/service/source/StreamSourceServiceImpl.java | 3 ++-
7 files changed, 15 insertions(+), 7 deletions(-)
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
index 9997c2abd2..a84d3f5868 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
+++
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
@@ -61,7 +61,8 @@ public class StartupSortListener implements
SortOperateListener {
}
log.info("add startup group listener for groupId [{}]", groupId);
-return
InlongConstants.DATASYNC_REALTIME_MODE.equals(groupProcessForm.getGroupInfo().getInlongGroupMode());
+return
(InlongConstants.DATASYNC_REALTIME_MODE.equals(groupProcessForm.getGroupInfo().getInlongGroupMode())
+||
InlongConstants.DATASYNC_OFFLINE_MODE.equals(groupProcessForm.getGroupInfo().getInlongGroupMode()));
}
@Override
diff --git
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupCompleteListener.java
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupCompleteListener.java
index c96c12c9ff..c8c087320f 100644
---
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupCompleteListener.java
+++
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupCompleteListener.java
@@ -98,7 +98,8 @@ public class InitGroupCompleteListener implements
ProcessEventListener {
// update status of other related configs
if
(InlongConstants.DISABLE_CREATE_RESOURCE.equals(groupInfo.getEnableCreateResource()))
{
-if
(InlongConstants.DATASYNC_REALTIME_MODE.equals(groupInfo.getInlongGroupMode()))
{
+if
(InlongConstants.DATASYNC_REALTIME_MODE.equals(groupInfo.getInlongGroupMode())
+||
InlongConstants.DATASYNC_OFFLINE_MODE.equals(groupInfo.getInlongGroupMode())) {
sourceService.updateStatus(groupId, null,
SourceStatus.SOURCE_NORMAL.getCode(), operator);
} else {
sourceService.updateStatus(groupId, null,
SourceStatus.TO_BE_ISSUED_ADD.getCode(), operator);
diff --git
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/UpdateGroupCompleteListener.java
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/UpdateGroupCompleteListener.java
index a6dd8fc786..1a6e1f7fe1 100644
---
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/UpdateGroupCompleteListener.java
+++
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/UpdateGroupCompleteListener.java
@@ -96,7 +96,8 @@ public class UpdateGroupCompleteListener implements
ProcessEventListener {
}
// if the inlong group is dataSync mode, the stream source needs to be
processed.
-if
(InlongConstants.DATASYNC_REALTIME_MODE.equals(groupInfo.getInlongGroupMode()))
{
+if
(InlongConstants.DATASYNC_REALTIME_MODE.equals(groupInfo.getInlongGroupMode())
+||
InlongConstants.DATASYNC_OFFLINE_MODE.equals(groupInfo.getInlongGroupMode())) {
changeSource4DataSync(groupId, operateType, operator);
}
diff --git
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/queue/QueueResourceListener.java
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/queue/QueueResourceListener.java
index f566c29728..0ce551d1e6 100644
---
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/list
(inlong) branch dev-offline-sync updated (a793963543 -> b68bf5c5e5)
This is an automated email from the ASF dual-hosted git repository. aloyszhang pushed a change to branch dev-offline-sync in repository https://gitbox.apache.org/repos/asf/inlong.git discard a793963543 [INLONG-10247][Manager] Support schedule information management for offline sync (#10254) omit 65d3a928ea [INLONG-9862][Manager] Support submit flink job for offline sync (#9865) omit ff3c351401 [INLONG-9822][Manager] Support flink job runtime execution mode configuration (#9823) omit dfb22f4d8e [INLONG-9813][Manager] Support offline data sync management (#9814) omit 1e0fddc0a8 [INLONG-9781][Manager] Add offline sync task type definition (#9787) add 03c2160411 [INLONG-10225][Audit] Audit SDK provides the ability to automatically assign and manage Audit ID (#10232) add 40548ae01b [INLONG-10229][Sort] EsSink support unified configuration (#10237) add 5a331f1c86 [INLONG-10230][Sort] KafkaSink support unified configuration (#10235) add 97e12593e2 [INLONG-10233][Manager] Report heartbeat removal port restriction (#10234) add e4e24def90 [INLONG-10228][Sort] PulsarSink support unified configuration (#10236) add 6645261800 [INLONG-10245][Manager] Support setting audit version for file collection (#10246) add 2acf82d174 [INLONG-10238][Sort] MySQL connector support audit ID (#10239) add 0acdfe9519 [INLONG-10195][Sort] Hudi connector support audit ID (#10231) add 23728055c3 [INLONG-10242][Audit] Audit SDK compatible with InLong Manager to manage audit items (#10248) add 59ea439321 [INLONG-10249][Manager] Fix the problem of duplicate data appears during data preview (#10250) add 85fc8b02e6 [INLONG-9613][TubeMQ] Adjust FATAL type error return content, without carrying class name (#10251) add 6eeec6ee61 [INLONG-10252][Docker] Fix the bug can not get the real audit address (#10253) add d890a1933f [INLONG-10260][Sort] Correct wrong create table statement (#10262) add 2c890c29bb [INLONG-10263][Audit] Solve the conflict between the jdbc.url parameter of Audit Store and the container environment variable (#10264) add d18232b54a [INLONG-10241][Sort] TLog format requires the ability to parse and determine if the first segment is present (#10243) add b5261a3927 [INLONG-10224][Sort] Unified configuration check utils (#10255) add 94a99ac999 [INLONG-10266][Manager] Fix the problem of password is overwritten when adding field information (#10267) add 0113074789 [INLONG-10272][Sort] Unified configuration check utils support check latest config (#10273) add b22a3bdca8 [INLONG-10256][DashBoard] Modify the data source IP item of the file data source form in the data access module (#10258) add ca4bc8dce9 [INLONG-10268][Agent] Get the data version from the auditVersion field (#10269) add ca3fd42542 [INLONG-10270][Manager] Data source tasks allow for multiple IPs (#10271) add 9f829fac3b [INLONG-10274][Audit] The OpenAPI of Audit Service returns the average transmission time (#10275) add 879bf36017 [INLONG-10265][Manager] Fix the problem of correct wrong starrocks create table statement (#10276) add 4f9c4098eb [INLONG-10281][Agent] Real time file collection takes the current time as the data time (#10282) add e0a564b0b2 [INLONG-10277][Manager] Support calling API to refresh cluster config (#10278) add af0cc8255a [INLONG-10277][Manager] Fix the problem of migration failed when migrating multiple groups to the same tenant (#10285) add 863190ea29 [INLONG-10290][Manager] Prohibit groups that have not been successfully configured from obtaining dataproxy addresses (#10293) add c07316253e [INLONG-10292][SDK] Fix panic in connpool.UpdateEndpoints() of Golang SDK (#10295) add d43e1e84ec [INLONG-10291][SDK] Fix incorrect initializing of gnet in Golang SDK (#10294) add 4555ebfe1c [INLONG-10297][Sort] Fix mysql connector cannot submit to flink cluster (#10301) add e5ad544716 [INLONG-10306][Audit] Compatible with scenarios where the Audit Tag is empty (#10308) add 12efb53dae [INLONG-10302][Agent] Add an interface for limiting the number of instances obtained (#10303) add b0a70bda57 [INLONG-10300][Manager] Allow unsubmitted groups to modify mq type (#10304) add 229174675b [INLONG-10298][Agent] Delete useless code (#10299) add 9d94ef72df [INLONG-10305][Manager] Delete k8s related parameters in file collection (#10307) add 83ded2149d [INLONG-10313][DataProxy] Replace audit ID macro with audit API (#10315) add c6892951ed [INLONG-10319][Agent] Get audit Id from audit sdk (#10322) add 8f6ce51cd7 [INLONG-10314][DashBoard] Add an operation time to the operation log table (#10316) add ca0983fceb [INLONG-10318][Agent] Add PostgreSQL data source for Agent (#10320) add 83fc0711b8 [INLONG-10324][Manager] Fix the problem of useExtendedFields in Stream incorrectly overwritten (#10325) add 8171257a62
(inlong) branch dev-offline-sync updated: [INLONG-10395][Manager] Add interface for schedule client and engine (#10397)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch dev-offline-sync
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/dev-offline-sync by this push:
new 08c520d5e4 [INLONG-10395][Manager] Add interface for schedule client
and engine (#10397)
08c520d5e4 is described below
commit 08c520d5e41b2f94ad450e7f35967e8c48169371
Author: AloysZhang
AuthorDate: Thu Jun 13 15:02:14 2024 +0800
[INLONG-10395][Manager] Add interface for schedule client and engine
(#10397)
* [INLONG-10395][Manager] Add interface for schedule client and engine
---
inlong-manager/manager-schedule/pom.xml| 41
.../org/apache/inlong/schedule/ScheduleEngine.java | 54 ++
.../inlong/schedule/ScheduleEngineClient.java | 45 ++
inlong-manager/pom.xml | 1 +
4 files changed, 141 insertions(+)
diff --git a/inlong-manager/manager-schedule/pom.xml
b/inlong-manager/manager-schedule/pom.xml
new file mode 100644
index 00..f3a4e0a41f
--- /dev/null
+++ b/inlong-manager/manager-schedule/pom.xml
@@ -0,0 +1,41 @@
+
+
+http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+4.0.0
+
+org.apache.inlong
+inlong-manager
+1.13.0-SNAPSHOT
+
+
+manager-schedule
+
+
+${project.parent.parent.basedir}
+UTF-8
+
+
+
+
+org.apache.inlong
+manager-pojo
+${project.version}
+
+
+
diff --git
a/inlong-manager/manager-schedule/src/main/java/org/apache/inlong/schedule/ScheduleEngine.java
b/inlong-manager/manager-schedule/src/main/java/org/apache/inlong/schedule/ScheduleEngine.java
new file mode 100644
index 00..bfdb9a88ad
--- /dev/null
+++
b/inlong-manager/manager-schedule/src/main/java/org/apache/inlong/schedule/ScheduleEngine.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.schedule;
+
+import org.apache.inlong.manager.pojo.schedule.ScheduleInfo;
+
+/**
+ * Build-in schedule engine, provides basic schedule capabilities.
+ * */
+public interface ScheduleEngine {
+
+/**
+ * Start schedule engine.
+ * */
+void start();
+
+/**
+ * Handle schedule register.
+ * @param scheduleInfo schedule info to register
+ * */
+boolean handleRegister(ScheduleInfo scheduleInfo);
+
+/**
+ * Handle schedule unregister.
+ * @param scheduleInfo schedule info to unregister
+ * */
+boolean handleUnregister(ScheduleInfo scheduleInfo);
+
+/**
+ * Handle schedule update.
+ * @param scheduleInfo schedule info to update
+ * */
+boolean handleUpdate(ScheduleInfo scheduleInfo);
+
+/**
+ * Stop schedule engine.
+ * */
+void stop();
+}
diff --git
a/inlong-manager/manager-schedule/src/main/java/org/apache/inlong/schedule/ScheduleEngineClient.java
b/inlong-manager/manager-schedule/src/main/java/org/apache/inlong/schedule/ScheduleEngineClient.java
new file mode 100644
index 00..1e5ce3460c
--- /dev/null
+++
b/inlong-manager/manager-schedule/src/main/java/org/apache/inlong/schedule/ScheduleEngineClient.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *
(inlong) 05/09: [INLONG-10247][Manager] Support schedule information management for offline sync (#10254)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch dev-offline-sync
in repository https://gitbox.apache.org/repos/asf/inlong.git
commit 8bd3a20cb49f9dc3ddc83c0bcb769f95e793e4b6
Author: AloysZhang
AuthorDate: Thu May 23 18:18:31 2024 +0800
[INLONG-10247][Manager] Support schedule information management for
offline sync (#10254)
---
.../client/api/inner/client/ClientFactory.java | 2 +
.../api/inner/client/InLongScheduleClient.java | 67 +
.../client/api/service/InLongScheduleApi.java | 49 +++
.../inlong/manager/common/enums/ErrorCodeEnum.java | 3 +
.../manager/common/enums/OperationTarget.java | 4 +-
.../{OperationTarget.java => ScheduleStatus.java} | 54 ---
.../inlong/manager/dao/entity/ScheduleEntity.java | 61
.../manager/dao/mapper/ScheduleEntityMapper.java} | 36 ++---
.../resources/mappers/ScheduleEntityMapper.xml | 144 +++
.../manager/dao/mapper/ScheduleEntityTest.java | 118
.../inlong/manager/pojo/schedule/ScheduleInfo.java | 82 +++
.../manager/pojo/schedule/ScheduleInfoRequest.java | 76 ++
.../manager/service/schedule/ScheduleService.java | 71 ++
.../service/schedule/ScheduleServiceImpl.java | 155 +
.../main/resources/h2/apache_inlong_manager.sql| 31 +
.../manager-web/sql/apache_inlong_manager.sql | 28
inlong-manager/manager-web/sql/changes-1.13.0.sql | 30
.../web/controller/InLongSchedulerController.java | 88
18 files changed, 1043 insertions(+), 56 deletions(-)
diff --git
a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/inner/client/ClientFactory.java
b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/inner/client/ClientFactory.java
index 3c67f1fa71..01c9fb6473 100644
---
a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/inner/client/ClientFactory.java
+++
b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/inner/client/ClientFactory.java
@@ -55,6 +55,7 @@ public class ClientFactory {
private final AuditClient auditClient;
private final InlongTenantClient inlongTenantClient;
private final InlongTenantRoleClient inlongTenantRoleClient;
+private final InLongScheduleClient inLongScheduleClient;
public ClientFactory(ClientConfiguration configuration) {
groupClient = new InlongGroupClient(configuration);
@@ -74,5 +75,6 @@ public class ClientFactory {
auditClient = new AuditClient(configuration);
inlongTenantClient = new InlongTenantClient(configuration);
inlongTenantRoleClient = new InlongTenantRoleClient(configuration);
+inLongScheduleClient = new InLongScheduleClient(configuration);
}
}
diff --git
a/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/inner/client/InLongScheduleClient.java
b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/inner/client/InLongScheduleClient.java
new file mode 100644
index 00..86638dbae8
--- /dev/null
+++
b/inlong-manager/manager-client/src/main/java/org/apache/inlong/manager/client/api/inner/client/InLongScheduleClient.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.manager.client.api.inner.client;
+
+import org.apache.inlong.manager.client.api.ClientConfiguration;
+import org.apache.inlong.manager.client.api.service.InLongScheduleApi;
+import org.apache.inlong.manager.client.api.util.ClientUtils;
+import org.apache.inlong.manager.common.enums.ErrorCodeEnum;
+import org.apache.inlong.manager.common.util.Preconditions;
+import org.apache.inlong.manager.pojo.common.Response;
+import org.apache.inlong.manager.pojo.schedule.ScheduleInfo;
+import org.apache.inlong.manager.pojo.schedule.ScheduleInfoRequest;
+
+public class InLongScheduleClient {
+
+private InLongScheduleApi scheduleApi;
+
+public InLongScheduleClient(ClientConfiguration clientConfiguration) {
+scheduleApi =
Cli
(inlong) 09/09: [INLONG-10360][Manager] Combine schedule state transition with group operations (#10445)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch dev-offline-sync
in repository https://gitbox.apache.org/repos/asf/inlong.git
commit 2a08660a079b30298dd4803736ed515563072aca
Author: AloysZhang
AuthorDate: Wed Jun 19 15:41:34 2024 +0800
[INLONG-10360][Manager] Combine schedule state transition with group
operations (#10445)
Co-authored-by: fuweng11 <76141879+fuwen...@users.noreply.github.com>
---
.../inlong/manager/common/enums/ErrorCodeEnum.java | 2 +
.../manager/common/enums/ScheduleStatus.java | 16 +-
.../inlong/manager/pojo/group/InlongGroupInfo.java | 31
.../manager/pojo/group/InlongGroupRequest.java | 31
.../inlong/manager/pojo/schedule/ScheduleInfo.java | 29
.../manager/pojo/schedule/ScheduleInfoRequest.java | 28
...leEngineClient.java => NoopScheduleClient.java} | 45 +++---
.../manager/schedule/ScheduleClientFactory.java| 55 +++
.../inlong/manager/schedule/ScheduleEngine.java| 4 +-
.../manager/schedule/ScheduleEngineClient.java | 12 +-
.../manager/schedule/ScheduleEngineType.java} | 28 +---
.../schedule/quartz/QuartzScheduleClient.java | 17 +-
.../schedule/quartz/QuartzScheduleEngine.java | 21 +--
.../manager/schedule/util/ScheduleUtils.java | 4 +-
.../inlong/manager/schedule/quartz/MockJob.java| 3 +-
.../schedule/quartz/QuartzScheduleEngineTest.java | 2 +-
.../manager/schedule/util/ScheduleUtilsTest.java | 8 +-
inlong-manager/manager-service/pom.xml | 5 +
.../manager/service/group/InlongGroupService.java | 9 --
.../service/group/InlongGroupServiceImpl.java | 71 ++---
.../schedule/GroupScheduleResourceListener.java| 8 +-
.../manager/service/schedule/ScheduleOperator.java | 95 +++
.../service/schedule/ScheduleOperatorImpl.java | 175 +
.../manager/service/schedule/ScheduleService.java | 11 ++
.../service/schedule/ScheduleServiceImpl.java | 54 ++-
.../web/controller/InLongSchedulerController.java | 24 ++-
.../openapi/OpenInLongGroupController.java | 2 +-
.../src/main/resources/application-dev.properties | 6 +-
.../src/main/resources/application-prod.properties | 3 +
.../src/main/resources/application-test.properties | 3 +
.../src/main/resources/application.properties | 4 +
31 files changed, 696 insertions(+), 110 deletions(-)
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/ErrorCodeEnum.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/ErrorCodeEnum.java
index 6a8f9b4699..2a82325ad9 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/ErrorCodeEnum.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/ErrorCodeEnum.java
@@ -127,6 +127,8 @@ public enum ErrorCodeEnum {
SCHEDULE_NOT_FOUND(1700, "Schedule info not found"),
SCHEDULE_DUPLICATE(1701, "Schedule info already exist"),
+SCHEDULE_ENGINE_NOT_SUPPORTED(1702, "Schedule engine type not supported"),
+SCHEDULE_STATUS_TRANSITION_NOT_ALLOWED(1703, "Schedule status transition
is not allowed"),
WORKFLOW_EXE_FAILED(4000, "Workflow execution exception"),
WORKFLOW_APPROVER_NOT_FOUND(4001, "Workflow approver does not exist/no
operation authority"),
diff --git
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/ScheduleStatus.java
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/ScheduleStatus.java
index cb256a491c..2d936b5532 100644
---
a/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/ScheduleStatus.java
+++
b/inlong-manager/manager-common/src/main/java/org/apache/inlong/manager/common/enums/ScheduleStatus.java
@@ -19,11 +19,25 @@ package org.apache.inlong.manager.common.enums;
import lombok.Getter;
+/**
+ * Status for schedule info.
+ * This is the transient status of the schedule info.
+ * With specified operations, the status will change to corresponding value.
+ * StatusOperations
+ * NEW inlong group created with schedule info
+ * APPROVED the new inlong group approved by admin
+ * REGISTEREDschedule info registered to schedule engine
+ * UPDATED update schedule info for a group
+ * DELETED delete a group
+ * */
@Getter
public enum ScheduleStatus {
NEW(100, "new"),
-DELETED(40, "deleted");
+APPROVED(101, "approved"),
+REGISTERED(102, "registered"),
+UPDATED(103, "updated"),
+DELETED(99, "deleted");
private final Integer code;
private final String description;
diff --git
a/in
(inlong) 02/09: [INLONG-9813][Manager] Support offline data sync management (#9814)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch dev-offline-sync
in repository https://gitbox.apache.org/repos/asf/inlong.git
commit 2674a5645c1f0f0e821ec26efb1351c4297dd056
Author: AloysZhang
AuthorDate: Wed Mar 13 16:18:04 2024 +0800
[INLONG-9813][Manager] Support offline data sync management (#9814)
---
.../apache/inlong/manager/plugin/listener/StartupSortListener.java | 3 ++-
.../manager/service/listener/group/InitGroupCompleteListener.java | 3 ++-
.../manager/service/listener/group/UpdateGroupCompleteListener.java | 3 ++-
.../manager/service/listener/queue/QueueResourceListener.java | 6 --
.../manager/service/listener/stream/InitStreamCompleteListener.java | 3 ++-
.../inlong/manager/service/source/AbstractSourceOperator.java | 1 +
.../inlong/manager/service/source/StreamSourceServiceImpl.java | 3 ++-
7 files changed, 15 insertions(+), 7 deletions(-)
diff --git
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
index 9997c2abd2..a84d3f5868 100644
---
a/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
+++
b/inlong-manager/manager-plugins/base/src/main/java/org/apache/inlong/manager/plugin/listener/StartupSortListener.java
@@ -61,7 +61,8 @@ public class StartupSortListener implements
SortOperateListener {
}
log.info("add startup group listener for groupId [{}]", groupId);
-return
InlongConstants.DATASYNC_REALTIME_MODE.equals(groupProcessForm.getGroupInfo().getInlongGroupMode());
+return
(InlongConstants.DATASYNC_REALTIME_MODE.equals(groupProcessForm.getGroupInfo().getInlongGroupMode())
+||
InlongConstants.DATASYNC_OFFLINE_MODE.equals(groupProcessForm.getGroupInfo().getInlongGroupMode()));
}
@Override
diff --git
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupCompleteListener.java
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupCompleteListener.java
index c96c12c9ff..c8c087320f 100644
---
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupCompleteListener.java
+++
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/InitGroupCompleteListener.java
@@ -98,7 +98,8 @@ public class InitGroupCompleteListener implements
ProcessEventListener {
// update status of other related configs
if
(InlongConstants.DISABLE_CREATE_RESOURCE.equals(groupInfo.getEnableCreateResource()))
{
-if
(InlongConstants.DATASYNC_REALTIME_MODE.equals(groupInfo.getInlongGroupMode()))
{
+if
(InlongConstants.DATASYNC_REALTIME_MODE.equals(groupInfo.getInlongGroupMode())
+||
InlongConstants.DATASYNC_OFFLINE_MODE.equals(groupInfo.getInlongGroupMode())) {
sourceService.updateStatus(groupId, null,
SourceStatus.SOURCE_NORMAL.getCode(), operator);
} else {
sourceService.updateStatus(groupId, null,
SourceStatus.TO_BE_ISSUED_ADD.getCode(), operator);
diff --git
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/UpdateGroupCompleteListener.java
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/UpdateGroupCompleteListener.java
index a6dd8fc786..1a6e1f7fe1 100644
---
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/UpdateGroupCompleteListener.java
+++
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/group/UpdateGroupCompleteListener.java
@@ -96,7 +96,8 @@ public class UpdateGroupCompleteListener implements
ProcessEventListener {
}
// if the inlong group is dataSync mode, the stream source needs to be
processed.
-if
(InlongConstants.DATASYNC_REALTIME_MODE.equals(groupInfo.getInlongGroupMode()))
{
+if
(InlongConstants.DATASYNC_REALTIME_MODE.equals(groupInfo.getInlongGroupMode())
+||
InlongConstants.DATASYNC_OFFLINE_MODE.equals(groupInfo.getInlongGroupMode())) {
changeSource4DataSync(groupId, operateType, operator);
}
diff --git
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/queue/QueueResourceListener.java
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/listener/queue/QueueResourceListener.java
index f566c29728..0ce551d1e6 100644
---
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/list
(inlong) 06/09: [INLONG-10395][Manager] Add interface for schedule client and engine (#10397)
This is an automated email from the ASF dual-hosted git repository.
aloyszhang pushed a commit to branch dev-offline-sync
in repository https://gitbox.apache.org/repos/asf/inlong.git
commit 4d5c873efba71c25fb71858dff6d419712801f9b
Author: AloysZhang
AuthorDate: Thu Jun 13 15:02:14 2024 +0800
[INLONG-10395][Manager] Add interface for schedule client and engine
(#10397)
* [INLONG-10395][Manager] Add interface for schedule client and engine
---
inlong-manager/manager-schedule/pom.xml| 41
.../org/apache/inlong/schedule/ScheduleEngine.java | 54 ++
.../inlong/schedule/ScheduleEngineClient.java | 45 ++
inlong-manager/pom.xml | 1 +
4 files changed, 141 insertions(+)
diff --git a/inlong-manager/manager-schedule/pom.xml
b/inlong-manager/manager-schedule/pom.xml
new file mode 100644
index 00..f3a4e0a41f
--- /dev/null
+++ b/inlong-manager/manager-schedule/pom.xml
@@ -0,0 +1,41 @@
+
+
+http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+4.0.0
+
+org.apache.inlong
+inlong-manager
+1.13.0-SNAPSHOT
+
+
+manager-schedule
+
+
+${project.parent.parent.basedir}
+UTF-8
+
+
+
+
+org.apache.inlong
+manager-pojo
+${project.version}
+
+
+
diff --git
a/inlong-manager/manager-schedule/src/main/java/org/apache/inlong/schedule/ScheduleEngine.java
b/inlong-manager/manager-schedule/src/main/java/org/apache/inlong/schedule/ScheduleEngine.java
new file mode 100644
index 00..bfdb9a88ad
--- /dev/null
+++
b/inlong-manager/manager-schedule/src/main/java/org/apache/inlong/schedule/ScheduleEngine.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.schedule;
+
+import org.apache.inlong.manager.pojo.schedule.ScheduleInfo;
+
+/**
+ * Build-in schedule engine, provides basic schedule capabilities.
+ * */
+public interface ScheduleEngine {
+
+/**
+ * Start schedule engine.
+ * */
+void start();
+
+/**
+ * Handle schedule register.
+ * @param scheduleInfo schedule info to register
+ * */
+boolean handleRegister(ScheduleInfo scheduleInfo);
+
+/**
+ * Handle schedule unregister.
+ * @param scheduleInfo schedule info to unregister
+ * */
+boolean handleUnregister(ScheduleInfo scheduleInfo);
+
+/**
+ * Handle schedule update.
+ * @param scheduleInfo schedule info to update
+ * */
+boolean handleUpdate(ScheduleInfo scheduleInfo);
+
+/**
+ * Stop schedule engine.
+ * */
+void stop();
+}
diff --git
a/inlong-manager/manager-schedule/src/main/java/org/apache/inlong/schedule/ScheduleEngineClient.java
b/inlong-manager/manager-schedule/src/main/java/org/apache/inlong/schedule/ScheduleEngineClient.java
new file mode 100644
index 00..1e5ce3460c
--- /dev/null
+++
b/inlong-manager/manager-schedule/src/main/java/org/apache/inlong/schedule/ScheduleEngineClient.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.schedule;
+
+import org.apache.inlong.manager.pojo.schedule.ScheduleInfo;
