(inlong) branch master updated: [INLONG-10886][SDK] Transform support ACOS function (#10888)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new ced348d9fc [INLONG-10886][SDK] Transform support ACOS function (#10888)
ced348d9fc is described below

commit ced348d9fcadf9d4021d3e61677cae947b526baf
Author: Xincheng Huang <60057611+ying-...@users.noreply.github.com>
AuthorDate: Tue Aug 27 15:39:17 2024 +0800

[INLONG-10886][SDK] Transform support ACOS function (#10888)
---
 .../transform/process/function/AcosFunction.java   | 57 ++
 .../transform/process/operator/OperatorTools.java  |  2 +
 .../TestTransformArithmeticFunctionsProcessor.java | 21 
 3 files changed, 80 insertions(+)

diff --git 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/AcosFunction.java
 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/AcosFunction.java
new file mode 100644
index 00..332716e57b
--- /dev/null
+++ 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/AcosFunction.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sdk.transform.process.function;
+
+import org.apache.inlong.sdk.transform.decode.SourceData;
+import org.apache.inlong.sdk.transform.process.Context;
+import org.apache.inlong.sdk.transform.process.operator.OperatorTools;
+import org.apache.inlong.sdk.transform.process.parser.ValueParser;
+
+import net.sf.jsqlparser.expression.Function;
+
+import java.math.BigDecimal;
+
+/**
+ * AcosFunction
+ * description: acos(numeric)--returns the arc cosine of numeric
+ */
+public class AcosFunction implements ValueParser {
+
+private ValueParser numberParser;
+
+/**
+ * Constructor
+ * @param expr
+ */
+public AcosFunction(Function expr) {
+numberParser = 
OperatorTools.buildParser(expr.getParameters().getExpressions().get(0));
+}
+
+/**
+ * parse
+ * @param sourceData
+ * @param rowIndex
+ * @return
+ */
+@Override
+public Object parse(SourceData sourceData, int rowIndex, Context context) {
+Object numberObj = numberParser.parse(sourceData, rowIndex, context);
+BigDecimal numberValue = OperatorTools.parseBigDecimal(numberObj);
+return Math.acos(numberValue.doubleValue());
+}
+}
\ No newline at end of file
diff --git 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
index f7f023e748..1cb1b8589e 100644
--- 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
+++ 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
@@ -18,6 +18,7 @@
 package org.apache.inlong.sdk.transform.process.operator;
 
 import org.apache.inlong.sdk.transform.process.function.AbsFunction;
+import org.apache.inlong.sdk.transform.process.function.AcosFunction;
 import org.apache.inlong.sdk.transform.process.function.BinFunction;
 import org.apache.inlong.sdk.transform.process.function.CeilFunction;
 import org.apache.inlong.sdk.transform.process.function.ConcatFunction;
@@ -134,6 +135,7 @@ public class OperatorTools {
 functionMap.put("sin", SinFunction::new);
 functionMap.put("sinh", SinhFunction::new);
 functionMap.put("cos", CosFunction::new);
+functionMap.put("acos", AcosFunction::new);
 functionMap.put("tan", TanFunction::new);
 functionMap.put("bin", BinFunction::new);
 functionMap.put("year", func -> new 
DateExtractFunction(DateExtractFunctionType.YEAR, func));
diff --git 
a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java
 
b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlon

(inlong) branch master updated: [INLONG-10913][SDK] Optimize the python sdk build script (#10917)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new 171eb76a1c [INLONG-10913][SDK] Optimize the python sdk build script 
(#10917)
171eb76a1c is described below

commit 171eb76a1c5db7e17aaa7d24b9d1b5db9cb4c470
Author: yfsn666 <61183968+yfsn...@users.noreply.github.com>
AuthorDate: Tue Aug 27 16:04:07 2024 +0800

[INLONG-10913][SDK] Optimize the python sdk build script (#10917)
---
 .../dataproxy-sdk-twins/dataproxy-sdk-python/build.sh| 16 
 .../dataproxy-sdk-python/requirements.txt|  1 +
 2 files changed, 17 insertions(+)

diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/build.sh 
b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/build.sh
index f38f535dad..29759c6c9c 100755
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/build.sh
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/build.sh
@@ -51,6 +51,15 @@ if [ "$(printf '%s\n' "$PYTHON_REQUIRED" "$PYTHON_VERSION" | 
sort -V | head -n1)
 exit 1
 fi
 
+# Install Python packages from requirements.txt
+if [ -f $PY_SDK_DIR/requirements.txt ]; then
+echo "Installing Python packages from requirements.txt..."
+pip install -r $PY_SDK_DIR/requirements.txt
+else
+echo "Error: cannot find requirements.txt!"
+exit 1
+fi
+
 # Build pybind11(If the pybind11 has been compiled, this step will be skipped)
 if [ ! -d "$PY_SDK_DIR/pybind11/build" ]; then
 if [ -d "$PY_SDK_DIR/pybind11" ]; then
@@ -58,9 +67,16 @@ if [ ! -d "$PY_SDK_DIR/pybind11/build" ]; then
 fi
 git clone https://github.com/pybind/pybind11.git $PY_SDK_DIR/pybind11
 mkdir $PY_SDK_DIR/pybind11/build && cd $PY_SDK_DIR/pybind11/build
+
+# Add a trap command to delete the pybind11 folder if an error occurs
+trap 'echo "Error occurred during pybind11 build. Deleting pybind11 
folder..."; cd $PY_SDK_DIR; rm -r pybind11; exit 1' ERR
+
 cmake $PY_SDK_DIR/pybind11
 cmake --build $PY_SDK_DIR/pybind11/build --config Release --target check
 make -j 4
+
+# Remove the trap command if the build is successful
+trap - ERR
 else
 echo "Skipped build pybind11"
 fi
diff --git 
a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/requirements.txt 
b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/requirements.txt
new file mode 100644
index 00..e160152b52
--- /dev/null
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/requirements.txt
@@ -0,0 +1 @@
+pytest>=7.1.2
\ No newline at end of file

(inlong) branch master updated: [INLONG-10876][SDK] Transform support RAND() function (#10877)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new d5a4ebadce [INLONG-10876][SDK] Transform support RAND() function 
(#10877)
d5a4ebadce is described below

commit d5a4ebadcef064993164b11ee26bc27b1e98a95b
Author: emptyOVO <118812562+empty...@users.noreply.github.com>
AuthorDate: Tue Aug 27 19:34:43 2024 +0800

[INLONG-10876][SDK] Transform support RAND() function (#10877)

* fix: conflicts

* fix: prepare Random Object in the constructor, check expression list size 
at first, avoid to use NullPointerException

* fix: check expressions.size()==1

* fix: conficts

* fix: conficts
---
 .../transform/process/function/RandFunction.java   | 61 ++
 .../transform/process/operator/OperatorTools.java  |  2 +
 .../TestTransformArithmeticFunctionsProcessor.java | 28 ++
 3 files changed, 91 insertions(+)

diff --git 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/RandFunction.java
 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/RandFunction.java
new file mode 100644
index 00..2baebe038a
--- /dev/null
+++ 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/RandFunction.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sdk.transform.process.function;
+
+import org.apache.inlong.sdk.transform.decode.SourceData;
+import org.apache.inlong.sdk.transform.process.Context;
+import org.apache.inlong.sdk.transform.process.operator.OperatorTools;
+import org.apache.inlong.sdk.transform.process.parser.ValueParser;
+
+import net.sf.jsqlparser.expression.Expression;
+import net.sf.jsqlparser.expression.Function;
+
+import java.math.BigDecimal;
+import java.util.List;
+import java.util.Random;
+
+/**
+ * RandFunction
+ * description: Rand()--Returns a pseudo-random double precision value in the 
range [0.0, 1.0)
+ *  Rand(Integer)--Returns a pseudo-random double precision value 
in the range [0.0, 1.0) with an initial seed of Integer.
+ */
+public class RandFunction implements ValueParser {
+
+private ValueParser seedParser;
+
+private Random random;
+
+public RandFunction(Function expr) {
+random = new Random();
+if (expr.getParameters() != null) {
+List expressions = 
expr.getParameters().getExpressions();
+if (expressions != null && expressions.size() == 1) {
+seedParser = OperatorTools.buildParser(expressions.get(0));
+}
+}
+}
+@Override
+public Object parse(SourceData sourceData, int rowIndex, Context context) {
+if (seedParser != null) {
+Object seedObj = seedParser.parse(sourceData, rowIndex, context);
+BigDecimal seedValue = OperatorTools.parseBigDecimal(seedObj);
+random.setSeed(seedValue.intValue());
+}
+return random.nextDouble();
+}
+}
diff --git 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
index 1cb1b8589e..ed7ba57d4b 100644
--- 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
+++ 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
@@ -38,6 +38,7 @@ import 
org.apache.inlong.sdk.transform.process.function.LogFunction;
 import org.apache.inlong.sdk.transform.process.function.ModuloFunction;
 import org.apache.inlong.sdk.transform.process.function.NowFunction;
 import org.apache.inlong.sdk.transform.process.function.PowerFunction;
+import org.apache.inlong.sdk.transform.process.function.RandFunction;
 import org.apache.inlong.sdk.transform.process.function.ReplaceFunction;
 import org.apache.inlong.sdk.transform.process.f

(inlong) branch master updated: [INLONG-10920][SDK] Transform support LocalTime function (#10922)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new 916e6c1532 [INLONG-10920][SDK] Transform support LocalTime function 
(#10922)
916e6c1532 is described below

commit 916e6c1532ac4f740abb7b4c0babf76ef855baee
Author: Xincheng Huang <60057611+ying-...@users.noreply.github.com>
AuthorDate: Tue Aug 27 19:35:32 2024 +0800

[INLONG-10920][SDK] Transform support LocalTime function (#10922)
---
 .../process/function/LocalTimeFunction.java| 55 ++
 .../transform/process/operator/OperatorTools.java  |  3 ++
 .../TestTransformTemporalFunctionsProcessor.java   | 47 ++
 3 files changed, 105 insertions(+)

diff --git 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/LocalTimeFunction.java
 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/LocalTimeFunction.java
new file mode 100644
index 00..a3072901a3
--- /dev/null
+++ 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/LocalTimeFunction.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sdk.transform.process.function;
+
+import org.apache.inlong.sdk.transform.decode.SourceData;
+import org.apache.inlong.sdk.transform.process.Context;
+import org.apache.inlong.sdk.transform.process.operator.OperatorTools;
+import org.apache.inlong.sdk.transform.process.parser.ValueParser;
+
+import net.sf.jsqlparser.expression.Function;
+
+import java.time.LocalTime;
+import java.time.ZoneId;
+
+/**
+ *  LocalTimeFunction
+ *  description:
+ *  localTime([string1]) returns the current time in the specified time zone.
+ *  (by default: the current time in the system time zone)
+ */
+public class LocalTimeFunction implements ValueParser {
+
+private ValueParser stringParser;
+
+public LocalTimeFunction(Function expr) {
+if (expr.getParameters() != null) {
+stringParser = 
OperatorTools.buildParser(expr.getParameters().getExpressions().get(0));
+}
+}
+
+@Override
+public Object parse(SourceData sourceData, int rowIndex, Context context) {
+if (stringParser != null) {
+String zoneString = 
OperatorTools.parseString(stringParser.parse(sourceData, rowIndex, context));
+return LocalTime.now(ZoneId.of(zoneString)).withNano(0);
+} else {
+return LocalTime.now(ZoneId.systemDefault()).withNano(0);
+}
+}
+}
diff --git 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
index 9c6f0d0731..5dc282130b 100644
--- 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
+++ 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
@@ -31,6 +31,7 @@ import 
org.apache.inlong.sdk.transform.process.function.FloorFunction;
 import org.apache.inlong.sdk.transform.process.function.FromUnixTimeFunction;
 import org.apache.inlong.sdk.transform.process.function.LengthFunction;
 import org.apache.inlong.sdk.transform.process.function.LnFunction;
+import org.apache.inlong.sdk.transform.process.function.LocalTimeFunction;
 import org.apache.inlong.sdk.transform.process.function.LocateFunction;
 import org.apache.inlong.sdk.transform.process.function.Log10Function;
 import org.apache.inlong.sdk.transform.process.function.Log2Function;
@@ -118,6 +119,8 @@ public class OperatorTools {
 static {
 functionMap.put("concat", ConcatFunction::new);
 functionMap.put("now", NowFunction::new);
+functionMap.put("localtime", LocalTimeFunction::new);
+functionMap.put("currenttime", LocalTimeFunction::new);
 functionMap.put("power", PowerFunction::new);
 

(inlong) branch master updated: [INLONG-10869][SDK] Transform SQL supports case conversion of strings (#10878)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new de1189f732 [INLONG-10869][SDK] Transform SQL supports case conversion 
of strings (#10878)
de1189f732 is described below

commit de1189f732f221e6e8e88abf3f48cc2cf7a4ae04
Author: Zkplo <87751516+zk...@users.noreply.github.com>
AuthorDate: Tue Aug 27 19:35:14 2024 +0800

[INLONG-10869][SDK] Transform SQL supports case conversion of strings 
(#10878)

* [INLONG-10869][SDK] Transform SQL supports case conversion of strings

* [INLONG-10864][SDK] Increase the judgment of null pointers

* [INLONG-10869][SDK] Modified the handling logic for null and added 
relevant testing code

-

Co-authored-by: ZKpLo <14148880+zk...@user.noreply.gitee.com>
---
 .../transform/process/function/LowerFunction.java  | 46 ++
 .../transform/process/function/UpperFunction.java  | 46 ++
 .../transform/process/operator/OperatorTools.java  |  4 ++
 .../TestTransformStringFunctionsProcessor.java | 56 ++
 4 files changed, 152 insertions(+)

diff --git 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/LowerFunction.java
 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/LowerFunction.java
new file mode 100644
index 00..e6aa83c58c
--- /dev/null
+++ 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/LowerFunction.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sdk.transform.process.function;
+
+import org.apache.inlong.sdk.transform.decode.SourceData;
+import org.apache.inlong.sdk.transform.process.Context;
+import org.apache.inlong.sdk.transform.process.operator.OperatorTools;
+import org.apache.inlong.sdk.transform.process.parser.ValueParser;
+
+import net.sf.jsqlparser.expression.Function;
+
+/**
+ * LowerFunction
+ * description: LOWER(s): Convert all letters of the string s to lowercase 
letters
+ */
+public class LowerFunction implements ValueParser {
+
+private ValueParser stringParser;
+
+public LowerFunction(Function expr) {
+stringParser = 
OperatorTools.buildParser(expr.getParameters().getExpressions().get(0));
+}
+
+@Override
+public Object parse(SourceData sourceData, int rowIndex, Context context) {
+Object stringObj = stringParser.parse(sourceData, rowIndex, context);
+if (stringObj == null)
+return null;
+return stringObj.toString().toLowerCase();
+}
+}
diff --git 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/UpperFunction.java
 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/UpperFunction.java
new file mode 100644
index 00..393b1abde4
--- /dev/null
+++ 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/UpperFunction.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sdk.transform.process.function;
+
+import org.apache.inlong.sdk.transform.decode.SourceData;
+import org.

(inlong) branch master updated: [INLONG-10929][SDK] Support Inlong Transform function annotation (#10936)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new 02a3815b43 [INLONG-10929][SDK] Support Inlong Transform function 
annotation (#10936)
02a3815b43 is described below

commit 02a3815b43959a56a7236b8377f936e7edcd9465
Author: vernedeng 
AuthorDate: Wed Aug 28 21:16:59 2024 +0800

[INLONG-10929][SDK] Support Inlong Transform function annotation (#10936)

* [INLONG-10929][SDK] Support Inlong Transform function annotation

* fix conflict

* fix conflict
---
 inlong-sdk/transform-sdk/pom.xml   |   4 +
 .../transform/process/function/AbsFunction.java|   1 +
 .../transform/process/function/AcosFunction.java   |   1 +
 .../transform/process/function/BinFunction.java|   1 +
 .../transform/process/function/CeilFunction.java   |   1 +
 .../transform/process/function/ConcatFunction.java |   1 +
 .../transform/process/function/CosFunction.java|   1 +
 .../process/function/DateExtractFunction.java  |  58 +-
 .../process/function/DateFormatFunction.java   |   1 +
 .../transform/process/function/ExpFunction.java|   1 +
 .../transform/process/function/FloorFunction.java  |   1 +
 .../process/function/FromUnixTimeFunction.java |   1 +
 .../transform/process/function/FunctionTools.java  |  90 +++
 .../transform/process/function/LeftFunction.java   |   1 +
 .../transform/process/function/LengthFunction.java |   1 +
 .../sdk/transform/process/function/LnFunction.java |   1 +
 .../process/function/LocalTimeFunction.java|   1 +
 .../transform/process/function/LocateFunction.java |   1 +
 .../transform/process/function/Log10Function.java  |   1 +
 .../transform/process/function/Log2Function.java   |   1 +
 .../transform/process/function/LogFunction.java|   1 +
 .../transform/process/function/LowerFunction.java  |   1 +
 .../transform/process/function/Md5Function.java|   1 +
 .../transform/process/function/ModuloFunction.java |   1 +
 .../transform/process/function/NowFunction.java|   1 +
 .../transform/process/function/PowerFunction.java  |   1 +
 .../transform/process/function/RandFunction.java   |   1 +
 .../process/function/ReplaceFunction.java  |   1 +
 .../process/function/ReplicateFunction.java|   1 +
 .../process/function/ReverseFunction.java  |   1 +
 .../transform/process/function/RightFunction.java  |   1 +
 .../transform/process/function/RoundFunction.java  |   1 +
 .../transform/process/function/SignFunction.java   |   1 +
 .../transform/process/function/SinFunction.java|   1 +
 .../transform/process/function/SinhFunction.java   |   1 +
 .../transform/process/function/SqrtFunction.java   |   1 +
 .../process/function/SubstringFunction.java|   1 +
 .../transform/process/function/TanFunction.java|   1 +
 .../process/function/TimestampAddFunction.java |   1 +
 .../process/function/TimestampExtractFunction.java |  26 -
 .../process/function/ToBase64Function.java |   1 +
 .../transform/process/function/ToDateFunction.java |   1 +
 .../process/function/ToTimestampFunction.java  |   1 +
 .../{NowFunction.java => TransformFunction.java}   |  34 ++
 .../process/function/TranslateFunction.java|   1 +
 .../transform/process/function/TrimFunction.java   |   1 +
 .../process/function/UnixTimestampFunction.java|   1 +
 .../transform/process/function/UpperFunction.java  |   1 +
 .../transform/process/operator/OperatorTools.java  | 125 +
 .../TestTransformTemporalFunctionsProcessor.java   |  42 +++
 50 files changed, 251 insertions(+), 171 deletions(-)

diff --git a/inlong-sdk/transform-sdk/pom.xml b/inlong-sdk/transform-sdk/pom.xml
index 63659e0812..4770f0025d 100644
--- a/inlong-sdk/transform-sdk/pom.xml
+++ b/inlong-sdk/transform-sdk/pom.xml
@@ -58,6 +58,10 @@
 protobuf-java-util
 ${protobuf.version}
 
+
+org.reflections
+reflections
+
 
 
 
diff --git 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/AbsFunction.java
 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/AbsFunction.java
index d2b2ceace6..497a69c0f6 100644
--- 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/AbsFunction.java
+++ 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/AbsFunction.java
@@ -30,6 +30,7 @@ import java.math.BigDecimal;
  * AbsFunction
  * description: abs(numeric)--returns the absolute value of numeric
  */
+@TransformFunction(names = {"abs"})
 public class AbsFunction implements ValueParser {
 
 private ValueParser numberParser;
diff --git 
a/inlong-sdk/transform-sdk/src/main/ja

(inlong) branch master updated (e7878063ff -> de98601ce7)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


from e7878063ff [INLONG-10897][SDK] Transform support DATEDIFF function 
(#10925)
 add de98601ce7 [INLONG-10959][SDK] Transform support IFNULL function 
(#10960)

No new revisions were added by this update.

Summary of changes:
 .../{StrcmpFunction.java => IfNullFunction.java}   | 55 ++-
 .../TestTransformArithmeticFunctionsProcessor.java | 63 ++
 2 files changed, 93 insertions(+), 25 deletions(-)
 copy 
inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/{StrcmpFunction.java
 => IfNullFunction.java} (55%)

(inlong) branch master updated: [INLONG-10944][SDK] Support Inlong Transform parser annotation (#10945)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new d8a09abac2 [INLONG-10944][SDK] Support Inlong Transform parser 
annotation (#10945)
d8a09abac2 is described below

commit d8a09abac25615e66f7893e90858b726a6c0fa61
Author: emptyOVO <118812562+empty...@users.noreply.github.com>
AuthorDate: Sat Aug 31 18:45:36 2024 +0800

[INLONG-10944][SDK] Support Inlong Transform parser annotation (#10945)

* [INLONG-10944][SDK] Support Inlong Transform parser annotation

* fix: naming notations

* fix: remove the check of class==null

* fix: code standard check

* fix: naming Convention
---
 .../transform/process/operator/OperatorTools.java  | 55 +--
 .../transform/process/parser/AdditionParser.java   |  1 +
 .../sdk/transform/process/parser/ColumnParser.java |  1 +
 .../sdk/transform/process/parser/DateParser.java   |  1 +
 .../transform/process/parser/DivisionParser.java   |  1 +
 .../sdk/transform/process/parser/DoubleParser.java |  1 +
 .../sdk/transform/process/parser/LongParser.java   |  1 +
 .../sdk/transform/process/parser/ModuloParser.java |  1 +
 .../process/parser/MultiplicationParser.java   |  1 +
 .../process/parser/ParenthesisParser.java  |  1 +
 .../sdk/transform/process/parser/ParserTools.java  | 79 ++
 .../sdk/transform/process/parser/SignParser.java   |  1 +
 .../sdk/transform/process/parser/StringParser.java |  1 +
 .../process/parser/SubtractionParser.java  |  1 +
 .../transform/process/parser/TimestampParser.java  |  1 +
 .../{DoubleParser.java => TransformParser.java}| 27 +++-
 16 files changed, 105 insertions(+), 69 deletions(-)

diff --git 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
index 90405d687c..dbe12ff814 100644
--- 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
+++ 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
@@ -18,36 +18,14 @@
 package org.apache.inlong.sdk.transform.process.operator;
 
 import org.apache.inlong.sdk.transform.process.function.FunctionTools;
-import org.apache.inlong.sdk.transform.process.parser.AdditionParser;
 import org.apache.inlong.sdk.transform.process.parser.ColumnParser;
-import org.apache.inlong.sdk.transform.process.parser.DateParser;
-import org.apache.inlong.sdk.transform.process.parser.DivisionParser;
-import org.apache.inlong.sdk.transform.process.parser.DoubleParser;
-import org.apache.inlong.sdk.transform.process.parser.LongParser;
-import org.apache.inlong.sdk.transform.process.parser.ModuloParser;
-import org.apache.inlong.sdk.transform.process.parser.MultiplicationParser;
-import org.apache.inlong.sdk.transform.process.parser.ParenthesisParser;
-import org.apache.inlong.sdk.transform.process.parser.SignParser;
-import org.apache.inlong.sdk.transform.process.parser.StringParser;
-import org.apache.inlong.sdk.transform.process.parser.SubtractionParser;
-import org.apache.inlong.sdk.transform.process.parser.TimestampParser;
+import org.apache.inlong.sdk.transform.process.parser.ParserTools;
 import org.apache.inlong.sdk.transform.process.parser.ValueParser;
 
-import net.sf.jsqlparser.expression.DateValue;
-import net.sf.jsqlparser.expression.DoubleValue;
 import net.sf.jsqlparser.expression.Expression;
 import net.sf.jsqlparser.expression.Function;
-import net.sf.jsqlparser.expression.LongValue;
 import net.sf.jsqlparser.expression.NotExpression;
 import net.sf.jsqlparser.expression.Parenthesis;
-import net.sf.jsqlparser.expression.SignedExpression;
-import net.sf.jsqlparser.expression.StringValue;
-import net.sf.jsqlparser.expression.TimestampValue;
-import net.sf.jsqlparser.expression.operators.arithmetic.Addition;
-import net.sf.jsqlparser.expression.operators.arithmetic.Division;
-import net.sf.jsqlparser.expression.operators.arithmetic.Modulo;
-import net.sf.jsqlparser.expression.operators.arithmetic.Multiplication;
-import net.sf.jsqlparser.expression.operators.arithmetic.Subtraction;
 import net.sf.jsqlparser.expression.operators.conditional.AndExpression;
 import net.sf.jsqlparser.expression.operators.conditional.OrExpression;
 import net.sf.jsqlparser.expression.operators.relational.EqualsTo;
@@ -56,7 +34,6 @@ import 
net.sf.jsqlparser.expression.operators.relational.GreaterThanEquals;
 import net.sf.jsqlparser.expression.operators.relational.MinorThan;
 import net.sf.jsqlparser.expression.operators.relational.MinorThanEquals;
 import net.sf.jsqlparser.expression.operators.relational.NotEqualsTo;
-import net.sf.jsqlparser.schema.Column

(inlong) branch master updated: [INLONG-10902][SDK] Transform support HEX(numeric or string) function (#10904)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new 7aa0247228 [INLONG-10902][SDK] Transform support HEX(numeric or 
string) function (#10904)
7aa0247228 is described below

commit 7aa0247228a4439123e5096dc6ac669ec2167f3f
Author: emptyOVO <118812562+empty...@users.noreply.github.com>
AuthorDate: Sat Aug 31 18:32:45 2024 +0800

[INLONG-10902][SDK] Transform support HEX(numeric or string) function 
(#10904)

* [INLONG-10902][SDK] Transform support HEX(numeric or string) function

* fix: add functions to check data type

* fix: test data comment

* fix: add anotation

* fix: code conflicts

* fix: add isBigDecimal function to check whether it can be parsed to 
BigDecimal

-

Co-authored-by: AloysZhang 
---
 .../transform/process/function/HexFunction.java| 108 +
 .../TestTransformArithmeticFunctionsProcessor.java |  29 ++
 2 files changed, 137 insertions(+)

diff --git 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/HexFunction.java
 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/HexFunction.java
new file mode 100644
index 00..c641209162
--- /dev/null
+++ 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/HexFunction.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sdk.transform.process.function;
+
+import org.apache.inlong.sdk.transform.decode.SourceData;
+import org.apache.inlong.sdk.transform.process.Context;
+import org.apache.inlong.sdk.transform.process.operator.OperatorTools;
+import org.apache.inlong.sdk.transform.process.parser.ValueParser;
+
+import net.sf.jsqlparser.expression.Function;
+
+import java.math.BigDecimal;
+import java.math.RoundingMode;
+import java.util.Optional;
+import java.util.regex.Pattern;
+
+/**
+ * HexFunction
+ * description:
+ * - If the input argument is a numeric value (such as an integer), the HEX 
function converts the value to the corresponding hexadecimal string.
+ * - If the input argument is a string, the HEX function converts each 
character in the string to its corresponding hexadecimal ASCII encoding and 
returns the hexadecimal representation of the entire string.
+ */
+@TransformFunction(names = {"hex"})
+class HexFunction implements ValueParser {
+
+private static final Pattern BIG_DECIMAL_PATTERN = 
Pattern.compile("^[-+]?\\d+(\\.\\d+)?([eE][-+]?\\d+)?$");
+
+private ValueParser valueParser;
+
+public HexFunction(Function expr) {
+valueParser = 
OperatorTools.buildParser(expr.getParameters().getExpressions().get(0));
+}
+
+@Override
+public Object parse(SourceData sourceData, int rowIndex, Context context) {
+Object valueObj = valueParser.parse(sourceData, rowIndex, context);
+if (isBigDecimal(valueObj)) {
+return hex(OperatorTools.parseBigDecimal(valueObj)).toUpperCase();
+}
+return hex(OperatorTools.parseString(valueObj)).toUpperCase();
+}
+
+private boolean isBigDecimal(Object valueObj) {
+if (valueObj instanceof BigDecimal) {
+return true;
+}
+if (valueObj instanceof String) {
+String str = (String) valueObj;
+return BIG_DECIMAL_PATTERN.matcher(str).matches();
+}
+return false;
+}
+
+// Handle Integer type
+private String hex(int number) {
+return Integer.toHexString(number).toUpperCase();
+}
+
+// Handle long type
+private String hex(long number) {
+return Long.toHexString(number).toUpperCase();
+}
+
+// Handle String type
+private String hex(String input) {
+StringBuilder hexString = new StringBuilder();
+for (char c : input.toCharArray()) {
+hexString.append(Integer.toHexString((int) c).toUpperCase());
+}
+return hexString.

(inlong) branch master updated (de98601ce7 -> 9f9acbd7f5)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


from de98601ce7 [INLONG-10959][SDK] Transform support IFNULL function 
(#10960)
 add 9f9acbd7f5 [INLONG-10957][SDK] Improve some code structures (#10958)

No new revisions were added by this update.

Summary of changes:
 .../sdk/transform/decode/JsonSourceData.java   | 24 +-
 .../sdk/transform/decode/JsonSourceDecoder.java| 55 +++---
 2 files changed, 41 insertions(+), 38 deletions(-)

(inlong) branch master updated: [INLONG-10935][SDK] Transform support DAYNAME function (#10937)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new 6234b9ea01 [INLONG-10935][SDK] Transform support DAYNAME function 
(#10937)
6234b9ea01 is described below

commit 6234b9ea01391b536928c0c0126023a4f5e2ab49
Author: Xincheng Huang <60057611+ying-...@users.noreply.github.com>
AuthorDate: Mon Sep 2 10:50:14 2024 +0800

[INLONG-10935][SDK] Transform support DAYNAME function (#10937)

* [INLONG-10935][SDK] Transform support DAYNAME function

* fix the style

* fix the failed UT
---
 .../transform/process/function/DateExtractFunction.java| 14 +-
 .../sdk/transform/process/operator/OperatorTools.java  |  1 -
 .../process/TestTransformTemporalFunctionsProcessor.java   | 10 ++
 3 files changed, 23 insertions(+), 2 deletions(-)

diff --git 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/DateExtractFunction.java
 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/DateExtractFunction.java
index ff0eb6b01e..cac2273619 100644
--- 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/DateExtractFunction.java
+++ 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/DateExtractFunction.java
@@ -42,6 +42,7 @@ import java.util.Locale;
  * - dayofyear(date)--returns the day of a year (an integer between 1 and 366) 
from SQL date
  * - dayofmonth(date)--returns the day of a month (an integer between 1 and 
31) from SQL date
  * - dayofweek(date)--returns the day of a week (an integer between 1(Sunday) 
and 7(Saturday)) from SQL date
+ * - dayname(date)--returns the name of the day of the week from SQL date
  */
 public abstract class DateExtractFunction implements ValueParser {
 
@@ -50,7 +51,7 @@ public abstract class DateExtractFunction implements 
ValueParser {
 private static final TemporalField weekOfYearField = 
WeekFields.of(Locale.getDefault()).weekOfWeekBasedYear();
 
 public enum DateExtractFunctionType {
-YEAR, QUARTER, MONTH, WEEK, DAY_OF_YEAR, DAY_OF_MONTH, DAY_OF_WEEK
+YEAR, QUARTER, MONTH, WEEK, DAY_OF_YEAR, DAY_OF_MONTH, DAY_OF_WEEK, 
DAY_NAME
 }
 
 @TransformFunction(names = {"year"})
@@ -109,6 +110,14 @@ public abstract class DateExtractFunction implements 
ValueParser {
 }
 }
 
+@TransformFunction(names = {"day_name", "dayname"})
+public static class DayNameExtractFunction extends DateExtractFunction {
+
+public DayNameExtractFunction(Function expr) {
+super(DateExtractFunctionType.DAY_NAME, expr);
+}
+}
+
 public DateExtractFunction(DateExtractFunctionType type, Function expr) {
 this.type = type;
 List expressions = expr.getParameters().getExpressions();
@@ -142,6 +151,9 @@ public abstract class DateExtractFunction implements 
ValueParser {
 // dayofweek(between 1 and 7)
 case DAY_OF_WEEK:
 return localDate.getDayOfWeek().getValue() % 7 + 1;
+// dayname(between Sunday and Saturday)
+case DAY_NAME:
+return localDate.getDayOfWeek().name();
 default:
 return null;
 }
diff --git 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
index 5e69984564..02b24cdb6b 100644
--- 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
+++ 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
@@ -49,7 +49,6 @@ public class OperatorTools {
 public static final String ROOT_KEY = "$root";
 
 public static final String CHILD_KEY = "$child";
-
 public static ExpressionOperator buildOperator(Expression expr) {
 if (expr instanceof AndExpression) {
 return new AndOperator((AndExpression) expr);
diff --git 
a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformTemporalFunctionsProcessor.java
 
b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformTemporalFunctionsProcessor.java
index f2c73ee3d9..14c1fab04b 100644
--- 
a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformTemporalFunctionsProcessor.java
+++ 
b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformTemporalFunctionsProcessor.java
@@ -216,6 +216,16 @@ public class TestTransformTemporalFunctionsProcessor {
 List o

(inlong) branch master updated: [INLONG-10984][SDK] Transform support RADIANS(x) function (#10987)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new c08549eff5 [INLONG-10984][SDK] Transform support RADIANS(x) function 
(#10987)
c08549eff5 is described below

commit c08549eff5113d23b03f4436ca02a77949f4a49d
Author: emptyOVO <118812562+empty...@users.noreply.github.com>
AuthorDate: Tue Sep 3 10:23:03 2024 +0800

[INLONG-10984][SDK] Transform support RADIANS(x) function (#10987)

* [INLONG-10984][SDK] Transform support RADIANS(x) function

* fix: add description
---
 .../process/function/RadiansFunction.java  | 49 ++
 .../TestTransformArithmeticFunctionsProcessor.java | 17 
 2 files changed, 66 insertions(+)

diff --git 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/RadiansFunction.java
 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/RadiansFunction.java
new file mode 100644
index 00..635d18b9b8
--- /dev/null
+++ 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/RadiansFunction.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sdk.transform.process.function;
+
+import org.apache.inlong.sdk.transform.decode.SourceData;
+import org.apache.inlong.sdk.transform.process.Context;
+import org.apache.inlong.sdk.transform.process.operator.OperatorTools;
+import org.apache.inlong.sdk.transform.process.parser.ValueParser;
+
+import net.sf.jsqlparser.expression.Function;
+
+/**
+ * RadiansFunction
+ * description:
+ *  - RADIANS(x)--returns radians of x, Convert degrees to radians
+ */
+@TransformFunction(names = {"radians"})
+public class RadiansFunction implements ValueParser {
+
+private ValueParser degreeParser;
+
+public RadiansFunction(Function expr) {
+degreeParser = 
OperatorTools.buildParser(expr.getParameters().getExpressions().get(0));
+}
+
+@Override
+public Object parse(SourceData sourceData, int rowIndex, Context context) {
+Object degreeObj = degreeParser.parse(sourceData, rowIndex, context);
+if (degreeObj == null) {
+return null;
+}
+return 
Math.toRadians(OperatorTools.parseBigDecimal(degreeObj).doubleValue());
+}
+}
diff --git 
a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java
 
b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java
index 1e0f14025f..c36835e579 100644
--- 
a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java
+++ 
b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java
@@ -422,6 +422,23 @@ public class TestTransformArithmeticFunctionsProcessor {
 Assert.assertEquals(output2.get(0), "result=2.302585092994046");
 }
 
+@Test
+public void testRadiansFunction() throws Exception {
+String transformSql = "select radians(numeric1) from source";
+TransformConfig config = new TransformConfig(transformSql);
+TransformProcessor processor = TransformProcessor
+.create(config, 
SourceDecoderFactory.createCsvDecoder(csvSource),
+SinkEncoderFactory.createKvEncoder(kvSink));
+// case1: radians(10)
+List output1 = processor.transform("10|4|6|8", new 
HashMap<>());
+Assert.assertEquals(1, output1.size());
+Assert.assertEquals(output1.get(0), "result=0.17453292519943295");
+// case2: radians(18.97)
+List output2 = processor.transform("18.97|4|6|8", new 
HashMap<>());
+Assert.assertEquals(1, output2.size());
+Assert.assertEquals(output2.get(0), "result=0.33108895910332425");
+}
+
 @Test
 

(inlong) branch master updated: [INLONG-10982][SDK] Improve the test code structure (#10985)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new bc6e9610bb [INLONG-10982][SDK] Improve the test code structure (#10985)
bc6e9610bb is described below

commit bc6e9610bb84efe3e794cda5d79a7dda784b71e8
Author: Xincheng Huang <60057611+ying-...@users.noreply.github.com>
AuthorDate: Tue Sep 3 10:23:19 2024 +0800

[INLONG-10982][SDK] Improve the test code structure (#10985)
---
 .../transform/process/TestTransformProcessor.java  | 67 ++
 1 file changed, 16 insertions(+), 51 deletions(-)

diff --git 
a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformProcessor.java
 
b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformProcessor.java
index 20af097de6..3413f1aca3 100644
--- 
a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformProcessor.java
+++ 
b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformProcessor.java
@@ -44,13 +44,7 @@ public class TestTransformProcessor {
 
 @Test
 public void testCsv2Kv() throws Exception {
-List fields = new ArrayList<>();
-FieldInfo ftime = new FieldInfo();
-ftime.setName("ftime");
-fields.add(ftime);
-FieldInfo extinfo = new FieldInfo();
-extinfo.setName("extinfo");
-fields.add(extinfo);
+List fields = this.getTestFieldList("ftime", "extinfo");
 CsvSourceInfo csvSource = new CsvSourceInfo("UTF-8", '|', '\\', 
fields);
 KvSinkInfo kvSink = new KvSinkInfo("UTF-8", fields);
 String transformSql = "select ftime,extinfo from source where 
extinfo='ok'";
@@ -97,13 +91,7 @@ public class TestTransformProcessor {
 
 @Test
 public void testKv2Csv() throws Exception {
-List fields = new ArrayList<>();
-FieldInfo ftime = new FieldInfo();
-ftime.setName("ftime");
-fields.add(ftime);
-FieldInfo extinfo = new FieldInfo();
-extinfo.setName("extinfo");
-fields.add(extinfo);
+List fields = this.getTestFieldList("ftime", "extinfo");
 KvSourceInfo kvSource = new KvSourceInfo("UTF-8", fields);
 CsvSinkInfo csvSink = new CsvSinkInfo("UTF-8", '|', '\\', fields);
 String transformSql = "select ftime,extinfo from source where 
extinfo='ok'";
@@ -148,7 +136,7 @@ public class TestTransformProcessor {
 
 @Test
 public void testJson2Csv() throws Exception {
-List fields1 = this.getTestFieldList();
+List fields1 = this.getTestFieldList("sid", "packageID", 
"msgTime", "msg");
 JsonSourceInfo jsonSource1 = new JsonSourceInfo("UTF-8", "msgs");
 CsvSinkInfo csvSink1 = new CsvSinkInfo("UTF-8", '|', '\\', fields1);
 String transformSql1 = "select 
$root.sid,$root.packageID,$child.msgTime,$child.msg from source";
@@ -170,7 +158,7 @@ public class TestTransformProcessor {
 Assert.assertEquals(output1.get(0), 
"value1|value2|1713243918000|value4");
 Assert.assertEquals(output1.get(1), "value1|value2|1713243918000|v4");
 // case2
-List fields2 = this.getTestFieldList2();
+List fields2 = this.getTestFieldList("id", "itemId", 
"subItemId", "msg");
 JsonSourceInfo jsonSource2 = new JsonSourceInfo("UTF-8", "items");
 CsvSinkInfo csvSink2 = new CsvSinkInfo("UTF-8", '|', '\\', fields2);
 String transformSql2 =
@@ -205,7 +193,7 @@ public class TestTransformProcessor {
 
 @Test
 public void testJson2CsvForOne() throws Exception {
-List fields = this.getTestFieldList();
+List fields = this.getTestFieldList("sid", "packageID", 
"msgTime", "msg");
 JsonSourceInfo jsonSource = new JsonSourceInfo("UTF-8", "");
 CsvSinkInfo csvSink = new CsvSinkInfo("UTF-8", '|', '\\', fields);
 String transformSql = "select 
$root.sid,$root.packageID,$root.msgs(1).msgTime,$root.msgs(0).msg from source";
@@ -229,7 +217,7 @@ public class TestTransformProcessor {
 
 @Test
 public void testPb2Csv() throws Exception {
-List fields = this.getTestFieldList();
+List fields = this.getTestFieldList("sid", "packageID", 
"msgTime", "msg")

(inlong) branch master updated: [INLONG-10809][SDK] Improvements to TypeConverter field types and CompareValue in OperatorTools (#10817)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new dfd42dc88d [INLONG-10809][SDK] Improvements to TypeConverter field 
types and CompareValue in OperatorTools (#10817)
dfd42dc88d is described below

commit dfd42dc88d7f5bb95d66165ae5cfcf6c92921bd7
Author: Zkplo <87751516+zk...@users.noreply.github.com>
AuthorDate: Tue Sep 3 10:24:08 2024 +0800

[INLONG-10809][SDK] Improvements to TypeConverter field types and 
CompareValue in OperatorTools (#10817)

Co-authored-by: ZKpLo <14148880+zk...@user.noreply.gitee.com>
---
 .../inlong/sdk/transform/decode/CsvSourceData.java |  10 +-
 .../sdk/transform/decode/CsvSourceDecoder.java |   8 +-
 .../inlong/sdk/transform/decode/SourceData.java|   2 +-
 .../inlong/sdk/transform/pojo/FieldInfo.java   |   2 +-
 .../converter/DoubleConverter.java}|  16 +-
 .../converter/LongConverter.java}  |  16 +-
 .../sdk/transform/process/function/IfFunction.java |  55 
 .../transform/process/operator/OperatorTools.java  |  21 +-
 .../sdk/transform/process/parser/DoubleParser.java |   1 +
 .../TestTransformExpressionOperatorsProcessor.java | 354 +
 10 files changed, 445 insertions(+), 40 deletions(-)

diff --git 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/CsvSourceData.java
 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/CsvSourceData.java
index d4492b4b85..e0bd9f794c 100644
--- 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/CsvSourceData.java
+++ 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/CsvSourceData.java
@@ -28,14 +28,14 @@ import java.util.Map;
  */
 public class CsvSourceData implements SourceData {
 
-private List> rows = new ArrayList<>();
+private List> rows = new ArrayList<>();
 
-private Map currentRow;
+private Map currentRow;
 
 public CsvSourceData() {
 }
 
-public void putField(String fieldName, String fieldValue) {
+public void putField(String fieldName, Object fieldValue) {
 this.currentRow.put(fieldName, fieldValue);
 }
 
@@ -50,11 +50,11 @@ public class CsvSourceData implements SourceData {
 }
 
 @Override
-public String getField(int rowNum, String fieldName) {
+public Object getField(int rowNum, String fieldName) {
 if (rowNum >= this.rows.size()) {
 return null;
 }
-Map targetRow = this.rows.get(rowNum);
+Map targetRow = this.rows.get(rowNum);
 return targetRow.get(fieldName);
 }
 }
diff --git 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/CsvSourceDecoder.java
 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/CsvSourceDecoder.java
index fb95dadc43..7b3dedb637 100644
--- 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/CsvSourceDecoder.java
+++ 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/CsvSourceDecoder.java
@@ -76,9 +76,13 @@ public class CsvSourceDecoder implements 
SourceDecoder {
 int fieldIndex = 0;
 for (FieldInfo field : fields) {
 String fieldName = field.getName();
-String fieldValue = null;
+Object fieldValue = null;
 if (fieldIndex < fieldValues.length) {
-fieldValue = fieldValues[fieldIndex];
+try {
+fieldValue = 
field.getConverter().convert(fieldValues[fieldIndex]);
+} catch (Exception e) {
+throw new RuntimeException(e);
+}
 }
 sourceData.putField(fieldName, fieldValue);
 fieldIndex++;
diff --git 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/SourceData.java
 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/SourceData.java
index 2c39948f2d..cf5f9c0fbe 100644
--- 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/SourceData.java
+++ 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/SourceData.java
@@ -26,5 +26,5 @@ public interface SourceData {
 
 int getRowCount();
 
-String getField(int rowNum, String fieldName);
+Object getField(int rowNum, String fieldName);
 }
diff --git 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/pojo/FieldInfo.java
 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/pojo/FieldInfo.java
index 1027dad944..2a7834112a 100644
--- 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlo

(inlong) branch master updated: [INLONG-10979][SDK] Transform support PI() function (#10983)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new 0a394f49ea [INLONG-10979][SDK] Transform support PI() function (#10983)
0a394f49ea is described below

commit 0a394f49ea026b2536ef0b2a90e3a9183e4a51de
Author: emptyOVO <118812562+empty...@users.noreply.github.com>
AuthorDate: Tue Sep 3 10:25:35 2024 +0800

[INLONG-10979][SDK] Transform support PI() function (#10983)
---
 .../sdk/transform/process/function/PiFunction.java | 40 ++
 .../TestTransformArithmeticFunctionsProcessor.java | 13 +++
 2 files changed, 53 insertions(+)

diff --git 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/PiFunction.java
 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/PiFunction.java
new file mode 100644
index 00..722955ebf5
--- /dev/null
+++ 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/PiFunction.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sdk.transform.process.function;
+
+import org.apache.inlong.sdk.transform.decode.SourceData;
+import org.apache.inlong.sdk.transform.process.Context;
+import org.apache.inlong.sdk.transform.process.parser.ValueParser;
+
+import net.sf.jsqlparser.expression.Function;
+/**
+ * PiFunction
+ * returns the mathematical constant PI
+ */
+@TransformFunction(names = {"pi"})
+public class PiFunction implements ValueParser {
+
+public PiFunction(Function expr) {
+}
+
+@Override
+public Object parse(SourceData sourceData, int rowIndex, Context context) {
+return String.valueOf(Math.PI);
+}
+
+}
diff --git 
a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java
 
b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java
index c36835e579..4007a9a876 100644
--- 
a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java
+++ 
b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformArithmeticFunctionsProcessor.java
@@ -728,4 +728,17 @@ public class TestTransformArithmeticFunctionsProcessor {
 Assert.assertEquals(output5.get(0), "result=616263");
 }
 
+@Test
+public void testPiFunction() throws Exception {
+String transformSql1 = "select pi() from source";
+TransformConfig config1 = new TransformConfig(transformSql1);
+TransformProcessor processor1 = TransformProcessor
+.create(config1, 
SourceDecoderFactory.createCsvDecoder(csvSource),
+SinkEncoderFactory.createKvEncoder(kvSink));
+// case: pi()
+List output1 = processor1.transform("1007|4|6|8", new 
HashMap<>());
+Assert.assertEquals(1, output1.size());
+Assert.assertEquals(output1.get(0), "result=3.141592653589793");
+}
+
 }

(inlong) branch master updated: [INLONG-10971][SDK] Transform support INSERT function (#10972)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new 3e7f81d41c [INLONG-10971][SDK] Transform support INSERT function 
(#10972)
3e7f81d41c is described below

commit 3e7f81d41c0e1ab8294d93b374e54515d0b7936b
Author: Xincheng Huang <60057611+ying-...@users.noreply.github.com>
AuthorDate: Tue Sep 3 10:25:52 2024 +0800

[INLONG-10971][SDK] Transform support INSERT function (#10972)
---
 .../transform/process/function/InsertFunction.java | 101 +
 .../TestTransformStringFunctionsProcessor.java |  34 +++
 2 files changed, 135 insertions(+)

diff --git 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/InsertFunction.java
 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/InsertFunction.java
new file mode 100644
index 00..3473bc9d79
--- /dev/null
+++ 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/InsertFunction.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sdk.transform.process.function;
+
+import org.apache.inlong.sdk.transform.decode.SourceData;
+import org.apache.inlong.sdk.transform.process.Context;
+import org.apache.inlong.sdk.transform.process.operator.OperatorTools;
+import org.apache.inlong.sdk.transform.process.parser.ValueParser;
+
+import net.sf.jsqlparser.expression.Expression;
+import net.sf.jsqlparser.expression.Function;
+
+import java.util.List;
+/**
+ * InsertFunction
+ *
+ * Description:
+ * Returns a string where a specified substring is replaced by another string, 
starting at a given position and for a specified length.
+ * If the position is out of the string's bounds, the original string is 
returned.
+ * If the length exceeds the remaining length of the string from the given 
position, the replacement continues to the end of the string.
+ * If any argument is null, the function returns null.
+ *
+ * Arguments:
+ * - str: The original string.
+ * - pos: The position to start the replacement (1-based index).
+ * - len: The number of characters to replace.
+ * - newstr: The string to insert.
+ *
+ * Examples:
+ * - INSERT('12345678', 3, 4, 'word') = '12word78'
+ * - INSERT('12345678', -1, 4, 'word') = '12345678'
+ * - INSERT('12345678', 3, 100, 'word') = '12word'
+ */
+@TransformFunction(names = {"insert"})
+public class InsertFunction implements ValueParser {
+
+private ValueParser strParser;
+private ValueParser posParser;
+private ValueParser lenParser;
+private ValueParser newStrParser;
+
+public InsertFunction(Function expr) {
+List expressions = expr.getParameters().getExpressions();
+strParser = OperatorTools.buildParser(expressions.get(0));
+posParser = OperatorTools.buildParser(expressions.get(1));
+lenParser = OperatorTools.buildParser(expressions.get(2));
+newStrParser = OperatorTools.buildParser(expressions.get(3));
+}
+
+@Override
+public Object parse(SourceData sourceData, int rowIndex, Context context) {
+Object strObject = strParser.parse(sourceData, rowIndex, context);
+Object posObject = posParser.parse(sourceData, rowIndex, context);
+Object lenObject = lenParser.parse(sourceData, rowIndex, context);
+Object newStrObject = newStrParser.parse(sourceData, rowIndex, 
context);
+
+if (strObject == null || posObject == null || lenObject == null || 
newStrObject == null) {
+return null;
+}
+
+String str = OperatorTools.parseString(strObject);
+int pos = OperatorTools.parseBigDecimal(posObject).intValue();
+int len = OperatorTools.parseBigDecimal(lenObject).intValue();
+String newStr = OperatorTools.parseString(newStrObject);
+
+if (str == null || newStr == null) {
+return null;
+}
+
+

(inlong) branch master updated: [INLONG-10965][SDK] Transform support Bitwise operation (#10970)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new cb4b61b5a8 [INLONG-10965][SDK] Transform support Bitwise operation 
(#10970)
cb4b61b5a8 is described below

commit cb4b61b5a889cbb035b51ad1dd48f6a02790573f
Author: Zkplo <87751516+zk...@users.noreply.github.com>
AuthorDate: Tue Sep 3 10:26:19 2024 +0800

[INLONG-10965][SDK] Transform support Bitwise operation (#10970)

Co-authored-by: ZKpLo <14148880+zk...@user.noreply.gitee.com>
---
 .../transform/process/parser/BitwiseAndParser.java |  61 +++
 .../process/parser/BitwiseLeftShiftParser.java |  67 
 .../transform/process/parser/BitwiseOrParser.java  |  62 
 .../process/parser/BitwiseRightShiftParser.java|  66 
 .../transform/process/parser/BitwiseXorParser.java |  62 
 .../sdk/transform/process/parser/SignParser.java   |  26 ++-
 .../TestTransformArithmeticFunctionsProcessor.java | 175 +
 7 files changed, 513 insertions(+), 6 deletions(-)

diff --git 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/BitwiseAndParser.java
 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/BitwiseAndParser.java
new file mode 100644
index 00..b8920434ef
--- /dev/null
+++ 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/BitwiseAndParser.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sdk.transform.process.parser;
+
+import org.apache.inlong.sdk.transform.decode.SourceData;
+import org.apache.inlong.sdk.transform.process.Context;
+import org.apache.inlong.sdk.transform.process.operator.OperatorTools;
+
+import lombok.extern.slf4j.Slf4j;
+import net.sf.jsqlparser.expression.operators.arithmetic.BitwiseAnd;
+
+import java.math.BigInteger;
+
+/**
+ * BitwiseAndParser
+ */
+@Slf4j
+@TransformParser(values = BitwiseAnd.class)
+public class BitwiseAndParser implements ValueParser {
+
+private final ValueParser left;
+
+private final ValueParser right;
+
+public BitwiseAndParser(BitwiseAnd expr) {
+this.left = OperatorTools.buildParser(expr.getLeftExpression());
+this.right = OperatorTools.buildParser(expr.getRightExpression());
+}
+
+@Override
+public Object parse(SourceData sourceData, int rowIndex, Context context) {
+try {
+Object leftObj = this.left.parse(sourceData, rowIndex, context);
+Object rightObj = this.right.parse(sourceData, rowIndex, context);
+if (leftObj == null || rightObj == null) {
+return null;
+}
+BigInteger leftValue = 
OperatorTools.parseBigDecimal(leftObj).toBigInteger();
+BigInteger rightValue = 
OperatorTools.parseBigDecimal(rightObj).toBigInteger();
+return 
Long.toUnsignedString(leftValue.and(rightValue).longValue());
+} catch (Exception e) {
+log.error("Value parsing failed", e);
+return null;
+}
+}
+}
diff --git 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/BitwiseLeftShiftParser.java
 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/BitwiseLeftShiftParser.java
new file mode 100644
index 00..c68bc89e54
--- /dev/null
+++ 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/parser/BitwiseLeftShiftParser.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+

(inlong) branch master updated: [INLONG-10956][SDK] Transform SQL supports SHA encryption algorithm (#10996)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new e394d74076 [INLONG-10956][SDK] Transform SQL supports SHA encryption 
algorithm (#10996)
e394d74076 is described below

commit e394d740769730f6960e7022e0bc8f4dee1e2e9b
Author: Zkplo <87751516+zk...@users.noreply.github.com>
AuthorDate: Tue Sep 3 10:39:27 2024 +0800

[INLONG-10956][SDK] Transform SQL supports SHA encryption algorithm (#10996)

Co-authored-by: ZKpLo <14148880+zk...@user.noreply.gitee.com>
---
 .../transform/process/function/Sha2Function.java   | 75 +
 .../transform/process/function/ShaFunction.java| 54 +++
 .../TestTransformArithmeticFunctionsProcessor.java | 78 ++
 3 files changed, 207 insertions(+)

diff --git 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/Sha2Function.java
 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/Sha2Function.java
new file mode 100644
index 00..ad120712bf
--- /dev/null
+++ 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/Sha2Function.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sdk.transform.process.function;
+
+import org.apache.inlong.sdk.transform.decode.SourceData;
+import org.apache.inlong.sdk.transform.process.Context;
+import org.apache.inlong.sdk.transform.process.operator.OperatorTools;
+import org.apache.inlong.sdk.transform.process.parser.ValueParser;
+
+import net.sf.jsqlparser.expression.Expression;
+import net.sf.jsqlparser.expression.Function;
+import org.apache.commons.codec.digest.DigestUtils;
+
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
+import static org.apache.commons.codec.digest.MessageDigestAlgorithms.SHA_224;
+
+/**
+ * Sha2Function
+ * description: SHA2(str, hash_length): Calculates the SHA-2 family of hash 
functions (SHA-224, SHA-256, SHA-384, and SHA-512)
+ * return NULL If either argument is NULL or the hash length(224 256 384 512) 
is not one of the permitted values
+ * return a hash value containing the desired number of bits.
+ */
+@TransformFunction(names = {"sha2"})
+public class Sha2Function implements ValueParser {
+
+private final ValueParser msgParser;
+private final ValueParser lenParser;
+
+public Sha2Function(Function expr) {
+List expressions = expr.getParameters().getExpressions();
+msgParser = OperatorTools.buildParser(expressions.get(0));
+lenParser = OperatorTools.buildParser(expressions.get(1));
+}
+
+@Override
+public Object parse(SourceData sourceData, int rowIndex, Context context) {
+Object msgObj = msgParser.parse(sourceData, rowIndex, context);
+Object lenObj = lenParser.parse(sourceData, rowIndex, context);
+if (msgObj == null || lenObj == null) {
+return null;
+}
+String msg = msgObj.toString();
+int len = Integer.parseInt(lenObj.toString());
+switch (len) {
+case 0:
+case 256:
+return 
DigestUtils.sha256Hex(msg.getBytes(StandardCharsets.UTF_8));
+case 224:
+return new 
DigestUtils(SHA_224).digestAsHex(msg.getBytes(StandardCharsets.UTF_8));
+case 384:
+return 
DigestUtils.sha384Hex(msg.getBytes(StandardCharsets.UTF_8));
+case 512:
+return 
DigestUtils.sha512Hex(msg.getBytes(StandardCharsets.UTF_8));
+default:
+return null;
+}
+}
+}
diff --git 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/ShaFunction.java
 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/ShaFunction.java
new file mode 100644
index 00..7da5e6c3a9
--- /dev/null
+++ 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/proce

(inlong) branch master updated: [INLONG-10999][SDK] Support to return raw data by star sign in transformer SQL (#11004)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new edf93bd547 [INLONG-10999][SDK] Support to return raw data by star sign 
in transformer SQL (#11004)
edf93bd547 is described below

commit edf93bd547a09df6b1d7a6d57ea3d719d9f63f4f
Author: ChunLiang Lu 
AuthorDate: Tue Sep 3 21:49:46 2024 +0800

[INLONG-10999][SDK] Support to return raw data by star sign in transformer 
SQL (#11004)

* [INLONG-10999][SDK] Support to return raw data by star sign in 
transformer SQL

* add more UT Case

* fix code format problems

* fix pom.xml problem
---
 .../sdk/transform/encode/CsvSinkEncoder.java   |  6 +-
 .../inlong/sdk/transform/encode/KvSinkEncoder.java |  6 +-
 .../inlong/sdk/transform/encode/SinkEncoder.java   |  2 +
 .../sdk/transform/process/TransformProcessor.java  | 34 +++---
 .../ValueParserNode.java}  | 19 +++---
 .../transform/process/TestTransformProcessor.java  | 76 ++
 6 files changed, 122 insertions(+), 21 deletions(-)

diff --git 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/encode/CsvSinkEncoder.java
 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/encode/CsvSinkEncoder.java
index ce47a0072c..89f6f364a0 100644
--- 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/encode/CsvSinkEncoder.java
+++ 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/encode/CsvSinkEncoder.java
@@ -66,7 +66,11 @@ public class CsvSinkEncoder implements SinkEncoder {
 } else {
 for (String fieldName : sinkData.keyList()) {
 String fieldValue = sinkData.getField(fieldName);
-EscapeUtils.escapeContent(builder, delimiter, escapeChar, 
fieldValue);
+if (StringUtils.equals(fieldName, ALL_SOURCE_FIELD_SIGN)) {
+builder.append(fieldValue);
+} else {
+EscapeUtils.escapeContent(builder, delimiter, 
escapeChar, fieldValue);
+}
 builder.append(delimiter);
 }
 }
diff --git 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/encode/KvSinkEncoder.java
 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/encode/KvSinkEncoder.java
index 7460ec95c2..2822374c41 100644
--- 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/encode/KvSinkEncoder.java
+++ 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/encode/KvSinkEncoder.java
@@ -63,7 +63,11 @@ public class KvSinkEncoder implements SinkEncoder {
 if (fields == null || fields.size() == 0) {
 for (String fieldName : sinkData.keyList()) {
 String fieldValue = sinkData.getField(fieldName);
-
builder.append(fieldName).append(kvDelimiter).append(fieldValue).append(entryDelimiter);
+if (StringUtils.equals(fieldName, ALL_SOURCE_FIELD_SIGN)) {
+builder.append(fieldValue).append(entryDelimiter);
+} else {
+
builder.append(fieldName).append(kvDelimiter).append(fieldValue).append(entryDelimiter);
+}
 }
 } else {
 for (FieldInfo field : fields) {
diff --git 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/encode/SinkEncoder.java
 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/encode/SinkEncoder.java
index 7f845a99d6..a63f970295 100644
--- 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/encode/SinkEncoder.java
+++ 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/encode/SinkEncoder.java
@@ -27,6 +27,8 @@ import java.util.List;
  */
 public interface SinkEncoder {
 
+public static final String ALL_SOURCE_FIELD_SIGN = "*";
+
 Output encode(SinkData sinkData, Context context);
 
 List getFields();
diff --git 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/TransformProcessor.java
 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/TransformProcessor.java
index 9944268dda..acb7e62e07 100644
--- 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/TransformProcessor.java
+++ 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/TransformProcessor.java
@@ -31,23 +31,23 @@ import 
org.apache.inlong.sdk.transform.process.parser.ValueParser;
 import com.google.common.collect.ImmutableMap;
 import net.sf.jsqlparser.JSQLParserException;
 import net.sf.jsqlparser.parser.CCJSqlParserManage

(inlong) branch master updated: [INLONG-10963][SDK] Transform SQL support CONTAINS function. (#10964)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new 49b07ad683 [INLONG-10963][SDK] Transform SQL support CONTAINS 
function. (#10964)
49b07ad683 is described below

commit 49b07ad68382cbaf50db2b6d04a799c3abff97f3
Author: rachely <124853723+ybszzz...@users.noreply.github.com>
AuthorDate: Thu Sep 5 09:02:48 2024 +0800

[INLONG-10963][SDK] Transform SQL support CONTAINS function. (#10964)

* [INLONG-10963][SDK] Transform SQL support CONTAINS function.(#10963)

* [INLONG-10963][SDK] Transform SQL support CONTAINS function.(#10963)
---
 .../process/function/ContainsFunction.java | 56 ++
 .../TestTransformStringFunctionsProcessor.java | 32 +
 2 files changed, 88 insertions(+)

diff --git 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/ContainsFunction.java
 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/ContainsFunction.java
new file mode 100644
index 00..e2905eea07
--- /dev/null
+++ 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/ContainsFunction.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sdk.transform.process.function;
+
+import org.apache.inlong.sdk.transform.decode.SourceData;
+import org.apache.inlong.sdk.transform.process.Context;
+import org.apache.inlong.sdk.transform.process.operator.OperatorTools;
+import org.apache.inlong.sdk.transform.process.parser.ValueParser;
+
+import net.sf.jsqlparser.expression.Expression;
+import net.sf.jsqlparser.expression.Function;
+
+import java.util.List;
+
+/**
+ * ContainsFunction
+ * description: contains(left, right) - Returns a boolean.
+ * The value is True if right is found inside left, otherwise, returns False.
+ * Both left or right must be of STRING type.
+ */
+@TransformFunction(names = {"contains"})
+public class ContainsFunction implements ValueParser {
+
+private ValueParser leftStrParser;
+private ValueParser rightStrParser;
+
+public ContainsFunction(Function expr) {
+List expressions = expr.getParameters().getExpressions();
+leftStrParser = OperatorTools.buildParser(expressions.get(0));
+rightStrParser = OperatorTools.buildParser(expressions.get(1));
+}
+
+@Override
+public Object parse(SourceData sourceData, int rowIndex, Context context) {
+Object leftStrObj = leftStrParser.parse(sourceData, rowIndex, context);
+Object rightStrObj = rightStrParser.parse(sourceData, rowIndex, 
context);
+String leftStr = OperatorTools.parseString(leftStrObj);
+String rightStr = OperatorTools.parseString(rightStrObj);
+return (leftStr == null || rightStr == null) ? null : 
leftStr.contains(rightStr);
+}
+}
diff --git 
a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformStringFunctionsProcessor.java
 
b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformStringFunctionsProcessor.java
index a3099d09f2..f099d728dc 100644
--- 
a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformStringFunctionsProcessor.java
+++ 
b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestTransformStringFunctionsProcessor.java
@@ -724,4 +724,36 @@ public class TestTransformStringFunctionsProcessor {
 Assert.assertEquals("result=1278", output5.get(0));
 }
 
+@Test
+public void testContainsFunction() throws Exception {
+String transformSql = "select contains(string1, string2) from source";
+TransformConfig config = new TransformConfig(transformSql);
+TransformProcessor processor = TransformProcessor
+.create(config, 
SourceDecoderFactory.createCsvDecoder(csvSource),
+SinkEncoderFactory.createKvEncoder(kvS

(inlong) branch master updated: [INLONG-11037][SDK] Transform support ENCODE() and DECODE() function (#11041)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new 751f69dad8 [INLONG-11037][SDK] Transform support ENCODE() and DECODE() 
function (#11041)
751f69dad8 is described below

commit 751f69dad806dff68ce796399c5e7f0c7dbfdb95
Author: emptyOVO <118812562+empty...@users.noreply.github.com>
AuthorDate: Wed Sep 11 14:13:37 2024 +0800

[INLONG-11037][SDK] Transform support ENCODE() and DECODE() function 
(#11041)

* [INLONG-11037][SDK] Transform support ENCODE() and DECODE() function

* fix: add NP check

* fix: clear definition of the specific encoding type

* fix: add description
---
 .../transform/process/function/DecodeFunction.java | 95 +
 .../transform/process/function/EncodeFunction.java | 97 ++
 .../function/string/TestDecodeFunction.java| 80 ++
 .../function/string/TestEncodeFunction.java| 76 +
 4 files changed, 348 insertions(+)

diff --git 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/DecodeFunction.java
 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/DecodeFunction.java
new file mode 100644
index 00..042e6b4f98
--- /dev/null
+++ 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/DecodeFunction.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sdk.transform.process.function;
+
+import org.apache.inlong.sdk.transform.decode.SourceData;
+import org.apache.inlong.sdk.transform.process.Context;
+import org.apache.inlong.sdk.transform.process.operator.OperatorTools;
+import org.apache.inlong.sdk.transform.process.parser.ValueParser;
+
+import net.sf.jsqlparser.expression.Expression;
+import net.sf.jsqlparser.expression.Function;
+
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+/**
+ * DecodeFunction
+ * description: decode(binary, string)
+ *  Decode using the supplied character set (' US-ASCII ', 'ISO-8859-1', 
'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16').
+ *  If either parameter is empty, the result will also be empty.
+ */
+@TransformFunction(names = {"decode"})
+public class DecodeFunction implements ValueParser {
+
+private ValueParser binaryParser;
+
+private ValueParser characterSetParser;
+
+private static final Set SUPPORTED_CHARSETS;
+
+static {
+Set charsets = new HashSet<>();
+charsets.add(StandardCharsets.US_ASCII.name());
+charsets.add(StandardCharsets.ISO_8859_1.name());
+charsets.add(StandardCharsets.UTF_8.name());
+charsets.add(StandardCharsets.UTF_16.name());
+charsets.add(StandardCharsets.UTF_16BE.name());
+charsets.add(StandardCharsets.UTF_16LE.name());
+SUPPORTED_CHARSETS = Collections.unmodifiableSet(charsets);
+}
+
+public DecodeFunction(Function expr) {
+List expressions = expr.getParameters().getExpressions();
+if (expressions != null && expressions.size() == 2) {
+binaryParser = OperatorTools.buildParser(expressions.get(0));
+characterSetParser = OperatorTools.buildParser(expressions.get(1));
+}
+}
+
+@Override
+public Object parse(SourceData sourceData, int rowIndex, Context context) {
+Object binaryObj = binaryParser.parse(sourceData, rowIndex, context);
+Object characterObj = characterSetParser.parse(sourceData, rowIndex, 
context);
+if (binaryObj == null || characterObj == null) {
+return null;
+}
+String binaryString = OperatorTools.parseString(binaryObj);
+String characterSetValue = 
OperatorTools.parseString(characterObj).toUpperCase();
+return decode

(inlong) branch master updated (5b0eb6df9c -> e0a827c414)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


from 5b0eb6df9c [INLONG-11056][SDK] Transform support RAND_INTEGER() 
function (#11063)
 add e0a827c414 [INLONG-11035][SDK] Transform support TRUNCATE() function 
(#11040)

No new revisions were added by this update.

Summary of changes:
 .../process/function/TruncateFunction.java | 90 ++
 ...egerFunction.java => TestTruncateFunction.java} | 41 +-
 2 files changed, 113 insertions(+), 18 deletions(-)
 create mode 100644 
inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/TruncateFunction.java
 copy 
inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/function/arithmetic/{TestRandIntegerFunction.java
 => TestTruncateFunction.java} (65%)

(inlong) branch master updated: [INLONG-11105][SDK] Fix the empty string is converted to a "null" (#11106)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new 6203004031 [INLONG-11105][SDK] Fix the empty string is converted to a 
"null" (#11106)
6203004031 is described below

commit 6203004031d3109e32c6bb90231f5f3a5bacb84f
Author: vernedeng 
AuthorDate: Sun Sep 15 19:48:14 2024 +0800

[INLONG-11105][SDK] Fix the empty string is converted to a "null" (#11106)

* [INLONG-11105][SDK] Fix the empty string is converted into a "null"

* fix ut

* fix
---
 .../inlong/sdk/transform/process/TransformProcessor.java   |  6 +-
 .../sdk/transform/process/TestParseUrlFunctionProcessor.java   |  6 +++---
 .../transform/process/TestTransformChrFunctionProcessor.java   |  2 +-
 .../process/TestTransformConcatWsFunctionProcessor.java|  2 +-
 .../transform/process/function/arithmetic/TestBinFunction.java |  2 +-
 .../process/function/arithmetic/TestIfNullFunction.java|  2 +-
 .../transform/process/function/arithmetic/TestMd5Function.java |  2 +-
 .../process/function/arithmetic/TestSha2Function.java  |  4 ++--
 .../transform/process/function/arithmetic/TestShaFunction.java |  2 +-
 .../process/function/flowcontrol/TestNullIfFunction.java   |  4 ++--
 .../transform/process/function/string/TestAsciiFunction.java   |  2 +-
 .../process/function/string/TestBitLengthFunction.java |  2 +-
 .../process/function/string/TestCompressFunction.java  |  4 ++--
 .../process/function/string/TestFromBase64Function.java|  2 +-
 .../transform/process/function/string/TestLcaseFunction.java   |  2 +-
 .../transform/process/function/string/TestLeftFunction.java|  4 ++--
 .../transform/process/function/string/TestLengthFunction.java  |  2 +-
 .../transform/process/function/string/TestLocateFunction.java  |  2 +-
 .../transform/process/function/string/TestLowerFunction.java   |  2 +-
 .../transform/process/function/string/TestLpadFunction.java|  8 
 .../transform/process/function/string/TestLtrimFunction.java   |  2 +-
 .../transform/process/function/string/TestRightFunction.java   |  4 ++--
 .../transform/process/function/string/TestRpadFunction.java|  8 
 .../transform/process/function/string/TestRtrimFunction.java   |  2 +-
 .../transform/process/function/string/TestSpaceFunction.java   |  2 +-
 .../process/function/string/TestSplitIndexFunction.java| 10 +-
 .../transform/process/function/string/TestStrcmpFunction.java  |  2 +-
 .../transform/process/function/string/TestUcaseFunction.java   |  2 +-
 .../process/function/string/TestUnCompressFunction.java|  8 
 .../transform/process/function/string/TestUpperFunction.java   |  2 +-
 .../process/function/temporal/TestDateDiffFunction.java|  6 +++---
 31 files changed, 57 insertions(+), 53 deletions(-)

diff --git 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/TransformProcessor.java
 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/TransformProcessor.java
index 32024ec1be..6d650cfd4a 100644
--- 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/TransformProcessor.java
+++ 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/TransformProcessor.java
@@ -177,7 +177,11 @@ public class TransformProcessor {
 }
 try {
 Object fieldValue = parser.parse(sourceData, i, context);
-sinkData.addField(fieldName, String.valueOf(fieldValue));
+if (fieldValue == null) {
+sinkData.addField(fieldName, "");
+} else {
+sinkData.addField(fieldName, fieldValue.toString());
+}
 } catch (Throwable t) {
 sinkData.addField(fieldName, "");
 }
diff --git 
a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestParseUrlFunctionProcessor.java
 
b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestParseUrlFunctionProcessor.java
index 407fa72d0a..5850d50d4b 100644
--- 
a/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestParseUrlFunctionProcessor.java
+++ 
b/inlong-sdk/transform-sdk/src/test/java/org/apache/inlong/sdk/transform/process/TestParseUrlFunctionProcessor.java
@@ -111,7 +111,7 @@ public class TestParseUrlFunctionProcessor {
 List output5 = 
processor5.transform("http://facebook.com/path1/p.php?k1=v1&k2=v2#Ref1|QUERY|k1|cloud|1",
 new HashMap<>());
 Assert.assertEquals(1, output5.size());
-Assert.assertEquals(outp

(inlong) branch master updated: [INLONG-11110][SDK] Fix incorrect usage of isLocalVisit variable in client example of dataproxy-sdk module (#11116)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new 9d97dcbb30 [INLONG-0][SDK] Fix incorrect usage of isLocalVisit 
variable in client example of dataproxy-sdk module (#6)
9d97dcbb30 is described below

commit 9d97dcbb309897f64c4406ff96d818c4946cca6f
Author: Haotian Ma <60374114+qy-liu...@users.noreply.github.com>
AuthorDate: Tue Sep 17 11:03:03 2024 +0800

[INLONG-0][SDK] Fix incorrect usage of isLocalVisit variable in client 
example of dataproxy-sdk module (#6)
---
 .../sdk/dataproxy/example/HttpClientExample.java   | 23 ++---
 .../sdk/dataproxy/example/TcpClientExample.java| 29 --
 2 files changed, 18 insertions(+), 34 deletions(-)

diff --git 
a/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/example/HttpClientExample.java
 
b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/example/HttpClientExample.java
index 22274fcf7c..3999390f9b 100644
--- 
a/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/example/HttpClientExample.java
+++ 
b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/example/HttpClientExample.java
@@ -29,40 +29,31 @@ import java.util.concurrent.TimeUnit;
 public class HttpClientExample {
 
 public static void main(String[] args) {
-/*
- * 1. if 'isLocalVisit' is true use local config from file in 
${configBasePath} directory/${dataProxyGroupId}
- * .local such as : configBasePath = /data/inlong/dataproxy/conf 
dataProxyGroupId = test so config file is :
- * /data/inlong/dataproxy/conf/test.local and config context like this:
- * 
{"isInterVisit":1,"clusterId":"1","size":1,"switch":1,"address":[{"host":"127.0.0
- * .1","port":"46802"},{"host":"127.0.0.1","port":"46802"}]}
- *
- * 2. if 'isLocalVisit' is false sdk will get config from manager auto.
- */
 String inlongGroupId = "test_group_id";
 String inlongStreamId = "test_stream_id";
-String configBasePath = "/data/inlong/dataproxy/conf";
+String configBasePath = "";
 String inLongManagerAddr = "127.0.0.1";
-String inLongManagerPort = "8080";
+String inLongManagerPort = "8083";
 String localIP = "127.0.0.1";
 String messageBody = "inlong message body!";
 
 HttpProxySender sender = getMessageSender(localIP, inLongManagerAddr,
-inLongManagerPort, inlongGroupId, false, false,
+inLongManagerPort, inlongGroupId, true, false,
 configBasePath);
-
 sendHttpMessage(sender, inlongGroupId, inlongStreamId, messageBody);
+sender.close(); // close the sender
 }
 
 public static HttpProxySender getMessageSender(String localIP, String 
inLongManagerAddr,
 String inLongManagerPort, String inlongGroupId,
-boolean isLocalVisit, boolean isReadProxyIPFromLocal,
+boolean requestByHttp, boolean isReadProxyIPFromLocal,
 String configBasePath) {
 ProxyClientConfig proxyConfig = null;
 HttpProxySender sender = null;
 try {
-proxyConfig = new ProxyClientConfig(localIP, isLocalVisit, 
inLongManagerAddr,
+proxyConfig = new ProxyClientConfig(localIP, requestByHttp, 
inLongManagerAddr,
 Integer.valueOf(inLongManagerPort),
-inlongGroupId, "test", "123456");
+inlongGroupId, "admin", "inlong");// user and password of 
manager
 proxyConfig.setInlongGroupId(inlongGroupId);
 proxyConfig.setConfStoreBasePath(configBasePath);
 proxyConfig.setReadProxyIPFromLocal(isReadProxyIPFromLocal);
diff --git 
a/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/example/TcpClientExample.java
 
b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/example/TcpClientExample.java
index 993b7750f7..55b6cf6d99 100644
--- 
a/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/example/TcpClientExample.java
+++ 
b/inlong-sdk/dataproxy-sdk/src/main/java/org/apache/inlong/sdk/dataproxy/example/TcpClientExample.java
@@ -40,22 +40,12 @@ public class TcpClientExample {
  */
 public static void main(String[] args) throws InterruptedException {
 
-String inlongGroupId = "test_test";
-String inlongStreamId = "test_test";
-
-/*
- * 1. if i

(inlong) branch master updated: [INLONG-9907][Audit] Audit-service add codes of entities (#9911)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new 3746e3ffcc [INLONG-9907][Audit] Audit-service add codes of entities 
(#9911)
3746e3ffcc is described below

commit 3746e3ffcc97e476fb0fc526a81b298d144607d8
Author: doleyzi <43397300+dole...@users.noreply.github.com>
AuthorDate: Tue Apr 2 10:10:04 2024 +0800

[INLONG-9907][Audit] Audit-service add codes of entities (#9911)

* Audit-service add codes of entities

* Audit-service add codes of entities

* Audit-service add codes of entities

* Audit-service add codes of entities

* Audit-service add codes of entities
---
 .../src/main/java/entities/AuditCycle.java | 36 ++
 .../src/main/java/entities/Request.java| 31 +++
 .../src/main/java/entities/SourceEntities.java | 34 
 .../src/main/java/entities/StartEndTime.java   | 28 +
 .../src/main/java/entities/StatData.java   | 36 ++
 5 files changed, 165 insertions(+)

diff --git a/inlong-audit/audit-service/src/main/java/entities/AuditCycle.java 
b/inlong-audit/audit-service/src/main/java/entities/AuditCycle.java
new file mode 100644
index 00..0b0bb82ce5
--- /dev/null
+++ b/inlong-audit/audit-service/src/main/java/entities/AuditCycle.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package entities;
+
+/**
+ * Audit cycle
+ */
+public enum AuditCycle {
+
+MINUTE_5(5), MINUTE_10(10), MINUTE_30(30), HOUR(60), DAY(1440);
+
+private final int cycle;
+
+AuditCycle(int cycle) {
+this.cycle = cycle;
+}
+
+public int getValue() {
+return cycle;
+}
+}
diff --git a/inlong-audit/audit-service/src/main/java/entities/Request.java 
b/inlong-audit/audit-service/src/main/java/entities/Request.java
new file mode 100644
index 00..0e9ec8a4da
--- /dev/null
+++ b/inlong-audit/audit-service/src/main/java/entities/Request.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package entities;
+
+import lombok.Data;
+
+@Data
+public class Request {
+
+String startTime;
+String endTime;
+String auditId;
+String auditTag;
+String inlongGroupId;
+String inlongStreamId;
+}
diff --git 
a/inlong-audit/audit-service/src/main/java/entities/SourceEntities.java 
b/inlong-audit/audit-service/src/main/java/entities/SourceEntities.java
new file mode 100644
index 00..9e079a875b
--- /dev/null
+++ b/inlong-audit/audit-service/src/main/java/entities/SourceEntities.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS&quo

(inlong) branch master updated: [INLONG-9943][Audit] Audit-service add codes of jdbc sink (#9949)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new 50539a5660 [INLONG-9943][Audit] Audit-service add codes of jdbc sink 
(#9949)
50539a5660 is described below

commit 50539a566013f49a70499b57750e3fc27a099ebc
Author: doleyzi <43397300+dole...@users.noreply.github.com>
AuthorDate: Tue Apr 9 19:12:45 2024 +0800

[INLONG-9943][Audit] Audit-service add codes of jdbc sink (#9949)

* Audit-service add codes of jdbc sink for aggregate the data from the data 
source and store the aggregated data to the target storage

* Use lombok.AllArgsConstructor annotation to structure SinkConfig
---
 .../inlong/audit/config/ConfigConstants.java   |  12 +-
 .../apache/inlong/audit/config/SqlConstants.java   |  10 ++
 .../apache/inlong/audit/entities/SinkConfig.java   |  36 
 .../apache/inlong/audit/service/EtlService.java| 196 +
 .../org/apache/inlong/audit/sink/JdbcSink.java | 164 +
 5 files changed, 412 insertions(+), 6 deletions(-)

diff --git 
a/inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/config/ConfigConstants.java
 
b/inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/config/ConfigConstants.java
index bc4f0f9326..ecf64338e7 100644
--- 
a/inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/config/ConfigConstants.java
+++ 
b/inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/config/ConfigConstants.java
@@ -25,14 +25,14 @@ public class ConfigConstants {
 // Source config
 public static final String KEY_CLICKHOUSE_DRIVER = "clickhouse.driver";
 public static final String DEFAULT_CLICKHOUSE_DRIVER = 
"ru.yandex.clickhouse.ClickHouseDriver";
-public static final String KEY_CLICKHOUSE_URL = "clickhouse.url";
+public static final String KEY_CLICKHOUSE_JDBC_URL = "clickhouse.jdbc.url";
 public static final String KEY_CLICKHOUSE_USERNAME = "clickhouse.username";
 public static final String KEY_CLICKHOUSE_PASSWORD = "clickhouse.password";
 
 // DB config
 public static final String KEY_MYSQL_DRIVER = "mysql.driver";
 public static final String KEY_DEFAULT_MYSQL_DRIVER = 
"com.mysql.cj.jdbc.Driver";
-public static final String KEY_MYSQL_URL = "mysql.url";
+public static final String KEY_MYSQL_JDBC_URL = "mysql.jdbc.url";
 public static final String KEY_MYSQL_USERNAME = "mysql.username";
 public static final String KEY_MYSQL_PASSWORD = "mysql.password";
 
@@ -91,15 +91,15 @@ public class ConfigConstants {
 public static final String DEFAULT_REALTIME_SUMMARY_SOURCE_TABLE = 
"audit_data";
 public static final String KEY_REALTIME_SUMMARY_SINK_TABLE = 
"realtime.summary.sink.table";
 public static final String DEFAULT_REALTIME_SUMMARY_SINK_TABLE = 
"audit_data_temp";
-public static final String KEY_REALTIME_SUMMARY_BEFORE_TIMES = 
"realtime.summary.before.times";
-public static final int DEFAULT_REALTIME_SUMMARY_BEFORE_TIMES = 6;
+public static final String KEY_REALTIME_SUMMARY_STAT_BACK_TIMES = 
"realtime.summary.stat.back.times";
+public static final int DEFAULT_REALTIME_SUMMARY_STAT_BACK_TIMES = 6;
 
 public static final String KEY_DAILY_SUMMARY_SOURCE_TABLE = 
"daily.summary.source.table";
 public static final String DEFAULT_DAILY_SUMMARY_SOURCE_TABLE = 
"audit_data_temp";
 public static final String KEY_DAILY_SUMMARY_SINK_TABLE = 
"daily.summary.sink.table";
 public static final String DEFAULT_DAILY_SUMMARY_SINK_TABLE = 
"audit_data_day";
-public static final String KEY_DAILY_SUMMARY_BEFORE_TIMES = 
"daily.summary.before.times";
-public static final int DEFAULT_DAILY_SUMMARY_BEFORE_TIMES = 2;
+public static final String KEY_DAILY_SUMMARY_STAT_BACK_TIMES = 
"daily.summary.stat.back.times";
+public static final int DEFAULT_DAILY_SUMMARY_STAT_BACK_TIMES = 2;
 
 // HA selector config
 public static final String KEY_RELEASE_LEADER_INTERVAL = 
"release.leader.interval";
diff --git 
a/inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/config/SqlConstants.java
 
b/inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/config/SqlConstants.java
index a64cd2234d..1eee2f7ec9 100644
--- 
a/inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/config/SqlConstants.java
+++ 
b/inlong-audit/audit-service/src/main/java/org/apache/inlong/audit/config/SqlConstants.java
@@ -77,4 +77,14 @@ public class SqlConstants {
 "AND audit_id = ? \n" +
 "GROUP BY inlong_group

(inlong) branch master updated: [INLONG-9962][Manager] Data preview supports returning header and specific field information (#9967)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new dba59007b6 [INLONG-9962][Manager] Data preview supports returning 
header and specific field information (#9967)
dba59007b6 is described below

commit dba59007b6641d31d10a6157a8e07f628082a8f7
Author: fuweng11 <76141879+fuwen...@users.noreply.github.com>
AuthorDate: Wed Apr 10 21:39:37 2024 +0800

[INLONG-9962][Manager] Data preview supports returning header and specific 
field information (#9967)

* [INLONG-9962][Manager] Data preview supports returning header and 
specific field information

* [INLONG-9962][Manager] Data preview supports returning header and 
specific field information

* [INLONG-9962][Manager] Fix error

* [INLONG-9962][Manager] Fix error

* [INLONG-9962][Manager] Fix error
---
 .../apache/inlong/common/enums/DataTypeEnum.java   | 20 ++-
 .../dataproxy/config/holder/MetaConfigHolder.java  |  6 +-
 .../dataproxy/config/pojo/IdTopicConfig.java   |  7 ++-
 .../manager/pojo/consume/BriefMQMessage.java   | 11 
 .../service/datatype/CsvDataTypeOperator.java  | 63 +++
 .../manager/service/datatype/DataTypeOperator.java | 56 +
 .../service/datatype/DataTypeOperatorFactory.java  | 49 +++
 .../message/InlongMsgDeserializeOperator.java  | 33 +-
 .../service/message/RawMsgDeserializeOperator.java | 32 +-
 .../resource/queue/kafka/KafkaOperator.java| 10 ++--
 .../resource/queue/pulsar/PulsarOperator.java  |  4 ++
 .../resource/queue/tubemq/TubeMQOperator.java  | 10 ++--
 .../resource/queue/kafka/KafkaOperatorTest.java|  4 ++
 .../sort/standalone/config/pojo/type/DataType.java | 70 --
 .../sort/standalone/sink/kafka/KafkaIdConfig.java  | 12 ++--
 .../standalone/sink/pulsar/PulsarIdConfig.java | 12 ++--
 16 files changed, 256 insertions(+), 143 deletions(-)

diff --git 
a/inlong-common/src/main/java/org/apache/inlong/common/enums/DataTypeEnum.java 
b/inlong-common/src/main/java/org/apache/inlong/common/enums/DataTypeEnum.java
index 2d2d8d1997..f327508660 100644
--- 
a/inlong-common/src/main/java/org/apache/inlong/common/enums/DataTypeEnum.java
+++ 
b/inlong-common/src/main/java/org/apache/inlong/common/enums/DataTypeEnum.java
@@ -30,8 +30,10 @@ public enum DataTypeEnum {
 CANAL("canal"),
 DEBEZIUM_JSON("debezium_json"),
 RAW("raw"),
-
-;
+TEXT("text"),
+PB("pb"),
+JCE("jce"),
+UNKNOWN("n");
 
 private final String type;
 
@@ -48,6 +50,20 @@ public enum DataTypeEnum {
 throw new IllegalArgumentException("Unsupported data type for " + 
type);
 }
 
+public static DataTypeEnum convert(String value) {
+for (DataTypeEnum v : values()) {
+if (v.getType().equals(value)) {
+return v;
+}
+}
+return UNKNOWN;
+}
+
+@Override
+public String toString() {
+return this.name() + ":" + this.type;
+}
+
 public String getType() {
 return type;
 }
diff --git 
a/inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/config/holder/MetaConfigHolder.java
 
b/inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/config/holder/MetaConfigHolder.java
index 369e7b2212..96c5bd820a 100644
--- 
a/inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/config/holder/MetaConfigHolder.java
+++ 
b/inlong-dataproxy/dataproxy-source/src/main/java/org/apache/inlong/dataproxy/config/holder/MetaConfigHolder.java
@@ -18,6 +18,7 @@
 package org.apache.inlong.dataproxy.config.holder;
 
 import org.apache.inlong.common.constant.Constants;
+import org.apache.inlong.common.enums.DataTypeEnum;
 import org.apache.inlong.common.pojo.dataproxy.CacheClusterObject;
 import org.apache.inlong.common.pojo.dataproxy.CacheClusterSetObject;
 import org.apache.inlong.common.pojo.dataproxy.DataProxyCluster;
@@ -29,7 +30,6 @@ import org.apache.inlong.dataproxy.config.ConfigHolder;
 import org.apache.inlong.dataproxy.config.ConfigManager;
 import org.apache.inlong.dataproxy.config.pojo.CacheClusterConfig;
 import org.apache.inlong.dataproxy.config.pojo.CacheType;
-import org.apache.inlong.dataproxy.config.pojo.DataType;
 import org.apache.inlong.dataproxy.config.pojo.IdTopicConfig;
 import org.apache.inlong.dataproxy.consts.ConfigConstants;
 import org.apache.inlong.sdk.commons.protocol.InlongId;
@@ -407,8 +407,8 @@ public class MetaConfigHolder extends ConfigHolder {
 tmpConfig.setTenantAndNameSpace(tenant, nameSpace);
 tmpConfig.setTopicName(topicName);
 tmpConfig.setParams(idObject.getParams(

(inlong) branch branch-1.12 updated: [INLONG-10029][Release] Change the tag of Docker images to 1.12.0 (#10065)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch branch-1.12
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/branch-1.12 by this push:
 new 79d0ef9f4e [INLONG-10029][Release] Change the tag of Docker images to 
1.12.0 (#10065)
79d0ef9f4e is described below

commit 79d0ef9f4e8dcc4e4f1246fe7e1f0a93a2191d52
Author: baomingyu 
AuthorDate: Wed Apr 24 20:37:45 2024 +0800

[INLONG-10029][Release] Change the tag of Docker images to 1.12.0 (#10065)
---
 conf/inlong.conf |  2 +-
 docker/docker-compose/.env   |  2 +-
 docker/kubernetes/README.md  |  2 +-
 docker/kubernetes/values.yaml| 14 +++---
 inlong-agent/agent-docker/README.md  |  2 +-
 inlong-audit/audit-docker/README.md  |  2 +-
 inlong-dataproxy/dataproxy-docker/README.md  |  2 +-
 inlong-manager/manager-docker/README.md  |  2 +-
 inlong-tubemq/tubemq-docker/tubemq-all/README.md |  4 ++--
 inlong-tubemq/tubemq-docker/tubemq-manager/README.md |  2 +-
 10 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/conf/inlong.conf b/conf/inlong.conf
index 28a1556661..576f1f7d9d 100644
--- a/conf/inlong.conf
+++ b/conf/inlong.conf
@@ -27,7 +27,7 @@ spring_datasource_password=inlong
 
 ## Dashboard Configuration ##
 # dashboard docker image
-dashboard_docker_image=inlong/dashboard:latest
+dashboard_docker_image=inlong/dashboard:1.12.0
 # dashboard service port
 dashboard_mapping_port=80
 
diff --git a/docker/docker-compose/.env b/docker/docker-compose/.env
index dce60412ce..a12c307795 100644
--- a/docker/docker-compose/.env
+++ b/docker/docker-compose/.env
@@ -16,4 +16,4 @@
 #
 # you can choose the released version number, like 1.3.0
 # the latest tag corresponds to the master branch
-VERSION_TAG=latest
+VERSION_TAG=1.12.0
diff --git a/docker/kubernetes/README.md b/docker/kubernetes/README.md
index 3d946a973c..20946d1119 100644
--- a/docker/kubernetes/README.md
+++ b/docker/kubernetes/README.md
@@ -74,7 +74,7 @@ The configuration file is [values.yaml](values.yaml), and the 
following tables l
 |`timezone`
| `Asia/Shanghai`  |   
World time and date for cities in all time zones
   |
 |   `images.pullPolicy`
|  `IfNotPresent`  | Image 
pull policy. One of `Always`, `Never`, `IfNotPresent`   
   |
 | `images..repository`  
|  |
  Docker image repository for the component 
  |
-| `images..tag` 
| `latest` |
  Docker image tag for the component
  |
+| `images..tag` 
| `1.12.0` |
  Docker image tag for the component
  |
 | `.component`  
|  |
Component name  
  |
 |  `.replicas`  
|   `1`|
Replicas is the desired number of replicas of a given Template  
  |
 |`.podManagementPolicy` 
|  `OrderedReady`  |PodManagementPolicy controls how pods 
are created during initial scale up, when replacing pods on nodes, or when 
scaling down |
diff --git a/docker/kubernetes/values.yaml b/docker/kubernetes/values.yaml
index b6e8eec58f..6f4f5e03e8 100644
--- a/docker/kubernetes/values.yaml
+++ b/docker/kubernetes/values.yaml
@@ -28,25 +28,25 @@ components:
 images:
   agent:
 repository: inlong/agent
-tag: latest
+tag: 1.12.0
   dashboard:
 repository: inlong/dashboard
-tag: latest
+tag: 1.12.0
   dataproxy:
 repository: inlong/dataproxy
-tag: latest
+tag: 1.12.0
   tubemqManager:
 repository: inlong/tubemq-manager
-tag: latest
+tag: 1.12.0
   tubemqServer:
 repository: inlong/tubemq-all
-tag: latest

svn commit: r68756 - /dev/inlong/KEYS

[luchunliang]

Author: luchunliang
Date: Thu Apr 25 06:37:54 2024
New Revision: 68756

Log:
add gpg key for Mingyu Bao

Modified:
dev/inlong/KEYS

Modified: dev/inlong/KEYS
==
--- dev/inlong/KEYS (original)
+++ dev/inlong/KEYS Thu Apr 25 06:37:54 2024
@@ -729,3 +729,61 @@ quKwuqTCrVNL3v2T02z9lwyRZ3IgCAPSyuv8OXL8
 KpYnh4MTa5g/+nQ7o7xoQc/je8FW/ENitxcRpbYzJTSBOLon9VzDtrDql6uUCA==
 =sgEg
 -END PGP PUBLIC KEY BLOCK-
+pub   rsa4096 2024-04-25 [SC]
+  5F80F06035CD1DC2091AA2CD7BC99AC591D0E34A
+uid   [ 绝对 ] Mingyu Bao 
+sig 37BC99AC591D0E34A 2024-04-25  Mingyu Bao 
+sub   rsa4096 2024-04-25 [E]
+sig  7BC99AC591D0E34A 2024-04-25  Mingyu Bao 
+
+-BEGIN PGP PUBLIC KEY BLOCK-
+
+mQINBGYpywIBEADdMUC7eUHK0KLWdzDLmDL0iLjAPrddz5yGF5/2nA7V1sRta6SJ
+J4UUbQDMtZlLEAo5LFd/sTMH9rxmFi4jewgEzVRmuRrQwjsUG9VBrvqBcCpZ4Axq
+WarmJZ7ccvQlyU+2yROKolVSj+wqFL0THhlEBgEtcRTD6AZ4NzDp1WWjkKrqPFcx
+B7QM0FWrd3Rlr9j4owGNMC/rmQwQjQOF9qIup9wy6YWpDMTRf53Ja8sOm/DbJmg9
+Zf9b3CWro2J0xVdxtGldLUW2G1JXReIXaONPYjRvmudFfK1QMMMZpIUDxfx38q5Z
+zWGTPy2VDWQAWmdgqxH3sDR436Fq8n1G52NQDPfEYn6uqUaLWDk+ZiqZtL5P29ot
+Vccq7UNbUEgWnVJ+HLBlC5GbcMpAWlKGJr7FvMaPoVC65gVrouycnooSr1oxHa8I
+yRD8urRoXKwow30+fqOg8oG+ZPsQAqiOpSlb0MIHVIkmitpFb62XKQxu88kByg6s
+10L+AkwP5xQoW0NfBodORXCdT2aysUPA3a+2tUuTBafd2nBKSG1Jj1jqz4JuDTWa
+E+usk8Zfyk01RbW96awG2Y/9ZYMb8YpHcx8gxJAVtiQftYQut+m6XpjgmfRqLSIL
+ZjLFGowD9DzCJOCKBjvSKpie9wHxmM9C6p3zIGQH+xJJUpWbelNP3N5cswARAQAB
+tCFNaW5neXUgQmFvIDxiYW9taW5neXVAYXBhY2hlLm9yZz6JAk0EEwEIADcWIQRf
+gPBgNc0dwgkaos17yZrFkdDjSgUCZinLAgIbAwULCQgHAgYVCgkICwIDFgIBAh4B
+AheAAAoJEHvJmsWR0ONKGQQQALodUWzYJW4ZptisPORXLCCB+JuIKapFispQzIs+
+8KnpMcZUEPFhArf7zhZQ6PZ6VGOMPyQrZU7LNp3ajxjxqKYnqwHjN55l1NY8070m
+YuVHK9FL6YbHKnzDa1t3WB5u2WfKLOPlOF0/4sBPJnGk5D3VA9U2t9i8tfTJxSu8
+QFd+JdRKCcvv/NWHBNS1MKlhb2FsPYftcK04LQietkQMdjKLefFiP6zKkQqUlwvC
+5Jol+hyhN+u0dYLS1H8bH7Qcg48LkDdqISzcUbgq9rQKn5+F8/kxOokEYmb++tIR
+QcbX+7JvT+igG63OSFO/6S+vvw4vNBDF9YVr0HVYStuT1yLSwyn2YXO383YZUhEc
+8JOPwTS2qKMOHsNCretGgMTc4RWVcTyHCzAgyeHqQ3f5itj/MQk9/yqPdagdHGGZ
+e1HzFXgiSAkvcczZA8xy2ya5tTlDFNhPp5YR3P+8yC6YK2z6uZGL0Zc/2ULmqpeo
+OLz1kffOnQkH82iBNX8mB8u6Qq92bEw+sRInUmXdhf136gEUVuBlOYEbaY+PEH1S
+HbpAN4WvMrEAavh9QskFFV04SjNTqAoc6qSqSuTKHstpGHc6W3chzEYQYJAw+iho
+/mvpLz+E1ejAuZaXECj7tJDp1DmnYfLZye4ociZ5LpvFFp2VW0JvyZrEyltzwQFs
+fMb6uQINBGYpywIBEADIM0MKdRn5Fxqti+ThPq1WzFLw65wMae5F73g1S7/DBh4x
+1hNtujqzvwwkhQWdWD35o8NcA/himWB1qJdv1VznIEl9YC4gwuZU+zHjTWjdsgHa
+AOiiYA3BAHGaI784a34jHQKMuVqXy7lIikTpDl+4oCG3qFMUTCMvLdpgR2TGv90T
++np83PW78R1nhaEXzfKRjYYn4IjijbJryi00/aWsVO/DlteAjy7v82ptihF5yGB5
+QsapKfAFaQUdc0678qBzMLWw0GtwSdjXNwncGnYxd3hZTI2pFBpxKJzY0vR4/mc5
+OcQWlu2BB7ln2JEd5onfEHqLdfl4mXruJLtsa7EtVe4ueJsgYlm+cEG1AVuADcZ5
+s193TfzM6j0X6Yk92MtY2pCZ9KIWfrgJLnDknv997Ex9ksOth8lmsmqCKontsQn5
+jv8gICMwBO1rIrhmQsmLrWSePaRQdQXaKRP36JxnSwEQHGuUbVp2wq4mqAxQgQVi
+m/Q4ub+Q9l+Us4u3rJgNPkYDZUvAgCuqkY/MSBU9qO24Z6WPrLdOKqWtyCBhv4ek
+ftgLK6lMzFgeFwdhsmRAALJqi+6IbA77pYKRWKft3GuFrJ+/AXQCzVgrEQRRrr9O
+TROD/7wvV0gSjTem/q/3wa+Guj0mOw4lkCxKKpRPtERSDo5GaGK6E9TcZpGBLQAR
+AQABiQI2BBgBCAAgFiEEX4DwYDXNHcIJGqLNe8maxZHQ40oFAmYpywICGwwACgkQ
+e8maxZHQ40rvVxAAwrKiAJ0EcWAGdxqL8+SvM83Mi5WMdWQlz6jBnJSwEGOhxitM
+ehEhrPNxdYxyn57aptEcTkwHPWCkzFj+0VSL4XzQyzRK5FKmRv9PgXhovf8bobKX
+iMpK4rwfkQDEAglnneapmKE7W3t12DgodiKNh3BcPoo/WbOj4cxFagmzgD4yCsUG
++ADOzHNkZtTnrxRFN9G/2cquz64Zt/0Rvmr+GCYrGcxoCBRV7s7PzlXAr8nayBlB
+8blnUSG/4uI8V8pQUv7Kabn9kR4Q2zDaPvYoMMEBmkA+A4EqAncmI2VBHOoMj1E7
+LVMvkbW7FcyCuLtkGJOwnIGXm7Hjz4Xz94GoQgovMD4/Z3d0lGa/Kax/GOXvJ4Qy
+Ueh20IuhwNZxtkL/Lc0Y7BNieuH7KXvIrGuVh9skl+IS2X0H3aHA3ZAtKqlSHb5i
+B25sjGTmIOsMFHyYemfsew24F4U5GEUCzBRLGg9oAJqGwXjBEqPiJy6la5tDkse6
+T6s2itD9lZjtwe1YwuQo0YJ41XWS4gliWpUhzR83VM2emMPmmdoIyLuWRL3GLPFr
+u6J8I0GjFRGswV3WRoMDLNNGIjlXx3Iq4ft4572GFq5ztRBxpz5XxcTTgPT0d5mg
+Klg2wNGUlZsohb9B9yNpeOn/1EP2c+e0vmNBPkPLjDQzjxfkCmcQXBJBmPU=
+=TcGH
+-END PGP PUBLIC KEY BLOCK-

svn commit: r68762 - /release/inlong/KEYS

[luchunliang]

Author: luchunliang
Date: Thu Apr 25 08:02:51 2024
New Revision: 68762

Log:
add gpg key for Mingyu Bao

Modified:
release/inlong/KEYS

Modified: release/inlong/KEYS
==
--- release/inlong/KEYS (original)
+++ release/inlong/KEYS Thu Apr 25 08:02:51 2024
@@ -729,3 +729,61 @@ quKwuqTCrVNL3v2T02z9lwyRZ3IgCAPSyuv8OXL8
 KpYnh4MTa5g/+nQ7o7xoQc/je8FW/ENitxcRpbYzJTSBOLon9VzDtrDql6uUCA==
 =sgEg
 -END PGP PUBLIC KEY BLOCK-
+pub   rsa4096 2024-04-25 [SC]
+  5F80F06035CD1DC2091AA2CD7BC99AC591D0E34A
+uid   [ 绝对 ] Mingyu Bao 
+sig 37BC99AC591D0E34A 2024-04-25  Mingyu Bao 
+sub   rsa4096 2024-04-25 [E]
+sig  7BC99AC591D0E34A 2024-04-25  Mingyu Bao 
+
+-BEGIN PGP PUBLIC KEY BLOCK-
+
+mQINBGYpywIBEADdMUC7eUHK0KLWdzDLmDL0iLjAPrddz5yGF5/2nA7V1sRta6SJ
+J4UUbQDMtZlLEAo5LFd/sTMH9rxmFi4jewgEzVRmuRrQwjsUG9VBrvqBcCpZ4Axq
+WarmJZ7ccvQlyU+2yROKolVSj+wqFL0THhlEBgEtcRTD6AZ4NzDp1WWjkKrqPFcx
+B7QM0FWrd3Rlr9j4owGNMC/rmQwQjQOF9qIup9wy6YWpDMTRf53Ja8sOm/DbJmg9
+Zf9b3CWro2J0xVdxtGldLUW2G1JXReIXaONPYjRvmudFfK1QMMMZpIUDxfx38q5Z
+zWGTPy2VDWQAWmdgqxH3sDR436Fq8n1G52NQDPfEYn6uqUaLWDk+ZiqZtL5P29ot
+Vccq7UNbUEgWnVJ+HLBlC5GbcMpAWlKGJr7FvMaPoVC65gVrouycnooSr1oxHa8I
+yRD8urRoXKwow30+fqOg8oG+ZPsQAqiOpSlb0MIHVIkmitpFb62XKQxu88kByg6s
+10L+AkwP5xQoW0NfBodORXCdT2aysUPA3a+2tUuTBafd2nBKSG1Jj1jqz4JuDTWa
+E+usk8Zfyk01RbW96awG2Y/9ZYMb8YpHcx8gxJAVtiQftYQut+m6XpjgmfRqLSIL
+ZjLFGowD9DzCJOCKBjvSKpie9wHxmM9C6p3zIGQH+xJJUpWbelNP3N5cswARAQAB
+tCFNaW5neXUgQmFvIDxiYW9taW5neXVAYXBhY2hlLm9yZz6JAk0EEwEIADcWIQRf
+gPBgNc0dwgkaos17yZrFkdDjSgUCZinLAgIbAwULCQgHAgYVCgkICwIDFgIBAh4B
+AheAAAoJEHvJmsWR0ONKGQQQALodUWzYJW4ZptisPORXLCCB+JuIKapFispQzIs+
+8KnpMcZUEPFhArf7zhZQ6PZ6VGOMPyQrZU7LNp3ajxjxqKYnqwHjN55l1NY8070m
+YuVHK9FL6YbHKnzDa1t3WB5u2WfKLOPlOF0/4sBPJnGk5D3VA9U2t9i8tfTJxSu8
+QFd+JdRKCcvv/NWHBNS1MKlhb2FsPYftcK04LQietkQMdjKLefFiP6zKkQqUlwvC
+5Jol+hyhN+u0dYLS1H8bH7Qcg48LkDdqISzcUbgq9rQKn5+F8/kxOokEYmb++tIR
+QcbX+7JvT+igG63OSFO/6S+vvw4vNBDF9YVr0HVYStuT1yLSwyn2YXO383YZUhEc
+8JOPwTS2qKMOHsNCretGgMTc4RWVcTyHCzAgyeHqQ3f5itj/MQk9/yqPdagdHGGZ
+e1HzFXgiSAkvcczZA8xy2ya5tTlDFNhPp5YR3P+8yC6YK2z6uZGL0Zc/2ULmqpeo
+OLz1kffOnQkH82iBNX8mB8u6Qq92bEw+sRInUmXdhf136gEUVuBlOYEbaY+PEH1S
+HbpAN4WvMrEAavh9QskFFV04SjNTqAoc6qSqSuTKHstpGHc6W3chzEYQYJAw+iho
+/mvpLz+E1ejAuZaXECj7tJDp1DmnYfLZye4ociZ5LpvFFp2VW0JvyZrEyltzwQFs
+fMb6uQINBGYpywIBEADIM0MKdRn5Fxqti+ThPq1WzFLw65wMae5F73g1S7/DBh4x
+1hNtujqzvwwkhQWdWD35o8NcA/himWB1qJdv1VznIEl9YC4gwuZU+zHjTWjdsgHa
+AOiiYA3BAHGaI784a34jHQKMuVqXy7lIikTpDl+4oCG3qFMUTCMvLdpgR2TGv90T
++np83PW78R1nhaEXzfKRjYYn4IjijbJryi00/aWsVO/DlteAjy7v82ptihF5yGB5
+QsapKfAFaQUdc0678qBzMLWw0GtwSdjXNwncGnYxd3hZTI2pFBpxKJzY0vR4/mc5
+OcQWlu2BB7ln2JEd5onfEHqLdfl4mXruJLtsa7EtVe4ueJsgYlm+cEG1AVuADcZ5
+s193TfzM6j0X6Yk92MtY2pCZ9KIWfrgJLnDknv997Ex9ksOth8lmsmqCKontsQn5
+jv8gICMwBO1rIrhmQsmLrWSePaRQdQXaKRP36JxnSwEQHGuUbVp2wq4mqAxQgQVi
+m/Q4ub+Q9l+Us4u3rJgNPkYDZUvAgCuqkY/MSBU9qO24Z6WPrLdOKqWtyCBhv4ek
+ftgLK6lMzFgeFwdhsmRAALJqi+6IbA77pYKRWKft3GuFrJ+/AXQCzVgrEQRRrr9O
+TROD/7wvV0gSjTem/q/3wa+Guj0mOw4lkCxKKpRPtERSDo5GaGK6E9TcZpGBLQAR
+AQABiQI2BBgBCAAgFiEEX4DwYDXNHcIJGqLNe8maxZHQ40oFAmYpywICGwwACgkQ
+e8maxZHQ40rvVxAAwrKiAJ0EcWAGdxqL8+SvM83Mi5WMdWQlz6jBnJSwEGOhxitM
+ehEhrPNxdYxyn57aptEcTkwHPWCkzFj+0VSL4XzQyzRK5FKmRv9PgXhovf8bobKX
+iMpK4rwfkQDEAglnneapmKE7W3t12DgodiKNh3BcPoo/WbOj4cxFagmzgD4yCsUG
++ADOzHNkZtTnrxRFN9G/2cquz64Zt/0Rvmr+GCYrGcxoCBRV7s7PzlXAr8nayBlB
+8blnUSG/4uI8V8pQUv7Kabn9kR4Q2zDaPvYoMMEBmkA+A4EqAncmI2VBHOoMj1E7
+LVMvkbW7FcyCuLtkGJOwnIGXm7Hjz4Xz94GoQgovMD4/Z3d0lGa/Kax/GOXvJ4Qy
+Ueh20IuhwNZxtkL/Lc0Y7BNieuH7KXvIrGuVh9skl+IS2X0H3aHA3ZAtKqlSHb5i
+B25sjGTmIOsMFHyYemfsew24F4U5GEUCzBRLGg9oAJqGwXjBEqPiJy6la5tDkse6
+T6s2itD9lZjtwe1YwuQo0YJ41XWS4gliWpUhzR83VM2emMPmmdoIyLuWRL3GLPFr
+u6J8I0GjFRGswV3WRoMDLNNGIjlXx3Iq4ft4572GFq5ztRBxpz5XxcTTgPT0d5mg
+Klg2wNGUlZsohb9B9yNpeOn/1EP2c+e0vmNBPkPLjDQzjxfkCmcQXBJBmPU=
+=TcGH
+-END PGP PUBLIC KEY BLOCK-

svn commit: r68807 - /dev/inlong/1.12.0-RC0/

[luchunliang]

Author: luchunliang
Date: Fri Apr 26 13:16:45 2024
New Revision: 68807

Log:
prepare for 1.12.0 RC0

Added:
dev/inlong/1.12.0-RC0/
dev/inlong/1.12.0-RC0/apache-inlong-1.12.0-bin.tar.gz   (with props)
dev/inlong/1.12.0-RC0/apache-inlong-1.12.0-bin.tar.gz.asc   (with props)
dev/inlong/1.12.0-RC0/apache-inlong-1.12.0-bin.tar.gz.sha512
dev/inlong/1.12.0-RC0/apache-inlong-1.12.0-src.tar.gz   (with props)
dev/inlong/1.12.0-RC0/apache-inlong-1.12.0-src.tar.gz.asc   (with props)
dev/inlong/1.12.0-RC0/apache-inlong-1.12.0-src.tar.gz.sha512

Added: dev/inlong/1.12.0-RC0/apache-inlong-1.12.0-bin.tar.gz
==
Binary file - no diff available.

Propchange: dev/inlong/1.12.0-RC0/apache-inlong-1.12.0-bin.tar.gz
--
svn:mime-type = application/x-gzip

Added: dev/inlong/1.12.0-RC0/apache-inlong-1.12.0-bin.tar.gz.asc
==
Binary file - no diff available.

Propchange: dev/inlong/1.12.0-RC0/apache-inlong-1.12.0-bin.tar.gz.asc
--
svn:mime-type = application/pgp-signature

Added: dev/inlong/1.12.0-RC0/apache-inlong-1.12.0-bin.tar.gz.sha512
==
--- dev/inlong/1.12.0-RC0/apache-inlong-1.12.0-bin.tar.gz.sha512 (added)
+++ dev/inlong/1.12.0-RC0/apache-inlong-1.12.0-bin.tar.gz.sha512 Fri Apr 26 
13:16:45 2024
@@ -0,0 +1,4 @@
+apache-inlong-1.12.0-bin.tar.gz: D0B5F4AF D919EFFA 9D44737F 3736ED61 3696743E
+ 58CD4E3D 134130F3 98306F11 8AD42B34 6C5CCDD4
+ E5A39FD4 415E0E40 D4EB2017 77A34DC0 EBD2F3AA
+ 7C886062

Added: dev/inlong/1.12.0-RC0/apache-inlong-1.12.0-src.tar.gz
==
Binary file - no diff available.

Propchange: dev/inlong/1.12.0-RC0/apache-inlong-1.12.0-src.tar.gz
--
svn:mime-type = application/x-gzip

Added: dev/inlong/1.12.0-RC0/apache-inlong-1.12.0-src.tar.gz.asc
==
Binary file - no diff available.

Propchange: dev/inlong/1.12.0-RC0/apache-inlong-1.12.0-src.tar.gz.asc
--
svn:mime-type = application/pgp-signature

Added: dev/inlong/1.12.0-RC0/apache-inlong-1.12.0-src.tar.gz.sha512
==
--- dev/inlong/1.12.0-RC0/apache-inlong-1.12.0-src.tar.gz.sha512 (added)
+++ dev/inlong/1.12.0-RC0/apache-inlong-1.12.0-src.tar.gz.sha512 Fri Apr 26 
13:16:45 2024
@@ -0,0 +1,4 @@
+apache-inlong-1.12.0-src.tar.gz: B1805B53 437A00D9 4333A2FF 55DB5F86 E7B7B9C8
+ 3D900CDF F6461B98 95DB9348 75947EA5 3A866E90
+ 745A1CBD E6460CBF 1419F32E 82C9676A A504B540
+ 463F619A

(inlong) branch master updated: [INLONG-10099][Audit] Audit-store adds the general JDBC sink capability (#10104)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new d963e5c753 [INLONG-10099][Audit] Audit-store adds the general JDBC 
sink capability (#10104)
d963e5c753 is described below

commit d963e5c7537a8685df6812bbcf1b2d3b1b0a41a4
Author: doleyzi <43397300+dole...@users.noreply.github.com>
AuthorDate: Mon Apr 29 18:00:19 2024 +0800

[INLONG-10099][Audit] Audit-store adds the general JDBC sink capability 
(#10104)

* Audit-store adds the general JDBC sink capability

* Audit-store adds the general JDBC sink capability

* Modify configuration file of JDBC

* Modify configuration file of JDBC

* Modify configuration file of JDBC

* Adjust the import order
---
 inlong-audit/audit-docker/Dockerfile   |   7 +-
 inlong-audit/audit-docker/audit-docker.sh  |  26 ++-
 .../config/{StoreConfig.java => JdbcConfig.java}   |  47 +++--
 .../apache/inlong/audit/config/StoreConfig.java|   3 +
 .../entities/JdbcDataPo.java}  |  55 ++---
 .../audit/service/AuditMsgConsumerServer.java  |  18 +-
 .../inlong/audit/service/ClickHouseService.java|   7 +
 .../inlong/audit/service/ElasticsearchService.java |   7 +
 .../apache/inlong/audit/service/InsertData.java|   5 +
 .../apache/inlong/audit/service/JdbcService.java   | 224 +
 .../apache/inlong/audit/service/MySqlService.java  |   8 +
 .../inlong/audit/service/consume/BaseConsume.java  |  13 +-
 .../audit/service/consume/PulsarConsume.java   |   3 +-
 inlong-audit/conf/application.properties   |   6 +
 inlong-audit/conf/audit-service.properties |  16 --
 15 files changed, 366 insertions(+), 79 deletions(-)

diff --git a/inlong-audit/audit-docker/Dockerfile 
b/inlong-audit/audit-docker/Dockerfile
index d55ac7e2f6..6717eca684 100644
--- a/inlong-audit/audit-docker/Dockerfile
+++ b/inlong-audit/audit-docker/Dockerfile
@@ -35,7 +35,7 @@ ENV TUBE_AUDIT_TOPIC="inlong-audit"
 ENV AUDIT_DBNAME="apache_inlong_audit"
 # proxy/store/all, start audit module individually, or all
 ENV START_MODE="all"
-# mysql / clickhouse / elasticsearch
+# mysql / clickhouse / elasticsearch / starrocks
 ENV STORE_MODE=mysql
 # mysql
 ENV JDBC_URL=127.0.0.1:3306
@@ -52,6 +52,11 @@ ENV STORE_ES_PORT=9200
 ENV STORE_ES_AUTHENABLE=false
 ENV STORE_ES_USERNAME=elastic
 ENV STORE_ES_PASSWD=inlong
+# starrocks
+ENV STORE_SR_URL=127.0.0.1:9030
+ENV STORE_SR_USERNAME=default
+ENV STORE_SR_PASSWD=default
+ENV STORE_SR_DBNAME="apache_inlong_audit"
 # jvm
 ENV AUDIT_JVM_HEAP_OPTS="-XX:+UseContainerSupport 
-XX:InitialRAMPercentage=40.0 -XX:MaxRAMPercentage=80.0 
-XX:-UseAdaptiveSizePolicy"
 WORKDIR /opt/inlong-audit
diff --git a/inlong-audit/audit-docker/audit-docker.sh 
b/inlong-audit/audit-docker/audit-docker.sh
index 42c43c9322..777a80b801 100755
--- a/inlong-audit/audit-docker/audit-docker.sh
+++ b/inlong-audit/audit-docker/audit-docker.sh
@@ -21,8 +21,12 @@ file_path=$(cd "$(dirname "$0")"/../;pwd)
 store_conf_file=${file_path}/conf/application.properties
 # proxy config
 proxy_conf_file=${file_path}/conf/audit-proxy-${MQ_TYPE}.conf
-sql_file="${file_path}"/sql/apache_inlong_audit.sql
+sql_mysql_file="${file_path}"/sql/apache_inlong_audit_mysql.sql
 sql_ck_file="${file_path}"/sql/apache_inlong_audit_clickhouse.sql
+sql_sr_file="${file_path}"/sql/apache_inlong_audit_starrocks.sql
+
+# audit-service config
+service_conf_file=${file_path}/conf/audit-service.properties
 
 # replace the configuration for audit proxy
 sed -i 
"s/manager.hosts=.*$/manager.hosts=${MANAGER_OPENAPI_IP}:${MANAGER_OPENAPI_PORT}/g"
 "${store_conf_file}"
@@ -52,7 +56,7 @@ sed -i 
"s/127.0.0.1:3306\/apache_inlong_audit/${JDBC_URL}\/${AUDIT_DBNAME}/g" "$
 sed -i 
"s/spring.datasource.druid.username=.*$/spring.datasource.druid.username=${USERNAME}/g"
 "${store_conf_file}"
 sed -i 
"s/spring.datasource.druid.password=.*$/spring.datasource.druid.password=${PASSWORD}/g"
 "${store_conf_file}"
 # mysql file for audit
-sed -i "s/apache_inlong_audit/${AUDIT_DBNAME}/g" "${sql_file}"
+sed -i "s/apache_inlong_audit/${AUDIT_DBNAME}/g" "${sql_mysql_file}"
 # clickhouse
 sed -i 
"s/clickhouse.url=.*$/clickhouse.url=jdbc:clickhouse:\/\/${STORE_CK_URL}\/${STORE_CK_DBNAME}/g"
 "${store_conf_file}"
 sed -i "s/clickhouse.username=.*$/clickhouse.username=${STORE_CK_USERNAME}/g" 
"${store_conf_file}"
@@ -66,6 +70,18 @@ sed -i 
"s/elasticsearch.authEnable=.*$/elasticsearch.authEnable=${STORE_ES_AUTHE
 sed -i 
"s/elasticsearch.username=.*$/elasticsearc

svn commit: r68908 - /dev/inlong/1.12.0-RC0/ /release/inlong/1.12.0/

[luchunliang]

Author: luchunliang
Date: Thu May  2 00:54:56 2024
New Revision: 68908

Log:
Release 1.12.0

Added:
release/inlong/1.12.0/
  - copied from r68907, dev/inlong/1.12.0-RC0/
Removed:
dev/inlong/1.12.0-RC0/

svn commit: r69004 - /release/inlong/1.11.0/

[luchunliang]

Author: luchunliang
Date: Tue May  7 12:06:23 2024
New Revision: 69004

Log:
Delete 1.11.0

Removed:
release/inlong/1.11.0/

(inlong) branch master updated: [INLONG-10531][SDK] Add InLong dataproxy python SDK based on C++ SDK (#10538)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new d5189626ab [INLONG-10531][SDK] Add InLong dataproxy python SDK based 
on C++ SDK (#10538)
d5189626ab is described below

commit d5189626aba1a03448d55de84da10d69095dc7a1
Author: LeeWY <61183968+yfsn...@users.noreply.github.com>
AuthorDate: Sat Jun 29 18:24:35 2024 +0800

[INLONG-10531][SDK] Add InLong dataproxy python SDK based on C++ SDK 
(#10538)

* [INLONG-10531][SDK] Add InLong dataproxy python SDK based on C++ SDK

* [INLONG-10531][SDK] Improve the callback function part

* [INLONG-10531][SDK] Optimize the build script

-

Co-authored-by: jameswyli 
---
 .../dataproxy-sdk-python/CMakeLists.txt| 38 +++
 .../dataproxy-sdk-python/README.md | 41 
 .../dataproxy-sdk-python/build.sh  | 73 ++
 .../dataproxy-sdk-python/inlong_dataproxy.cpp  | 61 ++
 4 files changed, 213 insertions(+)

diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/CMakeLists.txt 
b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/CMakeLists.txt
new file mode 100644
index 00..80791b1c02
--- /dev/null
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/CMakeLists.txt
@@ -0,0 +1,38 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+cmake_minimum_required(VERSION 3.5)
+project(dataproxy-sdk-python)
+
+set(CMAKE_CXX_STANDARD 11)
+
+include_directories("./dataproxy-sdk-cpp/src/core")
+
+include_directories("./dataproxy-sdk-cpp/third_party/lib")
+include_directories("./dataproxy-sdk-cpp/third_party/lib64")
+
+add_subdirectory(pybind11)
+add_subdirectory(dataproxy-sdk-cpp)
+
+link_directories("./dataproxy-sdk-cpp/third_party/lib")
+link_directories("./dataproxy-sdk-cpp/third_party/lib64")
+
+pybind11_add_module(inlong_dataproxy inlong_dataproxy.cpp)
+
+target_link_libraries(inlong_dataproxy PRIVATE pybind11::module dataproxy_sdk)
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/README.md 
b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/README.md
new file mode 100644
index 00..d85a8632f3
--- /dev/null
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/README.md
@@ -0,0 +1,41 @@
+
+
+# DataProxy-SDK-Python
+Dataproxy-SDK Python version, used for sending data to InLong dataproxy.
+
+InLong Dataproxy Python SDK is a wrapper over the existing [C++ 
SDK](https://github.com/apache/inlong/tree/master/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp)
 and exposes all of the same features.
+
+## Prerequisites
+- CMake 3.5+
+- Python 3.6+
+
+## Build
+Go to the dataproxy-sdk-python root directory, and run
+
+```bash
+chmod +x ./build.sh
+./build.sh
+```
+
+After the build process finished, you can import the package (`import 
inlong_dataproxy`) in your python project to use InLong dataproxy.
+
+> **Note**: When the C++ SDK or the version of Python you're using is updated, 
you'll need to rebuild it by re-executing the `build.sh` script
diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/build.sh 
b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/build.sh
new file mode 100755
index 00..388edcbc61
--- /dev/null
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/build.sh
@@ -0,0 +1,73 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT 

(inlong) branch master updated: [INLONG-10607][SDK] Transform SQL support arithmetic functions(Including log10, log2, log and exp) (#10615)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new 5d4128d826 [INLONG-10607][SDK] Transform SQL support arithmetic 
functions(Including log10, log2, log and exp) (#10615)
5d4128d826 is described below

commit 5d4128d8266e745a5df8580e7e1a886308d20b12
Author: LeeWY <61183968+yfsn...@users.noreply.github.com>
AuthorDate: Sat Jul 13 16:36:20 2024 +0800

[INLONG-10607][SDK] Transform SQL support arithmetic functions(Including 
log10, log2, log and exp) (#10615)

* [INLONG-10607][SDK] Transform SQL support arithmetic functions(Including 
log10, log2, log and exp)

* [INLONG-10607][SDK] Rename some variables and optimize the test code

-

Co-authored-by: jameswyli 
---
 .../transform/process/function/AbsFunction.java|  6 +-
 .../{AbsFunction.java => ExpFunction.java} | 16 ++--
 .../sdk/transform/process/function/LnFunction.java |  6 +-
 .../{AbsFunction.java => Log10Function.java}   | 16 ++--
 .../{AbsFunction.java => Log2Function.java}| 16 ++--
 .../{AbsFunction.java => LogFunction.java} | 33 +++--
 .../transform/process/function/PowerFunction.java  | 12 +--
 .../transform/process/function/SqrtFunction.java   |  6 +-
 .../transform/process/operator/OperatorTools.java  |  8 ++
 .../TestTransformArithmeticFunctionsProcessor.java | 86 +++---
 10 files changed, 149 insertions(+), 56 deletions(-)

diff --git 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/AbsFunction.java
 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/AbsFunction.java
index cc56afb85e..a94d662eae 100644
--- 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/AbsFunction.java
+++ 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/AbsFunction.java
@@ -31,14 +31,14 @@ import java.math.BigDecimal;
  */
 public class AbsFunction implements ValueParser {
 
-private ValueParser number;
+private ValueParser numberParser;
 
 /**
  * Constructor
  * @param expr
  */
 public AbsFunction(Function expr) {
-number = 
OperatorTools.buildParser(expr.getParameters().getExpressions().get(0));
+numberParser = 
OperatorTools.buildParser(expr.getParameters().getExpressions().get(0));
 }
 
 /**
@@ -49,7 +49,7 @@ public class AbsFunction implements ValueParser {
  */
 @Override
 public Object parse(SourceData sourceData, int rowIndex) {
-Object numberObj = number.parse(sourceData, rowIndex);
+Object numberObj = numberParser.parse(sourceData, rowIndex);
 BigDecimal numberValue = OperatorTools.parseBigDecimal(numberObj);
 return numberValue.abs();
 }
diff --git 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/AbsFunction.java
 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/ExpFunction.java
similarity index 77%
copy from 
inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/AbsFunction.java
copy to 
inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/ExpFunction.java
index cc56afb85e..5f542413e2 100644
--- 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/AbsFunction.java
+++ 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/ExpFunction.java
@@ -26,19 +26,19 @@ import net.sf.jsqlparser.expression.Function;
 import java.math.BigDecimal;
 
 /**
- * AbsFunction
- * description: abs(numeric)--returns the absolute value of numeric
+ * ExpFunction
+ * description: exp(numeric)--returns e raised to the power of numeric
  */
-public class AbsFunction implements ValueParser {
+public class ExpFunction implements ValueParser {
 
-private ValueParser number;
+private ValueParser numberParser;
 
 /**
  * Constructor
  * @param expr
  */
-public AbsFunction(Function expr) {
-number = 
OperatorTools.buildParser(expr.getParameters().getExpressions().get(0));
+public ExpFunction(Function expr) {
+numberParser = 
OperatorTools.buildParser(expr.getParameters().getExpressions().get(0));
 }
 
 /**
@@ -49,8 +49,8 @@ public class AbsFunction implements ValueParser {
  */
 @Override
 public Object parse(SourceData sourceData, int rowIndex) {
-Object numberObj = number.parse(sourceData, rowIndex);
+Object numberObj = numberParser.parse(sourceData, rowIndex);
 BigDecimal numberValue = OperatorTools.parseBigDecimal(numberObj);
-return numberValue.abs();
+return Math.exp(numbe

(inlong) branch master updated: [INLONG-10652][SDK] Inlong Transform support for generics (#10672)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new 13481e140d [INLONG-10652][SDK] Inlong Transform support for generics 
(#10672)
13481e140d is described below

commit 13481e140d2114b375e4b43d6af3f27c58c76e70
Author: vernedeng 
AuthorDate: Fri Jul 19 09:45:37 2024 +0800

[INLONG-10652][SDK] Inlong Transform support for generics (#10672)

* [INLONG-10652][SDK] Inlong Transform support for generics

* fix UT
---
 .../sdk/transform/decode/CsvSourceDecoder.java |   2 +-
 .../sdk/transform/decode/JsonSourceDecoder.java|   2 +-
 .../sdk/transform/decode/KvSourceDecoder.java  |   2 +-
 .../sdk/transform/decode/PbSourceDecoder.java  |  14 +-
 .../inlong/sdk/transform/decode/SourceDecoder.java |   5 +-
 ...ourceDecoder.java => SourceDecoderFactory.java} |  26 ++-
 .../sdk/transform/encode/CsvSinkEncoder.java   |   2 +-
 .../inlong/sdk/transform/encode/KvSinkEncoder.java |   2 +-
 .../inlong/sdk/transform/encode/SinkEncoder.java   |   4 +-
 .../{SinkEncoder.java => SinkEncoderFactory.java}  |  18 +-
 .../inlong/sdk/transform/pojo/TransformConfig.java |  45 +
 .../sdk/transform/process/TransformProcessor.java  |  98 +++
 .../sdk/transform/pojo/TestTransformConfig.java| 118 -
 .../TestTransformArithmeticFunctionsProcessor.java |  66 +---
 .../transform/process/TestTransformProcessor.java  | 187 +++--
 15 files changed, 208 insertions(+), 383 deletions(-)

diff --git 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/CsvSourceDecoder.java
 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/CsvSourceDecoder.java
index 14a505d422..daddfd36d7 100644
--- 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/CsvSourceDecoder.java
+++ 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/CsvSourceDecoder.java
@@ -30,7 +30,7 @@ import java.util.Map;
  * CsvSourceDecoder
  * 
  */
-public class CsvSourceDecoder implements SourceDecoder {
+public class CsvSourceDecoder implements SourceDecoder {
 
 protected CsvSourceInfo sourceInfo;
 private Charset srcCharset = Charset.defaultCharset();
diff --git 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/JsonSourceDecoder.java
 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/JsonSourceDecoder.java
index 57bf6a9982..13c363912a 100644
--- 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/JsonSourceDecoder.java
+++ 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/JsonSourceDecoder.java
@@ -34,7 +34,7 @@ import java.util.Map;
  * JsonSourceDecoder
  * 
  */
-public class JsonSourceDecoder implements SourceDecoder {
+public class JsonSourceDecoder implements SourceDecoder {
 
 protected JsonSourceInfo sourceInfo;
 private Charset srcCharset = Charset.defaultCharset();
diff --git 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/KvSourceDecoder.java
 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/KvSourceDecoder.java
index 03b40c9f1c..77a4fef8b4 100644
--- 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/KvSourceDecoder.java
+++ 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/KvSourceDecoder.java
@@ -30,7 +30,7 @@ import java.util.Map;
  * KvSourceDecoder
  * 
  */
-public class KvSourceDecoder implements SourceDecoder {
+public class KvSourceDecoder implements SourceDecoder {
 
 protected KvSourceInfo sourceInfo;
 private Charset srcCharset = Charset.defaultCharset();
diff --git 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/PbSourceDecoder.java
 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/PbSourceDecoder.java
index 5ac13cf28f..6c8a919e24 100644
--- 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/PbSourceDecoder.java
+++ 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/decode/PbSourceDecoder.java
@@ -38,7 +38,7 @@ import java.util.concurrent.ConcurrentHashMap;
  * PbSourceDecoder
  * 
  */
-public class PbSourceDecoder implements SourceDecoder {
+public class PbSourceDecoder implements SourceDecoder {
 
 private static final Logger LOG = 
LoggerFactory.getLogger(PbSourceDecoder.class);
 
@@ -150,16 +150,4 @@ public class PbSourceDecoder implements SourceDecoder {
 return null;
 }
 }
-
-/**
- * decode
- * @param srcString
- * @param extParams
- * @return
- */
-@Override
-public SourceData decode(String srcString, Map ext

(inlong) branch master updated: [INLONG-10748][Bug] Fix some null pointer dereference risks in the code. (#10753)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new 30ce46c39b [INLONG-10748][Bug] Fix some null pointer dereference risks 
in the code. (#10753)
30ce46c39b is described below

commit 30ce46c39bb581a30c0d2610d467983b385db7ea
Author: LeeWY <61183968+yfsn...@users.noreply.github.com>
AuthorDate: Tue Aug 6 19:02:55 2024 +0800

[INLONG-10748][Bug] Fix some null pointer dereference risks in the code. 
(#10753)

* [INLONG-10748] Fix some null pointer dereference risks in the code.

* [INLONG-10748] Optimize the processing logic of `StringUtils.java`.

-

Co-authored-by: jameswyli 
---
 .../org/apache/inlong/agent/plugin/sources/PulsarSource.java |  2 +-
 .../inlong/agent/plugin/sources/file/AbstractSource.java | 12 +++-
 .../apache/inlong/audit/service/AuditMsgConsumerServer.java  |  4 +++-
 .../manager/service/source/StreamSourceServiceImpl.java  |  2 +-
 .../inlong/sdk/dataproxy/pb/network/TcpChannelGroup.java |  2 +-
 .../org/apache/inlong/sdk/commons/protocol/ProxyEvent.java   |  8 +---
 .../apache/inlong/sdk/sort/fetcher/kafka/KafkaSeeker.java|  1 +
 .../inlong/sort/base/format/JsonDynamicSchemaFormat.java |  2 +-
 .../inlong/sort/pulsar/table/PulsarRowDataConverter.java |  6 --
 .../sort/pulsar/table/source/PulsarRowDataConverter.java |  6 --
 .../org/apache/inlong/sort/formats/util/StringUtils.java |  4 
 11 files changed, 32 insertions(+), 17 deletions(-)

diff --git 
a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/PulsarSource.java
 
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/PulsarSource.java
index f7c63acba4..9ca7a5aaf4 100644
--- 
a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/PulsarSource.java
+++ 
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/PulsarSource.java
@@ -132,7 +132,7 @@ public class PulsarSource extends AbstractSource {
 }
 return consumer;
 } catch (PulsarClientException | IllegalArgumentException e) {
-if (consumer == null) {
+if (consumer != null) {
 try {
 consumer.close();
 } catch (PulsarClientException ex) {
diff --git 
a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/file/AbstractSource.java
 
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/file/AbstractSource.java
index a477d84093..1fb96f7ff7 100644
--- 
a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/file/AbstractSource.java
+++ 
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/file/AbstractSource.java
@@ -178,12 +178,14 @@ public abstract class AbstractSource implements Source {
 continue;
 }
 emptyCount = 0;
-for (int i = 0; i < lines.size(); i++) {
-boolean suc4Queue = 
waitForPermit(AGENT_GLOBAL_READER_QUEUE_PERMIT, lines.get(i).getData().length);
-if (!suc4Queue) {
-break;
+if (lines != null) {
+for (int i = 0; i < lines.size(); i++) {
+boolean suc4Queue = 
waitForPermit(AGENT_GLOBAL_READER_QUEUE_PERMIT, lines.get(i).getData().length);
+if (!suc4Queue) {
+break;
+}
+putIntoQueue(lines.get(i));
 }
-putIntoQueue(lines.get(i));
 }
 
MemoryManager.getInstance().release(AGENT_GLOBAL_READER_SOURCE_PERMIT, 
BATCH_READ_LINE_TOTAL_LEN);
 if (AgentUtils.getCurrentTime() - lastPrintTime > 
CORE_THREAD_PRINT_INTERVAL_MS) {
diff --git 
a/inlong-audit/audit-store/src/main/java/org/apache/inlong/audit/service/AuditMsgConsumerServer.java
 
b/inlong-audit/audit-store/src/main/java/org/apache/inlong/audit/service/AuditMsgConsumerServer.java
index acd43a7ed6..51ed0caf60 100644
--- 
a/inlong-audit/audit-store/src/main/java/org/apache/inlong/audit/service/AuditMsgConsumerServer.java
+++ 
b/inlong-audit/audit-store/src/main/java/org/apache/inlong/audit/service/AuditMsgConsumerServer.java
@@ -97,7 +97,9 @@ public class AuditMsgConsumerServer implements 
InitializingBean {
 if (storeConfig.isJdbc()) {
 jdbcService.start();
 }
-mqConsume.start();
+if (mqConsume != null) {
+mqConsume.start();
+}
 }
 
 /**
diff --git 
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/source/StreamSourceServiceImpl.java
 
b/inlong-manager/manager-service/src/main/java/or

(inlong) branch master updated: [INLONG-10618][SDK] Transform SQL support common functions(Including substring, locate, to_date and date_format) (#10744)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new 8a27664530 [INLONG-10618][SDK] Transform SQL support common 
functions(Including substring, locate, to_date and date_format) (#10744)
8a27664530 is described below

commit 8a2766453099e281e3a4d9f490207737a9f7cda1
Author: LeeWY <61183968+yfsn...@users.noreply.github.com>
AuthorDate: Tue Aug 6 19:03:46 2024 +0800

[INLONG-10618][SDK] Transform SQL support common functions(Including 
substring, locate, to_date and date_format) (#10744)

* [INLONG-10618][SDK] Transform SQL support common functions(Including 
substring, locate, to_date and date_format)

* [INLONG-10618][SDK] Fix unit test failure, set time zone to Shanghai

* [INLONG-10618][SDK] Make the DateTimeFormatter object reusable to avoid 
creating multiple identical DateTimeFormatter objects.

* [INLONG-10618][SDK] Make the SimpleDateFormat object reusable to avoid 
creating multiple identical SimpleDateFormat objects.

-

Co-authored-by: jameswyli 
---
 .../process/function/DateFormatFunction.java   |  90 +++
 .../transform/process/function/LocateFunction.java |  87 +++
 .../process/function/SubstringFunction.java|  81 ++
 .../transform/process/function/ToDateFunction.java |  93 
 .../transform/process/operator/OperatorTools.java  |   8 ++
 .../TestTransformStringFunctionsProcessor.java | 121 
 .../TestTransformTemporalFunctionsProcessor.java   | 123 +
 7 files changed, 603 insertions(+)

diff --git 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/DateFormatFunction.java
 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/DateFormatFunction.java
new file mode 100644
index 00..9e233ff882
--- /dev/null
+++ 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/DateFormatFunction.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sdk.transform.process.function;
+
+import org.apache.inlong.sdk.transform.decode.SourceData;
+import org.apache.inlong.sdk.transform.process.Context;
+import org.apache.inlong.sdk.transform.process.operator.OperatorTools;
+import org.apache.inlong.sdk.transform.process.parser.ValueParser;
+
+import net.sf.jsqlparser.expression.Expression;
+import net.sf.jsqlparser.expression.Function;
+
+import java.math.BigDecimal;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * DateFormatFunction
+ * description: date_format(timestamp,format)--converts timestamp(in seconds) 
to a value of string in the format
+ * specified by the date format string. The format string is compatible with 
Java’s SimpleDateFormat
+ */
+public class DateFormatFunction implements ValueParser {
+
+private ValueParser timestampParser;
+private ValueParser formatParser;
+private static final Map SIMPLE_DATE_FORMATS = 
new ConcurrentHashMap<>();
+
+/**
+ * Constructor
+ *
+ * @param expr
+ */
+public DateFormatFunction(Function expr) {
+List expressions = expr.getParameters().getExpressions();
+timestampParser = OperatorTools.buildParser(expressions.get(0));
+formatParser = OperatorTools.buildParser(expressions.get(1));
+}
+
+/**
+ * parse
+ *
+ * @param sourceData
+ * @param rowIndex
+ * @return
+ */
+@Override
+public Object parse(SourceData sourceData, int rowIndex, Context context) {
+Object timestampObj = timestampParser.parse(sourceData, rowIndex, 
context);
+Object formatObj = formatParser.parse(sourceData, rowIndex, context);
+BigDecimal timestamp = OperatorTools.parseBigDecimal(timestampObj);
+String format = OperatorTools.parseString(formatObj

(inlong) branch master updated: [INLONG-10784][Sort] Refactor the structure of inlongmsg-rowdata (#10786)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new 7148c1cfaa [INLONG-10784][Sort] Refactor the structure of 
inlongmsg-rowdata (#10786)
7148c1cfaa is described below

commit 7148c1cfaa6208116a776694564bd0506825eeb0
Author: XiaoYou201 
AuthorDate: Wed Aug 14 12:11:56 2024 +0800

[INLONG-10784][Sort] Refactor the structure of inlongmsg-rowdata (#10786)

* [INLONG-10784][Sort] Refactor the structure of inlongmsg-rowdata

* [INLONG-10784][Sort] fix format

* [INLONG-10784][Sort] fix format

* [INLONG-10784][Sort] fix format

* [INLONG-10784][Sort] Rename row module structure

* Revert "[INLONG-10784][Sort] Rename row module structure"

This reverts commit 9ee654784ca6e213b642a584e7e5d78572925366.

* Revert "[INLONG-10784][Sort] fix format"

This reverts commit 32dcc771c1c2a53f14602537e38f7922ca620d8c.

* Revert "[INLONG-10784][Sort] fix format"

This reverts commit dd45c0fc26d6f7073d6d214ef36dc0d95ac22903.

* Revert "[INLONG-10784][Sort] fix format"

This reverts commit 4d1e2b0e8b74effa096a7612d2aa954882a519a2.

* Revert "[INLONG-10784][Sort] Refactor the structure of inlongmsg-rowdata"

This reverts commit a69075dfbb77e28fb2e3b6b7b6e41cbf89eb06de.

* [INLONG-10784][Sort] Refactor the Row module structure
---
 .../AbstractInLongMsgFormatDeserializer.java |  5 -
 .../AbstractInLongMsgMixedFormatConverter.java   |  2 +-
 .../AbstractInLongMsgMixedFormatDeserializer.java|  4 +++-
 .../inlongmsg/{ => row}/InLongMsgDecodingFormat.java |  5 +++--
 .../{ => row}/InLongMsgDeserializationSchema.java|  5 +++--
 .../inlongmsg/{ => row}/InLongMsgFormatFactory.java  |  2 +-
 .../{ => row}/InLongMsgMixedFormatConverter.java |  2 +-
 .../InLongMsgMixedFormatConverterBuilder.java| 10 +-
 .../InLongMsgMixedFormatConverterValidator.java  |  4 ++--
 .../InLongMsgMixedFormatDeserializerValidator.java   |  2 +-
 .../{ => row}/InLongMsgMixedFormatFactory.java   |  2 +-
 .../InLongMsgTextMixedFormatDeserializerBuilder.java |  2 +-
 .../formats/inlongmsg/{ => row}/InLongMsgUtils.java  |  7 ++-
 .../org.apache.flink.table.factories.Factory |  2 +-
 .../inlongmsg/InLongMsgFormatFactoryTest.java|  3 ++-
 .../formats/inlongmsg/InLongMsgRowDataSerDeTest.java |  1 +
 .../formats/inlongmsgbinlog/InLongMsgBinlog.java |  4 ++--
 .../InLongMsgBinlogFormatBuilder.java|  8 
 .../InLongMsgBinlogFormatDeserializer.java   |  4 ++--
 .../InLongMsgBinlogFormatFactory.java| 14 +++---
 .../InLongMsgBinlogMixedFormatConverter.java |  2 +-
 .../InLongMsgBinlogMixedFormatDeserializer.java  |  4 ++--
 .../inlongmsgbinlog/InLongMsgBinlogUtils.java| 20 ++--
 .../inlongmsgbinlog/InLongMsgBinlogValidator.java|  4 ++--
 .../InLongMsgBinlogFormatDeserializerTest.java   |  4 ++--
 .../sort/formats/inlongmsgcsv/InLongMsgCsv.java  |  6 +++---
 .../inlongmsgcsv/InLongMsgCsvFormatDeserializer.java | 16 
 .../inlongmsgcsv/InLongMsgCsvFormatFactory.java  | 18 +-
 .../InLongMsgCsvMixedFormatConverter.java|  6 +++---
 .../InLongMsgCsvMixedFormatDeserializer.java |  6 +++---
 .../sort/formats/inlongmsgcsv/InLongMsgCsvUtils.java | 18 +-
 .../formats/inlongmsgcsv/InLongMsgCsvValidator.java  |  2 +-
 .../InLongMsgCsvFormatDeserializerTest.java  |  4 ++--
 .../inlongmsgcsv/InLongMsgCsvFormatFactoryTest.java  |  2 +-
 .../inlong/sort/formats/inlongmsgkv/InLongMsgKv.java |  6 +++---
 .../inlongmsgkv/InLongMsgKvFormatDeserializer.java   | 16 
 .../inlongmsgkv/InLongMsgKvFormatFactory.java| 18 +-
 .../inlongmsgkv/InLongMsgKvMixedFormatConverter.java |  6 +++---
 .../InLongMsgKvMixedFormatDeserializer.java  |  6 +++---
 .../sort/formats/inlongmsgkv/InLongMsgKvUtils.java   | 18 +-
 .../formats/inlongmsgkv/InLongMsgKvValidator.java|  2 +-
 .../inlongmsgkv/InLongMsgKvFormatFactoryTest.java|  4 ++--
 .../formats/inlongmsgtlogcsv/InLongMsgTlogCsv.java   |  2 +-
 .../InLongMsgTlogCsvFormatDeserializer.java  | 12 ++--
 .../InLongMsgTlogCsvFormatFactory.java   | 16 
 .../InLongMsgTlogCsvMixedFormatConverter.java|  6 +++---
 .../InLongMsgTlogCsvMixedFormatDeserializer.java |  6 +++---
 .../inlongmsgtlogcsv/InLongMsgTlogCsvUtils.java  | 12 ++--
 .../inlongmsgtlogcsv/InLongMsgTlogCsvValidator.java  |  2 +-
 .../InLongMsgTlogCsvFormatDeserializerTest.java  |  4 ++--
 .../InLongMsgTlogCsvF

(inlong) branch master updated: [INLONG-10780][SDK] Optimize memory management for DataProxy CPP SDK (#10792)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new f01d49347e [INLONG-10780][SDK] Optimize memory management for 
DataProxy CPP SDK (#10792)
f01d49347e is described below

commit f01d49347e6932dea5ae95b37111a5f9bdde0244
Author: doleyzi <43397300+dole...@users.noreply.github.com>
AuthorDate: Wed Aug 14 16:33:32 2024 +0800

[INLONG-10780][SDK] Optimize memory management for DataProxy CPP SDK 
(#10792)
---
 .../dataproxy-sdk-cpp/src/config/sdk_conf.cc   | 46 +---
 .../dataproxy-sdk-cpp/src/config/sdk_conf.h|  3 +
 .../dataproxy-sdk-cpp/src/manager/buffer_manager.h | 85 ++
 .../dataproxy-sdk-cpp/src/utils/capi_constant.h|  2 +
 4 files changed, 126 insertions(+), 10 deletions(-)

diff --git 
a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/config/sdk_conf.cc 
b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/config/sdk_conf.cc
index 319262adc2..68cc122c4c 100644
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/config/sdk_conf.cc
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/config/sdk_conf.cc
@@ -38,6 +38,9 @@ bool SdkConfig::ParseConfig(const std::string &config_path) {
   // Guaranteed to only parse the configuration file once
   if (!__sync_bool_compare_and_swap(&parsed_, false, true)) {
 LOG_INFO("ParseConfig has  parsed .");
+if (++instance_num_ > max_instance_) {
+  return false;
+}
 return true;
   }
 
@@ -92,7 +95,7 @@ void SdkConfig::defaultInit() {
   load_balance_interval_ = constants::kLoadBalanceInterval;
   heart_beat_interval_ = constants::kHeartBeatInterval;
   enable_balance_ = constants::kEnableBalance;
-  isolation_level_=constants::IsolationLevel::kLevelSecond;
+  isolation_level_ = constants::IsolationLevel::kLevelSecond;
 
   // cache parameter
   send_buf_size_ = constants::kSendBufSize;
@@ -132,6 +135,10 @@ void SdkConfig::defaultInit() {
   enable_setaffinity_ = constants::kEnableSetAffinity;
   mask_cpu_affinity_ = constants::kMaskCPUAffinity;
   extend_field_ = constants::kExtendField;
+
+  need_auth_ = constants::kNeedAuth;
+  max_instance_ = constants::kMaxInstance;
+  instance_num_ = 1;
 }
 
 void SdkConfig::InitThreadParam(const rapidjson::Value &doc) {
@@ -212,6 +219,14 @@ void SdkConfig::InitCacheParam(const rapidjson::Value 
&doc) {
   } else {
 max_stream_id_num_ = constants::kMaxGroupIdNum;
   }
+
+  // max_cache_num
+  if (doc.HasMember("max_cache_num") && doc["max_cache_num"].IsInt() && 
doc["max_cache_num"].GetInt() >= 0) {
+const rapidjson::Value &obj = doc["max_cache_num"];
+max_cache_num_ = obj.GetInt();
+  } else {
+max_cache_num_ = constants::kMaxCacheNum;
+  }
 }
 
 void SdkConfig::InitZipParam(const rapidjson::Value &doc) {
@@ -431,9 +446,10 @@ void SdkConfig::InitAuthParm(const rapidjson::Value &doc) {
   } else {
 need_auth_ = constants::kNeedAuth;
 LOG_INFO("need_auth is not expect, then use default:%s" << need_auth_
- ? "true"
- : "false");
+ ? "true"
+ : "false");
   }
+
 }
 void SdkConfig::OthersParam(const rapidjson::Value &doc) {
   // ser_ip
@@ -475,12 +491,20 @@ void SdkConfig::OthersParam(const rapidjson::Value &doc) {
   } else {
 extend_field_ = constants::kExtendField;
   }
+
+  // instance num
+  if (doc.HasMember("max_instance") && doc["max_instance"].IsInt() && 
doc["max_instance"].GetInt() > 0) {
+const rapidjson::Value &obj = doc["max_instance"];
+max_instance_ = obj.GetInt();
+  } else {
+max_instance_ = constants::kMaxInstance;
+  }
 }
 
-bool SdkConfig::GetLocalIPV4Address(std::string& err_info, std::string& 
localhost) {
+bool SdkConfig::GetLocalIPV4Address(std::string &err_info, std::string 
&localhost) {
   int32_t sockfd;
   int32_t ip_num = 0;
-  char  buf[1024] = {0};
+  char buf[1024] = {0};
   struct ifreq *ifreq;
   struct ifreq if_flag;
   struct ifconf ifconf;
@@ -493,7 +517,7 @@ bool SdkConfig::GetLocalIPV4Address(std::string& err_info, 
std::string& localhos
   }
 
   ioctl(sockfd, SIOCGIFCONF, &ifconf);
-  ifreq  = (struct ifreq *)buf;
+  ifreq = (struct ifreq *) buf;
   ip_num = ifconf.ifc_len / sizeof(struct ifreq);
   for (int32_t i = 0; i < ip_num; i++, ifreq++) {
 if (ifreq->ifr_flags != AF_INET) {
@@ -511,11 +535,11 @@ bool SdkConfig::GetLocalIPV4Address(std::string& 
err_info, std::string& localhos
   continue;
 }
 
-if (!strncmp(inet_ntoa(((struct 
sockaddr_in*)&(ifreq->ifr_addr))->sin_addr),
+if (!strncmp(inet_ntoa(((struct

(inlong) branch master updated: [INLONG-10803][SDK] Transform SQL support Round function (#10810)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new a46d4da26f [INLONG-10803][SDK] Transform SQL support Round function 
(#10810)
a46d4da26f is described below

commit a46d4da26fe04a8fc442d3ef009d140903c2ceab
Author: Zkplo <87751516+zk...@users.noreply.github.com>
AuthorDate: Tue Aug 20 10:57:20 2024 +0800

[INLONG-10803][SDK] Transform SQL support Round function (#10810)

* [INLONG-10803][SDK] Transform SQL support Round function

* [INLONG-10803][SDK] Transform SQL support Round function

-

Co-authored-by: ZKpLo <14148880+zk...@user.noreply.gitee.com>
---
 .../transform/process/function/RoundFunction.java  | 60 ++
 .../transform/process/operator/OperatorTools.java  |  2 +
 .../TestTransformArithmeticFunctionsProcessor.java | 31 +++
 3 files changed, 93 insertions(+)

diff --git 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/RoundFunction.java
 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/RoundFunction.java
new file mode 100644
index 00..ecad419635
--- /dev/null
+++ 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/function/RoundFunction.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.sdk.transform.process.function;
+
+import org.apache.inlong.sdk.transform.decode.SourceData;
+import org.apache.inlong.sdk.transform.process.Context;
+import org.apache.inlong.sdk.transform.process.operator.OperatorTools;
+import org.apache.inlong.sdk.transform.process.parser.ValueParser;
+
+import net.sf.jsqlparser.expression.Expression;
+import net.sf.jsqlparser.expression.Function;
+
+import java.math.BigDecimal;
+import java.math.RoundingMode;
+import java.util.List;
+
+/**
+ * RoundFunction
+ * description: ROUND(x [,y]) -- Return the nearest integer to x, with 
optional parameter y indicating the number of decimal places to be rounded. If 
omitted, return the integer.
+ */
+public class RoundFunction implements ValueParser {
+
+private ValueParser numberParser;
+private ValueParser reservedDigitsParser;
+
+public RoundFunction(Function expr) {
+List expressions = expr.getParameters().getExpressions();
+numberParser = OperatorTools.buildParser(expressions.get(0));
+if (expressions.size() == 2) {
+reservedDigitsParser = 
OperatorTools.buildParser(expressions.get(1));
+}
+}
+
+@Override
+public Object parse(SourceData sourceData, int rowIndex, Context context) {
+Object numberObj = numberParser.parse(sourceData, rowIndex, context);
+BigDecimal number = OperatorTools.parseBigDecimal(numberObj);
+if (reservedDigitsParser != null) {
+Object reservedDigitsObj = reservedDigitsParser.parse(sourceData, 
rowIndex, context);
+int reservedDigits = 
OperatorTools.parseBigDecimal(reservedDigitsObj).intValue();
+return number.setScale(reservedDigits, 
RoundingMode.HALF_UP).doubleValue();
+}
+return Math.round(number.doubleValue());
+}
+}
diff --git 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
index 4f29fba315..fe361263a4 100644
--- 
a/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
+++ 
b/inlong-sdk/transform-sdk/src/main/java/org/apache/inlong/sdk/transform/process/operator/OperatorTools.java
@@ -34,6 +34,7 @@ import 
org.apache.inlong.sdk.transform.process.function.Log2Function;
 import org.apache.inlong.sdk.transform.process.function.LogFunction;
 import org.apache.inlong.sdk.transform.process.function.NowFunction;
 import org.apache.inlong.sdk.transform.process.function.PowerFunction;
+import org.apache.inlong.sdk.transfor

(inlong) branch master updated: [INLONG-10811][SDK] Fix callback function calls lead to coredump in python sdk (#10812)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new f18e501fb0 [INLONG-10811][SDK] Fix callback function calls lead to 
coredump in python sdk (#10812)
f18e501fb0 is described below

commit f18e501fb08c294094f3a717089fa289ea009ed2
Author: yfsn666 <61183968+yfsn...@users.noreply.github.com>
AuthorDate: Tue Aug 20 17:10:08 2024 +0800

[INLONG-10811][SDK] Fix callback function calls lead to coredump in python 
sdk (#10812)

* [INLONG-10811][SDK] Fix callback function calls lead to coredump in 
python sdk

* [INLONG-10811][SDK] Fix the close_api waitms

* [INLONG-10811][SDK] Update pybind wrapper code

* [INLONG-10811][SDK] Update pybind wrapper code

* [INLONG-10811][SDK] Update pybind wrapper code

* [INLONG-10811][SDK] Update pybind wrapper code
---
 .../dataproxy-sdk-python/CMakeLists.txt| 15 ++--
 .../dataproxy-sdk-python/demo/send_demo.py |  6 +-
 .../dataproxy-sdk-python/inlong_dataproxy.cpp  | 85 ++
 3 files changed, 66 insertions(+), 40 deletions(-)

diff --git a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/CMakeLists.txt 
b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/CMakeLists.txt
index 80791b1c02..5f84a35b19 100644
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/CMakeLists.txt
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/CMakeLists.txt
@@ -22,17 +22,18 @@ project(dataproxy-sdk-python)
 
 set(CMAKE_CXX_STANDARD 11)
 
-include_directories("./dataproxy-sdk-cpp/src/core")
+include_directories("${CMAKE_CURRENT_SOURCE_DIR}/dataproxy-sdk-cpp/src/core")
 
-include_directories("./dataproxy-sdk-cpp/third_party/lib")
-include_directories("./dataproxy-sdk-cpp/third_party/lib64")
+include_directories("${CMAKE_CURRENT_SOURCE_DIR}/dataproxy-sdk-cpp/third_party/lib")
+include_directories("${CMAKE_CURRENT_SOURCE_DIR}/dataproxy-sdk-cpp/third_party/lib64")
+include_directories("${CMAKE_CURRENT_SOURCE_DIR}/dataproxy-sdk-cpp/release/lib")
 
 add_subdirectory(pybind11)
-add_subdirectory(dataproxy-sdk-cpp)
 
-link_directories("./dataproxy-sdk-cpp/third_party/lib")
-link_directories("./dataproxy-sdk-cpp/third_party/lib64")
+link_directories("${CMAKE_CURRENT_SOURCE_DIR}/dataproxy-sdk-cpp/third_party/lib")
+link_directories("${CMAKE_CURRENT_SOURCE_DIR}/dataproxy-sdk-cpp/third_party/lib64")
+link_directories("${CMAKE_CURRENT_SOURCE_DIR}/dataproxy-sdk-cpp/release/lib")
 
 pybind11_add_module(inlong_dataproxy inlong_dataproxy.cpp)
 
-target_link_libraries(inlong_dataproxy PRIVATE pybind11::module dataproxy_sdk)
+target_link_libraries(inlong_dataproxy PRIVATE 
"${CMAKE_CURRENT_SOURCE_DIR}/dataproxy-sdk-cpp/release/lib/dataproxy_sdk.a" 
liblog4cplusS.a libsnappy.a libcurl.a libssl.a libcrypto.a)
diff --git 
a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/demo/send_demo.py 
b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/demo/send_demo.py
index e8c33bc23a..75974baa95 100644
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/demo/send_demo.py
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/demo/send_demo.py
@@ -40,7 +40,7 @@ def main():
 # step1. init api
 init_status = inlong_api.init_api(sys.argv[1])
 if init_status:
-print("init error, error code is: " + init_status)
+print("init error, error code is: " + str(init_status))
 return
 
 print(">start sdk successfully")
@@ -62,12 +62,12 @@ def main():
 for i in range(count):
 send_status = inlong_api.send(inlong_group_id, inlong_stream_id, msg, 
len(msg), callback_func)
 if send_status:
-print("tc_api_send error, error code is: " + send_status)
+print("tc_api_send error, error code is: " + str(send_status))
 
 # step3. close api
 close_status = inlong_api.close_api(1)
 if close_status:
-print("close sdk error, error code is: " + close_status)
+print("close sdk error, error code is: " + str(close_status))
 else:
 print(">close sdk successfully")
 
diff --git 
a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/inlong_dataproxy.cpp 
b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/inlong_dataproxy.cpp
index 26c260f3f9..2fc583 100644
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/inlong_dataproxy.cpp
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-python/inlong_dataproxy.cpp
@@ -18,44 +18,69 @@
  */
 
 #include 
+#include 
 #include 
 #include 
+#include 
+#include 
+#include 
 
 namespace py = pybind11;
-using namespace inlong;
 
-class 

svn commit: r63986 - /dev/inlong/KEYS

[luchunliang]

Author: luchunliang
Date: Thu Sep 14 07:36:53 2023
New Revision: 63986

Log:
add gpg key for luchunliang

Modified:
dev/inlong/KEYS

Modified: dev/inlong/KEYS
==
--- dev/inlong/KEYS (original)
+++ dev/inlong/KEYS Thu Sep 14 07:36:53 2023
@@ -613,3 +613,119 @@ ksEXOF9fwmX5WZCJD05O70MlTMPAT5h7qyKhFiqV
 RyQXB/4OoQZSAE8glbCZIR1N9GVwyvOX67KGm500fsSAkqbdgyaJFiYo2a+x+Q==
 =p/H7
 -END PGP PUBLIC KEY BLOCK-
+pub   rsa4096 2023-09-14 [SC]
+  E2F26602285A528E4238ADA04CB3DCDE9F240C46
+uid   [ultimate] luchunliang 
+sig 34CB3DCDE9F240C46 2023-09-14  luchunliang 
+sub   rsa4096 2023-09-14 [E]
+sig  4CB3DCDE9F240C46 2023-09-14  luchunliang 
+
+-BEGIN PGP PUBLIC KEY BLOCK-
+
+mQINBGUCr78BEADRyKA+ly2gMfoVDNJbbd45lzU1uqhzZV1Bv10YlCrSDUUKSVRT
+V8c/g20cVpq6cDcsWXJL2BOGg2PeAcs2AEyxoCwIgzhF+2HiZVgNVxpsm3PGonkD
+G8b+/1CbGXCyCxDlUjMRzpmf+FzIH2798c5e7dpf4Viv4+QLGrX2mJ/pnKNq1ndQ
+D+F+zE+EaMImURqIoQEPZTnzk/cm8h2vTgdi8eHOamoHYYx+jWvYSanSmrhi+JNQ
+GVe4Wze2XTqMj1GgYqUAOJIpAiRg7kIFRamABAkX8uesjdk06f/1NfF5FwCO056y
+3zGVAfscNjMqoRnVulv7+JyonvMua4/OVcj+DWiDepIv6REZcwF92lHV6g611dGs
+jrFTrQjdrlD3Z8GyLiHEyVwV+QVHZHXbm7wZinUWYVsF4jxt+bsyomjb6QdGeG3J
+3r/bSx2rYKDNM3B9fJbWLZ5L8+i6t/+O+1oWIOfw20u7P9UAF9jXV5+tBGNgo1fY
+s+0Kwk7azkpxEETenN/nyxbIqKifL+yjJgqyr4AvRzFNvqpxzlaSfLVlVbuojBuq
+L7Uf/kwKdasZceTq7B+VveClvn6zxj9Tnb5+RB0C9H1fiN64SBm93JQzHiY8XBbM
+NSqPQ7hmkET9gjLVEPWLhC1bniGr2YWIej6dJkJwkL/CVv7wFIqIJBqEzwARAQAB
+tCRsdWNodW5saWFuZyA8bHVjaHVubGlhbmdAYXBhY2hlLm9yZz6JAk4EEwEIADgW
+IQTi8mYCKFpSjkI4raBMs9zenyQMRgUCZQKvvwIbAwULCQgHAgYVCgkICwIEFgID
+AQIeAQIXgAAKCRBMs9zenyQMRrc7EACw5Wbby8djYz65GSbMzZhBI/DpaWgZCxGn
+NuqZtqZU449lOj7S+v/3scr83dVQKR+sTkWECtbTmimtQRaN+7PIM8OeBOBAye4j
+x4rS4mlxh5CskW9V6pKtyZeEZObBmxB6YcuhpfGIoiVQwVjdZo1B/t/4UUpcAd/d
+6LPPj12l8o/XCvw2q8XtuALz23P1zykkARRZjl/T8PvdqXOmiVXawW/Fo2J9l098
+rTx2iRjnI3kNV73U+F+KD6vWI2UroZ7jtWaWfLrawmS1/wtCSskocWbPDAOzviXe
+XEUu/dZ9uEQaQPJCsnJ+sPgIiT7Xq1htJHqvza6v+fi5SPL9pWMNxiycDLrUa4ZN
+uUAtWprCEqsYsnvtQ6n6lvFAvg7jl/1JOcAUKlz5xjerXe5/NrC6Rao1AYBbGOQN
+xxYjkmG7oxrMVT+nu/j0ePKRSl/Dl8wQVbaee82XFA0wiCKFbJHc+kSc2Lao8aH2
+ta3yMX/M5ibR769FCK1fDFj2IGH2Fza6rGpNtU+c/fzT4mrWsOXSR6Tnm9sjDTdC
+TnC/ScvNNPnQ9+vVDmYmReIsUDEWd+lFPn9sLmFlk8IqCWnyOIyANdLM17O2JozN
+x+KO6/sQy+luojRtJZDDmtWnNttSPZja6gaC2aCPp2467Hykk9ZgK/0GsDqmwxyF
+bGlOTZVlH7kCDQRlAq+/ARAA2eo1wwupTNrzoYnQqaPLdlWNIX8cjF0Fx178Befg
+eXoO63hpIGqeiuMace3KxqzbKToxWN1CjCskhWBg5bVO65b+ndmYCxzBNYfY8avu
+58Y5ci9uXx1dBMg7wTngmPxkrYPkz5kxCaH90cEmxzd45X4bchm/T3gzeuuvdWun
+SoOjkazCTVTs2vaDo57ZqJxfIvfSi4cDy1KVaFOJje44baq6ORu3YokWV09CHE7q
+LmqimtiLCFzSLAG3PREjvIgHp+LnI+geZSh7dLKgXTumehMp5y0AKIpEKHcAejCD
++VWmz3vH5ZIBwqDJahMjFexs1m9oD2CObjhYxcmaWcT4lpWo5kPQ74Dpd3+Qtxdj
+nLNrkjDdUBr0wTRyu8V6j6jsSHzPOE8PGJayXY1YwWX1EyMQZrjeY1SBXa5x8jSE
+gE8+ePFCGakHK/+rulnC2EOEpEnN5jAtgPX8Zg7+PvhWlDXu/mGxxR3u3dldBWte
+/0YfaOppqeQmIr+4b9G73ZAOzcJsPJTYb6QBu2kGCHDkKdVngdPqr+eA+lOclRUG
+vDFjuLVkiRUS46kS6yEpZY410PG+qhUd665GfhRNcsLG7MyBrfKPAEc8EpAHk+ZI
+bIJjOYAFAziIUjjEtHTuWMSgECKkofUOEstKa+rZXTTfvQLgJEx0Cpd1vV6imh3I
+kx8AEQEAAYkCNgQYAQgAIBYhBOLyZgIoWlKOQjitoEyz3N6fJAxGBQJlAq+/AhsM
+AAoJEEyz3N6fJAxGeaIP/3cFTPpOUvY0xg1WbhDeyQ+fs0jW7bG1trSp/t1no2oI
+o8vWt77nTQW2Rw6Plw3wRHRAOxofew7+wkzyylzSvTBY8gN5dxF8oSNjuHLnn+xF
+o8nWlhqiJZyUEQNgCX6VfsiHKtqYc23y529mQbWxoGTZuzvZbefLq31vgcVBCH6S
+U1acev2sb1YdS0ib8g1p2V0ssSytl9Ynk1flGyc2eQ7n9XMgQlVFvFWHINc6APXP
+wYnNbDQeE/ZtIpiYocuyFMQA4JyY5mACUQ09dRSuZMMCcwjxP8JTOZPhddZIDsTn
+yWtPPDmQ0bGaq26nq4PlZ5Vcdb/EEocNHEeZ6V0RSNlX+MvBLxplDYefM0d2LcId
+sqRqT+InXnfzbNuoLPmVx1k5Fh8IoWT+jl0KAiNkoZVOJC5UOW1WVv5YVEpHdLqt
+XUYix3twwTnOfY/j4KTHGiGQwnIOLRQC9oW4yylehO8g6/H26UTALlEG8e/w0NxI
+rb0qVtGeNtZ+u7aVBqOfFLZTyW8Sr1OJdx3fjWBw/ANz6zDn13iQHgjqD85taOCn
+9Rj9piB1nFRnVyLPdLhEN/6k1iT1X5NPhRwFxv3Ik4ea9s+QejT135QTtMWh6Znb
+ijdshVABKTENK4shbEo04aAsZc0zeSOEVqds/Y+oj6ZnHZf8v7L48LQCeeTlrRD5
+=C1Zz
+-END PGP PUBLIC KEY BLOCK-
+pub   rsa4096 2023-09-14 [SC]
+  E2F26602285A528E4238ADA04CB3DCDE9F240C46
+uid   [ultimate] luchunliang 
+sig 34CB3DCDE9F240C46 2023-09-14  luchunliang 
+sub   rsa4096 2023-09-14 [E]
+sig  4CB3DCDE9F240C46 2023-09-14  luchunliang 
+
+-BEGIN PGP PUBLIC KEY BLOCK-
+
+mQINBGUCr78BEADRyKA+ly2gMfoVDNJbbd45lzU1uqhzZV1Bv10YlCrSDUUKSVRT
+V8c/g20cVpq6cDcsWXJL2BOGg2PeAcs2AEyxoCwIgzhF+2HiZVgNVxpsm3PGonkD
+G8b+/1CbGXCyCxDlUjMRzpmf+FzIH2798c5e7dpf4Viv4+QLGrX2mJ/pnKNq1ndQ
+D+F+zE+EaMImURqIoQEPZTnzk/cm8h2vTgdi8eHOamoHYYx+jWvYSanSmrhi+JNQ
+GVe4Wze2XTqMj1GgYqUAOJIpAiRg7kIFRamABAkX8uesjdk06f/1NfF5FwCO056y
+3zGVAfscNjMqoRnVulv7+JyonvMua4/OVcj+DWiDepIv6REZcwF92lHV6g611dGs
+jrFTrQjdrlD3Z8GyLiHEyVwV+QVHZHXbm7wZinUWYVsF4jxt+bsyomjb6QdGeG3J
+3r/bSx2rYKDNM3B9fJbWLZ5L8+i6t/+O+1oWIOfw20u7P9UAF9jXV5+tBGNgo1fY
+s+0Kwk7azkpxEETenN/nyxbIqKifL+yjJgqyr4AvRzFNvqpxzlaSfLVlVbuojBuq
+L7Uf/kwKdasZceTq7B+VveClvn6zxj9Tnb5+RB0C9H1fiN64SBm93JQzHiY8XBbM
+NSqPQ7hmkET9gjLVEPWLhC1bniGr2YWIej6dJkJwkL/CVv7wFIqIJBqEzwARAQAB

[inlong] branch branch-1.9 created (now ddbdb226d9)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a change to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/inlong.git


  at ddbdb226d9 [INLONG-8447][Manager] Optimize paging logic (#8448)

No new revisions were added by this update.

[inlong] branch branch-1.9 deleted (was ddbdb226d9)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a change to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/inlong.git


 was ddbdb226d9 [INLONG-8447][Manager] Optimize paging logic (#8448)

The revisions that were on this branch are still contained in
other references; therefore, this change does not discard any commits
from the repository.

[inlong] branch branch-1.9 created (now ddbdb226d9)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a change to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/inlong.git


  at ddbdb226d9 [INLONG-8447][Manager] Optimize paging logic (#8448)

No new revisions were added by this update.

[inlong] annotated tag 1.9.0-RC0 updated (ddbdb226d9 -> 0bfbfdfa91)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a change to annotated tag 1.9.0-RC0
in repository https://gitbox.apache.org/repos/asf/inlong.git


*** WARNING: tag 1.9.0-RC0 was modified! ***

from ddbdb226d9 (commit)
  to 0bfbfdfa91 (tag)
 tagging ddbdb226d9417b7b8d1edc410179c7cec7c2 (commit)
 replaces 0.9.0-incubating-RC1
  by 卢春亮
  on Fri Sep 15 09:56:03 2023 +0800

- Log -
Tagging the 1.9.0 first Releae Candidate (Candidates start at zero)
---


No new revisions were added by this update.

Summary of changes:

[inlong] branch branch-1.9 updated (ddbdb226d9 -> 25e03cedea)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a change to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/inlong.git


from ddbdb226d9 [INLONG-8447][Manager] Optimize paging logic (#8448)
 add 083b1dfb76 [INLONG-8914][DataProxy] Optimize DataProxy event 
statistics (#8915)
 add a63b1d9591 [INLONG-7908][Sort] PostgreSQL connector supports parallel 
read (#8664)
 add 9558057833 [INLONG-8916][SDK] Update SDK configuration file for 
dataproxy cpp sdk (#8917)
 add f5e9f35a62 [INLONG-8918][DataProxy] Change tube to tubemq  (#8919)
 add 9fbb47939a [INLONG-8403][Manager] Support resource migrate to another 
tenant (#8913)
 add 25e03cedea [INLONG-8845][Manager] Support Tencent Cloud Log Service 
data flow (#8892)

No new revisions were added by this update.

Summary of changes:
 bin/inlong-daemon  |   2 +-
 .../inlong/dataproxy/config/ConfigManager.java |   4 +-
 .../inlong/dataproxy/consts/ConfigConstants.java   |   2 -
 .../inlong/dataproxy/consts/StatConstants.java |   8 +-
 .../dataproxy/sink/mq/MessageQueueZoneSink.java|   4 +-
 .../dataproxy/sink/mq/kafka/KafkaHandler.java  |  24 +-
 .../dataproxy/sink/mq/pulsar/PulsarHandler.java|  41 +-
 .../inlong/dataproxy/sink/mq/tube/TubeHandler.java |  29 +-
 .../apache/inlong/dataproxy/source/BaseSource.java |  16 +-
 .../dataproxy/source/ServerMessageHandler.java |  10 +-
 .../inlong/dataproxy/source/SourceConstants.java   |   2 -
 .../source/httpMsg/HttpMessageHandler.java |  14 +-
 .../inlong/dataproxy/source/v0msg/CodecBinMsg.java |   8 +-
 .../dataproxy/source/v0msg/CodecTextMsg.java   |   4 +-
 .../inlong/dataproxy/source/UdpSourceTest.java |   1 -
 .../inlong/manager/common/consts/DataNodeType.java |   5 +
 .../manager/common/consts/InlongConstants.java |   2 +
 .../inlong/manager/common/consts/SinkType.java |   5 +
 .../manager/dao/mapper/DataNodeEntityMapper.java   |   7 +
 .../dao/mapper/InlongConsumeEntityMapper.java  |   6 +
 .../dao/mapper/InlongGroupEntityMapper.java|   4 +
 .../dao/mapper/TenantClusterTagEntityMapper.java   |   4 +
 .../resources/mappers/DataNodeEntityMapper.xml |  52 +-
 .../mappers/InlongConsumeEntityMapper.xml  |  22 +
 .../resources/mappers/InlongGroupEntityMapper.xml  |  11 +-
 .../mappers/TenantClusterTagEntityMapper.xml   |  13 +
 .../pojo/group/pulsar/InlongPulsarRequest.java |   2 +-
 .../pojo/group/pulsar/InlongPulsarTopicInfo.java   |   2 +-
 .../manager/pojo/node/cls/ClsDataNodeDTO.java  |  94 +++
 .../manager/pojo/node/cls/ClsDataNodeInfo.java |  81 ++
 .../manager/pojo/node/cls/ClsDataNodeRequest.java  |  71 ++
 .../inlong/manager/pojo/sink/cls/ClsSink.java  |  90 +++
 .../inlong/manager/pojo/sink/cls/ClsSinkDTO.java   |  76 ++
 .../cls/ClsSinkRequest.java}   |  41 +-
 inlong-manager/manager-service/pom.xml |   5 +-
 .../manager/service/group/GroupCheckService.java   |  13 -
 .../service/group/InlongGroupOperator4Pulsar.java  |   2 +-
 .../service/node/cls/ClsDataNodeOperator.java  | 111 +++
 .../resource/sink/cls/ClsResourceOperator.java | 183 +
 .../sink/pulsar/PulsarResourceOperator.java|   6 +
 .../manager/service/sink/cls/ClsSinkOperator.java  | 131 
 .../service/tenant/InlongTenantService.java|   2 +
 .../service/tenant/InlongTenantServiceImpl.java| 135 +++-
 .../web/controller/InlongTenantController.java |  10 +
 .../release/conf/config_example.json   |  40 +-
 .../sort/parser/AllMigratePostgreSQLTest.java  |   1 +
 .../sort/cdc/base/dialect/DataSourceDialect.java   |  13 +-
 .../connection/JdbcConnectionPoolFactory.java  |   1 -
 .../state/PendingSplitsStateSerializer.java|   3 +
 .../meta/offset/OffsetDeserializerSerializer.java  |   7 +-
 .../meta/split/FinishedSnapshotSplitInfo.java  |   3 +
 .../source/meta/split/SourceSplitSerializer.java   |  15 +-
 .../source/reader/IncrementalSourceReader.java |  11 +-
 .../reader/IncrementalSourceRecordEmitter.java |  20 +-
 .../cdc/base/source/reader/external/FetchTask.java |   4 +
 .../inlong/sort/cdc/base/util/RecordUtils.java |  13 +
 .../sort-connectors/postgres-cdc/pom.xml   |  42 +-
 .../postgresql/PostgresObjectFactory.java  | 135 
 .../io/debezium/connector/postgresql/Utils.java|  79 ++
 .../connector/postgresql/connection/Lsn.java   | 161 
 .../postgresql/connection/PostgresConnection.java  | 812 +
 .../external/IncrementalSourceStreamFetcher.java   | 253 +++
 .../sort/cdc/postgres/DebeziumSourceFunction.java  |   2 +-
 .../cdc/postgres/source/PostgresChunkSplitter.java | 394 ++
 .../source/PostgresConnectionPoolFactory.java  |  42 ++
 .../sort/cdc/postgres/source/PostgresDialect.java  | 186 +
 .../cdc/postgres/source/PostgresSourceBuilder.java | 260

[inlong] branch release-1.9.0 created (now 25e03cedea)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a change to branch release-1.9.0
in repository https://gitbox.apache.org/repos/asf/inlong.git


  at 25e03cedea [INLONG-8845][Manager] Support Tencent Cloud Log Service 
data flow (#8892)

No new revisions were added by this update.

[inlong] branch branch-1.9 updated: [INLONG-8878][Release] Bumped 1.9.0 branch version to 1.9.0

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/branch-1.9 by this push:
 new 5b08444da5 [INLONG-8878][Release] Bumped 1.9.0 branch version to 1.9.0
5b08444da5 is described below

commit 5b08444da5272fc1886bd0681aba2822bfa41c5e
Author: 卢春亮 <946240...@qq.com>
AuthorDate: Mon Sep 18 15:22:56 2023 +0800

[INLONG-8878][Release] Bumped 1.9.0 branch version to 1.9.0
---
 inlong-agent/agent-common/pom.xml | 2 +-
 inlong-agent/agent-core/pom.xml   | 2 +-
 inlong-agent/agent-docker/pom.xml | 2 +-
 inlong-agent/agent-plugins/pom.xml| 2 +-
 inlong-agent/agent-release/pom.xml| 2 +-
 inlong-agent/pom.xml  | 2 +-
 inlong-audit/audit-common/pom.xml | 2 +-
 inlong-audit/audit-docker/pom.xml | 2 +-
 inlong-audit/audit-proxy/pom.xml  | 2 +-
 inlong-audit/audit-release/pom.xml| 2 +-
 inlong-audit/audit-sdk/pom.xml| 2 +-
 inlong-audit/audit-store/pom.xml  | 2 +-
 inlong-audit/pom.xml  | 2 +-
 inlong-common/pom.xml | 2 +-
 inlong-dashboard/pom.xml  | 2 +-
 inlong-dataproxy/dataproxy-dist/pom.xml   | 2 +-
 inlong-dataproxy/dataproxy-docker/pom.xml | 2 +-
 inlong-dataproxy/dataproxy-source/pom.xml | 2 +-
 inlong-dataproxy/pom.xml  | 2 +-
 inlong-distribution/pom.xml   | 2 +-
 inlong-manager/manager-client-examples/pom.xml| 2 +-
 inlong-manager/manager-client-tools/pom.xml   | 2 +-
 inlong-manager/manager-client/pom.xml | 2 +-
 inlong-manager/manager-common/pom.xml | 2 +-
 inlong-manager/manager-dao/pom.xml| 2 +-
 inlong-manager/manager-docker/pom.xml | 2 +-
 inlong-manager/manager-plugins/base/pom.xml   | 2 +-
 inlong-manager/manager-plugins/manager-plugins-flink-v1.13/pom.xml| 2 +-
 inlong-manager/manager-plugins/manager-plugins-flink-v1.15/pom.xml| 2 +-
 inlong-manager/manager-plugins/pom.xml| 2 +-
 inlong-manager/manager-pojo/pom.xml   | 2 +-
 inlong-manager/manager-service/pom.xml| 2 +-
 inlong-manager/manager-test/pom.xml   | 2 +-
 inlong-manager/manager-web/pom.xml| 2 +-
 inlong-manager/manager-workflow/pom.xml   | 2 +-
 inlong-manager/pom.xml| 2 +-
 inlong-sdk/dataproxy-sdk/pom.xml  | 2 +-
 inlong-sdk/pom.xml| 2 +-
 inlong-sdk/sdk-common/pom.xml | 2 +-
 inlong-sdk/sort-sdk/pom.xml   | 4 ++--
 inlong-sort-standalone/pom.xml| 2 +-
 inlong-sort-standalone/sort-standalone-common/pom.xml | 2 +-
 inlong-sort-standalone/sort-standalone-dist/pom.xml   | 2 +-
 inlong-sort-standalone/sort-standalone-source/pom.xml | 2 +-
 inlong-sort/pom.xml   | 2 +-
 inlong-sort/sort-api/pom.xml  | 2 +-
 inlong-sort/sort-common/pom.xml   | 2 +-
 inlong-sort/sort-core/pom.xml | 2 +-
 inlong-sort/sort-dist/pom.xml | 2 +-
 inlong-sort/sort-end-to-end-tests/pom.xml | 2 +-
 inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.13/pom.xml | 2 +-
 inlong-sort/sort-end-to-end-tests/sort-end-to-end-tests-v1.15/pom.xml | 2 +-
 inlong-sort/sort-flink/base/pom.xml   | 2 +-
 inlong-sort/sort-flink/cdc-base/pom.xml   | 2 +-
 inlong-sort/sort-flink/pom.xml| 2 +-
 inlong-sort/sort-flink/sort-flink-v1.13/pom.xml   | 2 +-
 inlong-sort/sort-flink/sort-flink-v1.13/sort-connectors/doris/p

[inlong] branch INLONG-8879 created (now 01af51d2ba)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a change to branch INLONG-8879
in repository https://gitbox.apache.org/repos/asf/inlong.git


  at 01af51d2ba [INLONG-8879][Release] Change the tag of Docker images to 
1.9.0

This branch includes the following new commits:

 new 01af51d2ba [INLONG-8879][Release] Change the tag of Docker images to 
1.9.0

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.

[inlong] 01/01: [INLONG-8879][Release] Change the tag of Docker images to 1.9.0

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch INLONG-8879
in repository https://gitbox.apache.org/repos/asf/inlong.git

commit 01af51d2ba670e92d64cf6b58dd3032a96cbbead
Author: 卢春亮 <946240...@qq.com>
AuthorDate: Mon Sep 18 15:37:28 2023 +0800

[INLONG-8879][Release] Change the tag of Docker images to 1.9.0
---
 conf/inlong.conf |  2 +-
 docker/docker-compose/.env   |  2 +-
 docker/kubernetes/README.md  |  2 +-
 docker/kubernetes/values.yaml| 14 +++---
 inlong-agent/agent-docker/README.md  |  2 +-
 inlong-audit/audit-docker/README.md  |  2 +-
 inlong-dataproxy/dataproxy-docker/README.md  |  2 +-
 inlong-manager/manager-docker/README.md  |  4 ++--
 inlong-tubemq/tubemq-docker/tubemq-all/README.md |  4 ++--
 inlong-tubemq/tubemq-docker/tubemq-manager/README.md |  4 ++--
 10 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/conf/inlong.conf b/conf/inlong.conf
index 28a1556661..0328c5fd6e 100644
--- a/conf/inlong.conf
+++ b/conf/inlong.conf
@@ -27,7 +27,7 @@ spring_datasource_password=inlong
 
 ## Dashboard Configuration ##
 # dashboard docker image
-dashboard_docker_image=inlong/dashboard:latest
+dashboard_docker_image=inlong/dashboard:1.9.0
 # dashboard service port
 dashboard_mapping_port=80
 
diff --git a/docker/docker-compose/.env b/docker/docker-compose/.env
index dce60412ce..753db03bd5 100644
--- a/docker/docker-compose/.env
+++ b/docker/docker-compose/.env
@@ -16,4 +16,4 @@
 #
 # you can choose the released version number, like 1.3.0
 # the latest tag corresponds to the master branch
-VERSION_TAG=latest
+VERSION_TAG=1.9.0
diff --git a/docker/kubernetes/README.md b/docker/kubernetes/README.md
index 3d946a973c..305d4e24a5 100644
--- a/docker/kubernetes/README.md
+++ b/docker/kubernetes/README.md
@@ -74,7 +74,7 @@ The configuration file is [values.yaml](values.yaml), and the 
following tables l
 |`timezone`
| `Asia/Shanghai`  |   
World time and date for cities in all time zones
   |
 |   `images.pullPolicy`
|  `IfNotPresent`  | Image 
pull policy. One of `Always`, `Never`, `IfNotPresent`   
   |
 | `images..repository`  
|  |
  Docker image repository for the component 
  |
-| `images..tag` 
| `latest` |
  Docker image tag for the component
  |
+| `images..tag` 
| `1.9.0` | 
 Docker image tag for the component 
 |
 | `.component`  
|  |
Component name  
  |
 |  `.replicas`  
|   `1`|
Replicas is the desired number of replicas of a given Template  
  |
 |`.podManagementPolicy` 
|  `OrderedReady`  |PodManagementPolicy controls how pods 
are created during initial scale up, when replacing pods on nodes, or when 
scaling down |
diff --git a/docker/kubernetes/values.yaml b/docker/kubernetes/values.yaml
index b6e8eec58f..fbde4735fc 100644
--- a/docker/kubernetes/values.yaml
+++ b/docker/kubernetes/values.yaml
@@ -28,25 +28,25 @@ components:
 images:
   agent:
 repository: inlong/agent
-tag: latest
+tag: 1.9.0
   dashboard:
 repository: inlong/dashboard
-tag: latest
+tag: 1.9.0
   dataproxy:
 repository: inlong/dataproxy
-tag: latest
+tag: 1.9.0
   tubemqManager:
 repository: inlong/tubemq-manager
-tag: latest
+tag: 1.9.0
   tubemqServer:
 repository: inlong/tubemq-all
-tag: latest
+tag: 1.9.0
   manager:
 repository: inlong/manager
-tag: latest
+tag: 1.9.0
   audit:
 repository: inlong/audit
-tag: latest
+tag: 1.9.0
   mysql:
 repository: mysql
  

[inlong] branch branch-1.9 deleted (was 5b08444da5)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a change to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/inlong.git


 was 5b08444da5 [INLONG-8878][Release] Bumped 1.9.0 branch version to 1.9.0

This change permanently discards the following revisions:

 discard 5b08444da5 [INLONG-8878][Release] Bumped 1.9.0 branch version to 1.9.0

[inlong] branch branch-1.9 created (now d253a55520)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a change to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/inlong.git


  at d253a55520 [INLONG-8909][Dashboard] Support management of Tencent 
cloud log service data node (#8927)

No new revisions were added by this update.

[inlong] branch branch-1.9 updated: [INLONG-8875][Release] Update changes log for the 1.9.0 version (#8924)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/branch-1.9 by this push:
 new 482176d319 [INLONG-8875][Release] Update changes log for the 1.9.0 
version (#8924)
482176d319 is described below

commit 482176d319ad12a2cb4ef0c193c9d5bf016f6278
Author: 卢春亮 <946240...@qq.com>
AuthorDate: Mon Sep 18 18:53:12 2023 +0800

[INLONG-8875][Release] Update changes log for the 1.9.0 version (#8924)

* [INLONG-8875][Release] Update changes log for the 1.9.0 version

* fix review comment
---
 CHANGES.md | 449 -
 1 file changed, 233 insertions(+), 216 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index a92730d8bd..6dca8ec213 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -22,229 +22,246 @@
 # of release header or remove the below marker. This file is generated.
 # DO NOT REMOVE THIS MARKER; FOR INTERPOLATING CHANGES!-->
 
-# Release InLong 1.8.0 - Released (as of 2023-07-09)
+# Release InLong 1.9.0 - Released (as of 2023-09-18)
 ### Agent
-|ISSUE| Summary
|
-|:---:|:---|
-| [INLONG-8176](https://github.com/apache/inlong/issues/8176) | 
[Improve][Agent] Upgrade rocksdb version   |
-| [INLONG-8180](https://github.com/apache/inlong/issues/8180) | 
[Improve][Agent] Improve the efficiency and safety of log file reading | 
-| [INLONG-8183](https://github.com/apache/inlong/issues/8183) | 
[Improve][Agent] Optimize agent UT |
-| [INLONG-8244](https://github.com/apache/inlong/issues/8244) | [Bug][Agent] 
Thread leaks after the job is finished| 
-| [INLONG-8251](https://github.com/apache/inlong/issues/8251) | 
[Improve][Agent] Add global memory limit for file collect  | 
-| [INLONG-8334](https://github.com/apache/inlong/issues/8334) | 
[Improve][Agent] Optimize the file collection UT   | 
-| [INLONG-8339](https://github.com/apache/inlong/issues/8339) | 
[Improve][DataProxy][Agent] Enable audit by default in DataProxy and Agent | 
-| [INLONG-8347](https://github.com/apache/inlong/issues/8347) | 
[Improve][Agent] Optimize  the agent UT of testTimeOffset  |
-| [INLONG-8352](https://github.com/apache/inlong/issues/8352) | 
[Improve][Agent] Optimize the agent UT of testRestartTriggerJobRestore |
-| [INLONG-8376](https://github.com/apache/inlong/issues/8376) | 
[Improve][Agent] Optimize the agent UT of TestTriggerManager   |
+|ISSUE| Summary

 |
+|:---:|:|
+| [INLONG-8850](https://github.com/apache/inlong/issues/8850) | 
[Improve][Agent] Remove unregister of MetricRegister when taskmanager is 
initialized|
+| [INLONG-8655](https://github.com/apache/inlong/issues/8655) | [Bug][Agent] 
JobWrapper thread leaks when the job is stopped 
   |
+| [INLONG-8652](https://github.com/apache/inlong/issues/8652) | 
[Improve][Agent] Delete the capacity of setting blacklist   
|
+| [INLONG-8649](https://github.com/apache/inlong/issues/8649) | [Bug][Agent] 
Thread leaks for ProxySink when the DataProxy SDK init failed   
   |
+| [INLONG-8647](https://github.com/apache/inlong/issues/8647) | 
[Improve][Agent] Stop sending task snapshot to Manager module   
|
+| [INLONG-8645](https://github.com/apache/inlong/issues/8645) | 
[Improve][Agent] Delete the capacity of the loading trigger for local files 
|
+| [INLONG-8629](https://github.com/apache/inlong/issues/8629) | [Bug][Agent] 
Sending invalid data to DataProxy failed blocks normal data sending 
   |
+| [INLONG-8524](https://github.com/apache/inlong/issues/8524) | 
[Improve][Agent] Update the JVM Options for Agent   
|
+| [INLONG-8520](https://github.com/apache/inlong/issues/8520) | [Bug][Agent] 
File agent sen

[inlong] branch branch-1.9 updated: [INLONG-8908][Dashboard] Support management of Tencent cloud log service sink (#8933)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/branch-1.9 by this push:
 new 12df355930 [INLONG-8908][Dashboard] Support management of Tencent 
cloud log service sink (#8933)
12df355930 is described below

commit 12df35593093bcdce198b0d70dc46bfd882cffb2
Author: Lizhen <88174078+bluew...@users.noreply.github.com>
AuthorDate: Tue Sep 19 09:28:53 2023 +0800

[INLONG-8908][Dashboard] Support management of Tencent cloud log service 
sink (#8933)
---
 inlong-dashboard/src/plugins/sinks/defaults/Cls.ts | 183 +
 .../src/plugins/sinks/defaults/index.ts|   5 +
 inlong-dashboard/src/ui/locales/cn.json|  13 ++
 inlong-dashboard/src/ui/locales/en.json|  13 ++
 4 files changed, 214 insertions(+)

diff --git a/inlong-dashboard/src/plugins/sinks/defaults/Cls.ts 
b/inlong-dashboard/src/plugins/sinks/defaults/Cls.ts
new file mode 100644
index 00..4a55390022
--- /dev/null
+++ b/inlong-dashboard/src/plugins/sinks/defaults/Cls.ts
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { DataWithBackend } from '@/plugins/DataWithBackend';
+import { RenderRow } from '@/plugins/RenderRow';
+import { RenderList } from '@/plugins/RenderList';
+import i18n from '@/i18n';
+import EditableTable from '@/ui/components/EditableTable';
+import { SinkInfo } from '../common/SinkInfo';
+import { sourceFields } from '../common/sourceFields';
+import NodeSelect from '@/ui/components/NodeSelect';
+
+const { I18n } = DataWithBackend;
+const { FieldDecorator, SyncField } = RenderRow;
+const { ColumnDecorator } = RenderList;
+
+const targetTypes = ['int', 'long', 'float', 'double', 'string', 'date', 
'timestamp'].map(item => ({
+  label: item,
+  value: item,
+}));
+
+export default class ClsSink extends SinkInfo implements DataWithBackend, 
RenderRow, RenderList {
+  @FieldDecorator({
+type: 'input',
+rules: [{ required: true }],
+props: values => ({
+  disabled: [110, 130].includes(values?.status),
+}),
+  })
+  @ColumnDecorator()
+  @I18n('Topic Name')
+  @SyncField()
+  topicName: string;
+
+  @FieldDecorator({
+type: 'input',
+rules: [{ required: true }],
+props: values => ({
+  disabled: [110, 130].includes(values?.status),
+}),
+  })
+  @ColumnDecorator()
+  @I18n('meta.Sinks.Cls.Tag')
+  @SyncField()
+  tag: string;
+
+  @FieldDecorator({
+type: 'input',
+rules: [{ required: true }],
+props: values => ({
+  disabled: [110, 130].includes(values?.status),
+}),
+  })
+  @ColumnDecorator()
+  @SyncField()
+  @I18n('meta.Sinks.Cls.Tokenizer')
+  tokenizer: string;
+
+  @FieldDecorator({
+type: 'select',
+rules: [{ required: true }],
+props: values => ({
+  disabled: [110, 130].includes(values?.status),
+  options: [
+{
+  label: i18n.t('meta.Sinks.Cls.SaveTime.Week'),
+  value: 7,
+},
+{
+  label: i18n.t('meta.Sinks.Cls.SaveTime.Month'),
+  value: 30,
+},
+{
+  label: i18n.t('meta.Sinks.Cls.SaveTime.HalfAYear'),
+  value: 182,
+},
+{
+  label: i18n.t('meta.Sinks.Cls.SaveTime.OneYear'),
+  value: 365,
+},
+  ],
+}),
+  })
+  @I18n('meta.Sinks.Cls.SaveTime')
+  @SyncField()
+  saveTime: number;
+
+  @FieldDecorator({
+type: NodeSelect,
+rules: [{ required: true }],
+props: values => ({
+  disabled: [110, 130].includes(values?.status),
+  nodeType: 'CLS',
+}),
+  })
+  @I18n('meta.Sinks.DataNodeName')
+  @SyncField()
+  @ColumnDecorator()
+  dataNodeName: string;
+
+  @FieldDecorator({
+type: EditableTable,
+props: values => ({
+  

[inlong] 01/06: [INLONG-8936][DataProxy] The log's path for DataProxy is wrong (#8937)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/inlong.git

commit c6f2fabbd8e6aded0ac130a63a3f28c811385112
Author: Goson Zhang <4675...@qq.com>
AuthorDate: Tue Sep 19 17:35:19 2023 +0800

[INLONG-8936][DataProxy] The log's path for DataProxy is wrong (#8937)
---
 inlong-dataproxy/bin/dataproxy-ng | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/inlong-dataproxy/bin/dataproxy-ng 
b/inlong-dataproxy/bin/dataproxy-ng
index d04ed8f282..70a5a12abb 100755
--- a/inlong-dataproxy/bin/dataproxy-ng
+++ b/inlong-dataproxy/bin/dataproxy-ng
@@ -229,7 +229,7 @@ run_flume() {
 $EXEC $JAVA_HOME/bin/java $JAVA_OPTS -javaagent:${OTEL_AGENT} -cp 
"$DATAPROXY_CLASSPATH" \
   -Djava.library.path=$FLUME_JAVA_LIBRARY_PATH 
"$FLUME_APPLICATION_CLASS" $*
   else
-$EXEC $JAVA_HOME/bin/java -cp "$DATAPROXY_CLASSPATH" \
+$EXEC $JAVA_HOME/bin/java $JAVA_OPTS -cp "$DATAPROXY_CLASSPATH" \
   -Djava.library.path=$FLUME_JAVA_LIBRARY_PATH 
"$FLUME_APPLICATION_CLASS" $*
   fi
 }

[inlong] 03/06: [INLONG-8934][Distribution] Add Iceberg connectors into the release bundle (#8935)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/inlong.git

commit 97a93f169b41608d96ac37bf539c3427a27f1e7e
Author: vernedeng 
AuthorDate: Tue Sep 19 18:43:46 2023 +0800

[INLONG-8934][Distribution] Add Iceberg connectors into the release bundle 
(#8935)
---
 inlong-distribution/src/main/assemblies/sort-connectors-v1.15.xml | 8 
 .../java/org/apache/inlong/sort/iceberg/source/IcebergSource.java | 4 
 .../iceberg/source/reader/InlongIcebergSourceReaderMetrics.java   | 6 --
 3 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/inlong-distribution/src/main/assemblies/sort-connectors-v1.15.xml 
b/inlong-distribution/src/main/assemblies/sort-connectors-v1.15.xml
index db8baf0382..f61fbc0e9e 100644
--- a/inlong-distribution/src/main/assemblies/sort-connectors-v1.15.xml
+++ b/inlong-distribution/src/main/assemblies/sort-connectors-v1.15.xml
@@ -43,5 +43,13 @@
 
 0644
 
+
+
../inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/iceberg/target
+inlong-sort/connectors
+
+
sort-connector-iceberg-v1.15-${project.version}.jar
+
+0644
+
 
 
diff --git 
a/inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/iceberg/src/main/java/org/apache/inlong/sort/iceberg/source/IcebergSource.java
 
b/inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/iceberg/src/main/java/org/apache/inlong/sort/iceberg/source/IcebergSource.java
index 47f7d04b44..d5b58e050c 100644
--- 
a/inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/iceberg/src/main/java/org/apache/inlong/sort/iceberg/source/IcebergSource.java
+++ 
b/inlong-sort/sort-flink/sort-flink-v1.15/sort-connectors/iceberg/src/main/java/org/apache/inlong/sort/iceberg/source/IcebergSource.java
@@ -531,10 +531,6 @@ public class IcebergSource implements Source

[inlong] 04/06: [INLONG-8942][Distribution] Support merging multiple module jars to reduce distribution package size (#8943)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/inlong.git

commit 56286d00e3ab891500f6dbe9a7725fe3a01c3eb5
Author: 卢春亮 <946240...@qq.com>
AuthorDate: Thu Sep 21 10:42:39 2023 +0800

[INLONG-8942][Distribution] Support merging multiple module jars to reduce 
distribution package size (#8943)
---
 inlong-agent/bin/agent.sh  |  6 ++
 inlong-audit/bin/proxy-start.sh|  6 ++
 inlong-audit/bin/store-start.sh|  6 ++
 inlong-dataproxy/bin/dataproxy-start.sh|  6 ++
 inlong-distribution/pom.xml| 18 +-
 .../script/backup_module_dependencys.sh| 74 ++
 .../script/copy_module_dependencys.sh  | 36 +++
 .../script/prepare_module_dependencys.sh   | 31 +
 inlong-manager/manager-web/bin/startup.sh  |  6 ++
 inlong-tubemq/bin/env.sh   |  6 ++
 inlong-tubemq/tubemq-manager/bin/start-manager.sh  |  6 ++
 11 files changed, 200 insertions(+), 1 deletion(-)

diff --git a/inlong-agent/bin/agent.sh b/inlong-agent/bin/agent.sh
index 60f827af69..480b55a219 100755
--- a/inlong-agent/bin/agent.sh
+++ b/inlong-agent/bin/agent.sh
@@ -18,6 +18,12 @@
 
 BASE_DIR=$(dirname $0)/..
 
+# Prepare common dependency
+ROOT_DIR=$BASE_DIR/..
+if [ -e $ROOT_DIR/bin/prepare_module_dependencys.sh ]; then
+$ROOT_DIR/bin/prepare_module_dependencys.sh ./inlong-agent/lib
+fi
+
 source "${BASE_DIR}"/bin/agent-env.sh
 CONSOLE_OUTPUT_FILE="${LOG_DIR}/agent-out.log"
 
diff --git a/inlong-audit/bin/proxy-start.sh b/inlong-audit/bin/proxy-start.sh
index 47460ea9aa..0bd6e25879 100644
--- a/inlong-audit/bin/proxy-start.sh
+++ b/inlong-audit/bin/proxy-start.sh
@@ -30,6 +30,12 @@ BASE_DIR=$(
 # shellcheck disable=SC2164
 cd "$BASE_DIR"
 
+# Prepare common dependency
+ROOT_DIR=$BASE_DIR/..
+if [ -e $ROOT_DIR/bin/prepare_module_dependencys.sh ]; then
+$ROOT_DIR/bin/prepare_module_dependencys.sh ./inlong-audit/lib
+fi
+
 error() {
   local msg=$1
   local exit_code=$2
diff --git a/inlong-audit/bin/store-start.sh b/inlong-audit/bin/store-start.sh
index 2b048e5602..002b7c0f7e 100644
--- a/inlong-audit/bin/store-start.sh
+++ b/inlong-audit/bin/store-start.sh
@@ -31,6 +31,12 @@ BASE_DIR=$(dirname $SCRIPT_DIR)
 cd "$BASE_DIR"
 cd ../
 
+# Prepare common dependency
+ROOT_DIR=$BASE_DIR/../..
+if [ -e $ROOT_DIR/bin/prepare_module_dependencys.sh ]; then
+$ROOT_DIR/bin/prepare_module_dependencys.sh ./inlong-audit/lib
+fi
+
 PID=$(ps -ef | grep "audit-store" | grep -v grep | awk '{ print $2}')
 LOG_DIR="${BASE_DIR}/logs"
 
diff --git a/inlong-dataproxy/bin/dataproxy-start.sh 
b/inlong-dataproxy/bin/dataproxy-start.sh
index c16aff658c..a7aa2df133 100755
--- a/inlong-dataproxy/bin/dataproxy-start.sh
+++ b/inlong-dataproxy/bin/dataproxy-start.sh
@@ -21,6 +21,12 @@
 cd "$(dirname "$0")"/../conf || exit
 basedir="$(pwd)"
 
+# Prepare common dependency
+ROOT_DIR=$basedir/../..
+if [ -e $ROOT_DIR/bin/prepare_module_dependencys.sh ]; then
+$ROOT_DIR/bin/prepare_module_dependencys.sh ./inlong-dataproxy/lib
+fi
+
 error() {
   local msg=$1
   local exit_code=$2
diff --git a/inlong-distribution/pom.xml b/inlong-distribution/pom.xml
index 548ab7ab8a..1d3f404da5 100644
--- a/inlong-distribution/pom.xml
+++ b/inlong-distribution/pom.xml
@@ -70,7 +70,23 @@
 
 
 
+
+org.codehaus.mojo
+exec-maven-plugin
+3.1.0
+
+
${basedir}/script/backup_module_dependencys.sh
+
+
+
+uncompress
+
+exec
+
+install
+
+
+
 
 
-
 
diff --git a/inlong-distribution/script/backup_module_dependencys.sh 
b/inlong-distribution/script/backup_module_dependencys.sh
new file mode 100755
index 00..909def54b2
--- /dev/null
+++ b/inlong-distribution/script/backup_module_dependencys.sh
@@ -0,0 +1,74 @@
+#!/bin/bash
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is dist

[inlong] 05/06: [INLONG-8946][Manager] Optimize the audit ID method issued by the manager (#8947)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/inlong.git

commit ff1af8c1a3db817eaa01c41356b2f11ee033cde2
Author: haifxu 
AuthorDate: Thu Sep 21 12:52:14 2023 +0800

[INLONG-8946][Manager] Optimize the audit ID method issued by the manager 
(#8947)
---
 .../service/core/impl/AuditServiceImpl.java| 32 ++
 .../resource/sort/DefaultSortConfigOperator.java   |  9 +++---
 2 files changed, 25 insertions(+), 16 deletions(-)

diff --git 
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/AuditServiceImpl.java
 
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/AuditServiceImpl.java
index 36b7cb0043..86e4034c0c 100644
--- 
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/AuditServiceImpl.java
+++ 
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/AuditServiceImpl.java
@@ -48,7 +48,6 @@ import org.apache.inlong.manager.service.core.AuditService;
 import org.apache.inlong.manager.service.resource.sink.ck.ClickHouseConfig;
 import org.apache.inlong.manager.service.resource.sink.es.ElasticsearchApi;
 
-import com.google.common.collect.Sets;
 import org.apache.commons.collections.CollectionUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.ibatis.jdbc.SQL;
@@ -239,6 +238,7 @@ public class AuditServiceImpl implements AuditService {
 // for now, we use the first sink type only.
 // this is temporary behavior before multiple sinks in one stream is 
fully supported.
 String sinkNodeType = null;
+String sourceNodeType = null;
 Integer sinkId = request.getSinkId();
 StreamSinkEntity sinkEntity = null;
 List sinkEntityList = 
sinkEntityMapper.selectByRelatedId(groupId, streamId);
@@ -253,10 +253,22 @@ public class AuditServiceImpl implements AuditService {
 sinkNodeType = sinkEntity.getSinkType();
 }
 
-Set sinkAuditIds = Sets.newHashSet(getAuditId(sinkNodeType, 
true), getAuditId(sinkNodeType, false));
+InlongGroupEntity groupEntity = 
inlongGroupMapper.selectByGroupId(groupId);
+List sourceEntityList = 
sourceEntityMapper.selectByRelatedId(groupId, streamId, null);
+if (CollectionUtils.isNotEmpty(sourceEntityList)) {
+sourceNodeType = sourceEntityList.get(0).getSourceType();
+}
+
+Map auditIdMap = new HashMap<>();
+auditIdMap.put(getAuditId(sourceNodeType, false), sourceNodeType);
+auditIdMap.put(getAuditId(sinkNodeType, true), sinkNodeType);
 
 // properly overwrite audit ids by role and stream config
-request.setAuditIds(getAuditIds(groupId, streamId, sinkNodeType));
+request.setAuditIds(getAuditIds(groupId, streamId, null, 
sinkNodeType));
+
+if 
(InlongConstants.DATASYNC_MODE.equals(groupEntity.getInlongGroupMode())) {
+request.setAuditIds(getAuditIds(groupId, streamId, sourceNodeType, 
sinkNodeType));
+}
 
 List result = new ArrayList<>();
 AuditQuerySource querySource = 
AuditQuerySource.valueOf(auditQuerySource);
@@ -275,7 +287,7 @@ public class AuditServiceImpl implements AuditService {
 vo.setDelay(((BigDecimal) 
s.get("totalDelay")).longValue());
 return vo;
 }).collect(Collectors.toList());
-result.add(new AuditVO(auditId, auditSet, 
sinkAuditIds.contains(auditId) ? sinkNodeType : null));
+result.add(new AuditVO(auditId, auditSet, 
auditIdMap.getOrDefault(auditId, null)));
 } else if (AuditQuerySource.ELASTICSEARCH == querySource) {
 String index = String.format("%s_%s", 
request.getStartDate().replaceAll("-", ""), auditId);
 if (!elasticsearchApi.indexExists(index)) {
@@ -294,8 +306,7 @@ public class AuditServiceImpl implements AuditService {
 vo.setDelay((long) ((ParsedSum) 
bucket.getAggregations().asList().get(1)).getValue());
 return vo;
 }).collect(Collectors.toList());
-result.add(new AuditVO(auditId, auditSet,
-auditId.equals(getAuditId(sinkNodeType, true)) 
? sinkNodeType : null));
+result.add(new AuditVO(auditId, auditSet, 
auditIdMap.getOrDefault(auditId, null)));
 }
 }
 } else if (AuditQuerySource.CLICKHOUSE == querySource) {
@@ -312,8 +323,7 @@ public class AuditServiceImpl implements AuditService {
 vo.setDelay(resultSet.getLong("total_delay"));
 auditSet.add(vo);
 }

[inlong] 02/06: [INLONG-8939][Manager] Add a switch to initiate the delete data source task (#8940)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/inlong.git

commit f7169e351e4f18616e3db310657e464c5c9ae2ff
Author: fuweng11 <76141879+fuwen...@users.noreply.github.com>
AuthorDate: Wed Sep 20 13:02:19 2023 +0800

[INLONG-8939][Manager] Add a switch to initiate the delete data source task 
(#8940)
---
 .../inlong/manager/service/task/DeleteStreamSourceTask.java  | 12 
 .../src/main/resources/application-dev.properties|  2 ++
 .../src/main/resources/application-prod.properties   |  2 ++
 .../src/main/resources/application-test.properties   |  2 ++
 4 files changed, 14 insertions(+), 4 deletions(-)

diff --git 
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/task/DeleteStreamSourceTask.java
 
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/task/DeleteStreamSourceTask.java
index f4def6697f..9d3a83dff9 100644
--- 
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/task/DeleteStreamSourceTask.java
+++ 
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/task/DeleteStreamSourceTask.java
@@ -54,6 +54,8 @@ public class DeleteStreamSourceTask extends TimerTask 
implements InitializingBea
 private static final int INITIAL_DELAY_MINUTES = 5;
 private static final int INTERVAL_MINUTES = 60;
 
+@Value("${group.deleted.enabled:false}")
+private Boolean enabled;
 @Value("${group.deleted.batchSize:100}")
 private Integer batchSize;
 @Value("${group.deleted.latest.hours:10}")
@@ -66,11 +68,13 @@ public class DeleteStreamSourceTask extends TimerTask 
implements InitializingBea
 
 @Override
 public void afterPropertiesSet() {
-ThreadFactory threadFactory = new 
ThreadFactoryBuilder().setNameFormat("inlong-group-delete-%s").build();
-ScheduledExecutorService executor = new ScheduledThreadPoolExecutor(1, 
threadFactory, new AbortPolicy());
-executor.scheduleWithFixedDelay(this, INITIAL_DELAY_MINUTES, 
INTERVAL_MINUTES, TimeUnit.MINUTES);
+if (enabled) {
+ThreadFactory threadFactory = new 
ThreadFactoryBuilder().setNameFormat("inlong-group-delete-%s").build();
+ScheduledExecutorService executor = new 
ScheduledThreadPoolExecutor(1, threadFactory, new AbortPolicy());
+executor.scheduleWithFixedDelay(this, INITIAL_DELAY_MINUTES, 
INTERVAL_MINUTES, TimeUnit.MINUTES);
 
-log.info("success to start the delete stream source task");
+log.info("success to start the delete stream source task");
+}
 }
 
 @Override
diff --git 
a/inlong-manager/manager-web/src/main/resources/application-dev.properties 
b/inlong-manager/manager-web/src/main/resources/application-dev.properties
index 69d051a771..ff248053a4 100644
--- a/inlong-manager/manager-web/src/main/resources/application-dev.properties
+++ b/inlong-manager/manager-web/src/main/resources/application-dev.properties
@@ -105,3 +105,5 @@ source.cleansing.interval=600
 group.deleted.latest.hours=10
 # The maximum size when querying InlongGroupIds in batches, those 
InlongGroupIds will be used to delete the related StreamSources.
 group.deleted.batchSize=100
+# If turned on, the groups could be deleted periodically.
+group.deleted.enabled=false
diff --git 
a/inlong-manager/manager-web/src/main/resources/application-prod.properties 
b/inlong-manager/manager-web/src/main/resources/application-prod.properties
index 7e99baff71..69122ed272 100644
--- a/inlong-manager/manager-web/src/main/resources/application-prod.properties
+++ b/inlong-manager/manager-web/src/main/resources/application-prod.properties
@@ -104,3 +104,5 @@ source.cleansing.interval=600
 group.deleted.latest.hours=10
 # The maximum size when querying InlongGroupIds in batches, those 
InlongGroupIds will be used to delete the related StreamSources.
 group.deleted.batchSize=100
+# If turned on, the groups could be deleted periodically.
+group.deleted.enabled=false
diff --git 
a/inlong-manager/manager-web/src/main/resources/application-test.properties 
b/inlong-manager/manager-web/src/main/resources/application-test.properties
index 69d051a771..ff248053a4 100644
--- a/inlong-manager/manager-web/src/main/resources/application-test.properties
+++ b/inlong-manager/manager-web/src/main/resources/application-test.properties
@@ -105,3 +105,5 @@ source.cleansing.interval=600
 group.deleted.latest.hours=10
 # The maximum size when querying InlongGroupIds in batches, those 
InlongGroupIds will be used to delete the related StreamSources.
 group.deleted.batchSize=100
+# If turned on, the groups could be deleted periodically.
+group.deleted.enabled=false

[inlong] 06/06: [INLONG-8951][Manager] Support for configuring built-in fields for iceberg and starrocks (#8952)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/inlong.git

commit 2680927d7a25a5ab5ae2ec54550adc60c122eec8
Author: fuweng11 <76141879+fuwen...@users.noreply.github.com>
AuthorDate: Fri Sep 22 10:56:50 2023 +0800

[INLONG-8951][Manager] Support for configuring built-in fields for iceberg 
and starrocks (#8952)
---
 .../org/apache/inlong/common/enums/MetaField.java  |  7 ++-
 .../inlong/manager/pojo/sort/node/NodeFactory.java | 59 ++
 .../pojo/sort/node/base/ExtractNodeProvider.java   |  5 ++
 .../pojo/sort/node/base/LoadNodeProvider.java  |  4 ++
 .../manager/pojo/sort/node/base/NodeProvider.java  |  9 
 .../pojo/sort/node/provider/IcebergProvider.java   | 25 +
 .../pojo/sort/node/provider/StarRocksProvider.java | 26 ++
 .../manager/pojo/sort/util/FieldInfoUtils.java | 12 +
 .../resource/sort/DefaultSortConfigOperator.java   | 27 --
 .../src/main/resources/application-dev.properties  |  2 +
 .../src/main/resources/application-prod.properties |  2 +
 .../src/main/resources/application-test.properties |  2 +
 .../org/apache/inlong/sort/protocol/Metadata.java  |  1 +
 .../protocol/node/extract/IcebergExtractNode.java  | 33 +++-
 .../org/apache/inlong/sort/base/Constants.java |  2 +-
 .../sort/iceberg/IcebergReadableMetadata.java  |  4 +-
 .../starrocks/table/sink/utils/SchemaUtils.java| 21 
 17 files changed, 222 insertions(+), 19 deletions(-)

diff --git 
a/inlong-common/src/main/java/org/apache/inlong/common/enums/MetaField.java 
b/inlong-common/src/main/java/org/apache/inlong/common/enums/MetaField.java
index f3c1a388f5..e6d0bbc6cb 100644
--- a/inlong-common/src/main/java/org/apache/inlong/common/enums/MetaField.java
+++ b/inlong-common/src/main/java/org/apache/inlong/common/enums/MetaField.java
@@ -163,7 +163,12 @@ public enum MetaField {
 /**
  * Timestamp of the Kafka record, it is only used for Kafka.
  */
-TIMESTAMP;
+TIMESTAMP,
+
+/**
+ * Inlong data time for audit.
+ */
+AUDIT_DATA_TIME;
 
 public static MetaField forName(String name) {
 for (MetaField metaField : values()) {
diff --git 
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/node/NodeFactory.java
 
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/node/NodeFactory.java
index cc6a545490..74f2efae00 100644
--- 
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/node/NodeFactory.java
+++ 
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/node/NodeFactory.java
@@ -18,14 +18,23 @@
 package org.apache.inlong.manager.pojo.sort.node;
 
 import org.apache.inlong.manager.pojo.sink.StreamSink;
+import org.apache.inlong.manager.pojo.sort.node.base.ExtractNodeProvider;
+import org.apache.inlong.manager.pojo.sort.node.base.LoadNodeProvider;
+import org.apache.inlong.manager.pojo.sort.util.FieldInfoUtils;
+import org.apache.inlong.manager.pojo.sort.util.TransformNodeUtils;
 import org.apache.inlong.manager.pojo.source.StreamSource;
 import org.apache.inlong.manager.pojo.stream.StreamField;
+import org.apache.inlong.manager.pojo.transform.TransformResponse;
 import org.apache.inlong.sort.protocol.node.ExtractNode;
 import org.apache.inlong.sort.protocol.node.LoadNode;
+import org.apache.inlong.sort.protocol.node.Node;
+import org.apache.inlong.sort.protocol.node.transform.TransformNode;
 
 import com.google.common.collect.Lists;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.collections.CollectionUtils;
 
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -33,6 +42,7 @@ import java.util.stream.Collectors;
 /**
  * The node factory
  */
+@Slf4j
 public class NodeFactory {
 
 /**
@@ -61,4 +71,53 @@ public class NodeFactory {
 return 
LoadNodeProviderFactory.getLoadNodeProvider(sinkType).createLoadNode(v, 
constantFieldMap);
 }).collect(Collectors.toList());
 }
+
+/**
+ * Create extract node from the given source.
+ */
+public static ExtractNode createExtractNode(StreamSource sourceInfo) {
+if (sourceInfo == null) {
+return null;
+}
+String sourceType = sourceInfo.getSourceType();
+return 
ExtractNodeProviderFactory.getExtractNodeProvider(sourceType).createExtractNode(sourceInfo);
+}
+
+/**
+ * Create load node from the given sink.
+ */
+public static LoadNode createLoadNode(StreamSink sinkInfo, Map constantFieldMap) {
+if (sinkInfo == null) {
+return null;
+}
+String sinkType = sinkInfo.getSinkType();
+return 
LoadNodeProviderFactory.getLoadNodeProvider(sinkType).createLoadNode(sinkInfo, 
constantFieldMap);
+}
+
+/**
+ * Add built-in fie

[inlong] branch branch-1.9 updated (12df355930 -> 2680927d7a)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a change to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/inlong.git


from 12df355930 [INLONG-8908][Dashboard] Support management of Tencent 
cloud log service sink (#8933)
 new c6f2fabbd8 [INLONG-8936][DataProxy] The log's path for DataProxy is 
wrong (#8937)
 new f7169e351e [INLONG-8939][Manager] Add a switch to initiate the delete 
data source task (#8940)
 new 97a93f169b [INLONG-8934][Distribution] Add Iceberg connectors into the 
release bundle (#8935)
 new 56286d00e3 [INLONG-8942][Distribution] Support merging multiple module 
jars to reduce distribution package size (#8943)
 new ff1af8c1a3 [INLONG-8946][Manager] Optimize the audit ID method issued 
by the manager (#8947)
 new 2680927d7a [INLONG-8951][Manager] Support for configuring built-in 
fields for iceberg and starrocks (#8952)

The 6 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 inlong-agent/bin/agent.sh  |  6 ++
 inlong-audit/bin/proxy-start.sh|  6 ++
 inlong-audit/bin/store-start.sh|  6 ++
 .../org/apache/inlong/common/enums/MetaField.java  |  7 +-
 inlong-dataproxy/bin/dataproxy-ng  |  2 +-
 inlong-dataproxy/bin/dataproxy-start.sh|  6 ++
 inlong-distribution/pom.xml| 18 +-
 .../script/backup_module_dependencys.sh| 74 ++
 .../script/copy_module_dependencys.sh  | 20 --
 .../script/prepare_module_dependencys.sh   | 16 +++--
 .../src/main/assemblies/sort-connectors-v1.15.xml  |  8 +++
 .../inlong/manager/pojo/sort/node/NodeFactory.java | 59 +
 .../pojo/sort/node/base/ExtractNodeProvider.java   |  5 ++
 .../pojo/sort/node/base/LoadNodeProvider.java  |  4 ++
 .../manager/pojo/sort/node/base/NodeProvider.java  |  9 +++
 .../pojo/sort/node/provider/IcebergProvider.java   | 25 
 .../pojo/sort/node/provider/StarRocksProvider.java | 26 
 .../manager/pojo/sort/util/FieldInfoUtils.java | 12 
 .../service/core/impl/AuditServiceImpl.java| 32 ++
 .../resource/sort/DefaultSortConfigOperator.java   | 32 --
 .../service/task/DeleteStreamSourceTask.java   | 12 ++--
 inlong-manager/manager-web/bin/startup.sh  |  6 ++
 .../src/main/resources/application-dev.properties  |  4 ++
 .../src/main/resources/application-prod.properties |  4 ++
 .../src/main/resources/application-test.properties |  4 ++
 .../org/apache/inlong/sort/protocol/Metadata.java  |  1 +
 .../protocol/node/extract/IcebergExtractNode.java  | 33 +-
 .../org/apache/inlong/sort/base/Constants.java |  2 +-
 .../sort/iceberg/IcebergReadableMetadata.java  |  4 +-
 .../inlong/sort/iceberg/source/IcebergSource.java  |  4 --
 .../reader/InlongIcebergSourceReaderMetrics.java   |  6 +-
 .../starrocks/table/sink/utils/SchemaUtils.java| 21 +++---
 inlong-tubemq/bin/env.sh   |  6 ++
 inlong-tubemq/tubemq-manager/bin/start-manager.sh  |  6 ++
 34 files changed, 430 insertions(+), 56 deletions(-)
 create mode 100755 inlong-distribution/script/backup_module_dependencys.sh
 copy inlong-sort-standalone/bin/sort-stop.sh => 
inlong-distribution/script/copy_module_dependencys.sh (65%)
 mode change 100644 => 100755
 copy inlong-sort-standalone/bin/sort-start.sh => 
inlong-distribution/script/prepare_module_dependencys.sh (76%)
 mode change 100644 => 100755

[inlong] branch branch-1.9 updated: [INLONG-8953][Sort] Fix IcebergSource defaults the StartSnapshot to the latest (#8954)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/branch-1.9 by this push:
 new 4ef2b3d1e6 [INLONG-8953][Sort] Fix IcebergSource defaults the 
StartSnapshot to the latest (#8954)
4ef2b3d1e6 is described below

commit 4ef2b3d1e61f6e3195b11a4477d92db01c150528
Author: vernedeng 
AuthorDate: Thu Sep 21 10:10:14 2023 +0800

[INLONG-8953][Sort] Fix IcebergSource defaults the StartSnapshot to the 
latest (#8954)
---
 .../apache/inlong/sort/protocol/constant/IcebergConstant.java  | 10 ++
 .../inlong/sort/protocol/node/extract/IcebergExtractNode.java  |  3 +++
 2 files changed, 13 insertions(+)

diff --git 
a/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/constant/IcebergConstant.java
 
b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/constant/IcebergConstant.java
index 676f7f4435..2cce35fb9b 100644
--- 
a/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/constant/IcebergConstant.java
+++ 
b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/constant/IcebergConstant.java
@@ -34,6 +34,7 @@ public class IcebergConstant {
 public static final String WAREHOUSE_KEY = "warehouse";
 public static final String START_SNAPSHOT_ID = "start-snapshot-id";
 public static final String STREAMING = "streaming";
+public static final String STARTING_STRATEGY_KEY = "starting-strategy";
 
 /**
  * Iceberg supported catalog type
@@ -65,4 +66,13 @@ public class IcebergConstant {
 throw new IllegalArgumentException(String.format("Unsupport 
catalogType:%s", name));
 }
 }
+
+public enum StreamingStartingStrategy {
+TABLE_SCAN_THEN_INCREMENTAL,
+INCREMENTAL_FROM_LATEST_SNAPSHOT,
+INCREMENTAL_FROM_EARLIEST_SNAPSHOT,
+INCREMENTAL_FROM_SNAPSHOT_ID,
+INCREMENTAL_FROM_SNAPSHOT_TIMESTAMP;
+
+}
 }
diff --git 
a/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/extract/IcebergExtractNode.java
 
b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/extract/IcebergExtractNode.java
index cbceb5485a..e87c111743 100644
--- 
a/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/extract/IcebergExtractNode.java
+++ 
b/inlong-sort/sort-common/src/main/java/org/apache/inlong/sort/protocol/node/extract/IcebergExtractNode.java
@@ -115,6 +115,7 @@ public class IcebergExtractNode extends ExtractNode 
implements InlongMetric, Met
 options.put(IcebergConstant.TABLE_KEY, tableName);
 options.put(IcebergConstant.CATALOG_TYPE_KEY, catalogType.name());
 options.put(IcebergConstant.CATALOG_NAME_KEY, catalogName);
+// support streaming only
 options.put(IcebergConstant.STREAMING, "true");
 options.put(IcebergConstant.STARTING_STRATEGY_KEY,
 
IcebergConstant.StreamingStartingStrategy.TABLE_SCAN_THEN_INCREMENTAL.name());
@@ -126,6 +127,8 @@ public class IcebergExtractNode extends ExtractNode 
implements InlongMetric, Met
 }
 if (null != startSnapShotId) {
 options.put(IcebergConstant.START_SNAPSHOT_ID, 
startSnapShotId.toString());
+options.put(IcebergConstant.STARTING_STRATEGY_KEY,
+
IcebergConstant.StreamingStartingStrategy.INCREMENTAL_FROM_SNAPSHOT_ID.name());
 }
 return options;
 }

[inlong-website] branch master updated: [INLONG-846][Release] Modify download & release notes for 1.9.0 (#857)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong-website.git


The following commit(s) were added to refs/heads/master by this push:
 new 639dc5df34 [INLONG-846][Release] Modify download & release notes for 
1.9.0 (#857)
639dc5df34 is described below

commit 639dc5df34bed109a26d62448230126055a0d808
Author: 卢春亮 
AuthorDate: Mon Sep 25 17:03:03 2023 +0800

[INLONG-846][Release] Modify download & release notes for 1.9.0 (#857)

* [INLONG-846][Release] Modify download & release notes for 1.9.0

* merge bugfix
---
 download/index.md  |  10 +-
 download/release-1.8.0.md  |   8 +-
 download/release-1.9.0.md  | 273 +
 .../index.md   |  10 +-
 .../release-1.8.0.md   |  14 +-
 .../release-1.9.0.md   | 273 +
 6 files changed, 567 insertions(+), 21 deletions(-)

diff --git a/download/index.md b/download/index.md
index 911f3f11e2..9aa1a710a0 100644
--- a/download/index.md
+++ b/download/index.md
@@ -2,15 +2,15 @@
 title: Download
 ---
 
-## Current version 1.8.0
+## Current version 1.9.0
 You can download the latest version here.
 
 | Name  | Version |  Date  |   


  Downloads 

|
 
|:-:|:---:|:--:|:-:|
-|  InLong Source Code   |  1.8.0  | July. 20, 2023 |   
  
[[SRC](https://downloads.apache.org/inlong/1.8.0/apache-inlong-1.8.0-src.tar.gz)]

[[ASC](https://downloads.apache.org/inlong/1.8.0/apache-inlong-1.8.0-src.tar.gz.asc)]

[[SHA512](https://downloads.apache.org/inlong/1.8.0/apache-inlong-1.8.0-src.tar.gz.sha512)]
 |
-|  InLong Binary File   |  1.8.0  | July. 20, 2023 |   
  
[[BIN](https://downloads.apache.org/inlong/1.8.0/apache-inlong-1.8.0-bin.tar.gz)]

[[ASC](https://downloads.apache.org/inlong/1.8.0/apache-inlong-1.8.0-bin.tar.gz.asc)]

[[SHA512](https://downloads.apache.org/inlong/1.8.0/apache-inlong-1.8.0-bin.tar.gz.sha512)]
 |
-| Connector Binary File For Flink v1.13 |  1.8.0  | July. 20, 2023 | 
[[BIN](https://downloads.apache.org/inlong/1.8.0/apache-inlong-1.8.0-sort-connectors-flink-v1.13.tar.gz)]

[[ASC](https://downloads.apache.org/inlong/1.8.0/apache-inlong-1.8.0-sort-connectors-flink-v1.13.tar.gz.asc)]

[[SHA512](https://downloads.apache.org/inlong/1.8.0/apache-inlong-1.8.0-sort-connectors-flink-v1.13.tar.gz.sha512)]
 |
-| Connector Binary File For Flink v1.15 |  1.8.0  | July. 20, 2023 | 
[[BIN](https://downloads.apache.org/inlong/1.8.0/apache-inlong-1.8.0-sort-connectors-flink-v1.15.tar.gz)]

[[ASC](https://downloads.apache.org/inlong/1.8.0/apache-inlong-1.8.0-sort-connectors-flink-v1.15.tar.gz.asc)]

[[SHA512](https://downloads.apache.org/inlong/1.8.0/apache-inlong-1.8.0-sort-connectors-flink-v1.15.tar.gz.sha512)]
 |
+|  InLong Source Code   |  1.9.0  | July. 20, 2023 |   
  
[[SRC](https://downloads.apache.org/inlong/1.9.0/apache-inlong-1.9.0-src.tar.gz)]

[[ASC](https://downloads.apache.org/inlong/1.9.0/apache-inlong-1.9.0-src.tar.gz.asc)]

[[SHA512](https://downloads.apache.org/inlong/1.9.0/apache-inlong-1.9.0-src.tar.gz.sha512)]
 |
+|  InLong Binary File   |  1.9.0  | July. 20, 2023 |   
  
[[BIN](https://downloads.apache.org/inlong/1.9.0/apache-inlong-1.9.0-bin.tar.gz)]

[[ASC](https://downloads.apache.org/inlong/1.9.0/apache-inlong-1.9.0-bin.tar.gz.asc)]

[[SHA512](https://downloads.apache.org/inlong/1.9.0/apache-inlong-1.9.0-bin.tar.gz.sha512)]
 |
+| Connector Binary File For Flink v1.13 |  1.9.0  | July. 20, 2023 | 

[inlong] branch branch-1.9 updated: [INLONG-8965][Doc] Update the description for swagger API (#8966)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch branch-1.9
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/branch-1.9 by this push:
 new 7a32e287dd [INLONG-8965][Doc] Update the description for swagger API 
(#8966)
7a32e287dd is described below

commit 7a32e287dd42609af649f2390a3c5a46b059f34a
Author: Charles Zhang 
AuthorDate: Thu Sep 21 17:28:30 2023 +0800

[INLONG-8965][Doc] Update the description for swagger API (#8966)
---
 .../java/org/apache/inlong/manager/web/config/Swagger2Config.java | 4 ++--
 .../apache/inlong/manager/web/controller/InlongRoleController.java| 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git 
a/inlong-manager/manager-web/src/main/java/org/apache/inlong/manager/web/config/Swagger2Config.java
 
b/inlong-manager/manager-web/src/main/java/org/apache/inlong/manager/web/config/Swagger2Config.java
index c278cba10f..7854c48016 100644
--- 
a/inlong-manager/manager-web/src/main/java/org/apache/inlong/manager/web/config/Swagger2Config.java
+++ 
b/inlong-manager/manager-web/src/main/java/org/apache/inlong/manager/web/config/Swagger2Config.java
@@ -53,8 +53,8 @@ public class Swagger2Config {
 
 private ApiInfo apiInfo() {
 return new ApiInfoBuilder()
-.title("Apache InLong Manager API Doc")
-.description("Apache InLong Manager API Doc")
+.title("Apache InLong REST API")
+.description("The Document For Apache InLong REST API")
 .version("2.0")
 .build();
 }
diff --git 
a/inlong-manager/manager-web/src/main/java/org/apache/inlong/manager/web/controller/InlongRoleController.java
 
b/inlong-manager/manager-web/src/main/java/org/apache/inlong/manager/web/controller/InlongRoleController.java
index d209d65bb8..5a692629da 100644
--- 
a/inlong-manager/manager-web/src/main/java/org/apache/inlong/manager/web/controller/InlongRoleController.java
+++ 
b/inlong-manager/manager-web/src/main/java/org/apache/inlong/manager/web/controller/InlongRoleController.java
@@ -43,7 +43,7 @@ import org.springframework.web.bind.annotation.RestController;
 
 @RestController
 @RequestMapping("/api")
-@Api(tags = "INLONG-USER-API")
+@Api(tags = "INLONG-User-API")
 public class InlongRoleController {
 
 @Autowired

[inlong] branch release-1.9.0 deleted (was 25e03cedea)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a change to branch release-1.9.0
in repository https://gitbox.apache.org/repos/asf/inlong.git


 was 25e03cedea [INLONG-8845][Manager] Support Tencent Cloud Log Service 
data flow (#8892)

The revisions that were on this branch are still contained in
other references; therefore, this change does not discard any commits
from the repository.

[inlong] branch release-1.9.0 created (now 7a32e287dd)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a change to branch release-1.9.0
in repository https://gitbox.apache.org/repos/asf/inlong.git


  at 7a32e287dd [INLONG-8965][Doc] Update the description for swagger API 
(#8966)

No new revisions were added by this update.

[inlong] annotated tag 1.9.0-RC0 deleted (was 0bfbfdfa91)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a change to annotated tag 1.9.0-RC0
in repository https://gitbox.apache.org/repos/asf/inlong.git


*** WARNING: tag 1.9.0-RC0 was deleted! ***

   tag was  0bfbfdfa91

The revisions that were on this annotated tag are still contained in
other references; therefore, this change does not discard any commits
from the repository.

[inlong] annotated tag 1.9.0-RC0 updated (7a32e287dd -> fa2161372e)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a change to annotated tag 1.9.0-RC0
in repository https://gitbox.apache.org/repos/asf/inlong.git


*** WARNING: tag 1.9.0-RC0 was modified! ***

from 7a32e287dd (commit)
  to fa2161372e (tag)
 tagging 7a32e287dd42609af649f2390a3c5a46b059f34a (commit)
 replaces 0.9.0-incubating-RC1
  by luchunliang
  on Wed Sep 27 00:41:55 2023 +0800

- Log -
Tagging the 1.9.0 first Releae Candidate (Candidates start at zero)
-BEGIN PGP SIGNATURE-

iQIzBAABCAAdFiEE4vJmAihaUo5COK2gTLPc3p8kDEYFAmUTCdMACgkQTLPc3p8k
DEaVYA/9EXo8mkJxEm1WaSRHf0wAPmcEQpyZySwXJqHZNYXFwbngDeBA2QefEg7h
bLr6LHxZcY5duhxaF9LagknzD4vda9SAkSmVZ9syGNw2FTt76Oikrssa7X1YdE6e
7ggmuouY5oG1JzDRnIhM9VlhQRJMJkQZa3HAFxpnDWusxKbwvK9y6LjH+cmEzpdC
fMIEgOF/W8S7GA7I6pHBgd6CdMXCQivVt/ZELsueP5caoArmukH7z4bYbavd5r15
nLAdBLk+ijGBKgVRhhGNOREWmWi82j452Z1xEbj1QVv7qYS0qr64XVTeAbZGwS9a
LI+0hu7IU4OpGAOCJEY5iK3g3IC0Z7j96mF5qOMibrfFWf67evhPcNn4rTedT1KW
wxktAXg4J23ZSeVd5ZTcoPwN0dcuKSfWNwIpNEn6nX5cmgZqKcpkmH/VNwIC0K/5
JsYOaKCBc9XsIWd9nAYnCJJFIZCv1xuQp1xjtffCMe5bbF1Vd3bBCkv154wR6WSh
sv3IUsGmGHRH4vJkfK8SXHDuu4SKBDfIahS75iBWQXyKFInoqJocfQnXXIwIvBNd
VEDNHbRwy7SPxqRhWWMZ/MITwiR8mLgbf6Hp7DccNNSTmXo6qsg+Fz+tPgGqf0ib
DyOXSyXvzqvEPUWNvEonVTDzmGputhgIeoxi4soRII6/eMnwomU=
=jYIg
-END PGP SIGNATURE-
---


No new revisions were added by this update.

Summary of changes:

svn commit: r64216 - /dev/inlong/1.9.0-RC0/

[luchunliang]

Author: luchunliang
Date: Wed Sep 27 04:20:17 2023
New Revision: 64216

Log:
prepare for 1.9.0 RC0

Added:
dev/inlong/1.9.0-RC0/
dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-bin.tar.gz   (with props)
dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-bin.tar.gz.asc   (with props)
dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-bin.tar.gz.sha512
dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-sort-connectors-flink-v1.13.tar.gz 
  (with props)

dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-sort-connectors-flink-v1.13.tar.gz.asc 
  (with props)

dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-sort-connectors-flink-v1.13.tar.gz.sha512
dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-sort-connectors-flink-v1.15.tar.gz 
  (with props)

dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-sort-connectors-flink-v1.15.tar.gz.asc 
  (with props)

dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-sort-connectors-flink-v1.15.tar.gz.sha512
dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-src.tar.gz   (with props)
dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-src.tar.gz.asc   (with props)
dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-src.tar.gz.sha512

Added: dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-bin.tar.gz
==
Binary file - no diff available.

Propchange: dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-bin.tar.gz
--
svn:mime-type = application/x-gzip

Added: dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-bin.tar.gz.asc
==
Binary file - no diff available.

Propchange: dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-bin.tar.gz.asc
--
svn:mime-type = application/pgp-signature

Added: dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-bin.tar.gz.sha512
==
--- dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-bin.tar.gz.sha512 (added)
+++ dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-bin.tar.gz.sha512 Wed Sep 27 
04:20:17 2023
@@ -0,0 +1,4 @@
+apache-inlong-1.9.0-bin.tar.gz: 1E98020B 2A4F7175 5DCE4314 0ADDF181 AB400736
+9E49C46A DA63C8EA D3F9F9E2 4A266775 2EC9420D
+74A64981 DAAA120B EDFA355E 87AA396C 1F45BDF3
+C2629AE4

Added: 
dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-sort-connectors-flink-v1.13.tar.gz
==
Binary file - no diff available.

Propchange: 
dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-sort-connectors-flink-v1.13.tar.gz
--
svn:mime-type = application/x-gzip

Added: 
dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-sort-connectors-flink-v1.13.tar.gz.asc
==
Binary file - no diff available.

Propchange: 
dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-sort-connectors-flink-v1.13.tar.gz.asc
--
svn:mime-type = application/pgp-signature

Added: 
dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-sort-connectors-flink-v1.13.tar.gz.sha512
==
--- 
dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-sort-connectors-flink-v1.13.tar.gz.sha512
 (added)
+++ 
dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-sort-connectors-flink-v1.13.tar.gz.sha512
 Wed Sep 27 04:20:17 2023
@@ -0,0 +1,3 @@
+apache-inlong-1.9.0-sort-connectors-flink-v1.13.tar.gz: 
+5A25CB1C 40C75F25 193DF68E 9B431A47 B8EE2E5E 545565BD AAB7ABDA C277B5F2 
9369432D
+ E259D7E9 AB2251F7 62B6362F 03D999B8 EAB7F7CD 9EF4F0C1 5AF8639F

Added: 
dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-sort-connectors-flink-v1.15.tar.gz
==
Binary file - no diff available.

Propchange: 
dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-sort-connectors-flink-v1.15.tar.gz
--
svn:mime-type = application/x-gzip

Added: 
dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-sort-connectors-flink-v1.15.tar.gz.asc
==
Binary file - no diff available.

Propchange: 
dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-sort-connectors-flink-v1.15.tar.gz.asc
--
svn:mime-type = application/pgp-signature

Added: 
dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-sort-connectors-flink-v1.15.tar.gz.sha512
==
--- 
dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-sort-connectors-flink-v1.15.tar.gz.sha512
 (added)
+++ 
dev/inlong/1.9.0-RC0/apache

svn commit: r64217 - /dev/inlong/1.9.0-RC0/

[luchunliang]

Author: luchunliang
Date: Wed Sep 27 08:18:12 2023
New Revision: 64217

Log:
prepare for 1.9.0 RC0

Modified:
dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-bin.tar.gz
dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-bin.tar.gz.asc
dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-bin.tar.gz.sha512
dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-sort-connectors-flink-v1.13.tar.gz

dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-sort-connectors-flink-v1.13.tar.gz.asc

dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-sort-connectors-flink-v1.13.tar.gz.sha512
dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-sort-connectors-flink-v1.15.tar.gz

dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-sort-connectors-flink-v1.15.tar.gz.asc

dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-sort-connectors-flink-v1.15.tar.gz.sha512
dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-src.tar.gz.asc

Modified: dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-bin.tar.gz
==
Binary files - no diff available.

Modified: dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-bin.tar.gz.asc
==
Binary files - no diff available.

Modified: dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-bin.tar.gz.sha512
==
--- dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-bin.tar.gz.sha512 (original)
+++ dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-bin.tar.gz.sha512 Wed Sep 27 
08:18:12 2023
@@ -1,4 +1,4 @@
-apache-inlong-1.9.0-bin.tar.gz: 1E98020B 2A4F7175 5DCE4314 0ADDF181 AB400736
-9E49C46A DA63C8EA D3F9F9E2 4A266775 2EC9420D
-74A64981 DAAA120B EDFA355E 87AA396C 1F45BDF3
-C2629AE4
+apache-inlong-1.9.0-bin.tar.gz: E8DAF132 4BCD6FC0 928FD981 8068EBAF 91B85BFF
+8E0F9ED7 9A135D2C C8F642F9 08E4098F 6D3751D9
+AC56E8C9 785803F5 F3D6180F C6B366E0 93DDF8E9
+2EFD61D8

Modified: 
dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-sort-connectors-flink-v1.13.tar.gz
==
Binary files - no diff available.

Modified: 
dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-sort-connectors-flink-v1.13.tar.gz.asc
==
Binary files - no diff available.

Modified: 
dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-sort-connectors-flink-v1.13.tar.gz.sha512
==
--- 
dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-sort-connectors-flink-v1.13.tar.gz.sha512
 (original)
+++ 
dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-sort-connectors-flink-v1.13.tar.gz.sha512
 Wed Sep 27 08:18:12 2023
@@ -1,3 +1,3 @@
 apache-inlong-1.9.0-sort-connectors-flink-v1.13.tar.gz: 
-5A25CB1C 40C75F25 193DF68E 9B431A47 B8EE2E5E 545565BD AAB7ABDA C277B5F2 
9369432D
- E259D7E9 AB2251F7 62B6362F 03D999B8 EAB7F7CD 9EF4F0C1 5AF8639F
+5C4EF26A 458D7B20 6FD87B62 7B9D626B 75291B2C 552B340D E8D80B33 00341390 
567DD65A
+ 383FEE3A 0A1D7619 EF2D31A4 E44E072B FD2FE9C5 636080F7 CABAA1DF

Modified: 
dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-sort-connectors-flink-v1.15.tar.gz
==
Binary files - no diff available.

Modified: 
dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-sort-connectors-flink-v1.15.tar.gz.asc
==
Binary files - no diff available.

Modified: 
dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-sort-connectors-flink-v1.15.tar.gz.sha512
==
--- 
dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-sort-connectors-flink-v1.15.tar.gz.sha512
 (original)
+++ 
dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-sort-connectors-flink-v1.15.tar.gz.sha512
 Wed Sep 27 08:18:12 2023
@@ -1,3 +1,3 @@
 apache-inlong-1.9.0-sort-connectors-flink-v1.15.tar.gz: 
-236B6197 C1BC428A BB198F9F F8E97DF3 4918E6F3 B0FC35C0 482CAA65 4637A2FC 
E6AC545B
- 15861ABF E82D1DF8 068C5A8E 38926727 E200F0A5 363CAF9F 5745EF6D
+84E325F5 34FF6738 24F2DED3 F7B3D3DF EFCBC2CF 552F5DED CB7EA2FF 330324BC 
72F3D341
+ EA2EDA47 65ED6D6E A1ABFD92 A32CF177 D7489136 939F160C 0F0B5061

Modified: dev/inlong/1.9.0-RC0/apache-inlong-1.9.0-src.tar.gz.asc
==
Binary files - no diff available.

svn commit: r64237 - /dev/inlong/KEYS

[luchunliang]

Author: luchunliang
Date: Thu Sep 28 07:34:23 2023
New Revision: 64237

Log:
add gpg key for YOUR_NAME

Modified:
dev/inlong/KEYS

Modified: dev/inlong/KEYS
==
--- dev/inlong/KEYS (original)
+++ dev/inlong/KEYS Thu Sep 28 07:34:23 2023
@@ -671,3 +671,61 @@ rb0qVtGeNtZ+u7aVBqOfFLZTyW8Sr1OJdx3fjWBw
 ijdshVABKTENK4shbEo04aAsZc0zeSOEVqds/Y+oj6ZnHZf8v7L48LQCeeTlrRD5
 =C1Zz
 -END PGP PUBLIC KEY BLOCK-
+pub   rsa4096 2023-09-14 [SC]
+  E2F26602285A528E4238ADA04CB3DCDE9F240C46
+uid   [ultimate] luchunliang 
+sig 34CB3DCDE9F240C46 2023-09-14  luchunliang 
+sub   rsa4096 2023-09-14 [E]
+sig  4CB3DCDE9F240C46 2023-09-14  luchunliang 
+
+-BEGIN PGP PUBLIC KEY BLOCK-
+
+mQINBGUCr78BEADRyKA+ly2gMfoVDNJbbd45lzU1uqhzZV1Bv10YlCrSDUUKSVRT
+V8c/g20cVpq6cDcsWXJL2BOGg2PeAcs2AEyxoCwIgzhF+2HiZVgNVxpsm3PGonkD
+G8b+/1CbGXCyCxDlUjMRzpmf+FzIH2798c5e7dpf4Viv4+QLGrX2mJ/pnKNq1ndQ
+D+F+zE+EaMImURqIoQEPZTnzk/cm8h2vTgdi8eHOamoHYYx+jWvYSanSmrhi+JNQ
+GVe4Wze2XTqMj1GgYqUAOJIpAiRg7kIFRamABAkX8uesjdk06f/1NfF5FwCO056y
+3zGVAfscNjMqoRnVulv7+JyonvMua4/OVcj+DWiDepIv6REZcwF92lHV6g611dGs
+jrFTrQjdrlD3Z8GyLiHEyVwV+QVHZHXbm7wZinUWYVsF4jxt+bsyomjb6QdGeG3J
+3r/bSx2rYKDNM3B9fJbWLZ5L8+i6t/+O+1oWIOfw20u7P9UAF9jXV5+tBGNgo1fY
+s+0Kwk7azkpxEETenN/nyxbIqKifL+yjJgqyr4AvRzFNvqpxzlaSfLVlVbuojBuq
+L7Uf/kwKdasZceTq7B+VveClvn6zxj9Tnb5+RB0C9H1fiN64SBm93JQzHiY8XBbM
+NSqPQ7hmkET9gjLVEPWLhC1bniGr2YWIej6dJkJwkL/CVv7wFIqIJBqEzwARAQAB
+tCRsdWNodW5saWFuZyA8bHVjaHVubGlhbmdAYXBhY2hlLm9yZz6JAk4EEwEIADgW
+IQTi8mYCKFpSjkI4raBMs9zenyQMRgUCZQKvvwIbAwULCQgHAgYVCgkICwIEFgID
+AQIeAQIXgAAKCRBMs9zenyQMRrc7EACw5Wbby8djYz65GSbMzZhBI/DpaWgZCxGn
+NuqZtqZU449lOj7S+v/3scr83dVQKR+sTkWECtbTmimtQRaN+7PIM8OeBOBAye4j
+x4rS4mlxh5CskW9V6pKtyZeEZObBmxB6YcuhpfGIoiVQwVjdZo1B/t/4UUpcAd/d
+6LPPj12l8o/XCvw2q8XtuALz23P1zykkARRZjl/T8PvdqXOmiVXawW/Fo2J9l098
+rTx2iRjnI3kNV73U+F+KD6vWI2UroZ7jtWaWfLrawmS1/wtCSskocWbPDAOzviXe
+XEUu/dZ9uEQaQPJCsnJ+sPgIiT7Xq1htJHqvza6v+fi5SPL9pWMNxiycDLrUa4ZN
+uUAtWprCEqsYsnvtQ6n6lvFAvg7jl/1JOcAUKlz5xjerXe5/NrC6Rao1AYBbGOQN
+xxYjkmG7oxrMVT+nu/j0ePKRSl/Dl8wQVbaee82XFA0wiCKFbJHc+kSc2Lao8aH2
+ta3yMX/M5ibR769FCK1fDFj2IGH2Fza6rGpNtU+c/fzT4mrWsOXSR6Tnm9sjDTdC
+TnC/ScvNNPnQ9+vVDmYmReIsUDEWd+lFPn9sLmFlk8IqCWnyOIyANdLM17O2JozN
+x+KO6/sQy+luojRtJZDDmtWnNttSPZja6gaC2aCPp2467Hykk9ZgK/0GsDqmwxyF
+bGlOTZVlH7kCDQRlAq+/ARAA2eo1wwupTNrzoYnQqaPLdlWNIX8cjF0Fx178Befg
+eXoO63hpIGqeiuMace3KxqzbKToxWN1CjCskhWBg5bVO65b+ndmYCxzBNYfY8avu
+58Y5ci9uXx1dBMg7wTngmPxkrYPkz5kxCaH90cEmxzd45X4bchm/T3gzeuuvdWun
+SoOjkazCTVTs2vaDo57ZqJxfIvfSi4cDy1KVaFOJje44baq6ORu3YokWV09CHE7q
+LmqimtiLCFzSLAG3PREjvIgHp+LnI+geZSh7dLKgXTumehMp5y0AKIpEKHcAejCD
++VWmz3vH5ZIBwqDJahMjFexs1m9oD2CObjhYxcmaWcT4lpWo5kPQ74Dpd3+Qtxdj
+nLNrkjDdUBr0wTRyu8V6j6jsSHzPOE8PGJayXY1YwWX1EyMQZrjeY1SBXa5x8jSE
+gE8+ePFCGakHK/+rulnC2EOEpEnN5jAtgPX8Zg7+PvhWlDXu/mGxxR3u3dldBWte
+/0YfaOppqeQmIr+4b9G73ZAOzcJsPJTYb6QBu2kGCHDkKdVngdPqr+eA+lOclRUG
+vDFjuLVkiRUS46kS6yEpZY410PG+qhUd665GfhRNcsLG7MyBrfKPAEc8EpAHk+ZI
+bIJjOYAFAziIUjjEtHTuWMSgECKkofUOEstKa+rZXTTfvQLgJEx0Cpd1vV6imh3I
+kx8AEQEAAYkCNgQYAQgAIBYhBOLyZgIoWlKOQjitoEyz3N6fJAxGBQJlAq+/AhsM
+AAoJEEyz3N6fJAxGeaIP/3cFTPpOUvY0xg1WbhDeyQ+fs0jW7bG1trSp/t1no2oI
+o8vWt77nTQW2Rw6Plw3wRHRAOxofew7+wkzyylzSvTBY8gN5dxF8oSNjuHLnn+xF
+o8nWlhqiJZyUEQNgCX6VfsiHKtqYc23y529mQbWxoGTZuzvZbefLq31vgcVBCH6S
+U1acev2sb1YdS0ib8g1p2V0ssSytl9Ynk1flGyc2eQ7n9XMgQlVFvFWHINc6APXP
+wYnNbDQeE/ZtIpiYocuyFMQA4JyY5mACUQ09dRSuZMMCcwjxP8JTOZPhddZIDsTn
+yWtPPDmQ0bGaq26nq4PlZ5Vcdb/EEocNHEeZ6V0RSNlX+MvBLxplDYefM0d2LcId
+sqRqT+InXnfzbNuoLPmVx1k5Fh8IoWT+jl0KAiNkoZVOJC5UOW1WVv5YVEpHdLqt
+XUYix3twwTnOfY/j4KTHGiGQwnIOLRQC9oW4yylehO8g6/H26UTALlEG8e/w0NxI
+rb0qVtGeNtZ+u7aVBqOfFLZTyW8Sr1OJdx3fjWBw/ANz6zDn13iQHgjqD85taOCn
+9Rj9piB1nFRnVyLPdLhEN/6k1iT1X5NPhRwFxv3Ik4ea9s+QejT135QTtMWh6Znb
+ijdshVABKTENK4shbEo04aAsZc0zeSOEVqds/Y+oj6ZnHZf8v7L48LQCeeTlrRD5
+=C1Zz
+-END PGP PUBLIC KEY BLOCK-

svn commit: r64358 - /dev/inlong/KEYS

[luchunliang]

Author: luchunliang
Date: Fri Oct  6 02:41:56 2023
New Revision: 64358

Log:
add gpg key for luchunliang

Modified:
dev/inlong/KEYS

Modified: dev/inlong/KEYS
==
--- dev/inlong/KEYS (original)
+++ dev/inlong/KEYS Fri Oct  6 02:41:56 2023
@@ -729,3 +729,61 @@ rb0qVtGeNtZ+u7aVBqOfFLZTyW8Sr1OJdx3fjWBw
 ijdshVABKTENK4shbEo04aAsZc0zeSOEVqds/Y+oj6ZnHZf8v7L48LQCeeTlrRD5
 =C1Zz
 -END PGP PUBLIC KEY BLOCK-
+pub   4096R/27A8CA6F 2023-10-05
+uid  luchunliang (luchunliang) 
+sig 327A8CA6F 2023-10-05  luchunliang (luchunliang) 

+sub   4096R/65374B45 2023-10-05
+sig  27A8CA6F 2023-10-05  luchunliang (luchunliang) 

+
+-BEGIN PGP PUBLIC KEY BLOCK-
+Version: GnuPG v2.0.22 (GNU/Linux)
+
+mQINBGUe52wBEAChGDorkJ6lEKF2PJTAFA1WeNHr8iAidYrf+YzAmVbualvLXI4w
+jR0oDKULeXilaMuohI0pMmjjGLTB/A0AkbK4c8sFWLJ+/1YqXJs/8RHR9bKOEfZc
+vfwMMUe6DrcVnUkLsU7FUhY6bCOB95e49EBloiP/Kgjd82Cy4CDh485vZ6cUredw
+jVQ3LZOg4XeOwmqsJr/JqH4cB1s8vdSjWE7q0wHUFOdBPSLkjNnlZEufoTt+vtnt
+KVVKFTZL7v5PRv+COhgViuZq3OwM450D5JZGH+On/wCgXKLTt0Qk5Rc3rEsm+ihT
+FRAq+JSz1E4/FFoS9nomH/Vbc8DHSIBABsO5i92vLH+lpyU1XhG6c1Wh3huoW7fG
+hSDLb5QQa47Kg3Dbvh5njSVisMKiIT3dvVjdRq1BC2ERaaJ6nkwIxaaVXwNq66m9
+J0C9rIvq66ddsz94gB1i4k9L7AJz0+5RDRVUY3pXTERxHWPWH0eAQlCK+F/t67ZX
+c3JHC8qy15VOUUdo4N2ZirITGX6JMf5UFrlLndIHYecQnewoNJrG609tfFn07/Bl
+a6gYfhFTBiI+SXYSP8Qv35fe5ndlPa6xVYDBxwYMVTxdb2jNDZwtO7q+rjxSJfE6
+DsFAnQ+/876ws5+KypvxTtmcx6uny4wC3heFyKXOZ1IWz3t1L4Lo2/ZbbQARAQAB
+tDJsdWNodW5saWFuZyAobHVjaHVubGlhbmcpIDxsdWNodW5saWFuZ0BhcGFjaGUu
+b3JnPokCOQQTAQIAIwUCZR7nbAIbAwcLCQgHAwIBBhUIAgkKCwQWAgMBAh4BAheA
+AAoJEF7WM2UnqMpvUFoP/A3GZztgmu4IOE5A39cEgwR17WdDWi6nI+xdEn2srbrX
+3uxDhijkGZsEFzJdtKoN1baG2niErlEaUOWVGG1Cc6GcntaTfe4sibLuLa0O+18v
+++g9wZpPPo3Py0xC45VdYg2fxT2eepKLirm02nV3eaQswNj9iXv/8a+9vhwc+ytp
+z0OD5WYzk/sZHuUgyRePdMgcHdksBR4ZXJnAacqcUNFNsUkSq265e6JkGN3z8khv
++9pWZ8b00ANGuwX8F5iFACiLJoyJ9Q/sZgK3sVFJ84jCEUj1/QFPOBzO+TiMG9+0
+a+oplVmaCtnN3iDYolVvtDM02+0Y2HlbqAB5dDaX2e8jQzTyIg2lZiFqrWqCeupS
+iSYDV+GexfINOkHdYBiY09NzamvIAiziXSMAO+43R+h7cEa+z0gjvuj7pDIIHhT8
+k8zczL9lWBDnFD1ERc8IBjxXmijTXG7pu2+hLBbt0Ns6AvZNY+3UwFP1yK6sDYKs
+dIio5GKN+YezQjIL3AJjx9Zq8RQC7KyWwV4fTZN6A6oJTNWU/USsz4sIsjnuAhyk
+x5WX2PRvwAbHT6PiEcI6cqBcaNj8iGscHsDNyzzjV/9IuegzZRM4tsur8YqzSAxL
+UPUpMq5LM8Qx4bsgjxNcTZg6NKZSspvfeue+a24VKLHUzIoiuDcl/Ol/8UcPcDSf
+uQINBGUe52wBEADYAHatQoBfvJJONAVByqDFEWZ9et6fq+8eIUvAjZKyJXyUM8OC
+NxZ4njxJNyMncYJ/xrbtdt+DD9uzjJicWsHmnXVAixWX+awmnelMiVWn4wGY2uB4
+N5/DTs2K5CG+Vigew5Ohu4d63AcXpiUKsVLmNyfiaAkurRn/HWsHHYBDTKuVyZTf
+43DhV7G8uqCVYXGHmafkiWD6y4jC5A86OUjCWZAuZtSjQY843TxMUAWeDHh4vVQi
+o8FP44CsfdsxIYO7cHqMFPP4boKVnAphf10Z5g+0wBbdZdS8TgOT8vvf5/gfAXFg
+yqdpKVMVaEloFSD9DBpmIF3vfYMrzVGhjAt/2bg0C6XkpibrA3ne/REIm1zebA0R
+VyT2eakS85ZiCIfabP9gr+IKU8x8epsupdyqmXe+zbK+wfDjg6dsWzS08zdXwBFT
+OYBnxw7v9qJKvXrH9KnhYiVFZgStLyRjrjK/lFqUVkLrqeNrnqne4KHPW+u8iuRf
+McKtpjQx7gGddXAnrA1ZnkygWZy5oS/WATNlBZOUy8cHn7DJLxgVKStacNexXddk
+PeLh3ZpFeAcVBFuyDSQf+N4Oa3wUZs/YbOq7YtIQG1iW0XmhY0vfeUGZQOhhMgFQ
+eYOBvmPe4yN8jQy7ggfB8Qh5gdQlOMo8eFl49AODi8cplF7GYl62OYr9SQARAQAB
+iQIfBBgBAgAJBQJlHudsAhsMAAoJEF7WM2UnqMpvFvMP/03f/s8huec6vganXOc7
+ruHbys+FtzB2UF+I5FOnFVaduuP9XfBCCM2MZNKu3wIhET5ckDzy+f74vuVvWIrE
+WfNToiaY4BI6N3r8YyalqYb43u9wH9VlP/RC3Te2Xsug0FggcP87OCCQ0EsNt9tS
+1fhZVtnXlIGx0O6iLabEnuET6rFLG6tR7GLLuZ+OexI9JTFGBIeMFvOn5nPWNn3H
+AqDQ+LGbeast94rufnoJU0AtjNTvfO6Lgktu1PIOcHZbWvF8hbLa47mQ32dPCR9z
+++1UGdK05RNSV/j0IpUbFnZuGTXfbNEDrh4EQbNV25a4hObFb4BKOIwjfz24VYFe
+s8pAHJSumYIDmWcOeKKZWDZnBNV1djUXWNgZ8TpvtVc/NNC0G+p3JNtkakhIiIBb
+lpad34mxmngt82hcF3lySRH4BymBm8kwsncXJPOEBgxYE100abWBR6g+OWLucEJN
+3ABDo5Sab3PHV8mjZkx9GK4NgBPVCClR6QUguBZyxdyJU5cOdS75yuqAoSrLB1aK
+F8JeHtpTFaWkDd8deg3lXXvVOTgrkUVPg5lnYM2rIWxK5aBiRNw2KSNpu3g33yy2
+m5qhltE6JRoufPpS8pkr2diaywmaMBX2vN50tzItcmR27gkUYk0Yb20zBWiAFJOc
+bNJDLI7qL8HfkLfD41nt8K5b
+=uhQ3
+-END PGP PUBLIC KEY BLOCK-

[inlong] branch master updated: [INLONG-9122][Agent] Add task and instance action (#9123)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new 636822bbaf [INLONG-9122][Agent] Add task and instance action (#9123)
636822bbaf is described below

commit 636822bbaf27770f48f3575107ff7287e0c255c3
Author: justinwwhuang 
AuthorDate: Thu Oct 26 19:20:14 2023 +0800

[INLONG-9122][Agent] Add task and instance action (#9123)
---
 .../inlong/agent/core/instance/ActionType.java | 24 
 .../inlong/agent/core/instance/InstanceAction.java | 33 ++
 .../apache/inlong/agent/core/task/ActionType.java  | 24 
 .../apache/inlong/agent/core/task/TaskAction.java  | 33 ++
 4 files changed, 114 insertions(+)

diff --git 
a/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/instance/ActionType.java
 
b/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/instance/ActionType.java
new file mode 100644
index 00..7db8634bc3
--- /dev/null
+++ 
b/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/instance/ActionType.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.agent.core.instance;
+
+public enum ActionType {
+ADD,
+FINISH,
+DELETE
+}
diff --git 
a/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/instance/InstanceAction.java
 
b/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/instance/InstanceAction.java
new file mode 100644
index 00..424c5f6eaa
--- /dev/null
+++ 
b/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/instance/InstanceAction.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.agent.core.instance;
+
+import org.apache.inlong.agent.conf.InstanceProfile;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@AllArgsConstructor
+@NoArgsConstructor
+public class InstanceAction {
+
+private ActionType actionType;
+private InstanceProfile profile;
+}
\ No newline at end of file
diff --git 
a/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/task/ActionType.java
 
b/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/task/ActionType.java
new file mode 100644
index 00..4345b35336
--- /dev/null
+++ 
b/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/task/ActionType.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permis

[inlong] branch master updated (636822bbaf -> 3ee9921001)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


from 636822bbaf [INLONG-9122][Agent] Add task and instance action (#9123)
 add 3ee9921001 [INLONG-9120][Agent] Add offset db to store the offset data 
(#9121)

No new revisions were added by this update.

Summary of changes:
 .../inlong/agent/constant/AgentConstants.java  |  3 +
 .../inlong/agent/constant/CommonConstants.java | 13 +++-
 .../java/org/apache/inlong/agent/db/OffsetDb.java  | 83 ++
 3 files changed, 96 insertions(+), 3 deletions(-)
 create mode 100755 
inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/db/OffsetDb.java

(inlong) branch master updated: [INLONG-9143][Agent] Add log file collect task (#9145)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new 119240cdbc [INLONG-9143][Agent] Add log file collect task (#9145)
119240cdbc is described below

commit 119240cdbc806cb0f98058625f460809b9b69ed5
Author: justinwwhuang 
AuthorDate: Mon Oct 30 11:10:55 2023 +0800

[INLONG-9143][Agent] Add log file collect task (#9145)

* [INLONG-9143][Agent] Add log file collect task

* [INLONG-9143][Agent] Add log file collect task

fix ut failed

* [INLONG-9143][Agent] Add log file collect task

Try to fix check failed

* [INLONG-9143][Agent] Add log file collect task

fix ut error
---
 .../apache/inlong/agent/plugin/task/CronTask.java  |  58 +++
 .../plugin/task/FormatDateLogFileCollectTask.java  |  27 ++
 .../inlong/agent/plugin/task/PathPattern.java  | 181 
 .../agent/plugin/task/filecollect/AgentErrMsg.java |  67 +++
 .../agent/plugin/task/filecollect/FileScanner.java | 174 +++
 .../task/filecollect/LogFileCollectTask.java   | 501 +
 .../agent/plugin/task/filecollect/TaskType.java|  39 ++
 .../agent/plugin/task/filecollect/WatchEntity.java | 358 +++
 .../inlong/agent/plugin/AgentBaseTestsHelper.java  |  50 +-
 .../agent/plugin/task/MockInstanceManager.java |  24 +
 .../agent/plugin/task/TestLogfileCollectTask.java  | 119 +
 .../src/test/resources/test/20230928_1.txt |   3 +
 12 files changed, 1593 insertions(+), 8 deletions(-)

diff --git 
a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/CronTask.java
 
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/CronTask.java
new file mode 100644
index 00..0216eb96c6
--- /dev/null
+++ 
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/CronTask.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.agent.plugin.task;
+
+import org.apache.inlong.agent.conf.TaskProfile;
+import org.apache.inlong.agent.db.Db;
+import org.apache.inlong.agent.plugin.file.Task;
+
+/**
+ * Generate job by crontab expression.
+ */
+public class CronTask extends Task {
+
+@Override
+public void init(Object srcManager, TaskProfile profile, Db basicDb) {
+
+}
+
+@Override
+public void run() {
+
+}
+
+@Override
+public void destroy() {
+
+}
+
+@Override
+public TaskProfile getProfile() {
+return null;
+}
+
+@Override
+public String getTaskId() {
+return null;
+}
+
+@Override
+public void addCallbacks() {
+
+}
+}
diff --git 
a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/FormatDateLogFileCollectTask.java
 
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/FormatDateLogFileCollectTask.java
new file mode 100644
index 00..b8837fc0df
--- /dev/null
+++ 
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/FormatDateLogFileCollectTask.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.agent.plugin.task;
+
+import org.apache.inlong.agent.plugin.task.filecollect.LogFileCollectTask;

(inlong) branch master updated: [INLONG-9155][Agent] Add file used proxy (#9156)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new c87d98c8dd [INLONG-9155][Agent] Add file used proxy (#9156)
c87d98c8dd is described below

commit c87d98c8dd50a611d9fa785757347c0f59367489
Author: justinwwhuang 
AuthorDate: Mon Oct 30 20:20:52 2023 +0800

[INLONG-9155][Agent] Add file used proxy (#9156)
---
 .../org/apache/inlong/agent/plugin/file/Sink.java  |  55 ++
 .../plugin/sinks/filecollect/AbstractSink.java |  90 +
 .../agent/plugin/sinks/filecollect/ProxySink.java  | 211 +
 3 files changed, 356 insertions(+)

diff --git 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/plugin/file/Sink.java
 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/plugin/file/Sink.java
new file mode 100755
index 00..0978ea8025
--- /dev/null
+++ 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/plugin/file/Sink.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.agent.plugin.file;
+
+import org.apache.inlong.agent.conf.InstanceProfile;
+import org.apache.inlong.agent.plugin.Message;
+
+/**
+ * Sink data to remote data center
+ */
+public interface Sink {
+
+/**
+ * Write data into data center
+ *
+ * @param message message
+ */
+void write(Message message);
+
+/**
+ * set source file name where the message is generated
+ *
+ * @param sourceFileName
+ */
+void setSourceName(String sourceFileName);
+
+/**
+ * init
+ *
+ * @param profile
+ */
+void init(InstanceProfile profile);
+
+/**
+ * destroy
+ */
+void destroy();
+
+boolean sinkFinish();
+}
diff --git 
a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sinks/filecollect/AbstractSink.java
 
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sinks/filecollect/AbstractSink.java
new file mode 100644
index 00..a92dd770e3
--- /dev/null
+++ 
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sinks/filecollect/AbstractSink.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.agent.plugin.sinks.filecollect;
+
+import org.apache.inlong.agent.conf.InstanceProfile;
+import org.apache.inlong.agent.message.filecollect.ProxyMessageCache;
+import org.apache.inlong.agent.metrics.AgentMetricItem;
+import org.apache.inlong.agent.metrics.AgentMetricItemSet;
+import org.apache.inlong.agent.plugin.file.Sink;
+import org.apache.inlong.common.metric.MetricRegister;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+
+import static 
org.apache.inlong.agent.constant.CommonConstants.DEFAULT_PROXY_BATCH_FLUSH_INTERVAL;
+import static 
org.apache.inlong.agent.constant.CommonConstants.DEFAULT_PROXY_INLONG_GROUP_ID;
+import static 
org.apache.inlong.agent.constant.CommonConstants.DEFAULT_PROXY_INLONG_STREAM_ID;
+import static 
org.apache.inlong.agent.constant.CommonConstants.PROXY_BATCH_FLUSH_INTERVAL;
+import static 
org.apache.inlong.agent.constant.CommonConstan

(inlong) branch master updated: [INLONG-9159][Agent] Add file instance (#9162)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new 6fc5ac83d3 [INLONG-9159][Agent] Add file instance (#9162)
6fc5ac83d3 is described below

commit 6fc5ac83d31e89718b2d3f22740ddc8935112543
Author: justinwwhuang 
AuthorDate: Tue Oct 31 11:10:56 2023 +0800

[INLONG-9159][Agent] Add file instance (#9162)

* [INLONG-9159][Agent] Add file instance

* [INLONG-9159][Agent] Add file instance
---
 .../inlong/agent/plugin/instance/FileInstance.java | 159 +
 .../agent/plugin/sinks/filecollect/ProxySink.java  |   6 +-
 2 files changed, 163 insertions(+), 2 deletions(-)

diff --git 
a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/instance/FileInstance.java
 
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/instance/FileInstance.java
new file mode 100644
index 00..5b12e8295e
--- /dev/null
+++ 
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/instance/FileInstance.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.agent.plugin.instance;
+
+import org.apache.inlong.agent.conf.InstanceProfile;
+import org.apache.inlong.agent.constant.TaskConstants;
+import org.apache.inlong.agent.core.instance.ActionType;
+import org.apache.inlong.agent.core.instance.InstanceAction;
+import org.apache.inlong.agent.core.instance.InstanceManager;
+import org.apache.inlong.agent.core.task.OffsetManager;
+import org.apache.inlong.agent.plugin.Instance;
+import org.apache.inlong.agent.plugin.Message;
+import org.apache.inlong.agent.plugin.file.Sink;
+import org.apache.inlong.agent.plugin.file.Source;
+import org.apache.inlong.agent.plugin.utils.file.FileDataUtils;
+import org.apache.inlong.agent.state.State;
+import org.apache.inlong.agent.utils.AgentUtils;
+import org.apache.inlong.agent.utils.ThreadUtils;
+import org.apache.inlong.common.enums.InstanceStateEnum;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ */
+public class FileInstance extends Instance {
+
+private static final Logger LOGGER = 
LoggerFactory.getLogger(FileInstance.class);
+private Source source;
+private Sink sink;
+private InstanceProfile profile;
+public static final int CORE_THREAD_SLEEP_TIME = 1;
+private InstanceManager instanceManager;
+private volatile boolean running = false;
+private volatile boolean inited = false;
+
+@Override
+public void init(Object srcManager, InstanceProfile srcProfile) {
+try {
+instanceManager = (InstanceManager) srcManager;
+profile = srcProfile;
+profile.set(TaskConstants.INODE_INFO, 
FileDataUtils.getInodeInfo(profile.getInstanceId()));
+LOGGER.info("task id: {} submit new instance {} profile detail 
{}.", profile.getTaskId(),
+profile.getInstanceId(), profile.toJsonStr());
+source = (Source) 
Class.forName(profile.getSourceClass()).newInstance();
+source.init(profile);
+sink = (Sink) Class.forName(profile.getSinkClass()).newInstance();
+sink.init(profile);
+inited = true;
+} catch (Throwable ex) {
+doChangeState(State.FATAL);
+LOGGER.error("init instance {} for task {} failed", 
profile.getInstanceId(), profile.getInstanceId(),
+ex);
+ThreadUtils.threadThrowableHandler(Thread.currentThread(), ex);
+}
+}
+
+@Override
+public void destroy() {
+if (!inited) {
+return;
+}
+doChangeState(State.SUCCEEDED);
+while (running) {
+AgentUtils.silenceSleepInMs(1);
+}
+this.source.destroy();
+this.sink.destroy();
+}
+
+@Override
+public void run() {
+Thread.currentThread().setName("file-instance-core-" + getTaskId() + 
"-" + getInstanceId());
+running = true;
+

(inlong) branch master updated: [INLONG-9165][Agent] Delete job related file (#9166)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new 5f2ad4bf90 [INLONG-9165][Agent] Delete job related file (#9166)
5f2ad4bf90 is described below

commit 5f2ad4bf905a162b0cf95753674994122b3051cd
Author: justinwwhuang 
AuthorDate: Tue Oct 31 16:11:00 2023 +0800

[INLONG-9165][Agent] Delete job related file (#9166)
---
 .../inlong/agent/common/AgentThreadFactory.java|   2 +-
 .../java/org/apache/inlong/agent/pojo/FileJob.java | 141 ---
 .../apache/inlong/agent/pojo/JobProfileDto.java| 426 -
 .../org/apache/inlong/agent/utils/AgentUtils.java  |  29 +-
 .../agent/common/TestAgentThreadFactory.java   |   4 +-
 .../apache/inlong/agent/core/HeartbeatManager.java |  18 -
 .../inlong/agent/core/task/PositionManager.java|   1 -
 .../inlong/agent/core/task/TestMemoryManager.java  |  11 +-
 .../inlong/agent/plugin/instance/FileInstance.java |   6 +-
 .../src/test/resources/fileAgent.trigger.json  |  27 --
 .../src/test/resources/fileAgentJob.json   |  24 --
 .../agent-plugins/src/test/resources/test/1.txt|   3 -
 .../agent-plugins/src/test/resources/test/2.txt|   2 -
 .../agent-plugins/src/test/resources/test/3.txt|   5 -
 .../agent-plugins/src/test/resources/test/a.txt|   3 -
 15 files changed, 20 insertions(+), 682 deletions(-)

diff --git 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/common/AgentThreadFactory.java
 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/common/AgentThreadFactory.java
index 5d756fc7a4..c9b9f3f2a6 100644
--- 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/common/AgentThreadFactory.java
+++ 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/common/AgentThreadFactory.java
@@ -33,7 +33,7 @@ public class AgentThreadFactory implements ThreadFactory {
 
 private static final Logger LOGGER = 
LoggerFactory.getLogger(AgentThreadFactory.class);
 
-public static final String NAMED_THREAD_PLACEHOLDER = "running-thread";
+public static final String NAMED_THREAD_PLACEHOLDER = 
"agent-thread-factory";
 
 private final AtomicInteger mThreadNum = new AtomicInteger(1);
 
diff --git 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/FileJob.java
 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/FileJob.java
deleted file mode 100644
index 33cdacf764..00
--- 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/FileJob.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.inlong.agent.pojo;
-
-import lombok.Data;
-
-import java.util.List;
-import java.util.Map;
-
-@Data
-public class FileJob {
-
-private String trigger;
-
-private Dir dir;
-private Thread thread;
-private int id;
-private String timeOffset;
-private String addictiveString;
-private String collectType;
-private Line line;
-
-// INCREMENT
-// FULL
-private String contentCollectType;
-
-private String envList;
-
-// JSON string, the content format is List>
-private String metaFields;
-
-private String dataSeparator;
-
-// JSON string, the content format is Map
-private String filterMetaByLabels;
-
-// JSON string, the content format is Map
-private String properties;
-
-// Monitor interval for file
-private Long monitorInterval;
-
-// Monitor switch, 1 true and 0 false
-private Integer monitorStatus;
-
-// Monitor expire time and the time in milliseconds
-private Long monitorExpire;
-
-@Data
-public static class Dir {
-
-private String patterns;
-
-private String blackList;
-}
-
-@Data
-public static class Running {
-
-private String core;
-}
-
-@Data
-public static class Thread {
-
-private Running running;
-}
-
-@Data
-public static class Line {
-
-private String endPattern;
-

(inlong) branch master updated: [INLONG-9190][Agent] Log file source clear buffer queue does not take effect (#9191)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new 58ecec5a9d [INLONG-9190][Agent] Log file source clear buffer queue 
does not take effect (#9191)
58ecec5a9d is described below

commit 58ecec5a9d57e6f00759825daccd2d7199abe5c7
Author: justinwwhuang 
AuthorDate: Wed Nov 1 15:21:33 2023 +0800

[INLONG-9190][Agent] Log file source clear buffer queue does not take 
effect (#9191)
---
 .../inlong/agent/plugin/sources/LogFileSource.java |  2 +-
 .../agent/plugin/sources/TestLogFileSource.java| 36 ++
 2 files changed, 31 insertions(+), 7 deletions(-)

diff --git 
a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/LogFileSource.java
 
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/LogFileSource.java
index cc9f3724e8..ea0e63c95f 100755
--- 
a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/LogFileSource.java
+++ 
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/LogFileSource.java
@@ -511,7 +511,7 @@ public class LogFileSource extends AbstractSource {
 }
 
 private void clearQueue(BlockingQueue queue) {
-if (queue != null) {
+if (queue == null) {
 return;
 }
 while (queue != null && !queue.isEmpty()) {
diff --git 
a/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/sources/TestLogFileSource.java
 
b/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/sources/TestLogFileSource.java
index 6c3257adc9..23b5027812 100644
--- 
a/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/sources/TestLogFileSource.java
+++ 
b/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/sources/TestLogFileSource.java
@@ -46,9 +46,11 @@ public class TestLogFileSource {
 
 private static final Logger LOGGER = 
LoggerFactory.getLogger(TestLogFileSource.class);
 private static final ClassLoader LOADER = 
TestLogFileSource.class.getClassLoader();
-private static LogFileSource source;
 private static AgentBaseTestsHelper helper;
 private static final Gson GSON = new Gson();
+private static final String[] check = {"hello line-end-symbol aa", "world 
line-end-symbol",
+"agent line-end-symbol"};
+private static InstanceProfile instanceProfile;
 
 @BeforeClass
 public static void setup() {
@@ -57,11 +59,15 @@ public class TestLogFileSource {
 helper = new 
AgentBaseTestsHelper(TestLogFileSource.class.getName()).setupAgentHome();
 String pattern = helper.getTestRootDir() + "/MMDD.log_[0-9]+";
 TaskProfile taskProfile = helper.getTaskProfile(1, pattern, false, 0L, 
0L, TaskStateEnum.RUNNING);
-InstanceProfile instanceProfile = taskProfile.createInstanceProfile("",
+instanceProfile = taskProfile.createInstanceProfile("",
 fileName, "20230928");
+
+}
+
+private LogFileSource getSource() {
 try {
 instanceProfile.set(TaskConstants.INODE_INFO, 
FileDataUtils.getInodeInfo(instanceProfile.getInstanceId()));
-source = new LogFileSource();
+LogFileSource source = new LogFileSource();
 Whitebox.setInternalState(source, "BATCH_READ_LINE_COUNT", 1);
 Whitebox.setInternalState(source, "BATCH_READ_LINE_TOTAL_LEN", 10);
 Whitebox.setInternalState(source, "PRINT_INTERVAL_MS", 0);
@@ -69,25 +75,31 @@ public class TestLogFileSource {
 Whitebox.setInternalState(source, "FINISH_READ_MAX_COUNT", 1);
 Whitebox.setInternalState(source, "READ_WAIT_TIMEOUT_MS", 10);
 source.init(instanceProfile);
+return source;
 } catch (Exception e) {
 LOGGER.error("source init error {}", e);
 Assert.assertTrue("source init error", false);
 }
+return null;
 }
 
 @AfterClass
 public static void teardown() throws Exception {
-source.destroy();
 helper.teardownAgentHome();
 }
 
 @Test
-public void testTaskManager() {
-String[] check = {"hello line-end-symbol aa", "world line-end-symbol", 
"agent line-end-symbol"};
+public void testLogFileSource() {
+testFullRead();
+testCleanQueue();
+}
+
+private void testFullRead() {
 int srcLen = 0;
 for (int i = 0; i < check.length; i++) {
 srcLen += check[i].getBytes(StandardCharsets.UTF_8).length;
 }
+LogFileSource source = getSource();
 await().atMost(2, TimeUnit.SECON

(inlong) branch master updated: [INLONG-9200][Agent] Fix bug: duplicate file collect instance (#9201)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new 7954b7f5c6 [INLONG-9200][Agent] Fix bug: duplicate file collect 
instance (#9201)
7954b7f5c6 is described below

commit 7954b7f5c642762b84f71e610e2d9be463a9a4e9
Author: justinwwhuang 
AuthorDate: Fri Nov 3 15:05:48 2023 +0800

[INLONG-9200][Agent] Fix bug: duplicate file collect instance (#9201)

* [INLONG-9200][Agent] Fix bug: duplicate file collect instance

* [INLONG-9200][Agent] Fix bug: duplicate file collect instance

* [INLONG-9200][Agent] Fix bug: duplicate file collect instance
---
 .../apache/inlong/agent/conf/InstanceProfile.java  | 10 -
 .../org/apache/inlong/agent/conf/TaskProfile.java  |  4 +-
 .../inlong/agent/constant/TaskConstants.java   |  2 +
 .../org/apache/inlong/agent/db/InstanceDb.java |  5 +--
 .../org/apache/inlong/agent/db/TaskProfileDb.java  |  5 +--
 .../agent/core/instance/InstanceManager.java   | 15 ++--
 .../inlong/agent/core/task/file/MemoryManager.java | 24 
 .../inlong/agent/core/task/file/TaskManager.java   |  4 +-
 .../agent/core/instance/TestInstanceManager.java   |  7 +++-
 .../agent/plugin/sinks/filecollect/ProxySink.java  |  3 +-
 .../inlong/agent/plugin/sources/LogFileSource.java |  3 +-
 .../task/filecollect/LogFileCollectTask.java   | 44 --
 .../sinks/filecollect/TestSenderManager.java   |  2 +-
 .../agent/plugin/sources/TestLogFileSource.java|  3 +-
 .../agent/plugin/sources/TestMqttConnect.java  |  3 +-
 15 files changed, 91 insertions(+), 43 deletions(-)

diff --git 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/conf/InstanceProfile.java
 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/conf/InstanceProfile.java
index 024b7674eb..5592008085 100644
--- 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/conf/InstanceProfile.java
+++ 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/conf/InstanceProfile.java
@@ -93,9 +93,17 @@ public class InstanceProfile extends AbstractConfiguration 
implements Comparable
 setInt(INSTANCE_STATE, state.ordinal());
 }
 
+public long getFileUpdateTime() {
+return getLong(TaskConstants.FILE_UPDATE_TIME, 0);
+}
+
+public void setFileUpdateTime(long lastUpdateTime) {
+setLong(TaskConstants.FILE_UPDATE_TIME, lastUpdateTime);
+}
+
 @Override
 public boolean allRequiredKeyExist() {
-return true;
+return hasKey(TaskConstants.FILE_UPDATE_TIME);
 }
 
 /**
diff --git 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/conf/TaskProfile.java
 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/conf/TaskProfile.java
index 1040afa88a..319f1abf56 100644
--- 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/conf/TaskProfile.java
+++ 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/conf/TaskProfile.java
@@ -106,7 +106,8 @@ public class TaskProfile extends AbstractConfiguration {
 return GSON.toJson(getConfigStorage());
 }
 
-public InstanceProfile createInstanceProfile(String instanceClass, String 
fileName, String dataTime) {
+public InstanceProfile createInstanceProfile(String instanceClass, String 
fileName, String dataTime,
+long fileUpdateTime) {
 InstanceProfile instanceProfile = 
InstanceProfile.parseJsonStr(toJsonStr());
 instanceProfile.setInstanceClass(instanceClass);
 instanceProfile.setInstanceId(fileName);
@@ -114,6 +115,7 @@ public class TaskProfile extends AbstractConfiguration {
 instanceProfile.setCreateTime(AgentUtils.getCurrentTime());
 instanceProfile.setModifyTime(AgentUtils.getCurrentTime());
 instanceProfile.setState(InstanceStateEnum.DEFAULT);
+instanceProfile.setFileUpdateTime(fileUpdateTime);
 return instanceProfile;
 }
 }
diff --git 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/constant/TaskConstants.java
 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/constant/TaskConstants.java
index bbf95cea76..fa2ac856fd 100755
--- 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/constant/TaskConstants.java
+++ 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/constant/TaskConstants.java
@@ -166,6 +166,8 @@ public class TaskConstants extends CommonConstants {
 
 public static final String INSTANCE_STATE = "instance.state";
 
+public static final String FILE_UPDATE_TIME = "fileUpdateTime";
+
 public static final String LAST_UPDATE_TIME = "lastUpdateTime";
 
 public static final String TRIGGER_ONLY_ONE_JOB = "job.standalone"; // 
TODO:delete it
diff --g

(inlong) branch master updated: [INLONG-9207][Agent] Fix bug: task manager stuck (#9208)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new 7cbe641d04 [INLONG-9207][Agent] Fix bug: task manager stuck (#9208)
7cbe641d04 is described below

commit 7cbe641d04479fe84de6ce66c3944107a3f45868
Author: justinwwhuang 
AuthorDate: Fri Nov 3 15:44:01 2023 +0800

[INLONG-9207][Agent] Fix bug: task manager stuck (#9208)
---
 .../inlong/agent/plugin/task/filecollect/LogFileCollectTask.java  | 7 +++
 .../apache/inlong/agent/plugin/task/TestLogfileCollectTask.java   | 8 +---
 .../src/test/resources/testScan/20230928_1/test_1.txt | 3 +++
 inlong-agent/agent-plugins/src/test/resources/testScan/temp.txt   | 3 +++
 4 files changed, 18 insertions(+), 3 deletions(-)

diff --git 
a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/filecollect/LogFileCollectTask.java
 
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/filecollect/LogFileCollectTask.java
index 11d9330274..54170a5290 100644
--- 
a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/filecollect/LogFileCollectTask.java
+++ 
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/filecollect/LogFileCollectTask.java
@@ -79,6 +79,7 @@ public class LogFileCollectTask extends Task {
 new ConcurrentHashMap<>();
 public static final long DAY_TIMEOUT_INTERVAL = 2 * 24 * 3600 * 1000;
 public static final int CORE_THREAD_SLEEP_TIME = 1000;
+public static final int CORE_THREAD_MAX_GAP_TIME_MS = 60 * 1000;
 private boolean retry;
 private long startTime;
 private long endTime;
@@ -87,6 +88,7 @@ public class LogFileCollectTask extends Task {
 private long lastScanTime = 0;
 public final long SCAN_INTERVAL = 1 * 60 * 1000;
 private volatile boolean runAtLeastOneTime = false;
+private volatile long coreThreadUpdateTime = 0;
 private volatile boolean running = false;
 
 @Override
@@ -196,6 +198,10 @@ public class LogFileCollectTask extends Task {
 
 private void releaseWatchers(Map watchers) {
 while (running) {
+if (AgentUtils.getCurrentTime() - coreThreadUpdateTime > 
CORE_THREAD_MAX_GAP_TIME_MS) {
+LOGGER.error("core thread not update, maybe it has broken");
+break;
+}
 AgentUtils.silenceSleepInMs(CORE_THREAD_SLEEP_TIME);
 }
 watchers.forEach((taskId, watcher) -> {
@@ -227,6 +233,7 @@ public class LogFileCollectTask extends Task {
 Thread.currentThread().setName("directory-task-core-" + getTaskId());
 running = true;
 while (!isFinished()) {
+coreThreadUpdateTime = AgentUtils.getCurrentTime();
 AgentUtils.silenceSleepInMs(CORE_THREAD_SLEEP_TIME);
 if (!initOK) {
 continue;
diff --git 
a/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/task/TestLogfileCollectTask.java
 
b/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/task/TestLogfileCollectTask.java
index 8a9ed4c021..29047919cc 100644
--- 
a/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/task/TestLogfileCollectTask.java
+++ 
b/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/task/TestLogfileCollectTask.java
@@ -61,6 +61,7 @@ public class TestLogfileCollectTask {
 private static final Gson GSON = new Gson();
 private static TaskManager manager;
 private static MockInstanceManager instanceManager = new 
MockInstanceManager();
+private static String tempResourceName;
 private static String resourceName;
 private static String fileName;
 private static String dataTime;
@@ -74,9 +75,10 @@ public class TestLogfileCollectTask {
 public static void setup() {
 helper = new 
AgentBaseTestsHelper(TestLogfileCollectTask.class.getName()).setupAgentHome();
 Db basicDb = TaskManager.initDb("/localdb");
-resourceName = LOADER.getResource("test/20230928_1.txt").getPath();
-File f = new File(resourceName);
-String pattern = f.getParent() + "/MMDD_[0-9]+.txt";
+resourceName = 
LOADER.getResource("testScan/20230928_1/test_1.txt").getPath();
+tempResourceName = LOADER.getResource("testScan/temp.txt").getPath();
+File f = new File(tempResourceName);
+String pattern = f.getParent() + "/MMDD_[0-9]+/test_[0-9]+.txt";
 TaskProfile taskProfile = helper.getTaskProfile(1, pattern, true, 0L, 
0L, TaskStateEnum.RUNNING);
 try {
 String startStr = "2023-09-20 00:00:00";
diff --git 
a/inlong-agent/agent-plugins/src/test/resources/testScan/20230928_1/test_1.tx

(inlong) branch master updated: [INLONG-9209][Manager] Support configuring predefined fields and issuing agents (#9210)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new ff41d6aa2b [INLONG-9209][Manager] Support configuring predefined 
fields and issuing agents (#9210)
ff41d6aa2b is described below

commit ff41d6aa2bd7e4ec21f5bae53b51b27a0d568d5d
Author: fuweng11 <76141879+fuwen...@users.noreply.github.com>
AuthorDate: Fri Nov 3 16:22:30 2023 +0800

[INLONG-9209][Manager] Support configuring predefined fields and issuing 
agents (#9210)

* [INLONG-9209][Manager] Support configuring predefined fields and issuing 
agents

* [INLONG-9209][Manager] Fix UT
---
 .../java/org/apache/inlong/common/pojo/agent/DataConfig.java | 1 +
 .../org/apache/inlong/manager/pojo/stream/BaseInlongStream.java  | 3 +++
 .../apache/inlong/manager/pojo/stream/InlongStreamExtParam.java  | 3 +++
 .../inlong/manager/service/core/impl/AgentServiceImpl.java   | 9 -
 4 files changed, 15 insertions(+), 1 deletion(-)

diff --git 
a/inlong-common/src/main/java/org/apache/inlong/common/pojo/agent/DataConfig.java
 
b/inlong-common/src/main/java/org/apache/inlong/common/pojo/agent/DataConfig.java
index 975f74d128..30bbafab5e 100644
--- 
a/inlong-common/src/main/java/org/apache/inlong/common/pojo/agent/DataConfig.java
+++ 
b/inlong-common/src/main/java/org/apache/inlong/common/pojo/agent/DataConfig.java
@@ -48,6 +48,7 @@ public class DataConfig {
 private Integer syncSend;
 private String syncPartitionKey;
 private Integer state;
+private String predefinedFields;
 private String extParams;
 /**
  * The task version.
diff --git 
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/stream/BaseInlongStream.java
 
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/stream/BaseInlongStream.java
index 1327c9ac7b..4abd2f836f 100644
--- 
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/stream/BaseInlongStream.java
+++ 
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/stream/BaseInlongStream.java
@@ -20,15 +20,18 @@ package org.apache.inlong.manager.pojo.stream;
 import io.swagger.annotations.ApiModel;
 import lombok.AllArgsConstructor;
 import lombok.Data;
+import lombok.NoArgsConstructor;
 
 /**
  * The base parameter class of InlongStream, support user extend their own 
business params.
  */
 @Data
 @AllArgsConstructor
+@NoArgsConstructor
 @ApiModel("Base info of inlong stream")
 public class BaseInlongStream {
 
 // you can add extend parameters in this class
+private String predefinedFields;
 
 }
diff --git 
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/stream/InlongStreamExtParam.java
 
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/stream/InlongStreamExtParam.java
index 0dba8034af..ad69b997c9 100644
--- 
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/stream/InlongStreamExtParam.java
+++ 
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/stream/InlongStreamExtParam.java
@@ -48,6 +48,9 @@ public class InlongStreamExtParam implements Serializable {
 @ApiModelProperty(value = "If use extended fields")
 private Boolean useExtendedFields = false;
 
+@ApiModelProperty(value = "Predefined fields")
+private String predefinedFields;
+
 /**
  * Pack extended attributes into ExtParams
  *
diff --git 
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/AgentServiceImpl.java
 
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/AgentServiceImpl.java
index 0d4ff83a37..d4731d4e55 100644
--- 
a/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/AgentServiceImpl.java
+++ 
b/inlong-manager/manager-service/src/main/java/org/apache/inlong/manager/service/core/impl/AgentServiceImpl.java
@@ -55,6 +55,7 @@ import 
org.apache.inlong.manager.pojo.cluster.agent.AgentClusterNodeDTO;
 import org.apache.inlong.manager.pojo.cluster.pulsar.PulsarClusterDTO;
 import org.apache.inlong.manager.pojo.group.pulsar.InlongPulsarDTO;
 import org.apache.inlong.manager.pojo.source.file.FileSourceDTO;
+import org.apache.inlong.manager.pojo.stream.InlongStreamInfo;
 import org.apache.inlong.manager.service.core.AgentService;
 import org.apache.inlong.manager.service.source.SourceSnapshotOperator;
 
@@ -101,6 +102,7 @@ import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import static org.apache.inlong.manager.common.consts.InlongConstants.DOT;
+import static 
org.apache.inlong.manager.pojo.stream.InlongStreamExtParam.unpackExtParams;
 
 /**
  * Agent service layer implementation
@@ -161,7 +163,7 @@ public class AgentServiceImpl implement

(inlong) branch master updated: [INLONG-9215][Agent] Add predefine fields (#9217)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new 3596862db0 [INLONG-9215][Agent] Add predefine fields (#9217)
3596862db0 is described below

commit 3596862db00db0a9ce11e406896a4f9d362ab3ee
Author: justinwwhuang 
AuthorDate: Fri Nov 3 17:39:01 2023 +0800

[INLONG-9215][Agent] Add predefine fields (#9217)

* [INLONG-9215][Agent] Add predefine fields

* [INLONG-9215][Agent] Add predefine fields

* [INLONG-9215][Agent] Add predefine fields
---
 .../apache/inlong/agent/conf/InstanceProfile.java  |  4 
 .../inlong/agent/constant/TaskConstants.java   |  1 +
 .../apache/inlong/agent/pojo/TaskProfileDto.java   |  4 +++-
 .../inlong/agent/plugin/sources/LogFileSource.java |  4 +++-
 .../inlong/agent/plugin/utils/MetaDataUtils.java   | 25 ++
 5 files changed, 36 insertions(+), 2 deletions(-)

diff --git 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/conf/InstanceProfile.java
 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/conf/InstanceProfile.java
index 5592008085..5c3e74fe86 100644
--- 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/conf/InstanceProfile.java
+++ 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/conf/InstanceProfile.java
@@ -101,6 +101,10 @@ public class InstanceProfile extends AbstractConfiguration 
implements Comparable
 setLong(TaskConstants.FILE_UPDATE_TIME, lastUpdateTime);
 }
 
+public String getPredefineFields() {
+return get(TaskConstants.PREDEFINE_FIELDS, "");
+}
+
 @Override
 public boolean allRequiredKeyExist() {
 return hasKey(TaskConstants.FILE_UPDATE_TIME);
diff --git 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/constant/TaskConstants.java
 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/constant/TaskConstants.java
index fa2ac856fd..facefd011c 100755
--- 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/constant/TaskConstants.java
+++ 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/constant/TaskConstants.java
@@ -80,6 +80,7 @@ public class TaskConstants extends CommonConstants {
 public static final String TASK_START_TIME = "task.fileTask.startTime";
 public static final String TASK_END_TIME = "task.fileTask.endTime";
 public static final String FILE_MAX_NUM = "task.fileTask.maxFileCount";
+public static final String PREDEFINE_FIELDS = "task.predefineFields";
 
 // Binlog job
 public static final String JOB_DATABASE_USER = "job.binlogJob.user";
diff --git 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/TaskProfileDto.java
 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/TaskProfileDto.java
index 964ac1cf02..0fcf7a95f4 100644
--- 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/TaskProfileDto.java
+++ 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/TaskProfileDto.java
@@ -154,7 +154,7 @@ public class TaskProfileDto {
 }
 
 if (null != taskConfig.getLineEndPattern()) {
-FileTask.Line line = new Line();
+Line line = new Line();
 line.setEndPattern(taskConfig.getLineEndPattern());
 fileTask.setLine(line);
 }
@@ -410,6 +410,7 @@ public class TaskProfileDto {
 task.setUuid(dataConfig.getUuid());
 task.setVersion(dataConfig.getVersion());
 task.setState(dataConfig.getState());
+task.setPredefinedFields(dataConfig.getPredefinedFields());
 
 // set sink type
 if (dataConfig.getDataReportType() == 
NORMAL_SEND_TO_DATAPROXY.ordinal()) {
@@ -516,6 +517,7 @@ public class TaskProfileDto {
 private String mqClusters;
 private String topicInfo;
 private String taskClass;
+private String predefinedFields;
 private Integer state;
 
 private FileTask fileTask;
diff --git 
a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/LogFileSource.java
 
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/LogFileSource.java
index 1e187e2bf4..c056b3aae1 100755
--- 
a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/LogFileSource.java
+++ 
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/LogFileSource.java
@@ -32,6 +32,7 @@ import org.apache.inlong.agent.plugin.Message;
 import org.apache.inlong.agent.plugin.file.Reader;
 import org.apache.inlong.agent.plugin.sources.file.AbstractSource;
 import 
org.apache.inlong.agent.plugin.sources.reader.file.KubernetesMetadataProvider;
+import

(inlong) branch master updated: [INLONG-9214][Agent] Limit max file count to collect once (#9216)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new a891202393 [INLONG-9214][Agent] Limit max file count to collect once 
(#9216)
a891202393 is described below

commit a8912023936b5742ef6113f975ca0fa1eb6e5a83
Author: justinwwhuang 
AuthorDate: Fri Nov 3 17:40:13 2023 +0800

[INLONG-9214][Agent] Limit max file count to collect once (#9216)
---
 .../apache/inlong/agent/core/instance/InstanceManager.java | 11 +++
 .../inlong/agent/core/instance/TestInstanceManager.java|  2 +-
 .../inlong/agent/plugin/task/filecollect/FileScanner.java  | 14 ++
 .../agent/plugin/task/filecollect/LogFileCollectTask.java  |  5 +++--
 .../org/apache/inlong/agent/plugin/utils/PluginUtils.java  |  3 +--
 5 files changed, 18 insertions(+), 17 deletions(-)

diff --git 
a/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/instance/InstanceManager.java
 
b/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/instance/InstanceManager.java
index e209535db8..de96b93bc1 100644
--- 
a/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/instance/InstanceManager.java
+++ 
b/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/instance/InstanceManager.java
@@ -21,7 +21,6 @@ import org.apache.inlong.agent.common.AbstractDaemon;
 import org.apache.inlong.agent.common.AgentThreadFactory;
 import org.apache.inlong.agent.conf.AgentConfiguration;
 import org.apache.inlong.agent.conf.InstanceProfile;
-import org.apache.inlong.agent.constant.AgentConstants;
 import org.apache.inlong.agent.db.Db;
 import org.apache.inlong.agent.db.InstanceDb;
 import org.apache.inlong.agent.plugin.Instance;
@@ -62,7 +61,7 @@ public class InstanceManager extends AbstractDaemon {
 new SynchronousQueue<>(),
 new AgentThreadFactory("instance-manager"));
 
-private final int taskMaxLimit;
+private final int instanceLimit;
 private final AgentConfiguration agentConf;
 private final String taskId;
 private volatile boolean runAtLeastOneTime = false;
@@ -71,12 +70,12 @@ public class InstanceManager extends AbstractDaemon {
 /**
  * Init task manager.
  */
-public InstanceManager(String taskId, Db basicDb) {
+public InstanceManager(String taskId, int instanceLimit, Db basicDb) {
 this.taskId = taskId;
 instanceDb = new InstanceDb(basicDb);
 this.agentConf = AgentConfiguration.getAgentConf();
 instanceMap = new ConcurrentHashMap<>();
-taskMaxLimit = agentConf.getInt(AgentConstants.JOB_NUMBER_LIMIT, 
AgentConstants.DEFAULT_JOB_NUMBER_LIMIT);
+this.instanceLimit = instanceLimit;
 actionQueue = new LinkedBlockingQueue<>(ACTION_QUEUE_CAPACITY);
 }
 
@@ -236,6 +235,10 @@ public class InstanceManager extends AbstractDaemon {
 }
 
 private void addInstance(InstanceProfile profile) {
+if (instanceMap.size() >= instanceLimit) {
+LOGGER.error("instanceMap size {} over limit {}", 
instanceMap.size(), instanceLimit);
+return;
+}
 LOGGER.info("add instance taskId {} instanceId {}", taskId, 
profile.getInstanceId());
 if (!shouldAddAgain(profile.getInstanceId(), 
profile.getFileUpdateTime())) {
 LOGGER.info("shouldAddAgain returns false skip taskId {} 
instanceId {}", taskId, profile.getInstanceId());
diff --git 
a/inlong-agent/agent-core/src/test/java/org/apache/inlong/agent/core/instance/TestInstanceManager.java
 
b/inlong-agent/agent-core/src/test/java/org/apache/inlong/agent/core/instance/TestInstanceManager.java
index e858743cb3..b1107bb2ab 100755
--- 
a/inlong-agent/agent-core/src/test/java/org/apache/inlong/agent/core/instance/TestInstanceManager.java
+++ 
b/inlong-agent/agent-core/src/test/java/org/apache/inlong/agent/core/instance/TestInstanceManager.java
@@ -50,7 +50,7 @@ public class TestInstanceManager {
 String pattern = helper.getTestRootDir() + "/MMDD_[0-9]+.txt";
 Db basicDb = TaskManager.initDb("/localdb");
 taskProfile = helper.getTaskProfile(1, pattern, false, 0L, 0L, 
TaskStateEnum.RUNNING);
-manager = new InstanceManager("1", basicDb);
+manager = new InstanceManager("1", 2, basicDb);
 manager.start();
 }
 
diff --git 
a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/filecollect/FileScanner.java
 
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/filecollect/FileScanner.java
index bfb1a7a80a..efa87e22dc 100644
--- 
a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/filecollect/FileScanner.java
+++ 
b/inlong-agent/agent-plugins/src/

(inlong) branch master updated: [INLONG-9244][Agent] Fix bug: miss file from next data time (#9245)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new f9c28bad17 [INLONG-9244][Agent] Fix bug: miss file from next data time 
(#9245)
f9c28bad17 is described below

commit f9c28bad17278d193038ae77a88dba62b1c8a40d
Author: justinwwhuang 
AuthorDate: Thu Nov 9 09:56:25 2023 +0800

[INLONG-9244][Agent] Fix bug: miss file from next data time (#9245)
---
 .../agent/plugin/task/filecollect/LogFileCollectTask.java | 11 ---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git 
a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/filecollect/LogFileCollectTask.java
 
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/filecollect/LogFileCollectTask.java
index 1629b89595..26c61efa7a 100644
--- 
a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/filecollect/LogFileCollectTask.java
+++ 
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/filecollect/LogFileCollectTask.java
@@ -325,7 +325,7 @@ public class LogFileCollectTask extends Task {
 for (Map.Entry> entry : 
eventMap.entrySet()) {
 Map sameDataTimeEvents = entry.getValue();
 if (sameDataTimeEvents.isEmpty()) {
-return;
+continue;
 }
 /*
  * Calculate whether the event needs to be processed at the 
current time based on its data time, business
@@ -442,10 +442,10 @@ public class LogFileCollectTask extends Task {
 
 private void handleFilePath(Path filePath, WatchEntity entity) {
 String newFileName = filePath.toFile().getAbsolutePath();
-LOGGER.info("[New File] {} {}", newFileName, 
entity.getOriginPattern());
+LOGGER.info("new file {} {}", newFileName, entity.getOriginPattern());
 Matcher matcher = entity.getPattern().matcher(newFileName);
 if (matcher.matches() || matcher.lookingAt()) {
-LOGGER.info("[Matched File] {} {}", newFileName, 
entity.getOriginPattern());
+LOGGER.info("matched file {} {}", newFileName, 
entity.getOriginPattern());
 String dataTime = getDataTimeFromFileName(newFileName, 
entity.getOriginPattern(),
 entity.getDateExpression());
 if (!checkFileNameForTime(newFileName, entity)) {
@@ -458,10 +458,14 @@ public class LogFileCollectTask extends Task {
 
 private void addToEvenMap(String fileName, String dataTime) {
 if (isInEventMap(fileName, dataTime)) {
+LOGGER.info("addToEvenMap isInEventMap returns true skip taskId {} 
dataTime {} fileName {}",
+taskProfile.getTaskId(), dataTime, fileName);
 return;
 }
 Long fileUpdateTime = FileUtils.getFileLastModifyTime(fileName);
 if (!instanceManager.shouldAddAgain(fileName, fileUpdateTime)) {
+LOGGER.info("addToEvenMap shouldAddAgain returns false skip taskId 
{} dataTime {} fileName {}",
+taskProfile.getTaskId(), dataTime, fileName);
 return;
 }
 Map sameDataTimeEvents = 
eventMap.computeIfAbsent(dataTime,
@@ -474,6 +478,7 @@ public class LogFileCollectTask extends Task {
 InstanceProfile instanceProfile = 
taskProfile.createInstanceProfile(DEFAULT_FILE_INSTANCE,
 fileName, dataTime, fileUpdateTime);
 sameDataTimeEvents.put(fileName, instanceProfile);
+LOGGER.info("add to eventMap taskId {} dataTime {} fileName {}", 
taskProfile.getTaskId(), dataTime, fileName);
 }
 
 private boolean checkFileNameForTime(String newFileName, WatchEntity 
entity) {

(inlong) branch master updated: [INLONG-9241][Agent] Print task and instance detail every ten seconds (#9243)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new 4552466dfe [INLONG-9241][Agent] Print task and instance detail every 
ten seconds (#9243)
4552466dfe is described below

commit 4552466dfefed046a2188571a3213570244c7d9e
Author: justinwwhuang 
AuthorDate: Thu Nov 9 09:57:24 2023 +0800

[INLONG-9241][Agent] Print task and instance detail every ten seconds 
(#9243)
---
 .../agent/core/instance/InstanceManager.java   | 28 ++
 .../inlong/agent/core/task/file/TaskManager.java   | 17 +
 .../agent/core/instance/TestInstanceManager.java   |  1 +
 3 files changed, 42 insertions(+), 4 deletions(-)

diff --git 
a/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/instance/InstanceManager.java
 
b/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/instance/InstanceManager.java
index de96b93bc1..d4a278a974 100644
--- 
a/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/instance/InstanceManager.java
+++ 
b/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/instance/InstanceManager.java
@@ -47,7 +47,9 @@ public class InstanceManager extends AbstractDaemon {
 
 private static final Logger LOGGER = 
LoggerFactory.getLogger(InstanceManager.class);
 private static final int ACTION_QUEUE_CAPACITY = 100;
-public static final int CORE_THREAD_SLEEP_TIME = 100;
+public volatile int CORE_THREAD_SLEEP_TIME_MS = 1000;
+public static final int CORE_THREAD_PRINT_TIME = 1;
+private long lastPrintTime = 0;
 // task in db
 private final InstanceDb instanceDb;
 // task in memory
@@ -109,7 +111,8 @@ public class InstanceManager extends AbstractDaemon {
 running = true;
 while (isRunnable()) {
 try {
-AgentUtils.silenceSleepInMs(CORE_THREAD_SLEEP_TIME);
+AgentUtils.silenceSleepInMs(CORE_THREAD_SLEEP_TIME_MS);
+printInstanceDetail();
 dealWithActionQueue(actionQueue);
 keepPaceWithDb();
 } catch (Throwable ex) {
@@ -122,6 +125,22 @@ public class InstanceManager extends AbstractDaemon {
 };
 }
 
+private void printInstanceDetail() {
+if (AgentUtils.getCurrentTime() - lastPrintTime > 
CORE_THREAD_PRINT_TIME) {
+LOGGER.info("instanceManager coreThread running! taskId {} action 
count {}", taskId,
+actionQueue.size());
+List instances = instanceDb.getInstances(taskId);
+for (int i = 0; i < instances.size(); i++) {
+InstanceProfile instance = instances.get(i);
+LOGGER.info(
+"instanceManager coreThread instance taskId {} index 
{} total {} instanceId {} state {}",
+taskId, i,
+instances.size(), instance.getInstanceId(), 
instance.getState());
+}
+lastPrintTime = AgentUtils.getCurrentTime();
+}
+}
+
 private void keepPaceWithDb() {
 traverseDbTasksToMemory();
 traverseMemoryTasksToDb();
@@ -215,7 +234,7 @@ public class InstanceManager extends AbstractDaemon {
 public void waitForTerminate() {
 super.waitForTerminate();
 while (running) {
-AgentUtils.silenceSleepInMs(CORE_THREAD_SLEEP_TIME);
+AgentUtils.silenceSleepInMs(CORE_THREAD_SLEEP_TIME_MS);
 }
 }
 
@@ -241,7 +260,8 @@ public class InstanceManager extends AbstractDaemon {
 }
 LOGGER.info("add instance taskId {} instanceId {}", taskId, 
profile.getInstanceId());
 if (!shouldAddAgain(profile.getInstanceId(), 
profile.getFileUpdateTime())) {
-LOGGER.info("shouldAddAgain returns false skip taskId {} 
instanceId {}", taskId, profile.getInstanceId());
+LOGGER.info("addInstance shouldAddAgain returns false skip taskId 
{} instanceId {}", taskId,
+profile.getInstanceId());
 return;
 }
 addToDb(profile);
diff --git 
a/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/task/file/TaskManager.java
 
b/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/task/file/TaskManager.java
index 9aa71a96ab..b4cb79a48c 100644
--- 
a/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/task/file/TaskManager.java
+++ 
b/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/task/file/TaskManager.java
@@ -56,7 +56,9 @@ public class TaskManager extends AbstractDaemon {
 private static final Logger LOGGER = 
LoggerFactory.getLogger(TaskManager.class);
 public static final int CONFIG_QUEUE_CAPACITY = 1;
 public sta

(inlong) branch master updated: [INLONG-9300][Agent] Divide data time into source time and sink time (#9301)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new f7827601fe [INLONG-9300][Agent] Divide data time into source time and 
sink time (#9301)
f7827601fe is described below

commit f7827601fec3f1d772a7065ba7e0afae23d998a6
Author: justinwwhuang 
AuthorDate: Thu Nov 16 17:32:22 2023 +0800

[INLONG-9300][Agent] Divide data time into source time and sink time (#9301)
---
 .../apache/inlong/agent/conf/InstanceProfile.java  | 18 +-
 .../org/apache/inlong/agent/conf/TaskProfile.java  | 22 +-
 .../inlong/agent/constant/TaskConstants.java   |  8 ++--
 .../message/filecollect/ProxyMessageCache.java |  2 +-
 .../org/apache/inlong/agent/pojo/FileTask.java |  3 +++
 .../apache/inlong/agent/pojo/TaskProfileDto.java   |  3 +++
 .../inlong/agent/core/AgentBaseTestsHelper.java|  4 +++-
 .../agent/core/instance/TestInstanceManager.java   |  4 ++--
 .../inlong/agent/plugin/sources/LogFileSource.java |  2 +-
 .../inlong/agent/plugin/AgentBaseTestsHelper.java  |  2 ++
 10 files changed, 55 insertions(+), 13 deletions(-)

diff --git 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/conf/InstanceProfile.java
 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/conf/InstanceProfile.java
index 5c3e74fe86..acc6444aba 100644
--- 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/conf/InstanceProfile.java
+++ 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/conf/InstanceProfile.java
@@ -155,18 +155,26 @@ public class InstanceProfile extends 
AbstractConfiguration implements Comparable
 set(TaskConstants.INSTANCE_ID, instanceId);
 }
 
-public void setDataTime(String dataTime) {
-set(TaskConstants.JOB_DATA_TIME, dataTime);
+public void setSourceDataTime(String dataTime) {
+set(TaskConstants.SOURCE_DATA_TIME, dataTime);
 }
 
-public String getDataTime() {
-return get(TaskConstants.JOB_DATA_TIME);
+public String getSourceDataTime() {
+return get(TaskConstants.SOURCE_DATA_TIME);
+}
+
+public void setSinkDataTime(Long dataTime) {
+setLong(TaskConstants.SINK_DATA_TIME, dataTime);
+}
+
+public Long getSinkDataTime() {
+return getLong(TaskConstants.SINK_DATA_TIME, 0);
 }
 
 @Override
 public int compareTo(InstanceProfile object) {
 int ret = ComparisonChain.start()
-.compare(getDataTime(), object.getDataTime())
+.compare(getSourceDataTime(), object.getSourceDataTime())
 .compare(FileUtils.getFileCreationTime(getInstanceId()),
 FileUtils.getFileCreationTime(object.getInstanceId()))
 .compare(FileUtils.getFileLastModifyTime(getInstanceId()),
diff --git 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/conf/TaskProfile.java
 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/conf/TaskProfile.java
index 319f1abf56..be9b8cd1f3 100644
--- 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/conf/TaskProfile.java
+++ 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/conf/TaskProfile.java
@@ -20,11 +20,17 @@ package org.apache.inlong.agent.conf;
 import org.apache.inlong.agent.constant.TaskConstants;
 import org.apache.inlong.agent.pojo.TaskProfileDto;
 import org.apache.inlong.agent.utils.AgentUtils;
+import org.apache.inlong.agent.utils.DateTransUtils;
 import org.apache.inlong.common.enums.InstanceStateEnum;
 import org.apache.inlong.common.enums.TaskStateEnum;
 import org.apache.inlong.common.pojo.agent.DataConfig;
 
 import com.google.gson.Gson;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.text.ParseException;
+import java.util.TimeZone;
 
 import static org.apache.inlong.agent.constant.TaskConstants.TASK_RETRY;
 import static org.apache.inlong.agent.constant.TaskConstants.TASK_STATE;
@@ -35,6 +41,7 @@ import static 
org.apache.inlong.agent.constant.TaskConstants.TASK_STATE;
 public class TaskProfile extends AbstractConfiguration {
 
 private static final Gson GSON = new Gson();
+private static final Logger logger = 
LoggerFactory.getLogger(TaskProfile.class);
 
 /**
  * Get a TaskProfile from a DataConfig
@@ -58,6 +65,10 @@ public class TaskProfile extends AbstractConfiguration {
 return get(TaskConstants.TASK_FILE_TIME_OFFSET);
 }
 
+public String getTimeZone() {
+return get(TaskConstants.TASK_FILE_TIME_ZONE);
+}
+
 public TaskStateEnum getState() {
 return TaskStateEnum.getTaskState(getInt(TASK_STATE));
 }
@@ -111,7 +122,16 @@ public class TaskProfile extends AbstractConfiguration {
 InstanceProfile instanceProfile = 
InstanceProfile.parseJsonStr(toJsonStr

(inlong) branch master updated: [INLONG-9308][Agent] The sink end of the file instance supports sending data with different streamIds (#9309)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new c3bdf56a92 [INLONG-9308][Agent] The sink end of the file instance 
supports sending data with different streamIds (#9309)
c3bdf56a92 is described below

commit c3bdf56a92d57a992e4639dbeb1f19e32c2d14d4
Author: justinwwhuang 
AuthorDate: Mon Nov 20 15:50:05 2023 +0800

[INLONG-9308][Agent] The sink end of the file instance supports sending 
data with different streamIds (#9309)
---
 .../org/apache/inlong/agent/conf/TaskProfile.java  |  10 +-
 .../{PackageAckInfo.java => OffsetAckInfo.java}|   7 +-
 .../agent/message/filecollect/ProxyMessage.java| 100 +
 .../message/filecollect/ProxyMessageCache.java |  72 +-
 .../agent/message/filecollect/SenderMessage.java   |   2 +-
 .../apache/inlong/agent/utils/DateTransUtils.java  |  10 +-
 .../agent/plugin/sinks/filecollect/ProxySink.java  | 155 ++---
 .../plugin/sinks/filecollect/SenderManager.java|  95 +
 .../inlong/agent/plugin/sources/LogFileSource.java |   8 +-
 .../agent/plugin/task/filecollect/WatchEntity.java |   2 +-
 .../agent/plugin/utils/file/NewDateUtils.java  |   6 +-
 .../sinks/filecollect/TestSenderManager.java   |  42 +++---
 12 files changed, 295 insertions(+), 214 deletions(-)

diff --git 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/conf/TaskProfile.java
 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/conf/TaskProfile.java
index be9b8cd1f3..cbad21e499 100644
--- 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/conf/TaskProfile.java
+++ 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/conf/TaskProfile.java
@@ -110,7 +110,8 @@ public class TaskProfile extends AbstractConfiguration {
 return hasKey(TaskConstants.TASK_ID) && 
hasKey(TaskConstants.TASK_SOURCE)
 && hasKey(TaskConstants.TASK_SINK) && 
hasKey(TaskConstants.TASK_CHANNEL)
 && hasKey(TaskConstants.TASK_GROUP_ID) && 
hasKey(TaskConstants.TASK_STREAM_ID)
-&& hasKey(TaskConstants.TASK_CYCLE_UNIT);
+&& hasKey(TaskConstants.TASK_CYCLE_UNIT)
+&& hasKey(TaskConstants.TASK_FILE_TIME_ZONE);
 }
 
 public String toJsonStr() {
@@ -125,10 +126,13 @@ public class TaskProfile extends AbstractConfiguration {
 instanceProfile.setSourceDataTime(dataTime);
 Long sinkDataTime = 0L;
 try {
-sinkDataTime = DateTransUtils.timeStrConvertTomillSec(dataTime, 
getCycleUnit(),
+sinkDataTime = DateTransUtils.timeStrConvertToMillSec(dataTime, 
getCycleUnit(),
 TimeZone.getTimeZone(getTimeZone()));
 } catch (ParseException e) {
-logger.error("createInstanceProfile error: ", e);
+logger.error("createInstanceProfile ParseException error: ", e);
+return null;
+} catch (Exception e) {
+logger.error("createInstanceProfile Exception error: ", e);
 return null;
 }
 instanceProfile.setSinkDataTime(sinkDataTime);
diff --git 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/message/filecollect/PackageAckInfo.java
 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/message/filecollect/OffsetAckInfo.java
similarity index 88%
rename from 
inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/message/filecollect/PackageAckInfo.java
rename to 
inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/message/filecollect/OffsetAckInfo.java
index 6efdbbdc1e..f6637955bc 100644
--- 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/message/filecollect/PackageAckInfo.java
+++ 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/message/filecollect/OffsetAckInfo.java
@@ -19,15 +19,12 @@ package org.apache.inlong.agent.message.filecollect;
 
 import lombok.AllArgsConstructor;
 import lombok.Data;
-import lombok.NoArgsConstructor;
 
 @Data
 @AllArgsConstructor
-@NoArgsConstructor
-public class PackageAckInfo {
+public class OffsetAckInfo {
 
-private Long index;
 private Long offset;
-private Integer len;
+private int len;
 private Boolean hasAck;
 }
diff --git 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/message/filecollect/ProxyMessage.java
 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/message/filecollect/ProxyMessage.java
new file mode 100644
index 00..7d9f4930ac
--- /dev/null
+++ 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/message/filecollect/ProxyMessage.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Found

(inlong) branch master updated: [INLONG-9310][Agent] Add extended handler in file source (#9311)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new fc8ef40ce0 [INLONG-9310][Agent] Add extended handler in file source 
(#9311)
fc8ef40ce0 is described below

commit fc8ef40ce0f59bae6b2876e1c9eabe8d0851d648
Author: justinwwhuang 
AuthorDate: Mon Nov 20 20:09:10 2023 +0800

[INLONG-9310][Agent] Add extended handler in file source (#9311)

* [INLONG-9310][Agent] Add extended handler in file source

* [INLONG-9310][Agent] Add extended handler in file source

* [INLONG-9310][Agent] Add extended handler in file source
---
 .../inlong/agent/constant/TaskConstants.java   |  3 ++
 .../agent/message/filecollect/ProxyMessage.java|  2 +-
 .../message/filecollect/ProxyMessageCache.java |  2 --
 .../inlong/agent/plugin/sources/LogFileSource.java | 15 +
 .../sources/file/extend/ExtendedHandler.java   | 39 ++
 5 files changed, 58 insertions(+), 3 deletions(-)

diff --git 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/constant/TaskConstants.java
 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/constant/TaskConstants.java
index 872c42319f..c501ec110b 100755
--- 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/constant/TaskConstants.java
+++ 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/constant/TaskConstants.java
@@ -82,6 +82,9 @@ public class TaskConstants extends CommonConstants {
 public static final String TASK_END_TIME = "task.fileTask.endTime";
 public static final String FILE_MAX_NUM = "task.fileTask.maxFileCount";
 public static final String PREDEFINE_FIELDS = "task.predefinedFields";
+public static final String FILE_SOURCE_EXTEND_CLASS = 
"task.fileTask.extendedClass";
+public static final String DEFAULT_FILE_SOURCE_EXTEND_CLASS =
+
"org.apache.inlong.agent.plugin.sources.file.extend.ExtendedHandler";
 
 // Binlog job
 public static final String JOB_DATABASE_USER = "job.binlogJob.user";
diff --git 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/message/filecollect/ProxyMessage.java
 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/message/filecollect/ProxyMessage.java
index 7d9f4930ac..e8b74f40b1 100644
--- 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/message/filecollect/ProxyMessage.java
+++ 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/message/filecollect/ProxyMessage.java
@@ -27,7 +27,7 @@ import static 
org.apache.inlong.agent.constant.CommonConstants.PROXY_KEY_GROUP_I
 import static 
org.apache.inlong.agent.constant.CommonConstants.PROXY_KEY_STREAM_ID;
 
 /**
- * Bus message with body, header, inlongGroupId and inlongStreamId.
+ * proxy message with body, header, inlongGroupId and inlongStreamId.
  */
 public class ProxyMessage implements Message {
 
diff --git 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/message/filecollect/ProxyMessageCache.java
 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/message/filecollect/ProxyMessageCache.java
index 9c0c84e0ef..66b3fb3b43 100644
--- 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/message/filecollect/ProxyMessageCache.java
+++ 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/message/filecollect/ProxyMessageCache.java
@@ -59,7 +59,6 @@ public class ProxyMessageCache {
 private final int cacheTimeout;
 // streamId -> list of proxyMessage
 private final ConcurrentHashMap> 
messageQueueMap;
-// private final LinkedBlockingQueue messageQueue;
 private final AtomicLong cacheSize = new AtomicLong(0);
 private long lastPrintTime = 0;
 private long dataTime;
@@ -77,7 +76,6 @@ public class ProxyMessageCache {
 DEFAULT_PROXY_INLONG_STREAM_ID_QUEUE_MAX_NUMBER);
 this.cacheTimeout = 
instanceProfile.getInt(PROXY_PACKAGE_MAX_TIMEOUT_MS, 
DEFAULT_PROXY_PACKAGE_MAX_TIMEOUT_MS);
 messageQueueMap = new ConcurrentHashMap<>();
-// this.messageQueue = new LinkedBlockingQueue<>(maxQueueNumber);
 try {
 dataTime = 
DateTransUtils.timeStrConvertToMillSec(instanceProfile.getSourceDataTime(),
 instanceProfile.get(TASK_CYCLE_UNIT));
diff --git 
a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/LogFileSource.java
 
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/LogFileSource.java
index e2b0b06517..d713fb4e8a 100755
--- 
a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/LogFileSource.java
+++ 
b/inlong-agent/agent-plugins/src/main/java/org/

(inlong) branch master updated: [INLONG-9312][Agent] Add data content style (#9313)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new 452951d9f4 [INLONG-9312][Agent] Add data content style (#9313)
452951d9f4 is described below

commit 452951d9f43c3e28063cf3cefe7ebdab79863599
Author: justinwwhuang 
AuthorDate: Tue Nov 21 18:42:10 2023 +0800

[INLONG-9312][Agent] Add data content style (#9313)

* [INLONG-9312][Agent] Add data content style

* [INLONG-9312][Agent] Add data content style
---
 .../src/main/java/org/apache/inlong/agent/constant/TaskConstants.java | 3 ++-
 .../src/main/java/org/apache/inlong/agent/pojo/FileTask.java  | 2 ++
 .../src/main/java/org/apache/inlong/agent/pojo/TaskProfileDto.java| 1 +
 .../java/org/apache/inlong/agent/plugin/AgentBaseTestsHelper.java | 4 
 .../org/apache/inlong/agent/plugin/sources/TestLogFileSource.java | 1 -
 5 files changed, 9 insertions(+), 2 deletions(-)

diff --git 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/constant/TaskConstants.java
 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/constant/TaskConstants.java
index c501ec110b..37dd2d5da3 100755
--- 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/constant/TaskConstants.java
+++ 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/constant/TaskConstants.java
@@ -73,7 +73,8 @@ public class TaskConstants extends CommonConstants {
 public static final String JOB_FILE_META_ENV_LIST = "job.fileTask.envList";
 public static final String JOB_FILE_META_FILTER_BY_LABELS = 
"job.fileTask.filterMetaByLabels";
 public static final String JOB_FILE_PROPERTIES = "job.fileTask.properties";
-public static final String JOB_FILE_DATA_SOURCE_COLUMN_SEPARATOR = 
"job.fileTask.dataSeparator";
+public static final String SOURCE_DATA_CONTENT_STYLE = 
"task.fileTask.dataContentStyle";
+public static final String SOURCE_DATA_SEPARATOR = 
"task.fileTask.dataSeparator";
 public static final String JOB_FILE_MONITOR_INTERVAL = 
"job.fileTask.monitorInterval";
 public static final String JOB_FILE_MONITOR_STATUS = 
"job.fileTask.monitorStatus";
 public static final String JOB_FILE_MONITOR_EXPIRE = 
"job.fileTask.monitorExpire";
diff --git 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/FileTask.java
 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/FileTask.java
index ec8ce9f47f..f31ec703c2 100644
--- 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/FileTask.java
+++ 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/FileTask.java
@@ -48,6 +48,8 @@ public class FileTask {
 // JSON string, the content format is List>
 private String metaFields;
 
+private String dataContentStyle;
+
 private String dataSeparator;
 
 // JSON string, the content format is Map
diff --git 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/TaskProfileDto.java
 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/TaskProfileDto.java
index 6d8cd16816..6ce6ba2d3d 100644
--- 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/TaskProfileDto.java
+++ 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/TaskProfileDto.java
@@ -138,6 +138,7 @@ public class TaskProfileDto {
 fileTask.setDir(dir);
 fileTask.setCollectType(taskConfig.getCollectType());
 fileTask.setContentCollectType(taskConfig.getContentCollectType());
+fileTask.setDataContentStyle(taskConfig.getDataContentStyle());
 fileTask.setDataSeparator(taskConfig.getDataSeparator());
 fileTask.setMaxFileCount(taskConfig.getMaxFileCount());
 fileTask.setRetry(taskConfig.getRetry());
diff --git 
a/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/AgentBaseTestsHelper.java
 
b/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/AgentBaseTestsHelper.java
index 2410c07ff3..465180cb8f 100755
--- 
a/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/AgentBaseTestsHelper.java
+++ 
b/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/AgentBaseTestsHelper.java
@@ -104,6 +104,10 @@ public class AgentBaseTestsHelper {
 fileTaskConfig.setRetry(retry);
 fileTaskConfig.setStartTime(startTime);
 fileTaskConfig.setEndTime(endTime);
+// mix: login|87601|968|67826|23579 or login|a=b&c=d&x=y&asdf
+fileTaskConfig.setDataContentStyle("mix");
+// 124 is the ASCII code of '|'
+fileTaskConfig.setDataSeparator("124");
 dataConfig.setExtParams(GSON.toJ

(inlong) branch master updated: [INLONG-9335][Agent] Bring cycle parameters when creating an instance (#9336)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new 8a97c353dd [INLONG-9335][Agent] Bring cycle parameters when creating 
an instance (#9336)
8a97c353dd is described below

commit 8a97c353ddcabf70e319d9e006293dda383d15ae
Author: justinwwhuang 
AuthorDate: Mon Nov 27 14:43:23 2023 +0800

[INLONG-9335][Agent] Bring cycle parameters when creating an instance 
(#9336)

* [INLONG-9335][Agent] Bring cycle parameters when creating an instance

* [INLONG-9335][Agent] Bring cycle parameters when creating an instance
---
 .../org/apache/inlong/agent/conf/TaskProfile.java  |  7 ++-
 .../inlong/agent/constant/CycleUnitType.java   | 25 
 .../inlong/agent/constant/FileTriggerType.java | 66 -
 .../apache/inlong/agent/utils/DateTransUtils.java  |  2 +-
 .../agent/core/instance/TestInstanceManager.java   |  6 +-
 .../agent/plugin/task/filecollect/FileScanner.java | 20 +++
 .../task/filecollect/LogFileCollectTask.java   | 68 +++---
 .../inlong/agent/plugin/utils/file/DateUtils.java  |  6 +-
 .../agent/plugin/utils/file/NewDateUtils.java  |  3 +
 .../sinks/filecollect/TestSenderManager.java   |  2 +-
 .../agent/plugin/sources/TestLogFileSource.java|  4 +-
 .../agent/plugin/sources/TestMqttConnect.java  |  3 +-
 .../inlong/agent/plugin/utils/TestUtils.java   |  1 +
 13 files changed, 100 insertions(+), 113 deletions(-)

diff --git 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/conf/TaskProfile.java
 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/conf/TaskProfile.java
index cbad21e499..de863a7aa0 100644
--- 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/conf/TaskProfile.java
+++ 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/conf/TaskProfile.java
@@ -62,7 +62,7 @@ public class TaskProfile extends AbstractConfiguration {
 }
 
 public String getTimeOffset() {
-return get(TaskConstants.TASK_FILE_TIME_OFFSET);
+return get(TaskConstants.TASK_FILE_TIME_OFFSET, "");
 }
 
 public String getTimeZone() {
@@ -118,7 +118,8 @@ public class TaskProfile extends AbstractConfiguration {
 return GSON.toJson(getConfigStorage());
 }
 
-public InstanceProfile createInstanceProfile(String instanceClass, String 
fileName, String dataTime,
+public InstanceProfile createInstanceProfile(String instanceClass, String 
fileName, String cycleUnit,
+String dataTime,
 long fileUpdateTime) {
 InstanceProfile instanceProfile = 
InstanceProfile.parseJsonStr(toJsonStr());
 instanceProfile.setInstanceClass(instanceClass);
@@ -126,7 +127,7 @@ public class TaskProfile extends AbstractConfiguration {
 instanceProfile.setSourceDataTime(dataTime);
 Long sinkDataTime = 0L;
 try {
-sinkDataTime = DateTransUtils.timeStrConvertToMillSec(dataTime, 
getCycleUnit(),
+sinkDataTime = DateTransUtils.timeStrConvertToMillSec(dataTime, 
cycleUnit,
 TimeZone.getTimeZone(getTimeZone()));
 } catch (ParseException e) {
 logger.error("createInstanceProfile ParseException error: ", e);
diff --git 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/constant/CycleUnitType.java
 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/constant/CycleUnitType.java
new file mode 100644
index 00..d12e825b85
--- /dev/null
+++ 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/constant/CycleUnitType.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.inlong.agent.constant;
+
+public class CycleUnitType {
+
+public static final String DAY = "D";
+public static final String HOUR = "h";
+public static final String REAL_TIME = "R";
+}
diff --git 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/constant/FileT

(inlong) branch master updated: [INLONG-9341][SDK] Optimize obtaining local IP information (#9342)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new fad216ff9a [INLONG-9341][SDK] Optimize obtaining local IP information 
(#9342)
fad216ff9a is described below

commit fad216ff9a3b3dca3138871e131dbc84efb47cef
Author: doleyzi <43397300+dole...@users.noreply.github.com>
AuthorDate: Tue Nov 28 10:25:38 2023 +0800

[INLONG-9341][SDK] Optimize obtaining local IP information (#9342)

* Optimize obtaining local IP information

* Optimize obtaining local IP information
---
 .../dataproxy-sdk-cpp/release/inc/sdk_conf.h   |  2 +-
 .../dataproxy-sdk-cpp/src/config/sdk_conf.cc   | 67 --
 .../dataproxy-sdk-cpp/src/core/api_imp.cc  |  3 +-
 3 files changed, 53 insertions(+), 19 deletions(-)

diff --git 
a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/release/inc/sdk_conf.h 
b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/release/inc/sdk_conf.h
index aa451caee9..578278c5fd 100644
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/release/inc/sdk_conf.h
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/release/inc/sdk_conf.h
@@ -40,7 +40,7 @@ private:
   void InitTcpParam(const rapidjson::Value &doc);
   void InitAuthParm(const rapidjson::Value &doc);
   void OthersParam(const rapidjson::Value &doc);
-  void InitLocalIp();
+  bool GetLocalIPV4Address(std::string& err_info, std::string& localhost);
 
   public:
   // cache parameter
diff --git 
a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/config/sdk_conf.cc 
b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/config/sdk_conf.cc
index 70cd2362e2..e9d679cd8c 100644
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/config/sdk_conf.cc
+++ b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/config/sdk_conf.cc
@@ -65,7 +65,14 @@ bool SdkConfig::ParseConfig(const std::string &config_path) {
   InitLogParam(doc);
   InitManagerParam(doc);
   InitTcpParam(doc);
-  InitLocalIp();
+
+  std::string err, local_ip;
+  if (GetLocalIPV4Address(err, local_ip)) {
+local_ip_ = local_ip;
+  } else {
+local_ip_ = constants::kSerIP;
+  }
+
   OthersParam(doc);
   InitAuthParm(doc);
 
@@ -469,24 +476,52 @@ void SdkConfig::OthersParam(const rapidjson::Value &doc) {
   }
 }
 
-void SdkConfig::InitLocalIp() {
-  struct sockaddr_in dest;
-  dest.sin_family = AF_INET;
-
-  int fd = socket(AF_INET, SOCK_DGRAM, 0);
-  if (fd <= 0) {
-throw std::runtime_error(std::string("socket failed") + strerror(errno));
+bool SdkConfig::GetLocalIPV4Address(std::string& err_info, std::string& 
localhost) {
+  int32_t sockfd;
+  int32_t ip_num = 0;
+  char  buf[1024] = {0};
+  struct ifreq *ifreq;
+  struct ifreq if_flag;
+  struct ifconf ifconf;
+
+  ifconf.ifc_len = sizeof(buf);
+  ifconf.ifc_buf = buf;
+  if ((sockfd = socket(AF_INET, SOCK_DGRAM, 0)) < 0) {
+err_info = "Open the local socket(AF_INET, SOCK_DGRAM) failure!";
+return false;
   }
 
-  struct ifreq ifreq_Buf;
-  strcpy(ifreq_Buf.ifr_name, "eth1"); // just check the eth1
-  if (-1 == ioctl(fd, SIOCGIFADDR, &ifreq_Buf)) {
-throw std::runtime_error(std::string("ioctl failed") + strerror(errno));
-  }
+  ioctl(sockfd, SIOCGIFCONF, &ifconf);
+  ifreq  = (struct ifreq *)buf;
+  ip_num = ifconf.ifc_len / sizeof(struct ifreq);
+  for (int32_t i = 0; i < ip_num; i++, ifreq++) {
+if (ifreq->ifr_flags != AF_INET) {
+  continue;
+}
+if (0 == strncmp(&ifreq->ifr_name[0], "lo", sizeof("lo"))) {
+  continue;
+}
+memcpy(&if_flag.ifr_name[0], &ifreq->ifr_name[0], sizeof(ifreq->ifr_name));
+if ((ioctl(sockfd, SIOCGIFFLAGS, (char *) &if_flag)) < 0) {
+  continue;
+}
+if ((if_flag.ifr_flags & IFF_LOOPBACK)
+|| !(if_flag.ifr_flags & IFF_UP)) {
+  continue;
+}
 
-  close(fd);
-  struct sockaddr_in *addr = (struct sockaddr_in *)&ifreq_Buf.ifr_addr;
-  local_ip_ = inet_ntoa(addr->sin_addr);
+if (!strncmp(inet_ntoa(((struct 
sockaddr_in*)&(ifreq->ifr_addr))->sin_addr),
+ "127.0.0.1", 7)) {
+  continue;
+}
+localhost = inet_ntoa(((struct sockaddr_in*)&(ifreq->ifr_addr))->sin_addr);
+close(sockfd);
+err_info = "Ok";
+return true;
+  }
+  close(sockfd);
+  err_info = "Not found the localHost in local OS";
+  return false;
 }
 
 void SdkConfig::ShowClientConfig() {
diff --git 
a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/core/api_imp.cc 
b/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/core/api_imp.cc
index 79706a511e..c4b493b068 100644
--- a/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-cpp/src/core/api_imp.cc
+++ b/inlong-s

(inlong) branch master updated: [INLONG-9364][Agent] Remove expired instance from db (#9365)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new f62b2e87a1 [INLONG-9364][Agent] Remove expired instance from db (#9365)
f62b2e87a1 is described below

commit f62b2e87a12d5c7c2d75417715d3edb422230dab
Author: justinwwhuang 
AuthorDate: Thu Nov 30 11:16:23 2023 +0800

[INLONG-9364][Agent] Remove expired instance from db (#9365)
---
 .../apache/inlong/agent/utils/DateTransUtils.java  | 40 
 .../agent/core/instance/InstanceManager.java   | 74 ++
 .../inlong/agent/core/task/file/TaskManager.java   | 22 +--
 .../agent/core/instance/TestInstanceManager.java   |  6 +-
 .../agent/plugin/task/filecollect/FileScanner.java |  4 +-
 .../task/filecollect/LogFileCollectTask.java   |  6 +-
 .../agent/plugin/utils/file/NewDateUtils.java  | 44 +
 .../inlong/agent/plugin/utils/TestUtils.java   | 14 ++--
 .../apache/inlong/common/enums/TaskStateEnum.java  |  4 +-
 9 files changed, 139 insertions(+), 75 deletions(-)

diff --git 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/utils/DateTransUtils.java
 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/utils/DateTransUtils.java
index 55182c7dd8..fe6257d64e 100644
--- 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/utils/DateTransUtils.java
+++ 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/utils/DateTransUtils.java
@@ -111,4 +111,44 @@ public class DateTransUtils {
 return retTime;
 }
 
+/**
+ * Calculate offset time based on offset
+ * The current offset will only be offset forward, or it can be offset 
backward to be compatible with the previous
+ * calculation method (subtraction).
+ * When it is offset backward, it returns negative;
+ * When offset forward, return positive
+ *
+ * @param timeOffset offset，such as -1d,-4h,-10m；
+ * @return
+ */
+public static long calcOffset(String timeOffset) {
+if (timeOffset.length() == 0) {
+return 0;
+}
+String offsetUnit = timeOffset.substring(timeOffset.length() - 1);
+int startIndex;
+int symbol;
+if (timeOffset.charAt(0) == '-') {
+symbol = -1;
+startIndex = 1;
+} else {
+symbol = 1;
+startIndex = 0;
+}
+
+String strOffset = timeOffset.substring(startIndex, 
timeOffset.length() - 1);
+if (strOffset.length() == 0) {
+return 0;
+}
+int offsetTime = Integer.parseInt(strOffset);
+if ("d".equalsIgnoreCase(offsetUnit)) {
+return offsetTime * 24 * 3600 * 1000 * symbol;
+} else if ("h".equalsIgnoreCase(offsetUnit)) {
+return offsetTime * 3600 * 1000 * symbol;
+} else if ("m".equalsIgnoreCase(offsetUnit)) {
+return offsetTime * 60 * 1000 * symbol;
+}
+return 0;
+}
+
 }
diff --git 
a/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/instance/InstanceManager.java
 
b/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/instance/InstanceManager.java
index a80ce8b53b..260e5a477f 100644
--- 
a/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/instance/InstanceManager.java
+++ 
b/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/instance/InstanceManager.java
@@ -21,16 +21,22 @@ import org.apache.inlong.agent.common.AbstractDaemon;
 import org.apache.inlong.agent.common.AgentThreadFactory;
 import org.apache.inlong.agent.conf.AgentConfiguration;
 import org.apache.inlong.agent.conf.InstanceProfile;
+import org.apache.inlong.agent.conf.TaskProfile;
+import org.apache.inlong.agent.constant.CycleUnitType;
 import org.apache.inlong.agent.db.Db;
 import org.apache.inlong.agent.db.InstanceDb;
+import org.apache.inlong.agent.db.TaskProfileDb;
 import org.apache.inlong.agent.plugin.Instance;
 import org.apache.inlong.agent.utils.AgentUtils;
+import org.apache.inlong.agent.utils.DateTransUtils;
 import org.apache.inlong.agent.utils.ThreadUtils;
 import org.apache.inlong.common.enums.InstanceStateEnum;
+import org.apache.inlong.common.enums.TaskStateEnum;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import java.util.Iterator;
 import java.util.List;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ConcurrentHashMap;
@@ -38,6 +44,7 @@ import java.util.concurrent.LinkedBlockingQueue;
 import java.util.concurrent.SynchronousQueue;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 
 /**
  * handle the instance created by task, including add, delete, update etc.
@@ -47,11 +54,14

(inlong) branch master updated: [INLONG-9369][Agent] Increase sending failure audit and real-time audit (#9370)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new af93ac24b7 [INLONG-9369][Agent] Increase sending failure audit and 
real-time audit (#9370)
af93ac24b7 is described below

commit af93ac24b72b90e488233a87e1a2fd62a7f9ad72
Author: justinwwhuang 
AuthorDate: Thu Nov 30 18:10:49 2023 +0800

[INLONG-9369][Agent] Increase sending failure audit and real-time audit 
(#9370)

* [INLONG-9369][Agent] Increase sending failure audit and real-time audit

* [INLONG-9369][Agent] Increase sending failure audit and real-time audit
---
 .../main/java/org/apache/inlong/agent/metrics/audit/AuditUtils.java | 4 
 .../apache/inlong/agent/plugin/sinks/filecollect/SenderManager.java | 6 ++
 .../java/org/apache/inlong/agent/plugin/sources/LogFileSource.java  | 4 
 3 files changed, 14 insertions(+)

diff --git 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/metrics/audit/AuditUtils.java
 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/metrics/audit/AuditUtils.java
index 68a4e64842..b4eda2ce60 100644
--- 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/metrics/audit/AuditUtils.java
+++ 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/metrics/audit/AuditUtils.java
@@ -42,6 +42,10 @@ public class AuditUtils {
 public static final int AUDIT_DEFAULT_MAX_CACHE_ROWS = 200;
 public static final int AUDIT_ID_AGENT_READ_SUCCESS = 3;
 public static final int AUDIT_ID_AGENT_SEND_SUCCESS = 4;
+public static final int AUDIT_ID_AGENT_READ_SUCCESS_REAL_TIME = 25;
+public static final int AUDIT_ID_AGENT_SEND_SUCCESS_REAL_TIME = 26;
+public static final int AUDIT_ID_AGENT_SEND_FAILED = 10004;
+public static final int AUDIT_ID_AGENT_SEND_FAILED_REAL_TIME = 10026;
 
 private static boolean IS_AUDIT = true;
 
diff --git 
a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sinks/filecollect/SenderManager.java
 
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sinks/filecollect/SenderManager.java
index 45fe9bc63e..d027672b18 100755
--- 
a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sinks/filecollect/SenderManager.java
+++ 
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sinks/filecollect/SenderManager.java
@@ -346,11 +346,17 @@ public class SenderManager {
 getMetricItem(groupId, 
streamId).pluginSendSuccessCount.addAndGet(msgCnt);
 AuditUtils.add(AuditUtils.AUDIT_ID_AGENT_SEND_SUCCESS, 
groupId, streamId,
 dataTime, message.getMsgCnt(), message.getTotalSize());
+
AuditUtils.add(AuditUtils.AUDIT_ID_AGENT_SEND_SUCCESS_REAL_TIME, groupId, 
streamId,
+AgentUtils.getCurrentTime(), message.getMsgCnt(), 
message.getTotalSize());
 } else {
 LOGGER.warn("send groupId {}, streamId {}, taskId {}, 
instanceId {}, dataTime {} fail with times {}, "
 + "error {}", groupId, streamId, taskId, instanceId, 
dataTime, retry, result);
 getMetricItem(groupId, 
streamId).pluginSendFailCount.addAndGet(msgCnt);
 putInResendQueue(new AgentSenderCallback(message, retry));
+AuditUtils.add(AuditUtils.AUDIT_ID_AGENT_SEND_FAILED, groupId, 
streamId,
+dataTime, message.getMsgCnt(), message.getTotalSize());
+
AuditUtils.add(AuditUtils.AUDIT_ID_AGENT_SEND_FAILED_REAL_TIME, groupId, 
streamId,
+AgentUtils.getCurrentTime(), message.getMsgCnt(), 
message.getTotalSize());
 }
 }
 
diff --git 
a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/LogFileSource.java
 
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/LogFileSource.java
index 7c719b1e47..0cc97afb8c 100755
--- 
a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/LogFileSource.java
+++ 
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/sources/LogFileSource.java
@@ -319,6 +319,8 @@ public class LogFileSource extends AbstractSource {
 if (overLen) {
 LOGGER.warn("readLines over len finally string len 
{}",
 new String(baos.toByteArray()).length());
+
AuditUtils.add(AuditUtils.AUDIT_ID_AGENT_READ_SUCCESS_REAL_TIME, inlongGroupId,
+inlongStreamId, 
AgentUtils.getCurrentTime(), 1, maxPackSize);
 }
 baos.reset();
 overLen = false;
@@ -382,6 +384,8 @@ publ

(inlong) branch master updated: [INLONG-9397][Agent] Do not directly delete the instance records of the local db when stopping the instances (#9398)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new 2122b972cb [INLONG-9397][Agent] Do not directly delete the instance 
records of the local db when stopping the instances (#9398)
2122b972cb is described below

commit 2122b972cb387d1cd024f94c1d17ea758d5853ba
Author: justinwwhuang 
AuthorDate: Mon Dec 4 12:33:08 2023 +0800

[INLONG-9397][Agent] Do not directly delete the instance records of the 
local db when stopping the instances (#9398)
---
 .../java/org/apache/inlong/agent/core/instance/InstanceManager.java | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git 
a/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/instance/InstanceManager.java
 
b/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/instance/InstanceManager.java
index 485f2dad56..e600c5b671 100644
--- 
a/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/instance/InstanceManager.java
+++ 
b/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/instance/InstanceManager.java
@@ -417,7 +417,7 @@ public class InstanceManager extends AbstractDaemon {
 
 private void stopAllInstances() {
 instanceMap.values().forEach((instance) -> {
-deleteInstance(instance.getInstanceId());
+deleteFromMemory(instance.getInstanceId());
 });
 instanceMap.clear();
 }

(inlong) branch master updated: [INLONG-9548][Agent] Supports HTTPS and can determine whether to enable it through configuration (#9549)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new b7ea9d55b4 [INLONG-9548][Agent] Supports HTTPS and can determine 
whether to enable it through configuration (#9549)
b7ea9d55b4 is described below

commit b7ea9d55b44effebeb9a1c4178dcb98bd71b1fe4
Author: justinwwhuang 
AuthorDate: Fri Jan 5 11:16:44 2024 +0800

[INLONG-9548][Agent] Supports HTTPS and can determine whether to enable it 
through configuration (#9549)

* [INLONG-9548][Agent] Supports HTTPS and can determine whether to enable 
it through configuration

* [INLONG-9548][Agent] Supports HTTPS and can determine whether to enable 
it through configuration

* [INLONG-9548][Agent] Supports HTTPS and can determine whether to enable 
it through configuration
---
 .../inlong/agent/constant/FetcherConstants.java|  7 ++-
 .../apache/inlong/agent/pojo/TaskProfileDto.java   |  9 ++--
 .../org/apache/inlong/agent/utils/HttpManager.java | 53 +++---
 .../inlong/agent/core/AgentBaseTestsHelper.java|  2 +
 .../agent/core/instance/TestInstanceManager.java   |  1 -
 .../agent/plugin/fetcher/ManagerFetcher.java   |  5 +-
 .../inlong/agent/plugin/sinks/SenderManager.java   | 21 ++---
 .../plugin/sinks/filecollect/SenderManager.java| 27 ---
 .../inlong/agent/plugin/AgentBaseTestsHelper.java  | 10 ++--
 .../sinks/filecollect/TestSenderManager.java   |  2 +-
 .../agent/plugin/sources/TestLogFileSource.java|  2 +-
 .../agent/plugin/task/TestLogfileCollectTask.java  |  2 +-
 .../inlong/sdk/dataproxy/ProxyClientConfig.java|  8 +++-
 13 files changed, 86 insertions(+), 63 deletions(-)

diff --git 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/constant/FetcherConstants.java
 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/constant/FetcherConstants.java
index 642fac298f..c336bce2cd 100644
--- 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/constant/FetcherConstants.java
+++ 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/constant/FetcherConstants.java
@@ -31,9 +31,12 @@ public class FetcherConstants {
 // default is 30s
 public static final int DEFAULT_AGENT_MANAGER_REQUEST_TIMEOUT = 30;
 
+// enable https
+public static final String ENABLE_HTTPS = "enable.https";
+public static final boolean DEFAULT_ENABLE_HTTPS = false;
+
 // required config
-public static final String AGENT_MANAGER_VIP_HTTP_HOST = 
"agent.manager.vip.http.host";
-public static final String AGENT_MANAGER_VIP_HTTP_PORT = 
"agent.manager.vip.http.port";
+public static final String AGENT_MANAGER_ADDR = "agent.manager.addr";
 
 public static final String AGENT_MANAGER_VIP_HTTP_PATH = 
"agent.manager.vip.http.managerIp.path";
 public static final String DEFAULT_AGENT_TDM_VIP_HTTP_PATH = 
"/agent/getManagerIpList";
diff --git 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/TaskProfileDto.java
 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/TaskProfileDto.java
index 6ce6ba2d3d..f4b78e3686 100644
--- 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/TaskProfileDto.java
+++ 
b/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/pojo/TaskProfileDto.java
@@ -29,8 +29,7 @@ import com.google.gson.Gson;
 import lombok.Data;
 
 import static java.util.Objects.requireNonNull;
-import static 
org.apache.inlong.agent.constant.FetcherConstants.AGENT_MANAGER_VIP_HTTP_HOST;
-import static 
org.apache.inlong.agent.constant.FetcherConstants.AGENT_MANAGER_VIP_HTTP_PORT;
+import static 
org.apache.inlong.agent.constant.FetcherConstants.AGENT_MANAGER_ADDR;
 import static org.apache.inlong.agent.constant.TaskConstants.SYNC_SEND_OPEN;
 import static 
org.apache.inlong.common.enums.DataReportTypeEnum.NORMAL_SEND_TO_DATAPROXY;
 
@@ -376,8 +375,7 @@ public class TaskProfileDto {
 Proxy proxy = new Proxy();
 Manager manager = new Manager();
 AgentConfiguration agentConf = AgentConfiguration.getAgentConf();
-manager.setHost(agentConf.get(AGENT_MANAGER_VIP_HTTP_HOST));
-manager.setPort(agentConf.get(AGENT_MANAGER_VIP_HTTP_PORT));
+manager.setAddr(agentConf.get(AGENT_MANAGER_ADDR));
 proxy.setInlongGroupId(dataConfigs.getInlongGroupId());
 proxy.setInlongStreamId(dataConfigs.getInlongStreamId());
 proxy.setManager(manager);
@@ -538,8 +536,7 @@ public class TaskProfileDto {
 @Data
 public static class Manager {
 
-private String port;
-private String host;
+private String addr;
 }
 
 @Data
diff --git 
a/inlong-agent/agent-common/src/main/java/org/apache/inlong/agent/utils/HttpManager

(inlong) branch master updated: [INLONG-9582][Agent] Add unit testing to instance manager to test their ability to recover tasks from DB (#9583)

[luchunliang]

This is an automated email from the ASF dual-hosted git repository.

luchunliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git


The following commit(s) were added to refs/heads/master by this push:
 new a87e7e1a3d [INLONG-9582][Agent] Add unit testing to instance manager 
to test their ability to recover tasks from DB (#9583)
a87e7e1a3d is described below

commit a87e7e1a3da7d75c136bce2cbb5096d8d49e2262
Author: justinwwhuang 
AuthorDate: Thu Jan 18 10:09:26 2024 +0800

[INLONG-9582][Agent] Add unit testing to instance manager to test their 
ability to recover tasks from DB (#9583)
---
 .../inlong/agent/core/instance/InstanceManager.java|  6 +-
 .../agent/core/instance/TestInstanceManager.java   | 18 --
 2 files changed, 21 insertions(+), 3 deletions(-)

diff --git 
a/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/instance/InstanceManager.java
 
b/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/instance/InstanceManager.java
index 3b74cf4e48..e980354fe2 100644
--- 
a/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/instance/InstanceManager.java
+++ 
b/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/instance/InstanceManager.java
@@ -132,6 +132,10 @@ public class InstanceManager extends AbstractDaemon {
 return taskId;
 }
 
+public InstanceDb getInstanceDb() {
+return instanceDb;
+}
+
 public Instance getInstance(String instanceId) {
 return instanceMap.get(instanceId);
 }
@@ -167,7 +171,7 @@ public class InstanceManager extends AbstractDaemon {
 
AuditUtils.add(AuditUtils.AUDIT_ID_AGENT_INSTANCE_MGR_HEARTBEAT, inlongGroupId, 
inlongStreamId,
 AgentUtils.getCurrentTime(), 1, 1);
 } catch (Throwable ex) {
-LOGGER.error("coreThread {}", ex);
+LOGGER.error("coreThread error: ", ex);
 ThreadUtils.threadThrowableHandler(Thread.currentThread(), 
ex);
 }
 runAtLeastOneTime = true;
diff --git 
a/inlong-agent/agent-core/src/test/java/org/apache/inlong/agent/core/instance/TestInstanceManager.java
 
b/inlong-agent/agent-core/src/test/java/org/apache/inlong/agent/core/instance/TestInstanceManager.java
index 34772636ad..8d267e71a1 100755
--- 
a/inlong-agent/agent-core/src/test/java/org/apache/inlong/agent/core/instance/TestInstanceManager.java
+++ 
b/inlong-agent/agent-core/src/test/java/org/apache/inlong/agent/core/instance/TestInstanceManager.java
@@ -23,6 +23,7 @@ import org.apache.inlong.agent.constant.AgentConstants;
 import org.apache.inlong.agent.core.AgentBaseTestsHelper;
 import org.apache.inlong.agent.core.task.file.TaskManager;
 import org.apache.inlong.agent.db.Db;
+import org.apache.inlong.agent.db.InstanceDb;
 import org.apache.inlong.agent.db.TaskProfileDb;
 import org.apache.inlong.agent.utils.AgentUtils;
 import org.apache.inlong.agent.utils.DateTransUtils;
@@ -57,9 +58,9 @@ public class TestInstanceManager {
 taskProfile = helper.getTaskProfile(1, pattern, false, 0L, 0L, 
TaskStateEnum.RUNNING, "GMT+6:00");
 Db taskBasicDb = 
TaskManager.initDb(AgentConstants.AGENT_LOCAL_DB_PATH_TASK);
 TaskProfileDb taskDb = new TaskProfileDb(taskBasicDb);
-manager = new InstanceManager("1", 2, basicDb, taskDb);
+taskDb.storeTask(taskProfile);
+manager = new InstanceManager("1", 20, basicDb, taskDb);
 manager.CORE_THREAD_SLEEP_TIME_MS = 100;
-manager.start();
 }
 
 @AfterClass
@@ -70,6 +71,19 @@ public class TestInstanceManager {
 
 @Test
 public void testInstanceManager() {
+InstanceDb instanceDb = manager.getInstanceDb();
+for (int i = 1; i <= 10; i++) {
+InstanceProfile profile = 
taskProfile.createInstanceProfile(MockInstance.class.getCanonicalName(),
+String.valueOf(i), taskProfile.getCycleUnit(), 
"2023092710",
+AgentUtils.getCurrentTime());
+instanceDb.storeInstance(profile);
+}
+manager.start();
+for (int i = 1; i <= 10; i++) {
+String instanceId = String.valueOf(i);
+await().atMost(1, TimeUnit.SECONDS).until(() -> 
manager.getInstance(instanceId) != null);
+
Assert.assertTrue(manager.getInstanceProfile(instanceId).getState() == 
InstanceStateEnum.DEFAULT);
+}
 long timeBefore = AgentUtils.getCurrentTime();
 InstanceProfile profile = 
taskProfile.createInstanceProfile(MockInstance.class.getCanonicalName(),
 helper.getTestRootDir() + "/2023092710_1.txt", 
taskProfile.getCycleUnit(), "2023092710",