This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 19681524bf [INLONG-9194][Agent] Calc time offset failed if the param
is "0" (#9195)
19681524bf is described below
commit 19681524bf2d9c66815487b5de88c1a89ea775db
Author: justinwwhuang <hww_jus...@163.com>
AuthorDate: Thu Nov 2 14:17:08 2023 +0800
[INLONG-9194][Agent] Calc time offset failed if the param is "0" (#9195)
---
.../org/apache/inlong/agent/core/task/file/TaskManager.java | 5 ++++-
.../inlong/agent/plugin/task/filecollect/FileScanner.java | 4 ++--
.../agent/plugin/task/filecollect/LogFileCollectTask.java | 4 ++--
.../apache/inlong/agent/plugin/utils/file/NewDateUtils.java | 13 ++++++++-----
.../org/apache/inlong/agent/plugin/utils/TestUtils.java | 12 ++++++++++++
5 files changed, 28 insertions(+), 10 deletions(-)
diff --git
a/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/task/file/TaskManager.java
b/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/task/file/TaskManager.java
index b52a51c90a..991be20c05 100644
---
a/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/task/file/TaskManager.java
+++
b/inlong-agent/agent-core/src/main/java/org/apache/inlong/agent/core/task/file/TaskManager.java
@@ -118,6 +118,10 @@ public class TaskManager extends AbstractDaemon {
while (configQueue.size() != 0) {
configQueue.poll();
}
+ for (int i = 0; i < taskProfiles.size(); i++) {
+ LOGGER.info("submitTaskProfiles index {} total {} {}", i,
taskProfiles.size(),
+ taskProfiles.get(i).toJsonStr());
+ }
configQueue.add(taskProfiles);
}
@@ -187,7 +191,6 @@ public class TaskManager extends AbstractDaemon {
* NEW and STOP only used in manager
*/
private void keepPaceWithManager(List<TaskProfile> taskProfiles) {
- LOGGER.info("deal with List<TaskProfile> {}", taskProfiles);
Map<String, TaskProfile> tasksFromManager = new ConcurrentHashMap<>();
taskProfiles.forEach((profile) -> {
TaskStateEnum state = profile.getState();
diff --git
a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/filecollect/FileScanner.java
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/filecollect/FileScanner.java
index eb25d60f82..bfb1a7a80a 100644
---
a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/filecollect/FileScanner.java
+++
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/filecollect/FileScanner.java
@@ -60,8 +60,8 @@ public class FileScanner {
long recoverTime, boolean isRetry) {
String cycleUnit = conf.getCycleUnit();
if (!isRetry) {
- failTime -= NewDateUtils.caclOffset(conf.getTimeOffset());
- recoverTime -= NewDateUtils.caclOffset(conf.getTimeOffset());
+ failTime -= NewDateUtils.calcOffset(conf.getTimeOffset());
+ recoverTime -= NewDateUtils.calcOffset(conf.getTimeOffset());
}
String startTime = NewDateUtils.millSecConvertToTimeStr(failTime,
cycleUnit);
diff --git
a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/filecollect/LogFileCollectTask.java
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/filecollect/LogFileCollectTask.java
index bde4a33361..26007d0290 100644
---
a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/filecollect/LogFileCollectTask.java
+++
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/task/filecollect/LogFileCollectTask.java
@@ -266,7 +266,7 @@ public class LogFileCollectTask extends Task {
private void scanExistingFile() {
originPatterns.forEach((originPattern) -> {
List<BasicFileInfo> fileInfos =
scanExistingFileByPattern(originPattern);
- LOGGER.debug("scan {} get file count {}", originPattern,
fileInfos.size());
+ LOGGER.info("scan {} get file count {}", originPattern,
fileInfos.size());
fileInfos.forEach((fileInfo) -> {
addToEvenMap(fileInfo.fileName, fileInfo.dataTime);
});
@@ -279,7 +279,7 @@ public class LogFileCollectTask extends Task {
if (!retry) {
long currentTime = System.currentTimeMillis();
// only scan two cycle, like two hours or two days
- long offset = NewDateUtils.caclOffset("-2" +
taskProfile.getCycleUnit());
+ long offset = NewDateUtils.calcOffset("-2" +
taskProfile.getCycleUnit());
startScanTime = currentTime - offset;
endScanTime = currentTime;
}
diff --git
a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/utils/file/NewDateUtils.java
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/utils/file/NewDateUtils.java
index 935a1e4314..c6d8082651 100644
---
a/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/utils/file/NewDateUtils.java
+++
b/inlong-agent/agent-plugins/src/main/java/org/apache/inlong/agent/plugin/utils/file/NewDateUtils.java
@@ -222,9 +222,9 @@ public class NewDateUtils {
// To handle the offset, add the time offset to the timeout period
if (timeOffset.startsWith("-")) {
- timeInterval += caclOffset(timeOffset);
+ timeInterval += calcOffset(timeOffset);
} else { // Process Backward Offset
- timeInterval -= caclOffset(timeOffset);
+ timeInterval -= calcOffset(timeOffset);
}
return isValidCreationTime(dataTime, timeInterval);
@@ -240,7 +240,7 @@ public class NewDateUtils {
* @param timeOffset offset，such as -1d,-4h,-10m；
* @return
*/
- public static long caclOffset(String timeOffset) {
+ public static long calcOffset(String timeOffset) {
String offsetUnit = timeOffset.substring(timeOffset.length() - 1);
int startIndex = timeOffset.charAt(0) == '-' ? 1 : 0;
// Default Backward Offset
@@ -250,8 +250,11 @@ public class NewDateUtils {
} else if (startIndex == 0) { // Forward offset
symbol = -1;
}
- int offsetTime = Integer
- .parseInt(timeOffset.substring(startIndex, timeOffset.length()
- 1));
+ String strOffset = timeOffset.substring(startIndex,
timeOffset.length() - 1);
+ if (strOffset.length() == 0) {
+ return 0;
+ }
+ int offsetTime = Integer.parseInt(strOffset);
if ("d".equalsIgnoreCase(offsetUnit)) {
return offsetTime * 24 * 3600 * 1000 * symbol;
} else if ("h".equalsIgnoreCase(offsetUnit)) {
diff --git
a/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/utils/TestUtils.java
b/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/utils/TestUtils.java
index 159240ab2c..c6ff2d6642 100644
---
a/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/utils/TestUtils.java
+++
b/inlong-agent/agent-plugins/src/test/java/org/apache/inlong/agent/plugin/utils/TestUtils.java
@@ -17,9 +17,12 @@
package org.apache.inlong.agent.plugin.utils;
+import org.apache.inlong.agent.plugin.utils.file.NewDateUtils;
import org.apache.inlong.common.metric.MetricRegister;
import org.apache.commons.io.FileUtils;
+import org.junit.Assert;
+import org.junit.Test;
import org.powermock.api.mockito.PowerMockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -41,6 +44,15 @@ public class TestUtils {
private static final Logger LOGGER =
LoggerFactory.getLogger(TestUtils.class);
private static final String RECORD = "This is the test line for file\n";
+ @Test
+ public void testCalcOffset() {
+ Assert.assertTrue(NewDateUtils.calcOffset("-1h") == 3600 * 1000);
+ Assert.assertTrue(NewDateUtils.calcOffset("1D") == -24 * 3600 * 1000);
+ Assert.assertTrue(NewDateUtils.calcOffset("0") == 0);
+ Assert.assertTrue(NewDateUtils.calcOffset("1") == 0);
+ Assert.assertTrue(NewDateUtils.calcOffset("10") == 0);
+ }
+
public static String getTestTriggerProfile() {
return "{\n"
+ " \"job\": {\n"
