This is an automated email from the ASF dual-hosted git repository.
dockerzhang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/inlong.git
The following commit(s) were added to refs/heads/master by this push:
new 039e681f11 [INLONG-9008][Manager] Set the ignoreParseError field to
null (#9810)
039e681f11 is described below
commit 039e681f112edf5de99bdbd7f9c458258cad5eda
Author: fuweng11 <76141879+fuwen...@users.noreply.github.com>
AuthorDate: Wed Mar 13 14:08:44 2024 +0800
[INLONG-9008][Manager] Set the ignoreParseError field to null (#9810)
---
.../inlong/manager/pojo/sort/node/base/ExtractNodeProvider.java | 2 +-
.../apache/inlong/manager/pojo/sort/node/provider/KafkaProvider.java | 2 +-
.../inlong/manager/pojo/sort/node/provider/PulsarProvider.java | 2 +-
.../apache/inlong/manager/pojo/sort/node/provider/RedisProvider.java | 2 +-
.../inlong/manager/pojo/sort/node/provider/TubeMqProvider.java | 2 +-
.../java/org/apache/inlong/manager/pojo/source/StreamSource.java | 3 +--
.../apache/inlong/manager/pojo/source/iceberg/IcebergSourceDTO.java | 5 +++++
.../org/apache/inlong/manager/pojo/source/kafka/KafkaSource.java | 2 +-
.../org/apache/inlong/manager/pojo/source/kafka/KafkaSourceDTO.java | 2 +-
.../apache/inlong/manager/pojo/source/kafka/KafkaSourceRequest.java | 2 +-
.../org/apache/inlong/manager/pojo/stream/InlongStreamExtParam.java | 2 +-
.../java/org/apache/inlong/manager/pojo/stream/InlongStreamInfo.java | 2 +-
.../org/apache/inlong/manager/pojo/stream/InlongStreamRequest.java | 2 +-
13 files changed, 17 insertions(+), 13 deletions(-)
diff --git
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/node/base/ExtractNodeProvider.java
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/node/base/ExtractNodeProvider.java
index d7e3c89648..81b9763d79 100644
---
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/node/base/ExtractNodeProvider.java
+++
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/node/base/ExtractNodeProvider.java
@@ -99,7 +99,7 @@ public interface ExtractNodeProvider extends NodeProvider {
String serializationType,
String wrapType,
String separatorStr,
- boolean ignoreParseErrors) {
+ Boolean ignoreParseErrors) {
Format format;
DataTypeEnum dataType = DataTypeEnum.forType(serializationType);
switch (dataType) {
diff --git
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/node/provider/KafkaProvider.java
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/node/provider/KafkaProvider.java
index 01649bea41..ecfa418f2a 100644
---
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/node/provider/KafkaProvider.java
+++
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/node/provider/KafkaProvider.java
@@ -66,7 +66,7 @@ public class KafkaProvider implements ExtractNodeProvider,
LoadNodeProvider {
kafkaSource.getSerializationType(),
kafkaSource.getWrapType(),
kafkaSource.getDataSeparator(),
- kafkaSource.isIgnoreParseErrors());
+ kafkaSource.getIgnoreParseError());
KafkaScanStartupMode startupMode =
parseStartupMode(kafkaSource.getAutoOffsetReset());
String topic = kafkaSource.getTopic();
diff --git
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/node/provider/PulsarProvider.java
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/node/provider/PulsarProvider.java
index 994ce8839a..a767f799f3 100644
---
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/node/provider/PulsarProvider.java
+++
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/node/provider/PulsarProvider.java
@@ -58,7 +58,7 @@ public class PulsarProvider implements ExtractNodeProvider {
Format format = parsingFormat(pulsarSource.getSerializationType(),
pulsarSource.getWrapType(),
pulsarSource.getDataSeparator(),
- pulsarSource.isIgnoreParseError());
+ pulsarSource.getIgnoreParseError());
PulsarScanStartupMode startupMode =
PulsarScanStartupMode.forName(pulsarSource.getScanStartupMode());
final String primaryKey = pulsarSource.getPrimaryKey();
diff --git
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/node/provider/RedisProvider.java
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/node/provider/RedisProvider.java
index 8bc083007a..3fa9ccf2be 100644
---
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/node/provider/RedisProvider.java
+++
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/node/provider/RedisProvider.java
@@ -218,7 +218,7 @@ public class RedisProvider implements ExtractNodeProvider,
LoadNodeProvider {
String formatName,
boolean wrapWithInlongMsg,
String separatorStr,
- boolean ignoreParseErrors) {
+ Boolean ignoreParseErrors) {
Format format;
DataTypeEnum dataType = DataTypeEnum.forType(formatName);
switch (dataType) {
diff --git
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/node/provider/TubeMqProvider.java
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/node/provider/TubeMqProvider.java
index 2942741a08..34723608a5 100644
---
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/node/provider/TubeMqProvider.java
+++
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/sort/node/provider/TubeMqProvider.java
@@ -52,7 +52,7 @@ public class TubeMqProvider implements ExtractNodeProvider {
source.getSerializationType(),
source.getWrapType(),
source.getDataSeparator(),
- source.isIgnoreParseError());
+ source.getIgnoreParseError());
Map<String, String> properties =
parseProperties(source.getProperties());
return new TubeMQExtractNode(
diff --git
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/source/StreamSource.java
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/source/StreamSource.java
index cb0ecb2897..9793189d26 100644
---
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/source/StreamSource.java
+++
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/source/StreamSource.java
@@ -117,9 +117,8 @@ public abstract class StreamSource extends StreamNode {
@ApiModelProperty("Sub source information of existing agents")
private List<SubSourceDTO> subSourceList;
- @Builder.Default
@ApiModelProperty(value = "Whether to ignore the parse errors of field
value, true as default")
- private boolean ignoreParseError = true;
+ private Boolean ignoreParseError;
public SourceRequest genSourceRequest() {
return null;
diff --git
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/source/iceberg/IcebergSourceDTO.java
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/source/iceberg/IcebergSourceDTO.java
index a8e4d6b5fe..e2c6598353 100644
---
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/source/iceberg/IcebergSourceDTO.java
+++
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/source/iceberg/IcebergSourceDTO.java
@@ -31,6 +31,8 @@ import org.apache.commons.lang3.StringUtils;
import javax.validation.constraints.NotNull;
+import java.util.Map;
+
/**
* Iceberg source info
*/
@@ -55,6 +57,9 @@ public class IcebergSourceDTO {
@ApiModelProperty("PrimaryKey")
private String primaryKey;
+ @ApiModelProperty("Properties for iceberg")
+ private Map<String, Object> properties;
+
/**
* Get the dto instance from the request
*/
diff --git
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/source/kafka/KafkaSource.java
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/source/kafka/KafkaSource.java
index 7140496ed5..061e9176af 100644
---
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/source/kafka/KafkaSource.java
+++
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/source/kafka/KafkaSource.java
@@ -76,7 +76,7 @@ public class KafkaSource extends StreamSource {
private String tablePattern;
@ApiModelProperty("ignore parse errors, true: ignore parse error; false:
not ignore parse error; default true")
- private boolean ignoreParseErrors;
+ private Boolean ignoreParseErrors;
@ApiModelProperty("Timestamp standard for binlog: SQL, ISO_8601")
private String timestampFormatStandard;
diff --git
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/source/kafka/KafkaSourceDTO.java
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/source/kafka/KafkaSourceDTO.java
index 462fa3b34f..082b1adf7c 100644
---
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/source/kafka/KafkaSourceDTO.java
+++
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/source/kafka/KafkaSourceDTO.java
@@ -82,7 +82,7 @@ public class KafkaSourceDTO {
private String tablePattern;
@ApiModelProperty("ignore parse errors, true: ignore parse error; false:
not ignore parse error; default true")
- private boolean ignoreParseErrors;
+ private Boolean ignoreParseErrors;
@ApiModelProperty("Timestamp standard for binlog: SQL, ISO_8601")
private String timestampFormatStandard;
diff --git
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/source/kafka/KafkaSourceRequest.java
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/source/kafka/KafkaSourceRequest.java
index 5c81d7f2a8..e9343253df 100644
---
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/source/kafka/KafkaSourceRequest.java
+++
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/source/kafka/KafkaSourceRequest.java
@@ -71,7 +71,7 @@ public class KafkaSourceRequest extends SourceRequest {
private String tablePattern;
@ApiModelProperty("ignore parse errors, true: ignore parse error; false:
not ignore parse error; default true")
- private boolean ignoreParseErrors = true;
+ private Boolean ignoreParseErrors;
@ApiModelProperty("Timestamp standard for binlog: SQL, ISO_8601")
private String timestampFormatStandard = "SQL";
diff --git
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/stream/InlongStreamExtParam.java
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/stream/InlongStreamExtParam.java
index bb67615167..0377b0d8f5 100644
---
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/stream/InlongStreamExtParam.java
+++
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/stream/InlongStreamExtParam.java
@@ -43,7 +43,7 @@ import java.io.Serializable;
public class InlongStreamExtParam implements Serializable {
@ApiModelProperty(value = "Whether to ignore the parse errors of field
value")
- private boolean ignoreParseError;
+ private Boolean ignoreParseError;
@ApiModelProperty(value = "If use extended fields")
private Boolean useExtendedFields = false;
diff --git
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/stream/InlongStreamInfo.java
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/stream/InlongStreamInfo.java
index 2920344bef..6fd8ce2d08 100644
---
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/stream/InlongStreamInfo.java
+++
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/stream/InlongStreamInfo.java
@@ -146,7 +146,7 @@ public class InlongStreamInfo extends BaseInlongStream {
private Boolean sinkMultipleEnable;
@ApiModelProperty(value = "Whether to ignore the parse errors of field
value")
- private Boolean ignoreParseError = true;
+ private Boolean ignoreParseError;
public InlongStreamRequest genRequest() {
return CommonBeanUtils.copyProperties(this, InlongStreamRequest::new);
diff --git
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/stream/InlongStreamRequest.java
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/stream/InlongStreamRequest.java
index 2269410b5e..b0c915a1ca 100644
---
a/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/stream/InlongStreamRequest.java
+++
b/inlong-manager/manager-pojo/src/main/java/org/apache/inlong/manager/pojo/stream/InlongStreamRequest.java
@@ -119,7 +119,7 @@ public class InlongStreamRequest extends BaseInlongStream {
private Integer version;
@ApiModelProperty(value = "Whether to ignore the parse errors of field
value")
- private boolean ignoreParseError = true;
+ private Boolean ignoreParseError;
@ApiModelProperty(value = "The compression type used for dataproxy and
sort side data transmission to reduce the network IO overhead")
private String inlongCompressType;
