Hisoka-X commented on code in PR #9303:
URL: https://github.com/apache/seatunnel/pull/9303#discussion_r2084522588
##########
seatunnel-connectors-v2/connector-file/connector-file-oss/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/oss/config/OssFileSinkOptions.java:
##########
@@ -17,4 +17,32 @@
package org.apache.seatunnel.connectors.seatunnel.file.oss.config;
-public class OssFileSinkOptions extends OssFileBaseOptions {}
+import org.apache.seatunnel.api.configuration.Option;
+import org.apache.seatunnel.api.configuration.Options;
+import org.apache.seatunnel.api.sink.DataSaveMode;
+import org.apache.seatunnel.api.sink.SchemaSaveMode;
+
+import java.util.Arrays;
+
+import static org.apache.seatunnel.api.sink.DataSaveMode.APPEND_DATA;
+import static org.apache.seatunnel.api.sink.DataSaveMode.DROP_DATA;
+import static
org.apache.seatunnel.api.sink.DataSaveMode.ERROR_WHEN_DATA_EXISTS;
+
+public class OssFileSinkOptions extends OssFileBaseOptions {
+
+ public static final Option<SchemaSaveMode> SCHEMA_SAVE_MODE =
+ Options.key("schema_save_mode")
+ .enumType(SchemaSaveMode.class)
+ .defaultValue(SchemaSaveMode.CREATE_SCHEMA_WHEN_NOT_EXIST)
+ .withDescription(
+ "Before the synchronization task begins, process
the existing path");
+
+ public static final Option<DataSaveMode> DATA_SAVE_MODE =
+ Options.key("data_save_mode")
+ .singleChoice(
+ DataSaveMode.class,
+ Arrays.asList(DROP_DATA, APPEND_DATA,
ERROR_WHEN_DATA_EXISTS))
+ .defaultValue(APPEND_DATA)
+ .withDescription(
+ "Before the synchronization task begins, different
processing of data files that already exist in the directory");
Review Comment:
Please reuse
https://github.com/apache/seatunnel/blob/4a2f3fa91590db9c1f0b3ca081d466cb54b24dae/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/config/FileBaseSinkOptions.java#L306-L321
##########
seatunnel-connectors-v2/connector-file/connector-file-oss/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/oss/sink/OssFileSink.java:
##########
@@ -17,28 +17,82 @@
package org.apache.seatunnel.connectors.seatunnel.file.oss.sink;
+import org.apache.seatunnel.shade.com.typesafe.config.Config;
+
+import org.apache.seatunnel.api.common.SeaTunnelAPIErrorCode;
import org.apache.seatunnel.api.configuration.ReadonlyConfig;
+import org.apache.seatunnel.api.sink.DataSaveMode;
+import org.apache.seatunnel.api.sink.DefaultSaveModeHandler;
+import org.apache.seatunnel.api.sink.SaveModeHandler;
+import org.apache.seatunnel.api.sink.SchemaSaveMode;
+import org.apache.seatunnel.api.sink.SupportSaveMode;
+import org.apache.seatunnel.api.table.catalog.Catalog;
import org.apache.seatunnel.api.table.catalog.CatalogTable;
+import org.apache.seatunnel.api.table.factory.CatalogFactory;
+import org.apache.seatunnel.common.config.CheckConfigUtil;
+import org.apache.seatunnel.common.config.CheckResult;
+import org.apache.seatunnel.common.constants.PluginType;
import org.apache.seatunnel.connectors.seatunnel.file.config.FileSystemType;
+import
org.apache.seatunnel.connectors.seatunnel.file.exception.FileConnectorException;
+import
org.apache.seatunnel.connectors.seatunnel.file.oss.config.OssFileSinkOptions;
import org.apache.seatunnel.connectors.seatunnel.file.oss.config.OssHadoopConf;
import
org.apache.seatunnel.connectors.seatunnel.file.sink.BaseMultipleTableFileSink;
import java.util.Optional;
-public class OssFileSink extends BaseMultipleTableFileSink {
+import static
org.apache.seatunnel.api.table.factory.FactoryUtil.discoverFactory;
+
+public class OssFileSink extends BaseMultipleTableFileSink implements
SupportSaveMode {
private final CatalogTable catalogTable;
+ private final ReadonlyConfig readonlyConfig;
+
public OssFileSink(ReadonlyConfig readonlyConfig, CatalogTable
catalogTable) {
super(OssHadoopConf.buildWithConfig(readonlyConfig), readonlyConfig,
catalogTable);
this.catalogTable = catalogTable;
+ this.readonlyConfig = readonlyConfig;
+ Config pluginConfig = readonlyConfig.toConfig();
+ CheckResult result =
+ CheckConfigUtil.checkAllExists(
+ pluginConfig,
+ OssFileSinkOptions.FILE_PATH.key(),
+ OssFileSinkOptions.BUCKET.key());
+ if (!result.isSuccess()) {
+ throw new FileConnectorException(
+ SeaTunnelAPIErrorCode.CONFIG_VALIDATION_FAILED,
+ String.format(
+ "PluginName: %s, PluginType: %s, Message: %s",
+ getPluginName(), PluginType.SINK,
result.getMsg()));
+ }
}
@Override
public String getPluginName() {
return FileSystemType.OSS.getFileSystemPluginName();
}
+ @Override
+ public Optional<SaveModeHandler> getSaveModeHandler() {
Review Comment:
why we should rewrite
https://github.com/apache/seatunnel/blob/4a2f3fa91590db9c1f0b3ca081d466cb54b24dae/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/sink/BaseMultipleTableFileSink.java#L132?
##########
seatunnel-connectors-v2/connector-file/connector-file-oss/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/oss/sink/OssFileSink.java:
##########
@@ -17,28 +17,82 @@
package org.apache.seatunnel.connectors.seatunnel.file.oss.sink;
+import org.apache.seatunnel.shade.com.typesafe.config.Config;
+
+import org.apache.seatunnel.api.common.SeaTunnelAPIErrorCode;
import org.apache.seatunnel.api.configuration.ReadonlyConfig;
+import org.apache.seatunnel.api.sink.DataSaveMode;
+import org.apache.seatunnel.api.sink.DefaultSaveModeHandler;
+import org.apache.seatunnel.api.sink.SaveModeHandler;
+import org.apache.seatunnel.api.sink.SchemaSaveMode;
+import org.apache.seatunnel.api.sink.SupportSaveMode;
+import org.apache.seatunnel.api.table.catalog.Catalog;
import org.apache.seatunnel.api.table.catalog.CatalogTable;
+import org.apache.seatunnel.api.table.factory.CatalogFactory;
+import org.apache.seatunnel.common.config.CheckConfigUtil;
+import org.apache.seatunnel.common.config.CheckResult;
+import org.apache.seatunnel.common.constants.PluginType;
import org.apache.seatunnel.connectors.seatunnel.file.config.FileSystemType;
+import
org.apache.seatunnel.connectors.seatunnel.file.exception.FileConnectorException;
+import
org.apache.seatunnel.connectors.seatunnel.file.oss.config.OssFileSinkOptions;
import org.apache.seatunnel.connectors.seatunnel.file.oss.config.OssHadoopConf;
import
org.apache.seatunnel.connectors.seatunnel.file.sink.BaseMultipleTableFileSink;
import java.util.Optional;
-public class OssFileSink extends BaseMultipleTableFileSink {
+import static
org.apache.seatunnel.api.table.factory.FactoryUtil.discoverFactory;
+
+public class OssFileSink extends BaseMultipleTableFileSink implements
SupportSaveMode {
private final CatalogTable catalogTable;
+ private final ReadonlyConfig readonlyConfig;
+
public OssFileSink(ReadonlyConfig readonlyConfig, CatalogTable
catalogTable) {
super(OssHadoopConf.buildWithConfig(readonlyConfig), readonlyConfig,
catalogTable);
this.catalogTable = catalogTable;
+ this.readonlyConfig = readonlyConfig;
+ Config pluginConfig = readonlyConfig.toConfig();
+ CheckResult result =
+ CheckConfigUtil.checkAllExists(
+ pluginConfig,
+ OssFileSinkOptions.FILE_PATH.key(),
+ OssFileSinkOptions.BUCKET.key());
+ if (!result.isSuccess()) {
+ throw new FileConnectorException(
+ SeaTunnelAPIErrorCode.CONFIG_VALIDATION_FAILED,
+ String.format(
+ "PluginName: %s, PluginType: %s, Message: %s",
+ getPluginName(), PluginType.SINK,
result.getMsg()));
Review Comment:
we don't need this now. Our config parser already help us to verify the
config.
##########
seatunnel-connectors-v2/connector-file/connector-file-oss/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/oss/sink/OssFileSinkFactory.java:
##########
@@ -57,6 +57,8 @@ public OptionRule optionRule() {
.required(OssFileSinkOptions.ACCESS_KEY)
.required(OssFileSinkOptions.ACCESS_SECRET)
.required(OssFileSinkOptions.ENDPOINT)
+ .required(OssFileSinkOptions.SCHEMA_SAVE_MODE)
+ .required(OssFileSinkOptions.DATA_SAVE_MODE)
Review Comment:
```suggestion
.optional(OssFileSinkOptions.SCHEMA_SAVE_MODE)
.optional(OssFileSinkOptions.DATA_SAVE_MODE)
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
