cw-longwang commented on issue #8909:
URL: https://github.com/apache/seatunnel/issues/8909#issuecomment-2789738247
conf
# Defining the runtime environment
env {
parallelism = 1
job.mode = "STREAMING"
checkpoint.interval=60000
}
source {
MySQL-CDC {
base-url =
"jdbc:mysql://ai-bigdatanew-instance-1.ca56eeecuskl.us-east-1.rds.amazonaws.com:3306/test"
username = "admin"
password = "abc123456"
table-names = ["test.test"]
startup.mode = "latest"
exactly_once = true
format = compatible_debezium_json
debezium = {
key.converter.schemas.enable = false
value.converter.schemas.enable = false
database.server.name = "mysql_cdc_1"
}
}
}
sink {
S3File {
bucket = "s3a://sparktest111cw"
path="/sea/test666"
tmp_path = "/tmp/seatunnel"
fs.s3a.endpoint="s3.us-east-1.amazonaws.com"
fs.s3a.aws.credentials.provider="org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider"
access_key="AKIAZ3MGNGECBK2XNCJE"
secret_key="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"
format = “compatible_debezium_json”
file_format_type = "parquet"
data_save_mode = "APPEND_DATA"
have_partition = true
partition_by = ["topic"]
partition_dir_expression = "${k0}=${v0}"
single_file_mode=false
is_enable_transaction=true
batch_size=2
custom_filename = true
file_name_expression = "${transactionId}_${now}"
filename_time_format = "yyyy.MM.dd"
}
}
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
