This is an automated email from the ASF dual-hosted git repository. zhangshenghang pushed a commit to branch docs-fix-batch-20261014 in repository https://gitbox.apache.org/repos/asf/seatunnel.git
commit a97e9e22fe38a880008ff973c62ab473b8da4201 Author: zhangshenghang <[email protected]> AuthorDate: Wed Oct 14 23:52:31 2026 +0800 [Docs] Fix CDC, InfluxDB, file and IM connector doc inaccuracies verified against source code --- docs/en/connectors/sink/InfluxDB.md | 7 ++++--- docs/en/connectors/sink/S3File.md | 1 + docs/en/connectors/sink/Socket.md | 2 +- docs/en/connectors/source/GcsFile.md | 8 ++++++++ docs/en/connectors/source/InfluxDB.md | 2 +- docs/en/connectors/source/MongoDB-CDC.md | 3 ++- docs/en/connectors/source/Oracle-CDC.md | 2 +- docs/en/connectors/source/PostgreSQL-CDC.md | 11 ++++++----- docs/en/connectors/source/S3File.md | 2 +- docs/zh/connectors/sink/Feishu.md | 2 +- docs/zh/connectors/sink/InfluxDB.md | 5 +++-- docs/zh/connectors/sink/S3File.md | 8 ++++---- docs/zh/connectors/sink/SftpFile.md | 1 + docs/zh/connectors/source/GcsFile.md | 8 ++++++++ docs/zh/connectors/source/InfluxDB.md | 2 +- docs/zh/connectors/source/MongoDB-CDC.md | 3 ++- docs/zh/connectors/source/PostgreSQL-CDC.md | 11 ++++++----- docs/zh/connectors/source/S3File.md | 2 +- 18 files changed, 52 insertions(+), 28 deletions(-) diff --git a/docs/en/connectors/sink/InfluxDB.md b/docs/en/connectors/sink/InfluxDB.md index b01122832f..8a18579d62 100644 --- a/docs/en/connectors/sink/InfluxDB.md +++ b/docs/en/connectors/sink/InfluxDB.md @@ -33,7 +33,7 @@ fields. | BIGINT | Written as an InfluxDB field, or as the timestamp when configured by `key_time`. | | FLOAT | Written as an InfluxDB field. | | DOUBLE | Written as an InfluxDB field. | -| STRING | Written as an InfluxDB field, a tag value, or a timestamp string when configured by `key_time`. | +| STRING | Written as an InfluxDB field, a tag value, or a numeric timestamp string when configured by `key_time`. | | TIMESTAMP | Can be used by `key_time`; converted with UTC zone to epoch milliseconds. | Other SeaTunnel types are not supported by the current InfluxDB sink serializer. @@ -86,8 +86,9 @@ The InfluxDB user password. Configure it together with `username` when authentic ### key_time [string] The field name in the upstream row that supplies the InfluxDB point timestamp. When omitted, the -sink uses the current processing time. Values configured here are accepted as numeric timestamps -or as an ISO-8601 timestamp string. +sink uses the current processing time. The referenced field must be a `BIGINT` epoch-milliseconds +value, a `STRING` containing a numeric epoch value, or a `TIMESTAMP` value, which is converted to +epoch milliseconds using UTC. ISO-8601 timestamp strings are not parsed. ### key_tags [array] diff --git a/docs/en/connectors/sink/S3File.md b/docs/en/connectors/sink/S3File.md index 483d62bcba..48ac7f5793 100644 --- a/docs/en/connectors/sink/S3File.md +++ b/docs/en/connectors/sink/S3File.md @@ -128,6 +128,7 @@ If write to `csv`, `text` file type, All column will be string. | is_enable_transaction | boolean | no | true | | | batch_size | int | no | 1000000 | | | compress_codec | string | no | none | | +| multi_table_sink_replica | int | no | 1 | The replica number of sink writers used for each table in a multi-table sink job. | | common-options | object | no | - | | | max_rows_in_memory | int | no | - | Only used when file_format is excel. | | sheet_max_rows | int | no | 1048576 | Only used when file_format is excel. | diff --git a/docs/en/connectors/sink/Socket.md b/docs/en/connectors/sink/Socket.md index 3f5120add6..fa5ed24548 100644 --- a/docs/en/connectors/sink/Socket.md +++ b/docs/en/connectors/sink/Socket.md @@ -27,7 +27,7 @@ Jackson's `MappingIterator`) rather than a line-oriented parser. Tools like `nc concatenated bytes, so they are useful for a quick single-row check but cannot split records on their own. -> For example, if the data from upstream is [`age: 12, name: jared`], the content send to socket server is the following: `{"name":"jared","age":17}` +> For example, if the data from upstream is [`age: 17, name: jared`], the content send to socket server is the following: `{"name":"jared","age":17}` ## Sink Options diff --git a/docs/en/connectors/source/GcsFile.md b/docs/en/connectors/source/GcsFile.md index 260b3147ed..550840aca8 100644 --- a/docs/en/connectors/source/GcsFile.md +++ b/docs/en/connectors/source/GcsFile.md @@ -64,10 +64,17 @@ The explicit `service_account_key_file` option takes precedence over the corresp | field_delimiter | string | no | `\001` | Field delimiter for text and CSV files. `delimiter` is an alias. | | row_delimiter | string | no | `\n` | Row delimiter for text files. | | skip_header_row_number | long | no | `0` | Number of initial text or CSV rows to skip. | +| csv_use_header_line | boolean | no | `false` | Use the header line to parse the file, only used when the file format is `csv` and the file contains an RFC 4180 header line. | | encoding | string | no | `UTF-8` | Character encoding for text, JSON, CSV, and XML files. | +| date_format | string | no | `yyyy-MM-dd` | Date type format, used to tell the connector how to convert string to date. | +| datetime_format | string | no | `yyyy-MM-dd HH:mm:ss` | Datetime type format, used to tell the connector how to convert string to datetime. | +| time_format | string | no | `HH:mm:ss` | Time type format, used to tell the connector how to convert string to time. | | parse_partition_from_path | boolean | no | `true` | Adds partition values parsed from paths such as `/year=2026/month=08`. | | recursive_file_scan | boolean | no | `true` | Whether to scan subdirectories recursively. | | file_filter_pattern | string | no | - | File name filter pattern. | +| file_filter_modified_start | string | no | - | File modification time filter. Filters files based on the last modification start time (inclusive). The default format is `yyyy-MM-dd HH:mm:ss`. | +| file_filter_modified_end | string | no | - | File modification time filter. Filters files based on the last modification end time (exclusive). The default format is `yyyy-MM-dd HH:mm:ss`. | +| sort_files_by_modification_time | boolean | no | `false` | Sort files by modification time in descending order. Enable this when reading evolving schemas to ensure schema inference uses the latest file. | | filename_extension | string | no | - | File extension filter, for example `csv` or `.json`. | | compress_codec | string | no | `none` | Compression codec for a single compressed file. | | archive_compress_codec | string | no | `none` | Archive compression codec. | @@ -95,6 +102,7 @@ The explicit `service_account_key_file` option takes precedence over the corresp | backup_path | string | conditional | - | Required for `post_sync_action=backup`. Must not overlap the source path. | | retention_max_age | string | no | - | Maximum age for SeaTunnel backup objects. | | retention_check_interval | string | no | `1H` | Backup retention scan interval. | +| metalake_type | string | no | `gravitino` | The type of metalake service, currently supports `gravitino`. | | common-options | | no | - | See [Source Common Options](../common-options/source-common-options.md). | ## Example diff --git a/docs/en/connectors/source/InfluxDB.md b/docs/en/connectors/source/InfluxDB.md index 711ef844b2..45f4b29b64 100644 --- a/docs/en/connectors/source/InfluxDB.md +++ b/docs/en/connectors/source/InfluxDB.md @@ -153,7 +153,7 @@ Upper bound of the `split_column` column when parallel scan is enabled. Lower bound of the `split_column` column when parallel scan is enabled. The split column range is divided into `partition_num` parts. If `partition_num = 1`, the connector -uses the whole range. If `partition_num` is less than `upper_bound - lower_bound`, the connector +uses the whole range. If `partition_num` is greater than `upper_bound - lower_bound`, the connector uses `(upper_bound - lower_bound)` partitions. For example, with `lower_bound = 1`, `upper_bound = 10`, `partition_num = 2`, and diff --git a/docs/en/connectors/source/MongoDB-CDC.md b/docs/en/connectors/source/MongoDB-CDC.md index f22e62d7ac..80591e8977 100644 --- a/docs/en/connectors/source/MongoDB-CDC.md +++ b/docs/en/connectors/source/MongoDB-CDC.md @@ -132,7 +132,8 @@ For specific types in MongoDB, we use Extended JSON format to map them to Seatun | stop.mode | Enum | No | NEVER | Optional stop mode for MongoDB CDC consumer. Valid values are `never` and `timestamp`. See the [Stop Mode](#stop-mode) section below. | | stop.timestamp | Long | No | - | Stop at the change-stream position derived from this epoch timestamp in milliseconds. Only used when `stop.mode` is `timestamp`. | | exactly_once | Boolean | No | false | Enable exactly-once semantics. Enabling this may increase memory usage during large table snapshot recovery. | -| debezium | Config | No | - | Pass-through Debezium properties used by the embedded engine. | +| format | Enum | No | DEFAULT | Optional output format for MongoDB CDC, valid enumerations are `DEFAULT`, `COMPATIBLE_DEBEZIUM_JSON`. | +| debezium | Config | No | {key.converter.schemas.enable=false, value.converter.schemas.enable=false} | Pass-through Debezium properties used by the embedded engine. | | common-options | | No | - | Source plugin common parameters. For details, see [Source Common Options](../common-options/source-common-options.md). | ### Startup Mode diff --git a/docs/en/connectors/source/Oracle-CDC.md b/docs/en/connectors/source/Oracle-CDC.md index 7cbe579abc..bd3457754a 100644 --- a/docs/en/connectors/source/Oracle-CDC.md +++ b/docs/en/connectors/source/Oracle-CDC.md @@ -97,7 +97,7 @@ GRANT SELECT ANY TRANSACTION TO logminer_user; GRANT SELECT ON V_$TRANSACTION TO logminer_user; ``` -##### Oracle 11g is not supported +##### The following privilege is not supported on Oracle 11g ```sql GRANT LOGMINING TO logminer_user; diff --git a/docs/en/connectors/source/PostgreSQL-CDC.md b/docs/en/connectors/source/PostgreSQL-CDC.md index 1cd408eaec..144147f5e8 100644 --- a/docs/en/connectors/source/PostgreSQL-CDC.md +++ b/docs/en/connectors/source/PostgreSQL-CDC.md @@ -98,6 +98,7 @@ ALTER TABLE your_table_name REPLICA IDENTITY FULL; | username | String | Yes | - | Name of the database to use when connecting to the database server. [...] | password | String | Yes | - | Password to use when connecting to the database server. [...] | database-names | List | No | - | Database names to monitor. [...] +| schema-name | List | No | - | Schema names of the database to monitor, for example: `["inventory"]`. [...] | table-names | List | Yes, if `table-pattern` is not used | - | Tables to monitor. Use the fully qualified `database.schema.table` format, for example: `postgres_cdc.inventory.orders`. [...] | table-pattern | String | Yes, if `table-names` is not used | - | Regular expression for tables to monitor. Use the fully qualified table name in the pattern, for example: `postgres_cdc\\.inventory\\..*`. `table-names` and `table-pattern` are mutually exclusive. [...] | table-names-config | List | No | - | Per-table config list. Example: `[{"table": "db1.schema1.table1","primaryKeys": ["key1"],"snapshotSplitColumn": "key2"}]`. Use `primaryKeys` for tables without a physical primary key. `snapshotSplitColumn` must be a unique key; otherwise SeaTunnel ignores it and selects a split column internally. [...] @@ -107,8 +108,8 @@ ALTER TABLE your_table_name REPLICA IDENTITY FULL; | snapshot.fetch.size | Integer | No | 1024 | The maximum fetch size for per poll when read table snapshot. [...] | slot.name | String | No | seatunnel | The PostgreSQL logical decoding slot name. Use a different slot name for each CDC job that reads from the same PostgreSQL instance. [...] | decoding.plugin.name | String | No | pgoutput | The name of the Postgres logical decoding plug-in installed on the server,Supported values are decoderbufs, wal2json, wal2json_rds, wal2json_streaming,wal2json_rds_streaming and pgoutput. [...] -| server-time-zone | String | No | UTC | The session time zone in database server. If not set, then ZoneId.systemDefault() is used to determine the server time zone. [...] -| connect.timeout.ms | Duration | No | 30000 | The maximum time that the connector should wait after trying to connect to the database server before timing out. [...] +| server-time-zone | String | No | - | The session time zone in database server. If not set, then ZoneId.systemDefault() is used to determine the server time zone. [...] +| connect.timeout.ms | Long | No | 30000 | The maximum time that the connector should wait after trying to connect to the database server before timing out. [...] | connect.max-retries | Integer | No | 3 | The max retry times that the connector should retry to build database server connection. [...] | connection.pool.size | Integer | No | 20 | The jdbc connection pool size. [...] | chunk-key.even-distribution.factor.upper-bound | Double | No | 100 | The upper bound of the chunk key distribution factor. This factor is used to determine whether the table data is evenly distributed. If the distribution factor is calculated to be less than or equal to this upper bound (i.e., (MAX(id) - MIN(id) + 1) / row count), the table chunks would be optimized for even distribution. Otherwise, if the distribution factor is greater, the table will be considered as une [...] @@ -236,7 +237,7 @@ source { username = "postgres" password = "postgres" database-names = ["postgres_cdc"] - schema-names = ["inventory"] + schema-name = ["inventory"] table-names = ["postgres_cdc.inventory.postgres_cdc_table_1"] url = "jdbc:postgresql://postgres_cdc_e2e:5432/postgres_cdc?loggerLevel=OFF" decoding.plugin.name = "decoderbufs" @@ -265,7 +266,7 @@ source { username = "postgres" password = "postgres" database-names = ["postgres_cdc"] - schema-names = ["inventory"] + schema-name = ["inventory"] table-names = ["postgres_cdc.inventory.postgres_cdc_table_1"] url = "jdbc:postgresql://postgres_cdc_e2e:5432/postgres_cdc?loggerLevel=OFF" decoding.plugin.name = "decoderbufs" @@ -307,7 +308,7 @@ source { username = "postgres" password = "postgres" database-names = ["postgres_cdc"] - schema-names = ["inventory"] + schema-name = ["inventory"] table-names = ["postgres_cdc.inventory.full_types_no_primary_key"] url = "jdbc:postgresql://postgres_cdc_e2e:5432/postgres_cdc?loggerLevel=OFF" decoding.plugin.name = "decoderbufs" diff --git a/docs/en/connectors/source/S3File.md b/docs/en/connectors/source/S3File.md index 9a253ac423..dde0ebe139 100644 --- a/docs/en/connectors/source/S3File.md +++ b/docs/en/connectors/source/S3File.md @@ -244,7 +244,7 @@ If you assign file type to `parquet` `orc`, schema option not required, connecto | hadoop_s3_properties | map | no | - | If you need to add other option, you could add it here and refer to this [link](https://hadoop.apache.org/docs/stable/hadoop-aws/tools/hadoop-aws/index.html) [...] | delimiter/field_delimiter | string | no | \001 | Field delimiter, used to tell connector how to slice and dice fields when reading text files. Default `\001`, the same as hive's default delimiter. [...] | row_delimiter | string | no | \n | Row delimiter, used to tell connector how to slice and dice rows when reading text files. Default `\n`. [...] -| parse_partition_from_path | boolean | no | true | Control whether parse the partition keys and values from file path. For example if you read a file from path `s3n://hadoop-cluster/tmp/seatunnel/parquet/name=tyrantlucifer/age=26`. Every record data from file will be added these two fields: name="tyrantlucifer", age=16 [...] +| parse_partition_from_path | boolean | no | true | Control whether parse the partition keys and values from file path. For example if you read a file from path `s3n://hadoop-cluster/tmp/seatunnel/parquet/name=tyrantlucifer/age=26`. Every record data from file will be added these two fields: name="tyrantlucifer", age=26 [...] | date_format | string | no | yyyy-MM-dd | Date type format, used to tell connector how to convert string to date, supported as the following formats:`yyyy-MM-dd` `yyyy.MM.dd` `yyyy/MM/dd`. default `yyyy-MM-dd` [...] | datetime_format | string | no | yyyy-MM-dd HH:mm:ss | Datetime type format, used to tell connector how to convert string to datetime, supported as the following formats:`yyyy-MM-dd HH:mm:ss` `yyyy.MM.dd HH:mm:ss` `yyyy/MM/dd HH:mm:ss` `yyyyMMddHHmmss` [...] | time_format | string | no | HH:mm:ss | Time type format, used to tell connector how to convert string to time, supported as the following formats:`HH:mm:ss` `HH:mm:ss.SSS` [...] diff --git a/docs/zh/connectors/sink/Feishu.md b/docs/zh/connectors/sink/Feishu.md index e0735cb579..3f6129cdf0 100644 --- a/docs/zh/connectors/sink/Feishu.md +++ b/docs/zh/connectors/sink/Feishu.md @@ -221,7 +221,7 @@ source { transform { Sql { plugin_input = "raw" - query = "SELECT 'text' AS msg_type, named_struct('text', concat('User ', name, ' is ', cast(age as string), ' years old')) AS content FROM raw" + query = "SELECT 'text' AS msg_type, MAP('text', concat('User ', name, ' is ', cast(age as string), ' years old')) AS content FROM raw" } } diff --git a/docs/zh/connectors/sink/InfluxDB.md b/docs/zh/connectors/sink/InfluxDB.md index 59ac3ebd23..e4386ebf17 100644 --- a/docs/zh/connectors/sink/InfluxDB.md +++ b/docs/zh/connectors/sink/InfluxDB.md @@ -32,7 +32,7 @@ import ChangeLog from '../changelog/connector-influxdb.md'; | BIGINT | 写为 InfluxDB field；配置为 `key_time` 时也可以作为时间戳。 | | FLOAT | 写为 InfluxDB field。 | | DOUBLE | 写为 InfluxDB field。 | -| STRING | 可写为 InfluxDB field、tag 值，或在配置为 `key_time` 时作为时间戳字符串。 | +| STRING | 可写为 InfluxDB field、tag 值，或在配置为 `key_time` 时作为数字时间戳字符串。 | | TIMESTAMP | 可用于 `key_time`，连接器会按 UTC 时区转换成 epoch 毫秒。 | 当前 InfluxDB sink 序列化器不支持其他 SeaTunnel 类型。 @@ -85,7 +85,8 @@ InfluxDB 用户密码。 ### key_time [string] 指定 SeaTunnelRow 中作为 InfluxDB measurement 时间戳的字段名。 -如果未指定，则使用处理时间作为时间戳；支持数值类型或 ISO-8601 时间戳字符串。 +如果未指定，则使用处理时间作为时间戳；该字段必须是 `BIGINT`（epoch 毫秒数值）、内容为数字的 `STRING`， +或 `TIMESTAMP` 类型（按 UTC 时区转换为 epoch 毫秒），不支持 ISO-8601 时间戳字符串。 ### key_tags [array] diff --git a/docs/zh/connectors/sink/S3File.md b/docs/zh/connectors/sink/S3File.md index d1990c51bc..fc84b21286 100644 --- a/docs/zh/connectors/sink/S3File.md +++ b/docs/zh/connectors/sink/S3File.md @@ -117,7 +117,7 @@ import ChangeLog from '../changelog/connector-file-s3.md'; | filename_time_format | string | 否 | "yyyy.MM.dd" | 仅当 custom_filename 为 true 时使用 | | file_format_type | string | 否 | "csv" | | | filename_extension | string | 否 | - | 使用自定义的文件扩展名覆盖默认的文件扩展名。例如：`.xml`、`.json`、`dat`、`.customtype` | -| field_delimiter | string | 否 | '\001' for text and ',' for csv | 仅当 file_format 为 text 时使用 | +| field_delimiter | string | 否 | '\001' for text and ',' for csv | 仅当 file_format 为 text 和 csv 时使用 | | row_delimiter | string | 否 | "\n" | 仅当 file_format 为 `text`、`csv`、`json` 时使用 | | have_partition | boolean | 否 | false | 是否需要处理分区。 | | partition_by | array | 否 | - | 仅当 have_partition 为 true 时使用 | @@ -127,6 +127,7 @@ import ChangeLog from '../changelog/connector-file-s3.md'; | is_enable_transaction | boolean | 否 | true | | | batch_size | int | 否 | 1000000 | | | compress_codec | string | 否 | none | | +| multi_table_sink_replica | int | 否 | 1 | 多表 sink 作业中每张表使用的 sink writer 副本数量。 | | common-options | object | 否 | - | | | max_rows_in_memory | int | 否 | - | 仅当 file_format 为 excel 时使用 | | sheet_max_rows | int | 否 | 1048576 | 仅当 file_format 为 excel 时使用，每个工作表允许写入的最大行数。 | @@ -145,7 +146,7 @@ import ChangeLog from '../changelog/connector-file-s3.md'; | data_save_mode | Enum | 否 | APPEND_DATA | 在开启同步任务之前，对目标路径中的数据文件进行不同的处理 | | enable_header_write | boolean | 否 | false | 仅当 file_format_type 为 text,csv 时使用。<br/> false: 不写入表头, true: 写入表头。 | | encoding | string | 否 | "UTF-8" | 仅当 file_format_type 为 json,text,csv,xml 时使用。 | -| merge_update_event | boolean | 否 | false | 仅当file_format_type为canal_json、debezium_json、maxwell_json. | +| merge_update_event | boolean | 否 | false | 仅当 file_format_type 为 canal_json、debezium_json、maxwell_json 时使用。<br/> 设置为 true 时，UPDATE_AFTER 和 UPDATE_BEFORE 事件会合并为 UPDATE 事件数据。 | ### path [string] @@ -200,7 +201,7 @@ hadoop_s3_properties { ### field_delimiter [string] -行数据中列之间的分隔符。仅在 `text` 文件格式中需要。 +行数据中列之间的分隔符。仅在 `text` 和 `csv` 文件格式中需要。 ### row_delimiter [string] @@ -376,7 +377,6 @@ source { } # 如果您想了解更多关于如何配置SeaTunnel以及查看完整的源插件列表， # 请访问 https://seatunnel.apache.org/docs/connectors/source -source { } transform { diff --git a/docs/zh/connectors/sink/SftpFile.md b/docs/zh/connectors/sink/SftpFile.md index 7eaa932ef0..0169c98e1e 100644 --- a/docs/zh/connectors/sink/SftpFile.md +++ b/docs/zh/connectors/sink/SftpFile.md @@ -14,6 +14,7 @@ import ChangeLog from '../changelog/connector-file-sftp.md'; 如果你使用SeaTunnel引擎，当你下载并安装SeaTunnel引擎时，它会自动集成hadoop jar包。您可以在${SEATUNNEL_HOME}/lib下找到jar包。 +::: ## 主要特性 diff --git a/docs/zh/connectors/source/GcsFile.md b/docs/zh/connectors/source/GcsFile.md index 0aa7f4a74e..28b9375806 100644 --- a/docs/zh/connectors/source/GcsFile.md +++ b/docs/zh/connectors/source/GcsFile.md @@ -54,10 +54,17 @@ import ChangeLog from '../changelog/connector-file-gcs.md'; | field_delimiter | string | 否 | `\001` | text 和 CSV 的字段分隔符，`delimiter` 是其别名。 | | row_delimiter | string | 否 | `\n` | text 文件的行分隔符。 | | skip_header_row_number | long | 否 | `0` | 跳过 text 或 CSV 文件开头的行数。 | +| csv_use_header_line | boolean | 否 | `false` | 是否使用标题行来解析文件，仅在 file_format 为 `csv` 且文件包含符合 RFC 4180 的标题行时使用。 | | encoding | string | 否 | `UTF-8` | text、JSON、CSV 和 XML 的字符编码。 | +| date_format | string | 否 | `yyyy-MM-dd` | 日期类型格式，用于告诉连接器如何将字符串转换为日期。 | +| datetime_format | string | 否 | `yyyy-MM-dd HH:mm:ss` | 时间戳类型格式，用于告诉连接器如何将字符串转换为时间戳。 | +| time_format | string | 否 | `HH:mm:ss` | 时间类型格式，用于告诉连接器如何将字符串转换为时间。 | | parse_partition_from_path | boolean | 否 | `true` | 从 `/year=2026/month=08` 形式的路径解析分区值。 | | recursive_file_scan | boolean | 否 | `true` | 是否递归扫描子目录。 | | file_filter_pattern | string | 否 | - | 文件名过滤模式。 | +| file_filter_modified_start | string | 否 | - | 按最后修改时间过滤文件。过滤修改时间晚于等于该时间的文件（含该时间），默认格式为 `yyyy-MM-dd HH:mm:ss`。 | +| file_filter_modified_end | string | 否 | - | 按最后修改时间过滤文件。过滤修改时间早于该时间的文件（不含该时间），默认格式为 `yyyy-MM-dd HH:mm:ss`。 | +| sort_files_by_modification_time | boolean | 否 | `false` | 是否按修改时间降序排序文件。读取不断演化的 schema 时启用此选项，可确保 schema 推断使用最新的文件。 | | filename_extension | string | 否 | - | 文件扩展名过滤，例如 `csv` 或 `.json`。 | | compress_codec | string | 否 | `none` | 单个压缩文件的压缩编码。 | | archive_compress_codec | string | 否 | `none` | 归档压缩编码。 | @@ -85,6 +92,7 @@ import ChangeLog from '../changelog/connector-file-gcs.md'; | backup_path | string | 条件必填 | - | `post_sync_action=backup` 时必填，且不能与源路径重叠。 | | retention_max_age | string | 否 | - | SeaTunnel 备份对象的最大保留时间。 | | retention_check_interval | string | 否 | `1H` | 备份保留扫描间隔。 | +| metalake_type | string | 否 | `gravitino` | Metalake 服务类型，目前支持 `gravitino`。 | | common-options | | 否 | - | 参见 [Source 通用配置](../common-options/source-common-options.md)。 | ## 示例 diff --git a/docs/zh/connectors/source/InfluxDB.md b/docs/zh/connectors/source/InfluxDB.md index 76bff9ed74..b4f35f3568 100644 --- a/docs/zh/connectors/source/InfluxDB.md +++ b/docs/zh/connectors/source/InfluxDB.md @@ -143,7 +143,7 @@ InfluxDB 密码。必须和 `username` 一起配置。 `split_column` 列的下界。启用并行范围读取时使用。 连接器会把 `split_column` 范围切成 `partition_num` 份；若 `partition_num = 1`，则使用整段范围；若 -`partition_num` 小于 `upper_bound - lower_bound`，则按 `upper_bound - lower_bound` 切分。 +`partition_num` 大于 `upper_bound - lower_bound`，则按 `upper_bound - lower_bound` 切分。 例如 `lower_bound = 1`、`upper_bound = 10`、`partition_num = 2`、 `sql = "select * from test where age > 0 and age < 10"` 会被切分成： diff --git a/docs/zh/connectors/source/MongoDB-CDC.md b/docs/zh/connectors/source/MongoDB-CDC.md index 7b60d22742..5ccd43a7b5 100644 --- a/docs/zh/connectors/source/MongoDB-CDC.md +++ b/docs/zh/connectors/source/MongoDB-CDC.md @@ -132,7 +132,8 @@ db.grantRolesToUser("<USER_NAME>", ["<ROLE_NAME>"]) | stop.mode | Enum | 否 | NEVER | MongoDB CDC 的停止模式，可选值为 `never` 和 `timestamp`。详见下方[停止模式](#停止模式)。 | | stop.timestamp | Long | 否 | - | 在该毫秒级时间戳对应的变更流位置停止。仅在 `stop.mode` 为 `timestamp` 时使用。 | | exactly_once | Boolean | 否 | false | 启用精确一次语义。开启后，大表快照阶段恢复时可能增加内存使用。 | -| debezium | Config | 否 | - | 透传给内嵌 Debezium 引擎的配置。 | +| format | Enum | 否 | DEFAULT | MongoDB CDC 的可选输出格式，有效枚举值为 `DEFAULT`、`COMPATIBLE_DEBEZIUM_JSON`。 | +| debezium | Config | 否 | {key.converter.schemas.enable=false, value.converter.schemas.enable=false} | 透传给内嵌 Debezium 引擎的配置。 | | common-options | | 否 | - | 源插件通用参数，请参考 [源通用选项](../common-options/source-common-options.md)。 | ### 启动模式 diff --git a/docs/zh/connectors/source/PostgreSQL-CDC.md b/docs/zh/connectors/source/PostgreSQL-CDC.md index 1b1f6906d4..17c45e6e04 100644 --- a/docs/zh/connectors/source/PostgreSQL-CDC.md +++ b/docs/zh/connectors/source/PostgreSQL-CDC.md @@ -95,6 +95,7 @@ ALTER TABLE your_table_name REPLICA IDENTITY FULL; | username | String | 是 | - | 连接到数据库服务器时使用的数据库名称。 [...] | password | String | 是 | - | 连接到数据库服务器时使用的密码。 [...] | database-names | List | 否 | - | 需要监控的数据库名称。 [...] +| schema-name | List | 否 | - | 需要监控的数据库 Schema 名称，例如：`["inventory"]`。 [...] | table-names | List | 二选一 | - | 需要监控的表。请使用完整的 `database.schema.table` 格式，例如：`postgres_cdc.inventory.orders`。 [...] | table-pattern | String | 二选一 | - | 需要监控的表名正则表达式。正则需要匹配完整表名，例如：`postgres_cdc\\.inventory\\..*`。`table-names` 和 `table-pattern` 互斥。 [...] | table-names-config | List | 否 | - | 表级配置列表。例如：`[{"table": "db1.schema1.table1","primaryKeys": ["key1"],"snapshotSplitColumn": "key2"}]`。无物理主键表可通过 `primaryKeys` 指定唯一键。`snapshotSplitColumn` 必须是唯一键，否则 SeaTunnel 会忽略该配置并自动选择拆分列。 [...] @@ -104,8 +105,8 @@ ALTER TABLE your_table_name REPLICA IDENTITY FULL; | snapshot.fetch.size | Integer | 否 | 1024 | 读取表快照时每次轮询的最大获取大小。 [...] | slot.name | String | 否 | seatunnel | PostgreSQL 逻辑解码槽名称。同一个 PostgreSQL 实例上如果有多个 CDC 任务，请为每个任务配置不同的 `slot.name`。 | | decoding.plugin.name | String | 否 | pgoutput | 安装在服务器上的 Postgres 逻辑解码插件的名称，支持的值有 decoderbufs、wal2json、wal2json_rds、wal2json_streaming、wal2json_rds_streaming 和 pgoutput。 [...] -| server-time-zone | String | 否 | UTC | 数据库服务器中的会话时区。如果未设置，则使用 ZoneId.systemDefault() 来确定服务器时区。 [...] -| connect.timeout.ms | Duration | 否 | 30000 | 连接器在尝试连接到数据库服务器后应等待的最大时间，以防超时。 [...] +| server-time-zone | String | 否 | - | 数据库服务器中的会话时区。如果未设置，则使用 ZoneId.systemDefault() 来确定服务器时区。 [...] +| connect.timeout.ms | Long | 否 | 30000 | 连接器在尝试连接到数据库服务器后应等待的最大时间，以防超时。 [...] | connect.max-retries | Integer | 否 | 3 | 连接器应重试建立数据库服务器连接的最大重试次数。 [...] | connection.pool.size | Integer | 否 | 20 | JDBC 连接池大小。 [...] | chunk-key.even-distribution.factor.upper-bound | Double | 否 | 100 | 块键分布因子的上限。此因子用于确定表数据是否均匀分布。如果计算出的分布因子小于或等于此上限（即 (MAX(id) - MIN(id) + 1) / 行数），则将优化表块以实现均匀分布。否则，如果分布因子更大，则将认为该表分布不均匀，并且如果估计的分片数量超过 `sample-sharding.threshold` 指定的值，则将使用基于采样的分片策略。默认值为 100.0。 | @@ -231,7 +232,7 @@ source { username = "postgres" password = "postgres" database-names = ["postgres_cdc"] - schema-names = ["inventory"] + schema-name = ["inventory"] table-names = ["postgres_cdc.inventory.postgres_cdc_table_1"] url = "jdbc:postgresql://postgres_cdc_e2e:5432/postgres_cdc?loggerLevel=OFF" decoding.plugin.name = "decoderbufs" @@ -260,7 +261,7 @@ source { username = "postgres" password = "postgres" database-names = ["postgres_cdc"] - schema-names = ["inventory"] + schema-name = ["inventory"] table-names = ["postgres_cdc.inventory.postgres_cdc_table_1"] url = "jdbc:postgresql://postgres_cdc_e2e:5432/postgres_cdc?loggerLevel=OFF" decoding.plugin.name = "decoderbufs" @@ -298,7 +299,7 @@ source { username = "postgres" password = "postgres" database-names = ["postgres_cdc"] - schema-names = ["inventory"] + schema-name = ["inventory"] table-names = ["postgres_cdc.inventory.full_types_no_primary_key"] url = "jdbc:postgresql://postgres_cdc_e2e:5432/postgres_cdc?loggerLevel=OFF" decoding.plugin.name = "decoderbufs" diff --git a/docs/zh/connectors/source/S3File.md b/docs/zh/connectors/source/S3File.md index 8dc9fd2752..4cddadf6a6 100644 --- a/docs/zh/connectors/source/S3File.md +++ b/docs/zh/connectors/source/S3File.md @@ -236,7 +236,7 @@ schema { | hadoop_s3_properties | map | 否 | - | 如果您需要添加其他选项，可以在此处添加并参考此[链接](https://hadoop.apache.org/docs/stable/hadoop-aws/tools/hadoop-aws/index.html) | | delimiter/field_delimiter | string | 否 | \001 | 字段分隔符，用于告诉连接器在读取文本文件时如何切分字段。默认`\001`，与hive的默认分隔符相同。 | | row_delimiter | string | 否 | \n | 行分隔符，用于告诉连接器在读取文本文件时如何切分行。默认`\n`。 | -| parse_partition_from_path | boolean | 否 | true | 控制是否从文件路径解析分区键和值。例如，如果您从路径`s3n://hadoop-cluster/tmp/seatunnel/parquet/name=tyrantlucifer/age=26`读取文件。文件中的每条记录数据都将添加这两个字段：name="tyrantlucifer"，age=16 | +| parse_partition_from_path | boolean | 否 | true | 控制是否从文件路径解析分区键和值。例如，如果您从路径`s3n://hadoop-cluster/tmp/seatunnel/parquet/name=tyrantlucifer/age=26`读取文件。文件中的每条记录数据都将添加这两个字段：name="tyrantlucifer"，age=26 | | date_format | string | 否 | yyyy-MM-dd | 日期类型格式，用于告诉连接器如何将字符串转换为日期，支持以下格式：`yyyy-MM-dd` `yyyy.MM.dd` `yyyy/MM/dd`。默认`yyyy-MM-dd` | | datetime_format | string | 否 | yyyy-MM-dd HH:mm:ss | 日期时间类型格式，用于告诉连接器如何将字符串转换为日期时间，支持以下格式：`yyyy-MM-dd HH:mm:ss` `yyyy.MM.dd HH:mm:ss` `yyyy/MM/dd HH:mm:ss` `yyyyMMddHHmmss` | | time_format | string | 否 | HH:mm:ss | 时间类型格式，用于告诉连接器如何将字符串转换为时间，支持以下格式：`HH:mm:ss` `HH:mm:ss.SSS` |
