This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/dev/pr-11487-be73009224882acdeff90f147bf0406650528aa9 in repository https://gitbox.apache.org/repos/asf/seatunnel.git
commit f07b3ccfad1a440c081401c3588fe9bd84187740 Author: FatMan <[email protected]> AuthorDate: Mon Nov 9 12:45:36 2026 +0000 [Feature][Connector-V2] Support snapshot-only startup mode for MySQL CDC (#11487) Co-authored-by: chen <[email protected]> Co-authored-by: David Zollo <[email protected]> Co-authored-by: Claude Sonnet 5.5 <[email protected]> --- docs/en/connectors/source/MySQL-CDC.md | 52 ++- docs/en/connectors/source/OceanBase-CDC.md | 2 + docs/en/connectors/source/Opengauss-CDC.md | 2 +- docs/en/connectors/source/PostgreSQL-CDC.md | 2 + docs/zh/connectors/source/MySQL-CDC.md | 52 ++- docs/zh/connectors/source/OceanBase-CDC.md | 2 + docs/zh/connectors/source/Opengauss-CDC.md | 2 +- docs/zh/connectors/source/PostgreSQL-CDC.md | 2 + .../cdc/base/source/IncrementalSource.java | 127 +++++-- .../enumerator/SnapshotOnlySplitAssigner.java | 31 +- .../source/reader/IncrementalSourceReader.java | 13 + .../cdc/base/source/IncrementalSourceTest.java | 397 +++++++++++++++++++++ .../enumerator/SnapshotOnlySplitAssignerTest.java | 97 +++++ .../IncrementalSourceReaderSnapshotOnlyTest.java | 81 +++++ .../source/reader/IncrementalSourceReaderTest.java | 19 +- .../config/MySqlIncrementalSourceOptions.java | 9 +- .../cdc/mysql/config/MySqlSnapshotOnlyGuards.java | 106 ++++++ .../source/MySqlIncrementalSourceFactory.java | 15 +- .../source/MySqlIncrementalSourceFactoryTest.java | 109 ++++++ .../OceanBaseIncrementalSourceFactoryTest.java | 20 ++ .../cdc/mysql/MysqlCDCSnapshotOnlyIT.java | 304 ++++++++++++++++ .../src/test/resources/mysqlcdc_snapshot_only.conf | 51 +++ 22 files changed, 1445 insertions(+), 50 deletions(-) diff --git a/docs/en/connectors/source/MySQL-CDC.md b/docs/en/connectors/source/MySQL-CDC.md index 1df8363e7f..5bf5f95047 100644 --- a/docs/en/connectors/source/MySQL-CDC.md +++ b/docs/en/connectors/source/MySQL-CDC.md @@ -194,14 +194,14 @@ When an initial consistent snapshot is made for large databases, your establishe | table-names | List | Conditionally required | - | Table names to monitor. Each value must include the database name, for example: `database_name.table_name`. Configure either `table-names` or `table-pattern`. [...] | table-pattern | String | Conditionally required | - | Regular expression for table names to capture. Each matched table name includes the database name, for example: `database.*\\.table_.*`. Configure either `table-names` or `table-pattern`. [...] | table-names-config | List | No | - | Per-table config list. For example: `[{"table": "db1.table1","primaryKeys": ["key1"],"snapshotSplitColumn": "key2"}]`. Use this when the table has no primary key, needs a custom primary key, or needs an explicit snapshot split column. `snapshotSplitColumn` should be a primary key or unique key. If a non-unique column is provided, SeaTunnel ignores it and automatically selects an appropriate split column interna [...] -| startup.mode | Enum | No | INITIAL | Optional startup mode for MySQL CDC consumer, valid enumerations are `initial`, `earliest`, `latest` , `specific` and `timestamp`. <br/> `initial`: Synchronize historical data at startup, and then synchronize incremental data.<br/> `earliest`: Startup from the earliest offset possible.<br/> `latest`: Startup from the latest offset.<br/> `specific`: Startup from user-supplied specific offsets.<br/> `timestamp`: [...] -| startup.specific-offset.file | String | No | - | Start from the specified binlog file name. **Note, This option is required when the `startup.mode` option used `specific`.** [...] -| startup.specific-offset.pos | Long | No | - | Start from the specified binlog file position. **Note, This option is required when the `startup.mode` option used `specific`.** [...] -| startup.specific-offset.gtid-set | String | No | - | Optional MySQL GTID set for `specific` startup mode. This option is used together with `startup.specific-offset.file` and `startup.specific-offset.pos`. [...] +| startup.mode | Enum | No | INITIAL | Optional startup mode for MySQL CDC consumer, valid enumerations are `initial`, `earliest`, `latest` , `specific`, `timestamp` and `snapshot-only`. <br/> `initial`: Synchronize historical data at startup, and then synchronize incremental data.<br/> `earliest`: Startup from the earliest offset possible.<br/> `latest`: Startup from the latest offset.<br/> `specific`: Startup from user-supplied specific offsets.<b [...] +| startup.specific-offset.file | String | No | - | Start from the specified binlog file name. **Note, This option is required when the `startup.mode` option used `specific`.** Rejected when `startup.mode` is `snapshot-only`. [...] +| startup.specific-offset.pos | Long | No | - | Start from the specified binlog file position. **Note, This option is required when the `startup.mode` option used `specific`.** Rejected when `startup.mode` is `snapshot-only`. [...] +| startup.specific-offset.gtid-set | String | No | - | Optional MySQL GTID set for `specific` startup mode. This option is used together with `startup.specific-offset.file` and `startup.specific-offset.pos`. Rejected when `startup.mode` is `snapshot-only`. [...] | startup.specific-offset.skip-events | Long | No | - | Number of binlog events to skip after the configured specific startup offset. When omitted, no events are skipped. This option can only be used when `startup.mode` is `specific`. [...] | startup.specific-offset.skip-rows | Long | No | - | Number of rows to skip after the configured specific startup offset. When omitted, no rows are skipped. This option can only be used when `startup.mode` is `specific`. [...] -| startup.timestamp | Long | No | - | Start from the specified timestamp, in milliseconds since Unix epoch. **Note, This option is required when the `startup.mode` option uses `timestamp`.** [...] -| stop.mode | Enum | No | NEVER | Optional stop mode for MySQL CDC consumer, valid enumerations are `never`, `latest` or `specific`. <br/> `never`: Real-time job don't stop the source.<br/> `latest`: Stop from the latest offset. When combined with a snapshot-taking startup mode (`initial`/`earliest`), the stop offset is resolved once the snapshot phase completes, so changes written to the source while the snapshot is running are still captured [...] +| startup.timestamp | Long | No | - | Start from the specified timestamp, in milliseconds since Unix epoch. **Note, This option is required when the `startup.mode` option uses `timestamp`.** Rejected when `startup.mode` is `snapshot-only`. [...] +| stop.mode | Enum | No | NEVER | Optional stop mode for MySQL CDC consumer, valid enumerations are `never`, `latest` or `specific`. <br/> `never`: Real-time job don't stop the source.<br/> `latest`: Stop from the latest offset. When combined with a snapshot-taking startup mode (`initial`/`earliest`), the stop offset is resolved once the snapshot phase completes, so changes written to the source while the snapshot is running are still captured [...] | stop.specific-offset.file | String | No | - | Stop from the specified binlog file name. **Note, This option is required when the `stop.mode` option used `specific`.** [...] | stop.specific-offset.pos | Long | No | - | Stop from the specified binlog file position. **Note, This option is required when the `stop.mode` option used `specific`.** [...] | snapshot.split.size | Integer | No | 8096 | The split size (number of rows) of table snapshot, captured tables are split into multiple splits when read the snapshot of table. [...] @@ -218,7 +218,7 @@ When an initial consistent snapshot is made for large databases, your establishe | inverse-sampling.rate | Integer | No | 1000 | The inverse of the sampling rate used in the sample sharding strategy. For example, if this value is set to 1000, it means a 1/1000 sampling rate is applied during the sampling process. This option provides flexibility in controlling the granularity of the sampling, thus affecting the final number of shards. It's especially useful when dealing with very large datasets where a lower sampling rate is preferred. T [...] | split.allow-sampling | Boolean | No | true | Whether to allow sampling-based sharding strategy. When set to false, the system will fall back to unevenly-sized chunk splitting (iterative query approach) regardless of the shard count. The default value is true. | | enable_concurrent_read | Boolean | No | true | Whether to enable concurrent read with split during the snapshot phase. When set to false, the source skips split analysis and reads the table as a single split, which is useful for tables without indexes. The default value is true. | -| exactly_once | Boolean | No | false | Enable exactly once semantic. [...] +| exactly_once | Boolean | No | false | Enable exactly once semantics. When `startup.mode` is `initial` or `snapshot-only`, it additionally enables bounded low-to-high-watermark binlog backfill during the snapshot phase. [...] | format | Enum | No | DEFAULT | Optional output format for MySQL CDC, valid enumerations are `DEFAULT`、`COMPATIBLE_DEBEZIUM_JSON`. [...] | schema-changes.enabled | Boolean | No | false | Schema evolution is disabled by default. Now we only support `add column`、`drop column`、`rename column` and `modify column`. [...] | schema-changes.include | List | No | - | Only the listed schema change event types are sent downstream (when `schema-changes.enabled = true`). Empty means all are eligible. See [Schema change event filtering](#schema-change-event-filtering). [...] @@ -536,6 +536,42 @@ source { } ``` +### Snapshot-Only Bootstrap (One-Time Backfill) + +Use `startup.mode = "snapshot-only"` for a bounded bootstrap job: it reads the configured snapshot data and then finishes cleanly, without switching into binlog streaming. This suits one-time backfill, initial warehouse/table bootstrap, and controlled migration stages where a continuous CDC task is not wanted. + +Because the job never enters continuous binlog streaming, `snapshot-only` mode is incompatible with `stop.mode` values other than `never` and the binlog-oriented startup offset options `startup.specific-offset.*` and `startup.timestamp`; configuring them together fails fast at startup. `exactly_once` remains available for bounded backfill during the snapshot phase and does not start continuous binlog streaming. + +When a `snapshot-only` job is restored from a checkpoint or savepoint, it reads exactly the tables recorded in the checkpoint: tables created or added to `table-names` after the checkpoint are not read, and a restore fails with an error naming the tables when a checkpointed table that still has unread snapshot data is no longer configured or no longer exists. A checkpointed table matches a configured table only when the database and table names are identical, including letter case. Submi [...] + +Consistency under concurrent writes depends on `exactly_once`: + +- With `exactly_once = false` (default), each snapshot split is read by a plain query at its own point in time and no binlog backfill is applied. If the source tables receive writes while the job runs, the bounded output is not a consistent point-in-time snapshot: a change can be reflected in some splits and not in others. Use this setting only when the source tables are quiesced during the job. +- With `exactly_once = true`, each split is corrected with the binlog events between its low and high watermarks, so the output of every split is consistent as of its own high watermark. Splits finish at different times, so the overall output is still not a single global point-in-time snapshot, and changes made after a split has finished are not read because the job never enters continuous binlog streaming. Enable it when the source tables can receive writes during the snapshot, and quie [...] + +When `exactly_once = true`, each snapshot reader opens its own MySQL replication connection for the bounded backfill, so configure a `server-id` range that covers the job parallelism (for example `server-id = "5656-5657"` for parallelism 2). Each reader uses the server id `start + subtask index` from the range. SeaTunnel does not check the range against the job parallelism when the job is submitted; a range that is too small is only reported when a reader starts, with an error saying tha [...] + +The following example reads a snapshot with parallelism 2 and bounded binlog backfill enabled. Replace the placeholders with your own values, and use [Config Variable Substitution](../../introduction/concepts/config.md#config-variable-substitution) instead of writing real credentials into the job file, for example by passing `-i mysql_password=<password>` when submitting the job or by setting the `mysql_password` environment variable. + +```hocon +env { + parallelism = 2 + job.mode = "BATCH" +} + +source { + MySQL-CDC { + url = "jdbc:mysql://<mysql_host>:3306/<database>" + username = "<username>" + password = "${mysql_password}" + table-names = ["<database>.<table>"] + startup.mode = "snapshot-only" + exactly_once = true + server-id = "5656-5657" + } +} +``` + ### Bounded Read: Stop at a Specific Binlog Offset Use `stop.mode = "specific"` to make the job a bounded read: it reads the binlog between the @@ -666,7 +702,7 @@ unique key, UPDATE and DELETE events cannot be applied safely downstream. ### How does the full snapshot phase work, and when does it switch to incremental reading? -On first startup, SeaTunnel takes a consistent full snapshot of the configured tables. After the snapshot completes, it automatically switches to reading binlog from the position recorded at the beginning of the snapshot, ensuring no events are lost during the transition. +With `startup.mode = "initial"`, SeaTunnel takes a consistent full snapshot of the configured tables and then switches to reading binlog from the position recorded at the beginning of the snapshot, ensuring no events are lost during the transition. With `startup.mode = "snapshot-only"`, the source finishes after the snapshot phase and never enters continuous binlog reading. ### Does MySQL CDC support DDL propagation? diff --git a/docs/en/connectors/source/OceanBase-CDC.md b/docs/en/connectors/source/OceanBase-CDC.md index 38c87ecf48..ce306adb0c 100644 --- a/docs/en/connectors/source/OceanBase-CDC.md +++ b/docs/en/connectors/source/OceanBase-CDC.md @@ -77,6 +77,8 @@ list. `table-pattern`, and `table-names-config`. - Startup modes, checkpoint / restore semantics, and schema evolution behavior are the same as `MySQL-CDC`. +- `startup.mode = "snapshot-only"` is inherited from `MySQL-CDC` with the same option restrictions. + See [Snapshot-Only Bootstrap](./MySQL-CDC.md#snapshot-only-bootstrap-one-time-backfill). ## Task Example diff --git a/docs/en/connectors/source/Opengauss-CDC.md b/docs/en/connectors/source/Opengauss-CDC.md index 4aae472f19..e6210bdb94 100644 --- a/docs/en/connectors/source/Opengauss-CDC.md +++ b/docs/en/connectors/source/Opengauss-CDC.md @@ -79,7 +79,7 @@ select 'ALTER TABLE ' || schemaname || '.' || tablename || ' REPLICA IDENTITY FU | table-names | List | Yes, if `table-pattern` is not used | - | Tables to monitor. Use the fully qualified `database.schema.table` format, for example: `opengauss_cdc.inventory.orders`. Use `database.schema.table` matching to scope capture to a specific schema inside a database. [...] | table-pattern | String | Yes, if `table-names` is not used | - | Regular expression for tables to monitor. Use the fully qualified table name in the pattern, for example: `opengauss_cdc\\.inventory\\..*`. `table-names` and `table-pattern` are mutually exclusive. [...] | table-names-config | List | No | - | Per-table config list. Example: `[{"table": "db1.schema1.table1","primaryKeys": ["key1"],"snapshotSplitColumn": "key2"}]`. Use `primaryKeys` for tables without a physical primary key. `snapshotSplitColumn` must be a unique key; otherwise SeaTunnel ignores it and selects a split column internally. [...] -| startup.mode | Enum | No | INITIAL | Optional startup mode for Opengauss CDC consumer, valid enumerations are `initial`, `snapshot-only`, `earliest` and `latest`. <br/> `initial`: Synchronize historical data at startup, and then synchronize incremental data.<br/> `snapshot-only`: Synchronize historical data at startup and finish as a bounded job without entering WAL streaming.<br/> `earliest`: Startup from the earliest offset possible.<br/> `late [...] +| startup.mode | Enum | No | INITIAL | Optional startup mode for Opengauss CDC consumer, valid enumerations are `initial`, `snapshot-only`, `earliest` and `latest`. <br/> `initial`: Synchronize historical data at startup, and then synchronize incremental data.<br/> `snapshot-only`: Synchronize historical data at startup and finish as a bounded job without entering WAL streaming. A restore reads only the tables recorded in the checkpoint, the same a [...] | stop.mode | Enum | No | NEVER | Optional stop mode for Opengauss CDC consumer. The only valid enumeration is `never`: the source keeps streaming WAL changes and never stops on its own once it reaches the incremental phase. [...] | snapshot.split.size | Integer | No | 8096 | The split size (number of rows) of table snapshot, captured tables are split into multiple splits when read the snapshot of table. [...] | snapshot.fetch.size | Integer | No | 1024 | The maximum fetch size for per poll when read table snapshot. [...] diff --git a/docs/en/connectors/source/PostgreSQL-CDC.md b/docs/en/connectors/source/PostgreSQL-CDC.md index 4215ebbbe3..d88b2bfede 100644 --- a/docs/en/connectors/source/PostgreSQL-CDC.md +++ b/docs/en/connectors/source/PostgreSQL-CDC.md @@ -295,6 +295,8 @@ sink { In `snapshot-only` mode, the connector skips WAL streaming entirely; configure `slot.name` if you need a dedicated slot for the snapshot read. +When a `snapshot-only` job is restored from a checkpoint or savepoint, it reads exactly the tables recorded in the checkpoint: tables created or added to `table-names` after the checkpoint are not read, and a restore fails with an error naming the tables when a checkpointed table that still has unread snapshot data is no longer configured or no longer exists. A checkpointed table matches a configured table only when the database, schema and table names are identical, including letter cas [...] + ### Read tables without a primary key Pick the path that matches what the source table guarantees: diff --git a/docs/zh/connectors/source/MySQL-CDC.md b/docs/zh/connectors/source/MySQL-CDC.md index 4c5354c5a4..a06ece9a64 100644 --- a/docs/zh/connectors/source/MySQL-CDC.md +++ b/docs/zh/connectors/source/MySQL-CDC.md @@ -193,14 +193,14 @@ show variables where variable_name in ('log_bin', 'binlog_format', 'binlog_row_i | table-names | List | 条件必填 | - | 要监控的表名,每个表名都需要包含库名,例如:`database_name.table_name`。`table-names` 和 `table-pattern` 二选一配置。 | | table-pattern | String | 条件必填 | - | 要捕获的表名正则表达式,匹配到的表名需要包含库名,例如:`database.*\\.table_.*`。`table-names` 和 `table-pattern` 二选一配置。 | | table-names-config | List | 否 | - | 按表单独配置。例如:`[{"table": "db1.table1","primaryKeys": ["key1"],"snapshotSplitColumn": "key2"}]`。当表没有主键、需要自定义主键,或需要指定快照拆分列时使用。`snapshotSplitColumn` 应该是主键或唯一键;如果指定了非唯一列,SeaTunnel 会忽略该配置并自动选择合适的拆分列。 | -| startup.mode | Enum | 否 | INITIAL | MySQL CDC 消费者的可选启动模式, 有效枚举值为 `initial`, `earliest`, `latest` , `specific` 和 `timestamp`. <br/> `initial`: 启动时同步历史数据, 然后同步增量数据.<br/> `earliest`: 从尽可能最早的偏移量开始启动.<br/> `latest`: 从最近的偏移量启动.<br/> `specific`: 从用户提供的特定偏移量开始启动.<br/> `timestamp`: 从用户提供的特定时间戳开始启动. | -| startup.specific-offset.file | String | 否 | - | 从指定的binlog日志文件名开始. **注意, 当使用 `startup.mode` 选项为 `specific` 时,此选项为必填项.** | -| startup.specific-offset.pos | Long | 否 | - | 从指定的binlog日志文件位置开始. **注意, 当使用 `startup.mode` 选项为 `specific` 时,此选项为必填项.** | -| startup.specific-offset.gtid-set | String | 否 | - | 当 `startup.mode` 为 `specific` 时,可选配置 MySQL GTID 集合. 该选项需要和 `startup.specific-offset.file`、`startup.specific-offset.pos` 一起使用. | +| startup.mode | Enum | 否 | INITIAL | MySQL CDC 消费者的可选启动模式, 有效枚举值为 `initial`, `earliest`, `latest` , `specific`, `timestamp` 和 `snapshot-only`. <br/> `initial`: 启动时同步历史数据, 然后同步增量数据.<br/> `earliest`: 从尽可能最早的偏移量开始启动.<br/> `latest`: 从最近的偏移量启动.<br/> `specific`: 从用户提供的特定偏移量开始启动.<br/> `timestamp`: 从用户提供的特定时间戳开始启动.<br/> `snapshot-only`: 仅读取快照数据,读取完成后任务有界结束,不会在快照完成后进入持续 binlog 流式读取。适用于一次性回填或初始化引导场景。该模式支持使用 `exactly_once` 在快照阶段执行有界回填,但不能与除 `never` 外的 `stop.mode [...] +| startup.specific-offset.file | String | 否 | - | 从指定的binlog日志文件名开始. **注意, 当使用 `startup.mode` 选项为 `specific` 时,此选项为必填项.** 当 `startup.mode` 为 `snapshot-only` 时不允许配置此选项. | +| startup.specific-offset.pos | Long | 否 | - | 从指定的binlog日志文件位置开始. **注意, 当使用 `startup.mode` 选项为 `specific` 时,此选项为必填项.** 当 `startup.mode` 为 `snapshot-only` 时不允许配置此选项. | +| startup.specific-offset.gtid-set | String | 否 | - | 当 `startup.mode` 为 `specific` 时,可选配置 MySQL GTID 集合. 该选项需要和 `startup.specific-offset.file`、`startup.specific-offset.pos` 一起使用. 当 `startup.mode` 为 `snapshot-only` 时不允许配置此选项. | | startup.specific-offset.skip-events | Long | 否 | - | 配置 specific 启动偏移量后需要跳过的 binlog event 数量. 未配置时不跳过任何 event. 该选项只能在 `startup.mode` 为 `specific` 时使用. | | startup.specific-offset.skip-rows | Long | 否 | - | 配置 specific 启动偏移量后需要跳过的行数. 未配置时不跳过任何行. 该选项只能在 `startup.mode` 为 `specific` 时使用. | -| startup.timestamp | Long | 否 | - | 从指定时间戳启动,单位为 Unix 纪元以来的毫秒数。**注意,当 `startup.mode` 为 `timestamp` 时,此选项必填。** | -| stop.mode | Enum | 否 | NEVER | MySQL CDC 消费者的可选停止模式, 有效枚举值为 `never`, `latest` 和 `specific`. <br/> `never`: 实时任务一直运行不停止.<br/> `latest`: 从最新的偏移量处停止. 与需要执行快照的启动模式(`initial`/`earliest`)组合使用时, 停止偏移量在快照阶段完成后解析, 因此快照运行期间写入源表的变更仍会被 binlog 阶段捕获, 不会丢失.<br/> `specific`: 从用户提供的特定偏移量处停止. | +| startup.timestamp | Long | 否 | - | 从指定时间戳启动,单位为 Unix 纪元以来的毫秒数。**注意,当 `startup.mode` 为 `timestamp` 时,此选项必填。** 当 `startup.mode` 为 `snapshot-only` 时不允许配置此选项。 | +| stop.mode | Enum | 否 | NEVER | MySQL CDC 消费者的可选停止模式, 有效枚举值为 `never`, `latest` 和 `specific`. <br/> `never`: 实时任务一直运行不停止.<br/> `latest`: 从最新的偏移量处停止. 与需要执行快照的启动模式(`initial`/`earliest`)组合使用时, 停止偏移量在快照阶段完成后解析, 因此快照运行期间写入源表的变更仍会被 binlog 阶段捕获, 不会丢失.<br/> `specific`: 从用户提供的特定偏移量处停止.<br/> 当 `startup.mode` 为 `snapshot-only` 时只允许配置为 `never`. | | stop.specific-offset.file | String | 否 | - | 从指定的binlog日志文件名停止. **注意, 当使用 `stop.mode` 选项为 `specific` 时,此选项为必填项.** | | stop.specific-offset.pos | Long | 否 | - | 从指定的binlog日志文件位置停止. **注意, 当使用 `stop.mode` 选项为 `specific` 时,此选项为必填项.** | | snapshot.split.size | Integer | 否 | 8096 | 表快照的分割大小(行数),读取表的快照时,被捕获的表会被分割成多个分割块. | @@ -217,7 +217,7 @@ show variables where variable_name in ('log_bin', 'binlog_format', 'binlog_row_i | inverse-sampling.rate | Integer | 否 | 1000 | 采样分片策略中使用的采样率的倒数. 例如, 如果该值设置为 1000, 则表示在采样过程中应用了 1/1000 的采样率. 此选项在控制采样的粒度方面提供了灵活性, 从而影响最终的分片数量. 在处理非常大的数据集时非常有用, 因为此时更倾向于使用较低的采样率. 默认值为 1000. | | split.allow-sampling | Boolean | 否 | true | 是否允许基于采样的分片策略。当设置为 false 时,无论预估分片数是否超过阈值,系统都将回退到非均匀分片方式(迭代查询方式)。默认值为 true。 | | enable_concurrent_read | Boolean | 否 | true | 是否在快照阶段启用基于分片的并发读取。当设置为 false 时,source 会跳过分片分析,并以单个 split 读取整张表,适合没有索引的表。默认值为 true。 | -| exactly_once | Boolean | 否 | false | 启用精确一次语义. | +| exactly_once | Boolean | 否 | false | 启用精确一次语义。当 `startup.mode` 为 `initial` 或 `snapshot-only` 时,还会在快照阶段额外启用有界的 low-to-high-watermark binlog 回填。 | | format | Enum | 否 | DEFAULT | MySQL CDC 的可选输出格式, 有效的枚举值为 `DEFAULT`、`COMPATIBLE_DEBEZIUM_JSON`. | | schema-changes.enabled | Boolean | 否 | false | 模式演进默认是禁用的. 当前我们只支持 `add column`、`drop column`、`rename column` 和 `modify column`. | | schema-changes.include | List | 否 | - | 仅向下游发送列出的 schema change 事件类型(需 `schema-changes.enabled = true`)。为空表示全部允许。详见 [Schema change 事件过滤](#schema-change-事件过滤)。 | @@ -528,6 +528,42 @@ source { } ``` +### 仅快照(一次性引导) + +当只需要做一次性全量引导(backfill)而不希望任务持续消费 binlog 时,使用 `startup.mode = "snapshot-only"`。任务会读取配置表的快照数据,读取完成后自然结束(有界任务),不会进入增量/binlog 阶段。 + +该模式不会进入持续 binlog 流式读取,因此与除 `never` 外的 `stop.mode` 以及 binlog 相关的 `startup.specific-offset.*`、`startup.timestamp` 选项互斥;同时配置这些选项会在启动时报错。`exactly_once` 仍可用于快照阶段的有界回填,但不会启动持续 binlog 流式读取。 + +`snapshot-only` 任务从 checkpoint 或 savepoint 恢复时,只读取 checkpoint 中记录的表:checkpoint 之后新建或新加入 `table-names` 的表不会被读取;如果 checkpoint 中仍有未读快照数据的某张表已不在配置中或已不存在,恢复会失败,报错中会列出这些表。只有数据库名和表名(包括大小写)完全相同时,checkpoint 中的表才与配置的表匹配。需要修改读取的表时,请提交新作业。 + +并发写入下的一致性取决于 `exactly_once`: + +- `exactly_once = false`(默认)时,每个快照分片在各自的时间点通过普通查询读取,不做 binlog 回填。如果任务运行期间源表有写入,有界输出并不是一个一致的时间点快照:同一个变更可能只体现在部分分片中。仅在任务运行期间源表无写入时使用该配置。 +- `exactly_once = true` 时,每个分片会用其低水位到高水位之间的 binlog 事件进行修正,因此每个分片的输出与其自身的高水位一致。各分片完成时间不同,整体输出仍不是单一的全局时间点快照;分片完成之后发生的变更不会被读取,因为任务不会进入持续 binlog 流式读取。源表在快照期间可能有写入时请开启该配置;如果要求整体输出对应同一时间点,请在任务运行期间暂停写入。 + +当 `exactly_once = true` 时,快照阶段的有界回填会为每个 reader 打开独立的 MySQL 复制连接,因此需要将 `server-id` 配置为覆盖任务并行度的范围(例如并行度为 2 时配置 `server-id = "5656-5657"`)。每个 reader 使用范围内的 `起始值 + subtask 序号` 作为 server id。SeaTunnel 不会在提交任务时校验该范围是否覆盖任务并行度;范围过小时,只会在 reader 启动时报错,提示 subtask ID 超出了 server id 范围。该范围还必须与复制同一个 MySQL 实例的其他任务、从库和工具互不重复:当另一个复制连接以相同的 server id 注册时,MySQL 会断开先前的连接,表现为回填过程中间歇性的读取失败。 + +下面的示例以并行度 2 读取快照并开启有界 binlog 回填。请将占位符替换为实际值,并使用[配置变量替换](../../introduction/concepts/config.md#配置变量替换)传入凭据,不要把真实凭据直接写入作业文件,例如提交任务时通过 `-i mysql_password=<password>` 传入,或设置 `mysql_password` 环境变量。 + +```hocon +env { + parallelism = 2 + job.mode = "BATCH" +} + +source { + MySQL-CDC { + url = "jdbc:mysql://<mysql_host>:3306/<database>" + username = "<username>" + password = "${mysql_password}" + table-names = ["<database>.<table>"] + startup.mode = "snapshot-only" + exactly_once = true + server-id = "5656-5657" + } +} +``` + ### 有界读取:在指定 Binlog 位置停止 使用 `stop.mode = "specific"` 可以将作业变为有界读取:作业读取启动偏移量(或启动时间戳) @@ -655,7 +691,7 @@ source options 示例那样,通过 `table-names-config.primaryKeys` 指定自 ### 全量快照阶段如何工作?何时切换为增量读取? -首次启动时,SeaTunnel 对已配置的表做一次一致性全量快照。快照完成后,自动从快照开始时记录的 binlog 位置切换为增量读取,确保切换过程中不丢失任何变更事件。 +使用 `startup.mode = "initial"` 时,SeaTunnel 会对已配置的表执行一致性全量快照,随后从快照开始时记录的 binlog 位置切换为增量读取,确保切换过程中不丢失变更事件。使用 `startup.mode = "snapshot-only"` 时,source 会在快照阶段完成后结束,不会进入持续 binlog 读取。 ### MySQL CDC 是否支持 DDL 传播? diff --git a/docs/zh/connectors/source/OceanBase-CDC.md b/docs/zh/connectors/source/OceanBase-CDC.md index c3e0ac6241..2504df833f 100644 --- a/docs/zh/connectors/source/OceanBase-CDC.md +++ b/docs/zh/connectors/source/OceanBase-CDC.md @@ -75,6 +75,8 @@ OceanBase Binlog Service 提供增量订阅能力。 - 首批版本只支持显式配置的表,即通过 `table-names`、`table-pattern`、 `table-names-config` 指定采集范围。 - 启动模式、checkpoint / restore 语义、模式演进行为与 `MySQL-CDC` 保持一致。 +- `startup.mode = "snapshot-only"` 继承自 `MySQL-CDC`,配置限制相同,详见 + [仅快照(一次性引导)](./MySQL-CDC.md#仅快照一次性引导)。 ## 任务示例 diff --git a/docs/zh/connectors/source/Opengauss-CDC.md b/docs/zh/connectors/source/Opengauss-CDC.md index d84bf7c1fc..7c1dcb41ec 100644 --- a/docs/zh/connectors/source/Opengauss-CDC.md +++ b/docs/zh/connectors/source/Opengauss-CDC.md @@ -76,7 +76,7 @@ select 'ALTER TABLE ' || schemaname || '.' || tablename || ' REPLICA IDENTITY FU | table-names | 列表 | 二选一 | - | 需要监控的表。请使用完整的 `database.schema.table` 格式,例如:`opengauss_cdc.inventory.orders`。 | | table-pattern | 字符串 | 二选一 | - | 需要监控的表名正则表达式。正则需要匹配完整表名,例如:`opengauss_cdc\\.inventory\\..*`。`table-names` 和 `table-pattern` 互斥。 | | table-names-config | 列表 | 否 | - | 表级配置列表。例如:`[{"table": "db1.schema1.table1","primaryKeys": ["key1"],"snapshotSplitColumn": "key2"}]`。无物理主键表可通过 `primaryKeys` 指定唯一键。`snapshotSplitColumn` 必须是唯一键,否则 SeaTunnel 会忽略该配置并自动选择拆分列。 | -| startup.mode | 枚举 | 否 | INITIAL | Opengauss CDC消费者的可选启动模式,有效枚举为 `initial`、`snapshot-only`、`earliest` 和 `latest`。<br/> `initial`: 启动时同步历史数据,然后同步增量数据。<br/> `snapshot-only`: 仅同步启动时的历史数据,然后以有界任务结束,不进入 WAL 流读取。<br/> `earliest`: 从可能的最早偏移量启动。<br/> `latest`: 从最新偏移量启动。<br/> `committed-offset` 仅 PostgreSQL 支持(它读取 OpenGauss 不提供的 `pg_replication_slots` 列),该连接器会在配置校验阶段拒绝此值。 | +| startup.mode | 枚举 | 否 | INITIAL | Opengauss CDC消费者的可选启动模式,有效枚举为 `initial`、`snapshot-only`、`earliest` 和 `latest`。<br/> `initial`: 启动时同步历史数据,然后同步增量数据。<br/> `snapshot-only`: 仅同步启动时的历史数据,然后以有界任务结束,不进入 WAL 流读取。从 checkpoint 恢复时只读取 checkpoint 中记录的表,与 [PostgreSQL CDC](PostgreSQL-CDC.md#仅运行一次性快照) 相同。<br/> `earliest`: 从可能的最早偏移量启动。<br/> `latest`: 从最新偏移量启动。<br/> `committed-offset` 仅 PostgreSQL 支持(它读取 OpenGauss 不提供的 `pg_replication_slots` 列),该连接器会在配置校验阶段拒绝此值。 [...] | stop.mode | 枚举 | 否 | NEVER | Opengauss CDC 消费者的可选停止模式。唯一有效的枚举值是 `never`:一旦进入增量阶段,数据源会持续读取 WAL 变更,不会自行停止。 | | snapshot.split.size | 整型 | 否 | 8096 | 表快照的分割大小(行数),在读取表的快照时,捕获的表被分割成多个split | | snapshot.fetch.size | 整型 | 否 | 1024 | 读取表快照时,每次轮询的最大读取大小 | diff --git a/docs/zh/connectors/source/PostgreSQL-CDC.md b/docs/zh/connectors/source/PostgreSQL-CDC.md index 6c9d700f3a..93457719ce 100644 --- a/docs/zh/connectors/source/PostgreSQL-CDC.md +++ b/docs/zh/connectors/source/PostgreSQL-CDC.md @@ -290,6 +290,8 @@ sink { `snapshot-only` 模式下,connector 完全跳过 WAL 流式读取;如果快照读取需要独立的复制槽,请配置 `slot.name`。 +`snapshot-only` 任务从 checkpoint 或 savepoint 恢复时,只读取 checkpoint 中记录的表:checkpoint 之后新建或新加入 `table-names` 的表不会被读取;如果 checkpoint 中仍有未读快照数据的某张表已不在配置中或已不存在,恢复会失败,报错中会列出这些表。只有数据库名、schema 名和表名(包括大小写)完全相同时,checkpoint 中的表才与配置的表匹配。恢复时在 `snapshot-only` 与其他启动模式之间切换同样会被拒绝。需要修改读取的表或启动模式时,请提交新作业。 + ### 读取没有主键的表 根据源表能够提供的保证来选择合适的路径: diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/source/IncrementalSource.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/source/IncrementalSource.java index 2b97a41ed9..bec6210cb3 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/source/IncrementalSource.java +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/source/IncrementalSource.java @@ -59,6 +59,7 @@ import org.apache.seatunnel.connectors.cdc.base.source.split.SnapshotSplit; import org.apache.seatunnel.connectors.cdc.base.source.split.SourceRecords; import org.apache.seatunnel.connectors.cdc.base.source.split.SourceSplitBase; import org.apache.seatunnel.connectors.cdc.base.source.split.state.SourceSplitStateBase; +import org.apache.seatunnel.connectors.cdc.base.utils.CatalogTableUtils; import org.apache.seatunnel.connectors.cdc.debezium.DebeziumDeserializationSchema; import org.apache.seatunnel.connectors.cdc.debezium.DeserializeFormat; import org.apache.seatunnel.connectors.seatunnel.common.source.reader.RecordEmitter; @@ -84,7 +85,6 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; import java.util.function.Supplier; import java.util.stream.Collectors; -import java.util.stream.Stream; /** * Base class of the incremental snapshot CDC sources. @@ -380,30 +380,33 @@ public abstract class IncrementalSource<T, C extends SourceConfig> } } C sourceConfig = configFactory.create(0); - Set<TableId> capturedTables = - new HashSet<>(dataSourceDialect.discoverDataCollections(sourceConfig)); + StartupMode startupMode = sourceConfig.getStartupConfig().getStartupMode(); + validateRestoredState(startupMode, checkpointState); final SplitAssigner splitAssigner; - if (checkpointState instanceof HybridPendingSplitsState) { - checkpointState = restore(capturedTables, (HybridPendingSplitsState) checkpointState); - SnapshotPhaseState checkpointSnapshotState = - ((HybridPendingSplitsState) checkpointState).getSnapshotPhaseState(); + if (checkpointState instanceof SnapshotPhaseState) { + SnapshotPhaseState snapshotState = (SnapshotPhaseState) checkpointState; + Set<TableId> capturedTables = getCheckpointCapturedTables(snapshotState); + checkUnfinishedTablesAreConfigured(snapshotState); SplitAssigner.Context<C> assignerContext = new SplitAssigner.Context<>( sourceConfig, capturedTables, - checkpointSnapshotState.getAssignedSplits(), - checkpointSnapshotState.getSplitCompletedOffsets()); + snapshotState.getAssignedSplits(), + snapshotState.getSplitCompletedOffsets()); splitAssigner = - new HybridSplitAssigner<>( + new SnapshotOnlySplitAssigner<>( assignerContext, enumeratorContext.currentParallelism(), - incrementalParallelism, - (HybridPendingSplitsState) checkpointState, - dataSourceDialect, - offsetFactory); - } else if (checkpointState instanceof SnapshotPhaseState) { - SnapshotPhaseState checkpointSnapshotState = (SnapshotPhaseState) checkpointState; + snapshotState, + dataSourceDialect); + } else if (checkpointState instanceof HybridPendingSplitsState) { + Set<TableId> capturedTables = + new HashSet<>(dataSourceDialect.discoverDataCollections(sourceConfig)); + HybridPendingSplitsState hybridCheckpointState = + restore(capturedTables, (HybridPendingSplitsState) checkpointState); + SnapshotPhaseState checkpointSnapshotState = + hybridCheckpointState.getSnapshotPhaseState(); SplitAssigner.Context<C> assignerContext = new SplitAssigner.Context<>( sourceConfig, @@ -411,12 +414,16 @@ public abstract class IncrementalSource<T, C extends SourceConfig> checkpointSnapshotState.getAssignedSplits(), checkpointSnapshotState.getSplitCompletedOffsets()); splitAssigner = - new SnapshotOnlySplitAssigner<>( + new HybridSplitAssigner<>( assignerContext, enumeratorContext.currentParallelism(), - checkpointSnapshotState, - dataSourceDialect); + incrementalParallelism, + hybridCheckpointState, + dataSourceDialect, + offsetFactory); } else if (checkpointState instanceof IncrementalPhaseState) { + Set<TableId> capturedTables = + new HashSet<>(dataSourceDialect.discoverDataCollections(sourceConfig)); SplitAssigner.Context<C> assignerContext = new SplitAssigner.Context<>( sourceConfig, capturedTables, new HashMap<>(), new HashMap<>()); @@ -433,14 +440,88 @@ public abstract class IncrementalSource<T, C extends SourceConfig> return new IncrementalSourceEnumerator(enumeratorContext, splitAssigner); } + private static void validateRestoredState( + StartupMode startupMode, PendingSplitsState checkpointState) { + boolean snapshotOnlyMode = startupMode == StartupMode.SNAPSHOT_ONLY; + boolean snapshotOnlyState = checkpointState instanceof SnapshotPhaseState; + if (snapshotOnlyMode != snapshotOnlyState) { + throw new IllegalStateException( + String.format( + "Cannot restore startup mode '%s' from checkpoint state '%s'. Changing " + + "startup.mode across a restore is not supported; submit a new job instead.", + startupMode, checkpointState.getClass().getSimpleName())); + } + } + + /** + * Rejects a snapshot-only restore whose checkpoint still has data to read from a table that is + * not configured or no longer exists. The checkpointed table set is authoritative on this path, + * so such a table would still be split and read, while the readers no longer capture or convert + * it, and the job would fail again on every restart from the same checkpoint. Tables whose + * splits are all finished are not read again and are not checked. + * + * <p>A table is configured when its id equals the id of a configured table, built as the + * dialects build the table map their readers look tables up in ({@link + * CatalogTableUtils#convertTables}). That lookup is exact, so the check does not fold case: a + * checkpointed {@code public.Orders} does not match a configured {@code public.orders}, which + * is another table in a case-sensitive database such as PostgreSQL. + */ + private void checkUnfinishedTablesAreConfigured(SnapshotPhaseState snapshotState) { + if (catalogTables == null || catalogTables.isEmpty()) { + return; + } + Set<TableId> configuredTables = CatalogTableUtils.convertTables(catalogTables).keySet(); + List<String> unconfiguredTables = + getUnfinishedTables(snapshotState).stream() + .filter(tableId -> !configuredTables.contains(tableId)) + .map(TableId::toString) + .sorted() + .collect(Collectors.toList()); + if (!unconfiguredTables.isEmpty()) { + throw new IllegalStateException( + String.format( + "Cannot restore the snapshot-only job: the checkpoint has unread data of " + + "tables that are not in the current table list or no longer exist: %s. " + + "Changing the captured tables across a restore is not supported; " + + "submit a new job instead.", + unconfiguredTables)); + } + } + + /** Tables of the checkpoint that are not split yet or have a split that is not finished. */ + private static Set<TableId> getUnfinishedTables(SnapshotPhaseState snapshotState) { + Set<TableId> unfinishedTables = new HashSet<>(snapshotState.getRemainingTables()); + snapshotState.getRemainingSplits().stream() + .map(SnapshotSplit::getTableId) + .forEach(unfinishedTables::add); + snapshotState.getAssignedSplits().values().stream() + .filter( + split -> + !snapshotState + .getSplitCompletedOffsets() + .containsKey(split.splitId())) + .map(SnapshotSplit::getTableId) + .forEach(unfinishedTables::add); + return unfinishedTables; + } + + private static Set<TableId> getCheckpointCapturedTables(SnapshotPhaseState snapshotState) { + Set<TableId> checkpointTables = new HashSet<>(snapshotState.getAlreadyProcessedTables()); + checkpointTables.addAll(snapshotState.getRemainingTables()); + snapshotState.getRemainingSplits().stream() + .map(SnapshotSplit::getTableId) + .forEach(checkpointTables::add); + snapshotState.getAssignedSplits().values().stream() + .map(SnapshotSplit::getTableId) + .forEach(checkpointTables::add); + return checkpointTables; + } + private HybridPendingSplitsState restore( Set<TableId> capturedTables, HybridPendingSplitsState checkpointState) { SnapshotPhaseState checkpointSnapshotState = checkpointState.getSnapshotPhaseState(); Set<TableId> checkpointCapturedTables = - Stream.concat( - checkpointSnapshotState.getAlreadyProcessedTables().stream(), - checkpointSnapshotState.getRemainingTables().stream()) - .collect(Collectors.toSet()); + getCheckpointCapturedTables(checkpointSnapshotState); Set<TableId> newTables = Sets.difference(capturedTables, checkpointCapturedTables); Set<TableId> deletedTables = Sets.difference(checkpointCapturedTables, capturedTables); diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/source/enumerator/SnapshotOnlySplitAssigner.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/source/enumerator/SnapshotOnlySplitAssigner.java index 713598fdf2..88de8311cb 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/source/enumerator/SnapshotOnlySplitAssigner.java +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/source/enumerator/SnapshotOnlySplitAssigner.java @@ -56,7 +56,27 @@ public class SnapshotOnlySplitAssigner<C extends SourceConfig> implements SplitA SnapshotPhaseState checkpoint, DataSourceDialect<C> dialect) { this.snapshotSplitAssigner = - new SnapshotSplitAssigner<>(context, currentParallelism, checkpoint, dialect); + new SnapshotSplitAssigner<>( + context, + currentParallelism, + markRemainingTablesCheckpointed(checkpoint), + dialect); + } + + private static SnapshotPhaseState markRemainingTablesCheckpointed( + SnapshotPhaseState checkpoint) { + if (checkpoint.isRemainingTablesCheckpointed()) { + return checkpoint; + } + return new SnapshotPhaseState( + checkpoint.getAlreadyProcessedTables(), + checkpoint.getRemainingSplits(), + checkpoint.getAssignedSplits(), + checkpoint.getSplitCompletedOffsets(), + checkpoint.isAssignerCompleted(), + checkpoint.getRemainingTables(), + checkpoint.isTableIdCaseSensitive(), + true); } @Override @@ -81,6 +101,15 @@ public class SnapshotOnlySplitAssigner<C extends SourceConfig> implements SplitA @Override public void addSplits(Collection<SourceSplitBase> splits) { + for (SourceSplitBase split : splits) { + if (!split.isSnapshotSplit()) { + throw new IllegalStateException( + String.format( + "Snapshot-only startup mode received incremental split '%s'. " + + "Changing startup.mode across a restore is not supported.", + split.splitId())); + } + } snapshotSplitAssigner.addSplits(splits); } diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/source/reader/IncrementalSourceReader.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/source/reader/IncrementalSourceReader.java index 4e577ee7cd..4023849041 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/source/reader/IncrementalSourceReader.java +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/main/java/org/apache/seatunnel/connectors/cdc/base/source/reader/IncrementalSourceReader.java @@ -28,6 +28,7 @@ import org.apache.seatunnel.api.table.type.SeaTunnelRowType; import org.apache.seatunnel.connectors.cdc.base.config.JdbcSourceConfig; import org.apache.seatunnel.connectors.cdc.base.config.SourceConfig; import org.apache.seatunnel.connectors.cdc.base.dialect.DataSourceDialect; +import org.apache.seatunnel.connectors.cdc.base.option.StartupMode; import org.apache.seatunnel.connectors.cdc.base.source.event.CompletedSnapshotPhaseEvent; import org.apache.seatunnel.connectors.cdc.base.source.event.CompletedSnapshotSplitsAckEvent; import org.apache.seatunnel.connectors.cdc.base.source.event.CompletedSnapshotSplitsReportEvent; @@ -159,6 +160,18 @@ public class IncrementalSourceReader<T, C extends SourceConfig> unfinishedSplits.add(split); } } else { + if (sourceConfig.getStartupConfig().getStartupMode() == StartupMode.SNAPSHOT_ONLY) { + // Snapshot-only is a bounded startup mode that never streams binlog. A reader + // can only be handed an incremental split if the restored checkpoint had + // already entered the binlog phase (startup.mode was changed to snapshot-only + // across a restore). Fail fast instead of streaming binlog forever, which would + // break the bounded contract and leave the job running indefinitely. + throw new IllegalStateException( + String.format( + "Snapshot-only startup mode received an incremental split '%s' on subtask %d. " + + "Changing startup.mode across a restore is not supported.", + split.splitId(), subtaskId)); + } IncrementalSplit incrementalSplit = split.asIncrementalSplit(); if (hasRestoredCheckpointMetadata(incrementalSplit)) { if (!capturedTablesDiscovered) { diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/test/java/org/apache/seatunnel/connectors/cdc/base/source/IncrementalSourceTest.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/test/java/org/apache/seatunnel/connectors/cdc/base/source/IncrementalSourceTest.java new file mode 100644 index 0000000000..fe3e379c38 --- /dev/null +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/test/java/org/apache/seatunnel/connectors/cdc/base/source/IncrementalSourceTest.java @@ -0,0 +1,397 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.cdc.base.source; + +import org.apache.seatunnel.api.configuration.Option; +import org.apache.seatunnel.api.configuration.Options; +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.source.SourceSplitEnumerator; +import org.apache.seatunnel.api.table.catalog.CatalogTable; +import org.apache.seatunnel.api.table.catalog.TableIdentifier; +import org.apache.seatunnel.api.table.catalog.TablePath; +import org.apache.seatunnel.api.table.catalog.TableSchema; +import org.apache.seatunnel.connectors.cdc.base.config.SourceConfig; +import org.apache.seatunnel.connectors.cdc.base.config.StartupConfig; +import org.apache.seatunnel.connectors.cdc.base.config.StopConfig; +import org.apache.seatunnel.connectors.cdc.base.dialect.DataSourceDialect; +import org.apache.seatunnel.connectors.cdc.base.option.SourceOptions; +import org.apache.seatunnel.connectors.cdc.base.option.StartupMode; +import org.apache.seatunnel.connectors.cdc.base.option.StopMode; +import org.apache.seatunnel.connectors.cdc.base.source.enumerator.state.IncrementalPhaseState; +import org.apache.seatunnel.connectors.cdc.base.source.enumerator.state.PendingSplitsState; +import org.apache.seatunnel.connectors.cdc.base.source.enumerator.state.SnapshotPhaseState; +import org.apache.seatunnel.connectors.cdc.base.source.event.SnapshotSplitWatermark; +import org.apache.seatunnel.connectors.cdc.base.source.offset.OffsetFactory; +import org.apache.seatunnel.connectors.cdc.base.source.split.SnapshotSplit; +import org.apache.seatunnel.connectors.cdc.base.source.split.SourceSplitBase; +import org.apache.seatunnel.connectors.cdc.debezium.DebeziumDeserializationSchema; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import io.debezium.relational.TableId; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +class IncrementalSourceTest { + + private static final Option<StartupMode> STARTUP_MODE = + Options.key(SourceOptions.STARTUP_MODE_KEY) + .enumType(StartupMode.class) + .defaultValue(StartupMode.INITIAL) + .withDescription("Test startup mode"); + + private static final Option<StopMode> STOP_MODE = + Options.key(SourceOptions.STOP_MODE_KEY) + .enumType(StopMode.class) + .defaultValue(StopMode.NEVER) + .withDescription("Test stop mode"); + + @Test + @SuppressWarnings("unchecked") + void testSnapshotOnlyRestoreUsesCheckpointedTablesWithoutCatalogDiscovery() throws Exception { + TableId processedTable = TableId.parse("db.processed"); + TableId remainingTable = TableId.parse("db.remaining"); + TableId remainingSplitTable = TableId.parse("db.remaining_split"); + TableId assignedSplitTable = TableId.parse("db.assigned_split"); + TableId newlyDiscoveredTable = TableId.parse("db.new"); + SnapshotSplit remainingSplit = + new SnapshotSplit("db.remaining_split.0", remainingSplitTable, null, null, null); + SnapshotSplit assignedSplit = + new SnapshotSplit("db.assigned_split.0", assignedSplitTable, null, null, null); + SnapshotPhaseState checkpointState = + new SnapshotPhaseState( + Collections.singletonList(processedTable), + Collections.singletonList(remainingSplit), + Collections.singletonMap(assignedSplit.splitId(), assignedSplit), + Collections.emptyMap(), + false, + Collections.singletonList(remainingTable), + false, + false); + TestIncrementalSource source = + new TestIncrementalSource( + ReadonlyConfig.fromMap( + Collections.singletonMap(STARTUP_MODE.key(), "snapshot-only")), + Collections.emptyList()); + Mockito.when(source.getDialect().discoverDataCollections(source.getSourceConfig())) + .thenReturn(Collections.singletonList(newlyDiscoveredTable)); + SourceSplitEnumerator.Context<SourceSplitBase> enumeratorContext = + Mockito.mock(SourceSplitEnumerator.Context.class); + Mockito.when(enumeratorContext.currentParallelism()).thenReturn(2); + + SourceSplitEnumerator<SourceSplitBase, PendingSplitsState> enumerator = + source.restoreEnumerator(enumeratorContext, checkpointState); + enumerator.open(); + + Mockito.verify(source.getDialect(), Mockito.never()) + .discoverDataCollections(source.getSourceConfig()); + SnapshotPhaseState restoredState = (SnapshotPhaseState) enumerator.snapshotState(1L); + Assertions.assertTrue(restoredState.isRemainingTablesCheckpointed()); + Assertions.assertEquals( + Collections.singletonList(remainingTable), restoredState.getRemainingTables()); + Assertions.assertFalse(restoredState.getRemainingTables().contains(newlyDiscoveredTable)); + } + + @Test + @SuppressWarnings("unchecked") + void testSnapshotOnlyRestoreRejectsTableRemovedFromConfig() { + // db.b still has a split in the checkpoint, but table-names no longer lists it: the + // readers would not capture it, so the restore must fail with the table name. + SnapshotSplit removedTableSplit = + new SnapshotSplit("db.b.0", TableId.parse("db.b"), null, null, null); + SnapshotPhaseState checkpointState = + new SnapshotPhaseState( + Collections.singletonList(TableId.parse("db.a")), + Collections.singletonList(removedTableSplit), + Collections.emptyMap(), + Collections.emptyMap(), + false, + Collections.emptyList(), + false, + false); + TestIncrementalSource source = + new TestIncrementalSource( + ReadonlyConfig.fromMap( + Collections.singletonMap(STARTUP_MODE.key(), "snapshot-only")), + Collections.singletonList(catalogTable(TablePath.of("db", "a")))); + SourceSplitEnumerator.Context<SourceSplitBase> enumeratorContext = + Mockito.mock(SourceSplitEnumerator.Context.class); + + IllegalStateException exception = + Assertions.assertThrows( + IllegalStateException.class, + () -> source.restoreEnumerator(enumeratorContext, checkpointState)); + Assertions.assertTrue(exception.getMessage().contains("[db.b]"), exception.getMessage()); + } + + @Test + void testSnapshotOnlyRestoreRejectsUnfinishedTableOfAnotherSchema() { + // A table that is not split yet, configured only in another schema + SnapshotPhaseState checkpointState = + snapshotOnlyState( + Collections.emptyMap(), + Collections.emptyMap(), + TableId.parse("db.public.c")); + TestIncrementalSource source = + snapshotOnlySource(catalogTable(TablePath.of("db", "other", "c"))); + + IllegalStateException exception = + Assertions.assertThrows( + IllegalStateException.class, + () -> source.restoreEnumerator(enumeratorContext(), checkpointState)); + Assertions.assertTrue( + exception.getMessage().contains("[db.public.c]"), exception.getMessage()); + } + + @Test + void testSnapshotOnlyRestoreAcceptsConfiguredTables() throws Exception { + // The id of a MySQL table (database.table) and of a PostgreSQL table + // (database.schema.table), as the dialects discover them and key their table maps + SnapshotPhaseState checkpointState = + snapshotOnlyState( + Collections.emptyMap(), + Collections.emptyMap(), + TableId.parse("db.a"), + TableId.parse("db.public.Orders")); + TestIncrementalSource source = + snapshotOnlySource( + catalogTable(TablePath.of("db", "a")), + catalogTable(TablePath.of("db", "public", "Orders"))); + + Assertions.assertNotNull(source.restoreEnumerator(enumeratorContext(), checkpointState)); + } + + @Test + void testSnapshotOnlyRestoreRejectsTableThatDiffersInCase() { + // public."Orders" and public.orders are two tables in PostgreSQL. The readers look a + // checkpointed table up by its exact id, so a configured name that differs in case, or + // lacks a qualifier of the id, does not match it. + SnapshotPhaseState checkpointState = + snapshotOnlyState( + Collections.emptyMap(), + Collections.emptyMap(), + TableId.parse("db.public.Orders"), + TableId.parse("db.b"), + TableId.parse("db.public.c")); + TestIncrementalSource source = + snapshotOnlySource( + catalogTable(TablePath.of("db", "public", "orders")), + catalogTable(TablePath.of("DB", "B")), + catalogTable(TablePath.of("db", "c"))); + + IllegalStateException exception = + Assertions.assertThrows( + IllegalStateException.class, + () -> source.restoreEnumerator(enumeratorContext(), checkpointState)); + Assertions.assertTrue( + exception.getMessage().contains("[db.b, db.public.Orders, db.public.c]"), + exception.getMessage()); + } + + @Test + void testSnapshotOnlyRestoreAcceptsFinishedTableThatIsNoLongerConfigured() throws Exception { + // db.done has only finished splits, so it is not read again and may have been removed + // from the table list or dropped; db.a still has an unfinished split and is configured + SnapshotSplit finishedSplit = + new SnapshotSplit("db.done.0", TableId.parse("db.done"), null, null, null); + SnapshotSplit unfinishedSplit = + new SnapshotSplit("db.a.0", TableId.parse("db.a"), null, null, null); + Map<String, SnapshotSplit> assignedSplits = new HashMap<>(); + assignedSplits.put(finishedSplit.splitId(), finishedSplit); + assignedSplits.put(unfinishedSplit.splitId(), unfinishedSplit); + SnapshotPhaseState checkpointState = + snapshotOnlyState( + assignedSplits, + Collections.singletonMap( + finishedSplit.splitId(), + new SnapshotSplitWatermark(finishedSplit.splitId(), null, null))); + TestIncrementalSource source = snapshotOnlySource(catalogTable(TablePath.of("db", "a"))); + + Assertions.assertNotNull(source.restoreEnumerator(enumeratorContext(), checkpointState)); + } + + /** + * A checkpoint of a snapshot-only job: the tables of the assigned splits are already split, the + * remaining tables are not split yet. + */ + private static SnapshotPhaseState snapshotOnlyState( + Map<String, SnapshotSplit> assignedSplits, + Map<String, SnapshotSplitWatermark> splitCompletedOffsets, + TableId... remainingTables) { + List<TableId> processedTables = + assignedSplits.values().stream() + .map(SnapshotSplit::getTableId) + .distinct() + .collect(Collectors.toList()); + return new SnapshotPhaseState( + processedTables, + Collections.emptyList(), + assignedSplits, + splitCompletedOffsets, + false, + Arrays.asList(remainingTables), + false, + true); + } + + private static TestIncrementalSource snapshotOnlySource(CatalogTable... catalogTables) { + return new TestIncrementalSource( + ReadonlyConfig.fromMap( + Collections.singletonMap(STARTUP_MODE.key(), "snapshot-only")), + Arrays.asList(catalogTables)); + } + + @SuppressWarnings("unchecked") + private static SourceSplitEnumerator.Context<SourceSplitBase> enumeratorContext() { + SourceSplitEnumerator.Context<SourceSplitBase> enumeratorContext = + Mockito.mock(SourceSplitEnumerator.Context.class); + Mockito.when(enumeratorContext.currentParallelism()).thenReturn(1); + return enumeratorContext; + } + + @Test + @SuppressWarnings("unchecked") + void testSnapshotOnlyRestoreRejectsIncrementalCheckpoint() { + // A job that already entered the binlog phase checkpoints an IncrementalPhaseState. + // Restoring it with startup.mode changed to snapshot-only must fail fast rather than resume + // binlog streaming, which would break the bounded snapshot-only contract. + TestIncrementalSource source = + new TestIncrementalSource( + ReadonlyConfig.fromMap( + Collections.singletonMap(STARTUP_MODE.key(), "snapshot-only")), + Collections.emptyList()); + SourceSplitEnumerator.Context<SourceSplitBase> enumeratorContext = + Mockito.mock(SourceSplitEnumerator.Context.class); + + Assertions.assertThrows( + IllegalStateException.class, + () -> source.restoreEnumerator(enumeratorContext, new IncrementalPhaseState()), + "snapshot-only restore must reject an incremental (binlog) checkpoint"); + } + + @Test + @SuppressWarnings("unchecked") + void testInitialRestoreRejectsSnapshotOnlyCheckpoint() { + TestIncrementalSource source = + new TestIncrementalSource( + ReadonlyConfig.fromMap(Collections.emptyMap()), Collections.emptyList()); + SourceSplitEnumerator.Context<SourceSplitBase> enumeratorContext = + Mockito.mock(SourceSplitEnumerator.Context.class); + SnapshotPhaseState checkpointState = + new SnapshotPhaseState( + Collections.emptyList(), + Collections.emptyList(), + Collections.emptyMap(), + Collections.emptyMap(), + false, + Collections.emptyList(), + false, + true); + + Assertions.assertThrows( + IllegalStateException.class, + () -> source.restoreEnumerator(enumeratorContext, checkpointState), + "initial restore must reject a snapshot-only checkpoint"); + } + + private static CatalogTable catalogTable(TablePath tablePath) { + return CatalogTable.of( + TableIdentifier.of("test", tablePath), + TableSchema.builder().build(), + Collections.emptyMap(), + Collections.emptyList(), + ""); + } + + private static class TestIncrementalSource extends IncrementalSource<Object, SourceConfig> { + private SourceConfig sourceConfig; + private DataSourceDialect<SourceConfig> dialect; + + private TestIncrementalSource(ReadonlyConfig options, List<CatalogTable> catalogTables) { + super(options, catalogTables); + } + + @Override + public Option<StartupMode> getStartupModeOption() { + return STARTUP_MODE; + } + + @Override + public Option<StopMode> getStopModeOption() { + return STOP_MODE; + } + + @Override + public SourceConfig.Factory<SourceConfig> createSourceConfigFactory(ReadonlyConfig config) { + sourceConfig = Mockito.mock(SourceConfig.class); + StartupConfig startupConfig = + new StartupConfig( + config.get(STARTUP_MODE), + config.get(SourceOptions.STARTUP_SPECIFIC_OFFSET_FILE), + config.get(SourceOptions.STARTUP_SPECIFIC_OFFSET_POS), + config.get(SourceOptions.STARTUP_TIMESTAMP)); + StopConfig stopConfig = new StopConfig(config.get(STOP_MODE), null, null, null); + Mockito.when(sourceConfig.getStartupConfig()).thenReturn(startupConfig); + Mockito.when(sourceConfig.getStopConfig()).thenReturn(stopConfig); + return ignored -> sourceConfig; + } + + @Override + public DebeziumDeserializationSchema<Object> createDebeziumDeserializationSchema( + ReadonlyConfig config) { + return null; + } + + @Override + public DataSourceDialect<SourceConfig> createDataSourceDialect(ReadonlyConfig config) { + dialect = Mockito.mock(DataSourceDialect.class); + return dialect; + } + + @Override + public OffsetFactory createOffsetFactory(ReadonlyConfig config) { + return Mockito.mock(OffsetFactory.class); + } + + @Override + public String getPluginName() { + return "test-cdc"; + } + + @Override + public Optional<String> driverName() { + return Optional.empty(); + } + + private SourceConfig getSourceConfig() { + return sourceConfig; + } + + private DataSourceDialect<SourceConfig> getDialect() { + return dialect; + } + } +} diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/test/java/org/apache/seatunnel/connectors/cdc/base/source/enumerator/SnapshotOnlySplitAssignerTest.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/test/java/org/apache/seatunnel/connectors/cdc/base/source/enumerator/SnapshotOnlySplitAssignerTest.java new file mode 100644 index 0000000000..09a90582fb --- /dev/null +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/test/java/org/apache/seatunnel/connectors/cdc/base/source/enumerator/SnapshotOnlySplitAssignerTest.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.cdc.base.source.enumerator; + +import org.apache.seatunnel.connectors.cdc.base.config.SourceConfig; +import org.apache.seatunnel.connectors.cdc.base.dialect.DataSourceDialect; +import org.apache.seatunnel.connectors.cdc.base.source.enumerator.state.SnapshotPhaseState; +import org.apache.seatunnel.connectors.cdc.base.source.split.IncrementalSplit; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import io.debezium.relational.TableId; + +import java.util.Collections; +import java.util.HashMap; + +class SnapshotOnlySplitAssignerTest { + + @Test + void testRestoreMarksRemainingTablesCheckpointedWithoutDiscovery() { + TableId checkpointTable = TableId.parse("db.checkpoint_table"); + SnapshotPhaseState legacyCheckpointState = + new SnapshotPhaseState( + Collections.emptyList(), + Collections.emptyList(), + Collections.emptyMap(), + Collections.emptyMap(), + false, + Collections.singletonList(checkpointTable), + false, + false); + DataSourceDialect<SourceConfig> dialect = Mockito.mock(DataSourceDialect.class); + SplitAssigner.Context<SourceConfig> context = + new SplitAssigner.Context<>( + null, + Collections.singleton(checkpointTable), + new HashMap<>(), + new HashMap<>()); + + SnapshotOnlySplitAssigner<SourceConfig> assigner = + new SnapshotOnlySplitAssigner<>(context, 2, legacyCheckpointState, dialect); + assigner.open(); + + Mockito.verify(dialect, Mockito.never()).discoverDataCollections(Mockito.any()); + SnapshotPhaseState restoredState = (SnapshotPhaseState) assigner.snapshotState(1L); + Assertions.assertTrue(restoredState.isRemainingTablesCheckpointed()); + Assertions.assertEquals( + Collections.singletonList(checkpointTable), restoredState.getRemainingTables()); + } + + @Test + void testRejectsRestoredIncrementalSplit() { + SnapshotPhaseState checkpointState = + new SnapshotPhaseState( + Collections.emptyList(), + Collections.emptyList(), + Collections.emptyMap(), + Collections.emptyMap(), + true, + Collections.emptyList(), + false, + true); + SplitAssigner.Context<SourceConfig> context = + new SplitAssigner.Context<>( + null, Collections.emptySet(), new HashMap<>(), new HashMap<>()); + SnapshotOnlySplitAssigner<SourceConfig> assigner = + new SnapshotOnlySplitAssigner<>(context, 1, checkpointState, null); + IncrementalSplit incrementalSplit = + new IncrementalSplit( + "db.table.stream", + Collections.singletonList(TableId.parse("db.table")), + null, + null, + Collections.emptyList()); + + Assertions.assertThrows( + IllegalStateException.class, + () -> assigner.addSplits(Collections.singletonList(incrementalSplit))); + } +} diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/test/java/org/apache/seatunnel/connectors/cdc/base/source/reader/IncrementalSourceReaderSnapshotOnlyTest.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/test/java/org/apache/seatunnel/connectors/cdc/base/source/reader/IncrementalSourceReaderSnapshotOnlyTest.java new file mode 100644 index 0000000000..edac9831d2 --- /dev/null +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/test/java/org/apache/seatunnel/connectors/cdc/base/source/reader/IncrementalSourceReaderSnapshotOnlyTest.java @@ -0,0 +1,81 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.cdc.base.source.reader; + +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.source.SourceReader; +import org.apache.seatunnel.connectors.cdc.base.config.SourceConfig; +import org.apache.seatunnel.connectors.cdc.base.config.StartupConfig; +import org.apache.seatunnel.connectors.cdc.base.dialect.DataSourceDialect; +import org.apache.seatunnel.connectors.cdc.base.option.StartupMode; +import org.apache.seatunnel.connectors.cdc.base.source.split.IncrementalSplit; +import org.apache.seatunnel.connectors.cdc.base.source.split.SourceRecords; +import org.apache.seatunnel.connectors.cdc.debezium.DebeziumDeserializationSchema; +import org.apache.seatunnel.connectors.seatunnel.common.source.reader.RecordEmitter; +import org.apache.seatunnel.connectors.seatunnel.common.source.reader.RecordsWithSplitIds; +import org.apache.seatunnel.connectors.seatunnel.common.source.reader.SourceReaderOptions; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import io.debezium.relational.TableId; + +import java.util.Collections; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; + +/** + * Covers the snapshot-only startup guard in {@link IncrementalSourceReader#addSplits}: a bounded + * snapshot-only job must reject a restored incremental (binlog) split instead of streaming forever. + */ +class IncrementalSourceReaderSnapshotOnlyTest { + + @Test + @SuppressWarnings("unchecked") + void testSnapshotOnlyRejectsRestoredIncrementalSplit() { + SourceConfig sourceConfig = Mockito.mock(SourceConfig.class); + Mockito.when(sourceConfig.getStartupConfig()) + .thenReturn(new StartupConfig(StartupMode.SNAPSHOT_ONLY, null, null, null)); + SourceReader.Context context = Mockito.mock(SourceReader.Context.class); + Mockito.when(context.getIndexOfSubtask()).thenReturn(0); + BlockingQueue<RecordsWithSplitIds<SourceRecords>> elementsQueue = + new LinkedBlockingQueue<>(); + IncrementalSourceReader<Object, SourceConfig> reader = + new IncrementalSourceReader<>( + Mockito.mock(DataSourceDialect.class), + elementsQueue, + () -> Mockito.mock(IncrementalSourceSplitReader.class), + Mockito.mock(RecordEmitter.class), + new SourceReaderOptions(ReadonlyConfig.fromMap(Collections.emptyMap())), + context, + sourceConfig, + Mockito.mock(DebeziumDeserializationSchema.class)); + IncrementalSplit incrementalSplit = + new IncrementalSplit( + "db.table.stream", + Collections.singletonList(TableId.parse("db.table")), + null, + null, + Collections.emptyList()); + + Assertions.assertThrows( + IllegalStateException.class, + () -> reader.addSplits(Collections.singletonList(incrementalSplit))); + } +} diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/test/java/org/apache/seatunnel/connectors/cdc/base/source/reader/IncrementalSourceReaderTest.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/test/java/org/apache/seatunnel/connectors/cdc/base/source/reader/IncrementalSourceReaderTest.java index 18bebd9213..e536cb79a8 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/test/java/org/apache/seatunnel/connectors/cdc/base/source/reader/IncrementalSourceReaderTest.java +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-base/src/test/java/org/apache/seatunnel/connectors/cdc/base/source/reader/IncrementalSourceReaderTest.java @@ -31,7 +31,9 @@ import org.apache.seatunnel.api.table.type.SeaTunnelDataType; import org.apache.seatunnel.api.table.type.SeaTunnelRowType; import org.apache.seatunnel.connectors.cdc.base.config.JdbcSourceConfig; import org.apache.seatunnel.connectors.cdc.base.config.SourceConfig; +import org.apache.seatunnel.connectors.cdc.base.config.StartupConfig; import org.apache.seatunnel.connectors.cdc.base.dialect.DataSourceDialect; +import org.apache.seatunnel.connectors.cdc.base.option.StartupMode; import org.apache.seatunnel.connectors.cdc.base.source.event.CompletedSnapshotSplitsAckEvent; import org.apache.seatunnel.connectors.cdc.base.source.event.CompletedSnapshotSplitsReportEvent; import org.apache.seatunnel.connectors.cdc.base.source.event.SnapshotSplitWatermark; @@ -98,6 +100,13 @@ class IncrementalSourceReaderTest { private static final TableId REMOVED_TABLE = new TableId("alpha_online", null, "account_interests"); + private static SourceConfig incrementalSourceConfig() { + SourceConfig sourceConfig = Mockito.mock(SourceConfig.class); + Mockito.when(sourceConfig.getStartupConfig()) + .thenReturn(new StartupConfig(StartupMode.INITIAL, null, null, null)); + return sourceConfig; + } + @Test void testFetcherFailureBeforeHandoffRestoresUnemittedBatch() throws Exception { byte[] checkpoint; @@ -416,7 +425,7 @@ class IncrementalSourceReaderTest { new IncrementalSourceRecordEmitter<>(schema, offsets, context), new SourceReaderOptions(ReadonlyConfig.fromMap(Collections.emptyMap())), context, - Mockito.mock(SourceConfig.class), + incrementalSourceConfig(), schema); } @@ -489,7 +498,7 @@ class IncrementalSourceReaderTest { @Test void testAddSplitsEnqueuesPrunedRestoredIncrementalSplit() { - SourceConfig sourceConfig = Mockito.mock(SourceConfig.class); + SourceConfig sourceConfig = incrementalSourceConfig(); DataSourceDialect<SourceConfig> dialect = Mockito.mock(DataSourceDialect.class, Mockito.CALLS_REAL_METHODS); Mockito.when(dialect.discoverDataCollections(sourceConfig)) @@ -514,7 +523,7 @@ class IncrementalSourceReaderTest { @Test void testAddSplitsKeepsRestoredSplitWhenDiscoveryReturnsEmpty() { - SourceConfig sourceConfig = Mockito.mock(SourceConfig.class); + SourceConfig sourceConfig = incrementalSourceConfig(); DataSourceDialect<SourceConfig> dialect = Mockito.mock(DataSourceDialect.class, Mockito.CALLS_REAL_METHODS); Mockito.when(dialect.discoverDataCollections(sourceConfig)) @@ -538,7 +547,7 @@ class IncrementalSourceReaderTest { @Test void testAddSplitsKeepsRestoredSplitWhenDiscoveryFails() { - SourceConfig sourceConfig = Mockito.mock(SourceConfig.class); + SourceConfig sourceConfig = incrementalSourceConfig(); DataSourceDialect<SourceConfig> dialect = Mockito.mock(DataSourceDialect.class, Mockito.CALLS_REAL_METHODS); Mockito.when(dialect.discoverDataCollections(sourceConfig)) @@ -562,7 +571,7 @@ class IncrementalSourceReaderTest { @Test void testAddSplitsDiscoversCapturedTablesOnlyOncePerBatch() { - SourceConfig sourceConfig = Mockito.mock(SourceConfig.class); + SourceConfig sourceConfig = incrementalSourceConfig(); DataSourceDialect<SourceConfig> dialect = Mockito.mock(DataSourceDialect.class, Mockito.CALLS_REAL_METHODS); Mockito.when(dialect.discoverDataCollections(sourceConfig)) diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-mysql/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/config/MySqlIncrementalSourceOptions.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-mysql/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/config/MySqlIncrementalSourceOptions.java index 03dd26a6e9..8bb91851a0 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-mysql/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/config/MySqlIncrementalSourceOptions.java +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-mysql/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/config/MySqlIncrementalSourceOptions.java @@ -67,11 +67,16 @@ public class MySqlIncrementalSourceOptions extends JdbcSourceOptions implements StartupMode.EARLIEST, StartupMode.LATEST, StartupMode.SPECIFIC, - StartupMode.TIMESTAMP)) + StartupMode.TIMESTAMP, + StartupMode.SNAPSHOT_ONLY)) .defaultValue(StartupMode.INITIAL) .withDescription( "Optional startup mode for CDC source, valid enumerations are " - + "\"initial\", \"earliest\", \"latest\" , \"specific\" or \"timestamp\""); + + "\"initial\", \"earliest\", \"latest\" , \"specific\", \"timestamp\" " + + "or \"snapshot-only\". \"snapshot-only\" reads snapshot data only and then " + + "finishes without entering continuous binlog streaming. It may " + + "perform bounded binlog backfill during the snapshot phase when " + + "exactly_once is enabled."); public static final SingleChoiceOption<StopMode> STOP_MODE = Options.key(SourceOptions.STOP_MODE_KEY) diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-mysql/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/config/MySqlSnapshotOnlyGuards.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-mysql/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/config/MySqlSnapshotOnlyGuards.java new file mode 100644 index 0000000000..daa737f8e3 --- /dev/null +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-mysql/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/config/MySqlSnapshotOnlyGuards.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.cdc.mysql.config; + +import org.apache.seatunnel.api.configuration.Option; +import org.apache.seatunnel.api.configuration.ReadonlyConfig; +import org.apache.seatunnel.api.configuration.util.ConditionExtension; +import org.apache.seatunnel.api.configuration.util.OptionValidationException; +import org.apache.seatunnel.connectors.cdc.base.option.StartupMode; +import org.apache.seatunnel.connectors.cdc.base.option.StopMode; + +/** + * Reusable {@link ConditionExtension} guards for the MySQL CDC {@code snapshot-only} startup mode. + * + * <p>A {@code snapshot-only} job reads the snapshot, may perform a bounded backfill and then + * finishes; it never enters continuous binlog streaming. Options that only make sense for binlog + * streaming are therefore rejected rather than silently ignored. The guards are meant to be wired + * as conditional value constraints that only apply when {@code startup.mode} is {@code + * snapshot-only}, so they do not check the startup mode themselves. + * + * <p>The {@code startup.specific-offset.*} options are not covered here: {@link + * MySqlSpecificOffsetGuards} already rejects them for every startup mode other than {@code + * specific}. + */ +public final class MySqlSnapshotOnlyGuards { + + private MySqlSnapshotOnlyGuards() {} + + public static ConditionExtension<StopMode> stopModeGuard(Option<StopMode> stopModeOption) { + return new StopModeGuard(stopModeOption.key()); + } + + public static ConditionExtension<Long> startupTimestampGuard(Option<Long> timestampOption) { + return new StartupTimestampGuard(timestampOption.key()); + } + + /** Only {@code stop.mode = never} is allowed, because the job never streams the binlog. */ + static final class StopModeGuard implements ConditionExtension<StopMode> { + private final String guardedKey; + + StopModeGuard(String guardedKey) { + this.guardedKey = guardedKey; + } + + @Override + public String description() { + return "must be 'never' when the startup mode is '" + StartupMode.SNAPSHOT_ONLY + "'"; + } + + @Override + public boolean evaluate(ReadonlyConfig config, StopMode value) { + if (value != null && value != StopMode.NEVER) { + throw new OptionValidationException( + String.format( + "'%s' startup mode is a bounded snapshot-only job and does not" + + " enter continuous change log streaming, so it is" + + " incompatible with '%s = %s'. Remove the stop mode" + + " option.", + StartupMode.SNAPSHOT_ONLY, guardedKey, value.name().toLowerCase())); + } + return true; + } + } + + /** The snapshot phase derives the backfill offsets, so a startup timestamp is not used. */ + static final class StartupTimestampGuard implements ConditionExtension<Long> { + private final String guardedKey; + + StartupTimestampGuard(String guardedKey) { + this.guardedKey = guardedKey; + } + + @Override + public String description() { + return "can not be used when the startup mode is '" + StartupMode.SNAPSHOT_ONLY + "'"; + } + + @Override + public boolean evaluate(ReadonlyConfig config, Long value) { + if (value == null) { + return true; + } + throw new OptionValidationException( + String.format( + "'%s' startup mode derives bounded backfill offsets from the snapshot" + + " phase and does not use startup change log offsets, so" + + " startup offset option '%s' is not allowed.", + StartupMode.SNAPSHOT_ONLY, guardedKey)); + } + } +} diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-mysql/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/source/MySqlIncrementalSourceFactory.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-mysql/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/source/MySqlIncrementalSourceFactory.java index 3d4d244e72..07e75e1263 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-mysql/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/source/MySqlIncrementalSourceFactory.java +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-mysql/src/main/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/source/MySqlIncrementalSourceFactory.java @@ -36,6 +36,7 @@ import org.apache.seatunnel.connectors.cdc.base.option.StopMode; import org.apache.seatunnel.connectors.cdc.base.source.BaseChangeStreamTableSourceFactory; import org.apache.seatunnel.connectors.cdc.base.utils.CatalogTableUtils; import org.apache.seatunnel.connectors.seatunnel.cdc.mysql.config.MySqlIncrementalSourceOptions; +import org.apache.seatunnel.connectors.seatunnel.cdc.mysql.config.MySqlSnapshotOnlyGuards; import org.apache.seatunnel.connectors.seatunnel.cdc.mysql.config.MySqlSourceConfigFactory; import org.apache.seatunnel.connectors.seatunnel.cdc.mysql.config.MySqlSpecificOffsetGuards; @@ -43,6 +44,7 @@ import com.google.auto.service.AutoService; import lombok.extern.slf4j.Slf4j; import java.io.Serializable; +import java.util.Arrays; import java.util.List; import java.util.Optional; @@ -104,7 +106,7 @@ public class MySqlIncrementalSourceFactory extends BaseChangeStreamTableSourceFa .optional(MySqlIncrementalSourceOptions.STOP_MODE) .conditional( MySqlIncrementalSourceOptions.STARTUP_MODE, - StartupMode.INITIAL, + Arrays.asList(StartupMode.INITIAL, StartupMode.SNAPSHOT_ONLY), SourceOptions.EXACTLY_ONCE) .conditional( MySqlIncrementalSourceOptions.STOP_MODE, @@ -115,6 +117,17 @@ public class MySqlIncrementalSourceFactory extends BaseChangeStreamTableSourceFa MySqlIncrementalSourceOptions.STARTUP_MODE, StartupMode.TIMESTAMP, SourceOptions.STARTUP_TIMESTAMP) + .conditional( + MySqlIncrementalSourceOptions.STARTUP_MODE, + StartupMode.SNAPSHOT_ONLY, + Conditions.extension( + MySqlIncrementalSourceOptions.STOP_MODE, + MySqlSnapshotOnlyGuards.stopModeGuard( + MySqlIncrementalSourceOptions.STOP_MODE)), + Conditions.extension( + SourceOptions.STARTUP_TIMESTAMP, + MySqlSnapshotOnlyGuards.startupTimestampGuard( + SourceOptions.STARTUP_TIMESTAMP))) .optional( SourceOptions.STARTUP_SPECIFIC_OFFSET_FILE, Conditions.extension( diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-mysql/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/source/MySqlIncrementalSourceFactoryTest.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-mysql/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/source/MySqlIncrementalSourceFactoryTest.java index d55b42aa19..5bd1c87d01 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-mysql/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/source/MySqlIncrementalSourceFactoryTest.java +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-mysql/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/source/MySqlIncrementalSourceFactoryTest.java @@ -19,8 +19,12 @@ package org.apache.seatunnel.connectors.seatunnel.cdc.mysql.source; import org.apache.seatunnel.api.configuration.ReadonlyConfig; import org.apache.seatunnel.api.configuration.util.ConfigValidator; +import org.apache.seatunnel.api.configuration.util.Expression; import org.apache.seatunnel.api.configuration.util.OptionRule; import org.apache.seatunnel.api.configuration.util.OptionValidationException; +import org.apache.seatunnel.api.configuration.util.RequiredOption; +import org.apache.seatunnel.connectors.cdc.base.option.SourceOptions; +import org.apache.seatunnel.connectors.cdc.base.option.StartupMode; import org.apache.seatunnel.connectors.seatunnel.cdc.mysql.config.MySqlIncrementalSourceOptions; import org.junit.jupiter.api.Assertions; @@ -28,8 +32,10 @@ import org.junit.jupiter.api.Test; import java.util.Arrays; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import static org.apache.seatunnel.connectors.cdc.base.option.SourceOptions.STARTUP_SPECIFIC_OFFSET_FILE; import static org.apache.seatunnel.connectors.cdc.base.option.SourceOptions.STARTUP_SPECIFIC_OFFSET_POS; @@ -53,6 +59,37 @@ public class MySqlIncrementalSourceFactoryTest { Assertions.assertNotNull((new MySqlIncrementalSourceFactory()).optionRule()); } + @Test + public void testSnapshotOnlyAllowsExactlyOnce() { + RequiredOption.ConditionalRequiredOptions exactlyOnceRule = + new MySqlIncrementalSourceFactory() + .optionRule().getRequiredOptions().stream() + .filter(RequiredOption.ConditionalRequiredOptions.class::isInstance) + .map(RequiredOption.ConditionalRequiredOptions.class::cast) + .filter( + rule -> + rule.getRequiredOption() + .contains(SourceOptions.EXACTLY_ONCE)) + .findFirst() + .orElseThrow( + () -> + new AssertionError( + "exactly_once conditional rule is missing")); + Set<Object> startupModes = new HashSet<>(); + Expression expression = exactlyOnceRule.getExpression(); + while (expression != null) { + Assertions.assertEquals( + MySqlIncrementalSourceOptions.STARTUP_MODE, + expression.getCondition().getOption()); + startupModes.add(expression.getCondition().getExpectValue()); + expression = expression.getNext(); + } + + Assertions.assertEquals( + new HashSet<>(Arrays.asList(StartupMode.INITIAL, StartupMode.SNAPSHOT_ONLY)), + startupModes); + } + @Test public void testFileAndPosOnlySpecificOffsetsRemainValid() { Map<String, Object> options = basicOptions(); @@ -187,6 +224,78 @@ public class MySqlIncrementalSourceFactoryTest { } } + @Test + public void testSnapshotOnlyWithoutStreamingOptionsIsValid() { + assertValid(snapshotOnlyOptions()); + + Map<String, Object> explicitNever = snapshotOnlyOptions(); + explicitNever.put(MySqlIncrementalSourceOptions.STOP_MODE.key(), "never"); + assertValid(explicitNever); + } + + @Test + public void testSnapshotOnlyRejectsStopMode() { + for (String stopMode : Arrays.asList("latest", "specific")) { + Map<String, Object> options = snapshotOnlyOptions(); + options.put(MySqlIncrementalSourceOptions.STOP_MODE.key(), stopMode); + if ("specific".equals(stopMode)) { + options.put(SourceOptions.STOP_SPECIFIC_OFFSET_FILE.key(), "mysql-bin.000004"); + options.put(SourceOptions.STOP_SPECIFIC_OFFSET_POS.key(), 8937L); + } + OptionValidationException error = assertValidationFails(options); + Assertions.assertTrue( + error.getMessage().contains("'stop.mode = " + stopMode + "'"), + () -> + "snapshot-only must reject stop.mode " + + stopMode + + ": " + + error.getMessage()); + } + } + + @Test + public void testSnapshotOnlyRejectsStartupTimestamp() { + Map<String, Object> options = snapshotOnlyOptions(); + options.put(SourceOptions.STARTUP_TIMESTAMP.key(), 1L); + + OptionValidationException error = assertValidationFails(options); + Assertions.assertTrue( + error.getMessage() + .contains( + "startup offset option '" + + SourceOptions.STARTUP_TIMESTAMP.key() + + "' is not allowed"), + () -> "snapshot-only must reject startup.timestamp: " + error.getMessage()); + } + + @Test + public void testSnapshotOnlyRejectsEverySpecificOffset() { + for (String optionKey : SPECIFIC_OFFSET_OPTION_KEYS) { + Map<String, Object> options = snapshotOnlyOptions(); + options.put(optionKey, sampleValue(optionKey)); + OptionValidationException error = assertValidationFails(options); + Assertions.assertTrue( + error.getMessage().contains("current mode is 'SNAPSHOT_ONLY'"), + () -> "snapshot-only must reject '" + optionKey + "': " + error.getMessage()); + } + } + + @Test + public void testTimestampModeStillRequiresStartupTimestamp() { + Map<String, Object> options = basicOptions(); + options.put(MySqlIncrementalSourceOptions.STARTUP_MODE.key(), "timestamp"); + assertValidationFails(options); + + options.put(SourceOptions.STARTUP_TIMESTAMP.key(), 1L); + assertValid(options); + } + + private static Map<String, Object> snapshotOnlyOptions() { + Map<String, Object> options = basicOptions(); + options.put(MySqlIncrementalSourceOptions.STARTUP_MODE.key(), "snapshot-only"); + return options; + } + private static Map<String, Object> basicOptions() { Map<String, Object> options = new HashMap<>(); options.put(MySqlIncrementalSourceOptions.USERNAME.key(), "root"); diff --git a/seatunnel-connectors-v2/connector-cdc/connector-cdc-oceanbase/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/oceanbase/source/OceanBaseIncrementalSourceFactoryTest.java b/seatunnel-connectors-v2/connector-cdc/connector-cdc-oceanbase/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/oceanbase/source/OceanBaseIncrementalSourceFactoryTest.java index cdf6bb91d0..38cf6080f9 100644 --- a/seatunnel-connectors-v2/connector-cdc/connector-cdc-oceanbase/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/oceanbase/source/OceanBaseIncrementalSourceFactoryTest.java +++ b/seatunnel-connectors-v2/connector-cdc/connector-cdc-oceanbase/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/oceanbase/source/OceanBaseIncrementalSourceFactoryTest.java @@ -143,6 +143,26 @@ public class OceanBaseIncrementalSourceFactoryTest { .validate(new OceanBaseIncrementalSourceFactory().optionRule())); } + /** Verify the inherited MySQL rule accepts snapshot-only and keeps its stop.mode guard. */ + @Test + public void testOptionRuleInheritsSnapshotOnlyGuards() { + OceanBaseIncrementalSourceFactory factory = new OceanBaseIncrementalSourceFactory(); + Map<String, Object> options = requiredOptions(); + options.put("startup.mode", "snapshot-only"); + ConfigValidator.of(ReadonlyConfig.fromMap(options)).validate(factory.optionRule()); + + options.put("stop.mode", "latest"); + OptionValidationException error = + Assertions.assertThrows( + OptionValidationException.class, + () -> + ConfigValidator.of(ReadonlyConfig.fromMap(options)) + .validate(factory.optionRule())); + Assertions.assertTrue( + error.getMessage().contains("'stop.mode = latest'"), + () -> "snapshot-only guard must run for OceanBase: " + error.getMessage()); + } + private static Map<String, Object> requiredOptions() { Map<String, Object> options = new HashMap<>(); options.put("username", "root"); diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/MysqlCDCSnapshotOnlyIT.java b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/MysqlCDCSnapshotOnlyIT.java new file mode 100644 index 0000000000..17a4844e24 --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/java/org/apache/seatunnel/connectors/seatunnel/cdc/mysql/MysqlCDCSnapshotOnlyIT.java @@ -0,0 +1,304 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.seatunnel.connectors.seatunnel.cdc.mysql; + +import org.apache.seatunnel.connectors.seatunnel.cdc.mysql.testutils.MySqlContainer; +import org.apache.seatunnel.connectors.seatunnel.cdc.mysql.testutils.MySqlVersion; +import org.apache.seatunnel.connectors.seatunnel.cdc.mysql.testutils.UniqueDatabase; +import org.apache.seatunnel.e2e.common.TestResource; +import org.apache.seatunnel.e2e.common.TestSuiteBase; +import org.apache.seatunnel.e2e.common.container.ContainerExtendedFactory; +import org.apache.seatunnel.e2e.common.container.EngineType; +import org.apache.seatunnel.e2e.common.container.TestContainer; +import org.apache.seatunnel.e2e.common.junit.DisabledOnContainer; +import org.apache.seatunnel.e2e.common.junit.TestContainerExtension; +import org.apache.seatunnel.e2e.common.util.DependencyJar; +import org.apache.seatunnel.e2e.common.util.JobIdGenerator; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.TestTemplate; +import org.testcontainers.containers.Container; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.lifecycle.Startables; +import org.testcontainers.utility.DockerLoggerFactory; + +import lombok.extern.slf4j.Slf4j; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.stream.Stream; + +import static org.awaitility.Awaitility.await; + +/** + * E2E for the MySQL-CDC {@code startup.mode = "snapshot-only"} bounded bootstrap mode. + * + * <p>The job runs in BATCH mode, so {@link TestContainer#executeJob} blocks until the job + * terminates. This is the core assertion: a snapshot-only job must finish on its own after the + * snapshot phase. If it wrongly transitioned into (or waited for) binlog streaming, the job would + * never complete and this test would time out. The test also deletes a row from a completed + * snapshot split while many other snapshot splits are still running, then verifies that the bounded + * job does not transition into continuous binlog streaming. + */ +@Slf4j +@DisabledOnContainer( + value = {}, + type = {EngineType.SPARK}, + disabledReason = "Currently SPARK do not support cdc") +public class MysqlCDCSnapshotOnlyIT extends TestSuiteBase implements TestResource { + + private static final String MYSQL_HOST = "mysql_cdc_e2e"; + private static final String MYSQL_USER_NAME = "mysqluser"; + private static final String MYSQL_USER_PASSWORD = "mysqlpw"; + private static final String MYSQL_DATABASE = "mysql_cdc"; + + private static final String SOURCE_TABLE = "mysql_cdc_e2e_source_table"; + private static final String SINK_TABLE = "mysql_cdc_e2e_sink_table"; + private static final int ADDITIONAL_SNAPSHOT_ROWS = 100; + + // Query only the primary key so the assertion does not depend on the full wide-row schema. + private static final String ID_QUERY_TEMPLATE = "select id from %s.%s order by id"; + + private final MySqlContainer MYSQL_CONTAINER = createMySqlContainer(MySqlVersion.V8_0); + + private final UniqueDatabase inventoryDatabase = + new UniqueDatabase( + MYSQL_CONTAINER, MYSQL_DATABASE, "mysqluser", "mysqlpw", MYSQL_DATABASE); + + private MySqlContainer createMySqlContainer(MySqlVersion version) { + return new MySqlContainer(version) + .withConfigurationOverride("docker/server-gtids/my.cnf") + .withSetupSQL("docker/setup.sql") + .withNetwork(NETWORK) + .withNetworkAliases(MYSQL_HOST) + .withDatabaseName(MYSQL_DATABASE) + .withUsername(MYSQL_USER_NAME) + .withPassword(MYSQL_USER_PASSWORD) + .withLogConsumer( + new Slf4jLogConsumer(DockerLoggerFactory.getLogger("mysql-docker-image"))); + } + + @TestContainerExtension + protected final ContainerExtendedFactory extendedFactory = + container -> + DependencyJar.ofClassName("com.mysql.cj.jdbc.Driver") + .copyTo(container, "/tmp/seatunnel/plugins/MySQL-CDC/lib"); + + @BeforeAll + @Override + public void startUp() { + log.info("Starting MySQL container for snapshot-only IT..."); + Startables.deepStart(Stream.of(MYSQL_CONTAINER)).join(); + log.info("MySQL container is started"); + inventoryDatabase.createAndInitialize(); + log.info("MySQL ddl execution is complete"); + } + + @TestTemplate + public void testMysqlCdcSnapshotOnlyFinishesAndSkipsContinuousBinlog(TestContainer container) + throws Exception { + clearTable(MYSQL_DATABASE, SOURCE_TABLE); + clearTable(MYSQL_DATABASE, SINK_TABLE); + + // Seed a deterministic set of snapshot rows. + insertRow(2); + insertRow(3); + insertRow(5); + for (int id = 10; id < 10 + ADDITIONAL_SNAPSHOT_ROWS; id++) { + insertRow(id); + } + List<List<Object>> snapshotIds = query(idQuery(MYSQL_DATABASE, SOURCE_TABLE)); + Assertions.assertEquals( + 3 + ADDITIONAL_SNAPSHOT_ROWS, + snapshotIds.size(), + "precondition: all snapshot rows seeded"); + List<List<Object>> firstCompletedSplitIds = snapshotIds.subList(0, 3); + + // Only the Zeta container submits and cancels a job by id. A Flink container is not shared + // and is removed after the test, which stops a job that is still running. + String jobId = + container.identifier().getEngineType() == EngineType.SEATUNNEL + ? String.valueOf(JobIdGenerator.newJobId()) + : null; + CompletableFuture<Container.ExecResult> jobFuture = + CompletableFuture.supplyAsync( + () -> { + try { + return jobId == null + ? container.executeJob("/mysqlcdc_snapshot_only.conf") + : container.executeJob( + "/mysqlcdc_snapshot_only.conf", jobId); + } catch (Exception e) { + throw new RuntimeException(e); + } + }); + + Container.ExecResult execResult; + try { + // Wait until the first snapshot splits are visible while many later splits are still + // outstanding. Mutating only these completed ranges avoids racing with snapshot reads + // and leaves a window to detect an accidental transition into incremental streaming. + await().atMost(120, TimeUnit.SECONDS) + .untilAsserted( + () -> { + Assertions.assertFalse( + jobFuture.isDone(), + "snapshot-only job finished before the runtime mutation check"); + List<List<Object>> sinkIds = + query(idQuery(MYSQL_DATABASE, SINK_TABLE)); + Assertions.assertTrue(sinkIds.containsAll(firstCompletedSplitIds)); + Assertions.assertTrue( + sinkIds.size() < snapshotIds.size(), + "runtime mutation must happen before all snapshot splits finish"); + }); + + // Produce a binlog change for an already emitted snapshot range while other snapshot + // splits are still running. A snapshot-only job must not start an incremental reader + // afterward. Do not change the primary key here because moving a row into a pending + // split would make it part of that split's later snapshot read. + executeSql("DELETE FROM " + MYSQL_DATABASE + "." + SOURCE_TABLE + " WHERE id = 2"); + + try { + execResult = jobFuture.get(120, TimeUnit.SECONDS); + } catch (Exception e) { + throw new RuntimeException("Wait snapshot-only job exit failed", e); + } + } finally { + cancelIfRunning(container, jobId, jobFuture); + } + Assertions.assertEquals( + 0, + execResult.getExitCode(), + "snapshot-only job must finish cleanly: " + execResult.getStderr()); + + // The sink must remain unchanged by the binlog activity performed during the job. + Assertions.assertIterableEquals( + snapshotIds, + query(idQuery(MYSQL_DATABASE, SINK_TABLE)), + "snapshot-only must not consume binlog changes while the snapshot job is running"); + } + + /** + * Stops the job when the test leaves before it finished, so that a failed assertion does not + * leave it running against the shared database. A job without an id is left to the removal of + * its container. A failure to cancel is only logged, so that it does not hide the failure that + * ended the test. + */ + private void cancelIfRunning( + TestContainer container, + String jobId, + CompletableFuture<Container.ExecResult> jobFuture) { + if (jobId == null || jobFuture.isDone()) { + return; + } + try { + Container.ExecResult cancelResult = container.cancelJob(jobId); + if (cancelResult.getExitCode() != 0) { + log.warn("Cancel job {} failed: {}", jobId, cancelResult.getStderr()); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + log.warn("Cancel job {} was interrupted", jobId, e); + } catch (Exception e) { + log.warn("Cancel job {} failed", jobId, e); + } + } + + private Connection getJdbcConnection() throws SQLException { + return DriverManager.getConnection( + MYSQL_CONTAINER.getJdbcUrl(), + MYSQL_CONTAINER.getUsername(), + MYSQL_CONTAINER.getPassword()); + } + + private List<List<Object>> query(String sql) { + try (Connection connection = getJdbcConnection(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(sql)) { + List<List<Object>> result = new ArrayList<>(); + int columnCount = resultSet.getMetaData().getColumnCount(); + while (resultSet.next()) { + ArrayList<Object> objects = new ArrayList<>(); + for (int i = 1; i <= columnCount; i++) { + objects.add(resultSet.getObject(i)); + } + result.add(objects); + } + return result; + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + private void executeSql(String sql) { + try (Connection connection = getJdbcConnection()) { + connection.createStatement().execute(sql); + } catch (SQLException e) { + throw new RuntimeException(e); + } + } + + private void insertRow(int id) { + executeSql( + "INSERT INTO " + + MYSQL_DATABASE + + "." + + SOURCE_TABLE + + " ( id, f_binary, f_blob, f_long_varbinary, f_longblob, f_tinyblob, f_varbinary, f_smallint,\n" + + " f_smallint_unsigned, f_mediumint, f_mediumint_unsigned, f_int, f_int_unsigned, f_integer,\n" + + " f_integer_unsigned, f_bigint, f_bigint_unsigned, f_numeric, f_decimal, f_float, f_double,\n" + + " f_double_precision, f_longtext, f_mediumtext, f_text, f_tinytext, f_varchar, f_date, f_datetime,\n" + + " f_timestamp, f_bit1, f_bit64, f_char, f_enum, f_mediumblob, f_long_varchar, f_real, f_time,\n" + + " f_tinyint, f_tinyint_unsigned, f_json, f_year )\n" + + "VALUES ( " + + id + + ", 0x61626374000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000,\n" + + " 0x68656C6C6F, 0x18000000789C0BC9C82C5600A244859CFCBC7485B2C4A2A4CCBCC4A24A00697308D4, NULL,\n" + + " 0x74696E79626C6F62, 0x48656C6C6F20776F726C64, 12345, 54321, 123456, 654321, 1234567, 7654321, 1234567, 7654321,\n" + + " 123456789, 987654321, 123, 789, 12.34, 56.78, 90.12, 'This is a long text field', 'This is a medium text field',\n" + + " 'This is a text field', 'This is a tiny text field', 'This is a varchar field', '2022-04-27', '2022-04-27 14:30:00',\n" + + " '2023-04-27 11:08:40', 1, b'0101010101010101010101010101010101010101010101010101010101010101', 'C', 'enum2',\n" + + " 0x1B000000789C0BC9C82C5600A24485DCD494CCD25C85A49CFC2485B4CCD49C140083FF099A, 'This is a long varchar field',\n" + + " 12.345, '14:30:00', -128, 255, '{ \"key\": \"value\" }', 1992 )"); + } + + private void clearTable(String database, String tableName) { + executeSql("truncate table " + database + "." + tableName); + } + + private String idQuery(String database, String tableName) { + return String.format(ID_QUERY_TEMPLATE, database, tableName); + } + + @Override + @AfterAll + public void tearDown() { + if (MYSQL_CONTAINER != null) { + MYSQL_CONTAINER.close(); + } + } +} diff --git a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/mysqlcdc_snapshot_only.conf b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/mysqlcdc_snapshot_only.conf new file mode 100644 index 0000000000..f3d3fe8aba --- /dev/null +++ b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-cdc-mysql-e2e/src/test/resources/mysqlcdc_snapshot_only.conf @@ -0,0 +1,51 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +env { + parallelism = 2 + job.mode = "BATCH" +} + +source { + MySQL-CDC { + plugin_output = "customers_mysql_cdc" + server-id = "5657-5658" + username = "st_user_source" + password = "mysqlpw" + table-names = ["mysql_cdc.mysql_cdc_e2e_source_table"] + url = "jdbc:mysql://mysql_cdc_e2e:3306/mysql_cdc" + startup.mode = "snapshot-only" + snapshot.split.size = 1 + exactly_once = true + } +} + +sink { + jdbc { + plugin_input = "customers_mysql_cdc" + url = "jdbc:mysql://mysql_cdc_e2e:3306/mysql_cdc" + driver = "com.mysql.cj.jdbc.Driver" + user = "st_user_sink" + password = "mysqlpw" + + generate_sink_sql = true + batch_size = 1 + database = mysql_cdc + table = mysql_cdc_e2e_sink_table + primary_keys = ["id"] + } +}
