This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/seatunnel.git
The following commit(s) were added to refs/heads/dev by this push:
new 9f741b9c64 [Fix][Format] Apply configured CSV defaults to null and
missing fields (#12837)
9f741b9c64 is described below
commit 9f741b9c64220db31bd94c62dd0749d0b71e9358
Author: Goutam Adwant <[email protected]>
AuthorDate: Thu Nov 5 14:04:23 2026 +0000
[Fix][Format] Apply configured CSV defaults to null and missing fields
(#12837)
Signed-off-by: Goutam Adwant <[email protected]>
---
docs/en/connectors/source/FtpFile.md | 5 +-
docs/en/connectors/source/HdfsFile.md | 2 +-
docs/en/connectors/source/LocalFile.md | 5 +-
docs/en/connectors/source/ObsFile.md | 2 +-
docs/en/connectors/source/OssFile.md | 2 +-
docs/en/connectors/source/OssJindoFile.md | 5 +-
docs/en/connectors/source/S3File.md | 2 +-
docs/en/connectors/source/SftpFile.md | 2 +-
docs/en/connectors/source/SmbFile.md | 2 +-
.../introduction/concepts/incompatible-changes.md | 11 +
docs/zh/connectors/source/FtpFile.md | 5 +-
docs/zh/connectors/source/HdfsFile.md | 2 +-
docs/zh/connectors/source/LocalFile.md | 5 +-
docs/zh/connectors/source/ObsFile.md | 2 +-
docs/zh/connectors/source/OssFile.md | 2 +-
docs/zh/connectors/source/OssJindoFile.md | 2 +-
docs/zh/connectors/source/S3File.md | 2 +-
docs/zh/connectors/source/SftpFile.md | 2 +-
docs/zh/connectors/source/SmbFile.md | 2 +-
.../introduction/concepts/incompatible-changes.md | 8 +
.../file/source/reader/CsvReadStrategy.java | 1 +
.../file/source/reader/CsvReadStrategyTest.java | 56 +++
.../src/test/resources/csv/default-values.csv | 3 +
.../e2e/connector/file/local/LocalFileIT.java | 5 +
.../src/test/resources/csv/default-values.csv | 3 +
.../resources/csv/default_values_to_assert.conf | 57 +++
.../format/csv/CsvDeserializationSchema.java | 251 ++++++++++++-
.../format/csv/CsvTextFormatSchemaTest.java | 402 +++++++++++++++++++++
28 files changed, 815 insertions(+), 33 deletions(-)
diff --git a/docs/en/connectors/source/FtpFile.md
b/docs/en/connectors/source/FtpFile.md
index d7302d7f0e..29e239f86e 100644
--- a/docs/en/connectors/source/FtpFile.md
+++ b/docs/en/connectors/source/FtpFile.md
@@ -516,8 +516,9 @@ The encoding of the file to read. This param will be parsed
by `Charset.forName(
### null_format [string]
-Only used when file_format_type is text.
-null_format to define which strings can be represented as null.
+Used when `file_format_type` is `text` or `csv`. Defines which field value
represents null.
+For CSV with a configured `schema.columns[].defaultValue`, a matching
`null_format` value or a column absent from the file header uses that default.
An empty CSV field still produces null unless it matches `null_format`; columns
without a default keep their existing behavior.
+CSV defaults are converted to the column type when the reader is created. A
default that cannot be converted fails the job even if no field needs it. DATE
and TIMESTAMP defaults may use a different supported format from the file data.
e.g: `\N`
diff --git a/docs/en/connectors/source/HdfsFile.md
b/docs/en/connectors/source/HdfsFile.md
index f7d0bfa94b..6be5933f9d 100644
--- a/docs/en/connectors/source/HdfsFile.md
+++ b/docs/en/connectors/source/HdfsFile.md
@@ -82,7 +82,7 @@ Read data from hdfs file system.
| compress_codec | string | no | none
| The compress codec of files
|
| archive_compress_codec | string | no | none
| Which archive compress codec the files used. Supported: `none` `zip` `tar`
`tar.gz` `gz`. |
| encoding | string | no | UTF-8
|
|
-| null_format | string | no | -
| Only used when file_format_type is `text` or `csv`. null_format to define
which strings can be represented as null. e.g: `\N`
|
+| null_format | string | no | -
| Used when `file_format_type` is `text` or `csv`. Defines which field value
represents null. For CSV with a configured `schema.columns[].defaultValue`, a
matching `null_format` value or a column absent from the file header uses that
default. An empty CSV field still produces null unless it matches
`null_format`; columns without a default keep their existing behavior. CSV
defaults are converted to the column [...]
| excel_use_header_line | boolean | no | false
| Use the first Excel row as column headers for name-based mapping. Only used
when file_format_type is `excel`.
|
| markdown_rag_metadata_enabled | boolean | no | false
| Whether to append RAG-oriented metadata columns when reading markdown files.
Only valid when file_format_type is `markdown`.
|
| pdf_rag_metadata_enabled | boolean | no | false
| Whether to append RAG-oriented metadata columns when reading PDF files. Only
valid when file_format_type is `pdf`.
|
diff --git a/docs/en/connectors/source/LocalFile.md
b/docs/en/connectors/source/LocalFile.md
index 395b7c8fbf..59f6a17981 100644
--- a/docs/en/connectors/source/LocalFile.md
+++ b/docs/en/connectors/source/LocalFile.md
@@ -483,8 +483,9 @@ The encoding of the file to read. This param will be parsed
by `Charset.forName(
### null_format [string]
-Only used when file_format_type is `text` or `csv`.
-null_format to define which strings can be represented as null.
+Used when `file_format_type` is `text` or `csv`. Defines which field value
represents null.
+For CSV with a configured `schema.columns[].defaultValue`, a matching
`null_format` value or a column absent from the file header uses that default.
An empty CSV field still produces null unless it matches `null_format`; columns
without a default keep their existing behavior.
+CSV defaults are validated when the reader is created. DATE and TIMESTAMP
defaults may use a different supported format from the file data.
e.g: `\N`
diff --git a/docs/en/connectors/source/ObsFile.md
b/docs/en/connectors/source/ObsFile.md
index b0db487279..0df2b5ef5d 100644
--- a/docs/en/connectors/source/ObsFile.md
+++ b/docs/en/connectors/source/ObsFile.md
@@ -88,7 +88,7 @@ It only supports hadoop version **2.9.X+**.
| compress_codec | string | no | none | Which compress codec the files used. |
| archive_compress_codec | string | no | none | Which archive compress codec
the files used. Supported: `none` `zip` `tar` `tar.gz` `gz`. |
| encoding | string | no | UTF-8 | File encoding, only used when
`file_format_type` is `json`, `text`, `csv`, or `xml`. |
-| null_format | string | no | - | Only used when file_format_type is text.
null_format to define which strings can be represented as null. e.g: `\N` |
+| null_format | string | no | - | Used when `file_format_type` is `text` or
`csv`. Defines which field value represents null. For CSV with a configured
`schema.columns[].defaultValue`, a matching `null_format` value or a column
absent from the file header uses that default. An empty CSV field still
produces null unless it matches `null_format`; columns without a default keep
their existing behavior. CSV defaults are converted to the column type when the
reader is created. A default that [...]
| binary_chunk_size | int | no | 1024 | Only used when file_format_type is
binary. The chunk size (in bytes) for reading binary files. Default is 1024
bytes. Larger values may improve performance for large files but use more
memory. |
| binary_complete_file_mode | boolean | no | false | Only used when
file_format_type is binary. Whether to read the complete file as a single chunk
instead of splitting into chunks. When enabled, the entire file content will be
read into memory at once. Default is false. |
| file_filter_pattern | string | no | - | Filter pattern, which used for
filtering files. |
diff --git a/docs/en/connectors/source/OssFile.md
b/docs/en/connectors/source/OssFile.md
index ad60d4b6dd..d7ed8c158f 100644
--- a/docs/en/connectors/source/OssFile.md
+++ b/docs/en/connectors/source/OssFile.md
@@ -212,7 +212,7 @@ If you assign file type to `parquet` `orc`, schema option
not required, connecto
| compress_codec | string | no | none |
Which compress codec the files used.
|
| archive_compress_codec | string | no | none |
Which archive compress codec the files used. Supported: `none` `zip` `tar`
`tar.gz` `gz`.
|
| encoding | string | no | UTF-8 | File
encoding, only used when `file_format_type` is `json`, `text`, `csv`, or `xml`.
|
-| null_format | string | no | - | Only
used when file_format_type is text. null_format to define which strings can be
represented as null. e.g: `\N`
|
+| null_format | string | no | - | Used
when `file_format_type` is `text` or `csv`. Defines which field value
represents null. For CSV with a configured `schema.columns[].defaultValue`, a
matching `null_format` value or a column absent from the file header uses that
default. An empty CSV field still produces null unless it matches
`null_format`; columns without a default keep their existing behavior. CSV
defaults are converted to the column type whe [...]
| binary_chunk_size | int | no | 1024 | Only
used when file_format_type is binary. The chunk size (in bytes) for reading
binary files. Default is 1024 bytes. Larger values may improve performance for
large files but use more memory.
|
| binary_complete_file_mode | boolean | no | false | Only
used when file_format_type is binary. Whether to read the complete file as a
single chunk instead of splitting into chunks. When enabled, the entire file
content will be read into memory at once. Default is false.
|
| discovery_mode | string | no | once | File
discovery mode. Supported values: `once` (default), `continuous`. Continuous
mode periodically scans the path and currently requires `sync_mode=update` and
`file_format_type=binary`. |
diff --git a/docs/en/connectors/source/OssJindoFile.md
b/docs/en/connectors/source/OssJindoFile.md
index 6b648bae14..46028cdaf8 100644
--- a/docs/en/connectors/source/OssJindoFile.md
+++ b/docs/en/connectors/source/OssJindoFile.md
@@ -454,8 +454,9 @@ The encoding of the file to read. This param will be parsed
by `Charset.forName(
### null_format [string]
-Only used when file_format_type is text.
-null_format to define which strings can be represented as null.
+Used when `file_format_type` is `text` or `csv`. Defines which field value
represents null.
+For CSV with a configured `schema.columns[].defaultValue`, a matching
`null_format` value or a column absent from the file header uses that default.
An empty CSV field still produces null unless it matches `null_format`; columns
without a default keep their existing behavior.
+CSV defaults are converted to the column type when the reader is created. A
default that cannot be converted fails the job even if no field needs it. DATE
and TIMESTAMP defaults may use a different supported format from the file data.
e.g: `\N`
diff --git a/docs/en/connectors/source/S3File.md
b/docs/en/connectors/source/S3File.md
index 6a9ea3f487..78844607ec 100644
--- a/docs/en/connectors/source/S3File.md
+++ b/docs/en/connectors/source/S3File.md
@@ -261,7 +261,7 @@ If you assign file type to `parquet` `orc`, schema option
not required, connecto
| enable_file_split | boolean | no | false
| Turn on logical file split to improve
parallelism for huge files. Only supported for `text`/`csv`/`json`/`parquet`
and non-compressed format.
|
| file_split_size | long | no | 134217728
| Split size in bytes when
`enable_file_split=true`. For `text`/`csv`/`json`, the split end will be
aligned to the next `row_delimiter`. For `parquet`, the split unit is RowGroup
and will never break a RowGroup.
|
| encoding | string | no | UTF-8
|
[...]
-| null_format | string | no | -
| Only used when file_format_type is text.
null_format to define which strings can be represented as null. e.g: `\N`
[...]
+| null_format | string | no | -
| Used when `file_format_type` is `text` or
`csv`. Defines which field value represents null. For CSV with a configured
`schema.columns[].defaultValue`, a matching `null_format` value or a column
absent from the file header uses that default. An empty CSV field still
produces null unless it matches `null_format`; columns without a default keep
their existing behavior. CSV defaul [...]
| binary_chunk_size | int | no | 1024
| Only used when file_format_type is binary.
The chunk size (in bytes) for reading binary files. Default is 1024 bytes.
Larger values may improve performance for large files but use more memory.
[...]
| binary_complete_file_mode | boolean | no | false
| Only used when file_format_type is binary.
Whether to read the complete file as a single chunk instead of splitting into
chunks. When enabled, the entire file content will be read into memory at once.
Default is false.
[...]
| discovery_mode | string | no | once
| File discovery mode. Supported values: `once`
(default), `continuous`. When `continuous`, the source periodically scans the
path and processes new or changed files as an unbounded source. Continuous mode
currently requires `sync_mode=update` and `file_format_type=binary`.
|
diff --git a/docs/en/connectors/source/SftpFile.md
b/docs/en/connectors/source/SftpFile.md
index eb04505c68..6eccb62533 100644
--- a/docs/en/connectors/source/SftpFile.md
+++ b/docs/en/connectors/source/SftpFile.md
@@ -113,7 +113,7 @@ The File does not have a specific type list, and we can
indicate which SeaTunnel
| compress_codec | String | No | None
| The compress codec of files and the details that supported as the
following shown: <br/> - txt: `lzo` `None` <br/> - json: `lzo` `None` <br/> -
csv: `lzo` `None` <br/> - orc: `lzo` `snappy` `lz4` `zlib` `None` <br/> -
parquet: `lzo` `snappy` `lz4` `gzip` `brotli` `zstd` `None` <br/> Tips: excel
type does Not support any compression format |
| archive_compress_codec | string | no | none
| Which archive compress codec the files used. Supported: `none` `zip` `tar`
`tar.gz` `gz`. |
| encoding | string | no | UTF-8
| File encoding, only used when `file_format_type` is `json`, `text`, `csv`,
or `xml`. |
-| null_format | string | no | -
| Only used when file_format_type is text. null_format to define which
strings can be represented as null. e.g: `\N`
|
+| null_format | string | no | -
| Used when `file_format_type` is `text` or `csv`. Defines which field value
represents null. For CSV with a configured `schema.columns[].defaultValue`, a
matching `null_format` value or a column absent from the file header uses that
default. An empty CSV field still produces null unless it matches
`null_format`; columns without a default keep their existing behavior. CSV
defaults are converted to the colum [...]
| binary_chunk_size | int | no | 1024
| Only used when file_format_type is binary. The chunk size (in bytes) for
reading binary files. Default is 1024 bytes. Larger values may improve
performance for large files but use more memory.
|
| binary_complete_file_mode | boolean | no | false
| Only used when file_format_type is binary. Whether to read the complete
file as a single chunk instead of splitting into chunks. When enabled, the
entire file content will be read into memory at once. Default is false.
|
| discovery_mode | string | no | once
| File discovery mode. Supported values: `once` (default), `continuous`.
When `continuous`, the source keeps scanning the path and processes new/changed
files at runtime (unbounded). In the current implementation, `continuous`
requires `sync_mode=update` (binary only).
|
diff --git a/docs/en/connectors/source/SmbFile.md
b/docs/en/connectors/source/SmbFile.md
index f4c7a6f3e3..52e0a403f4 100644
--- a/docs/en/connectors/source/SmbFile.md
+++ b/docs/en/connectors/source/SmbFile.md
@@ -104,7 +104,7 @@ The File does not have a specific type list, and we can
indicate which SeaTunnel
| compress_codec | String | No | None
| The compress codec of files
|
| archive_compress_codec | String | No | none
| Which archive compress codec the files used. Supported: `none` `zip` `tar`
`tar.gz` `gz`. |
| encoding | String | No | UTF-8
| The encoding of the file to read
|
-| null_format | String | No | -
| Only used when file_format_type is text. Define which strings can be
represented as null, e.g. `\N`. |
+| null_format | String | No | -
| Used when `file_format_type` is `text` or `csv`. Defines which field value
represents null. For CSV with a configured `schema.columns[].defaultValue`, a
matching `null_format` value or a column absent from the file header uses that
default. An empty CSV field still produces null unless it matches
`null_format`; columns without a default keep their existing behavior. CSV
defaults are converted to the colum [...]
| binary_chunk_size | int | no | 1024 | Only used when file_format_type is
binary. The chunk size (in bytes) for reading binary files. Larger values may
improve performance for large files but use more memory. |
| binary_complete_file_mode | boolean | no | false | Only used when
file_format_type is binary. Whether to read the complete file as a single chunk
instead of splitting into chunks. |
| filename_extension | String | No | -
| Filter filename extension, which used for filtering files with specific
extension. Example: `csv` `.txt` `json` `.xml`. |
diff --git a/docs/en/introduction/concepts/incompatible-changes.md
b/docs/en/introduction/concepts/incompatible-changes.md
index e98a52c541..5ca2420243 100644
--- a/docs/en/introduction/concepts/incompatible-changes.md
+++ b/docs/en/introduction/concepts/incompatible-changes.md
@@ -464,6 +464,17 @@ or state format migration is required.
### Format Changes
+- **CSV source defaults for missing and configured null fields**: When a CSV
source declares
+ `schema.columns[].defaultValue`, a column absent from the file header or a
field matching
+ `null_format` now receives that default instead of null. Jobs without
configured defaults are
+ unchanged, and an empty field remains null unless it matches `null_format`.
If downstream logic
+ intentionally relied on null despite a configured default, remove that
default from the source
+ schema before upgrading.
+ Configured defaults are now converted to the column type when the CSV reader
is created.
+ A default that cannot be converted, such as `CURRENT_TIMESTAMP` for a
TIMESTAMP column, now
+ fails the job at reader creation instead of being ignored, even if no field
needs the default.
+ Defaults must be literal values, not SQL expressions. Fix or remove invalid
defaults before upgrading.
+
- **Breaking Change: JSON serialization of numeric fields now follows the
runtime value type**
- **Affected component**: `seatunnel-formats/seatunnel-format-json`
(`RowToJsonConverters`) - affects every connector that serializes rows with the
JSON format (for example Kafka, RabbitMQ, Pulsar, and file JSON sinks)
- **Description**: Previously, a field declared as a numeric type in the
catalog (`TINYINT`, `SMALLINT`, `INT`, `BIGINT`, `FLOAT`, `DOUBLE`, `DECIMAL`)
was serialized by blindly casting the runtime value to the Java type implied by
the declared type (for example `(long) value` for `BIGINT`). In multi-table
jobs (for example CDC jobs writing JSON to RabbitMQ/Kafka) where several tables
share one catalog schema but carry different physical column types, a `String`
or `BigDecimal` runtime [...]
diff --git a/docs/zh/connectors/source/FtpFile.md
b/docs/zh/connectors/source/FtpFile.md
index be15ba625f..9b34e9e625 100644
--- a/docs/zh/connectors/source/FtpFile.md
+++ b/docs/zh/connectors/source/FtpFile.md
@@ -489,8 +489,9 @@ POI 引擎允许读取的最大 Excel 文件大小,单位为字节。默认值
### null_format [string]
-仅在文件格式类型为 text 时使用。
-用于定义哪些字符串可以表示为 null。
+当 `file_format_type` 为 `text` 或 `csv` 时使用。指定表示 null 的字段值。
+对于配置了 `schema.columns[].defaultValue` 的 CSV,如果字段值匹配
`null_format`,或文件表头缺少该列,则使用配置的默认值。空 CSV 字段仍为 null,除非它与 `null_format`
匹配;未配置默认值的列保持原有行为。
+CSV 默认值在创建读取器时转换为列类型。无法转换的默认值会导致作业失败,即使没有字段需要使用该默认值。DATE 和 TIMESTAMP
默认值可以使用与文件数据不同的受支持格式。
例如:`\N`
diff --git a/docs/zh/connectors/source/HdfsFile.md
b/docs/zh/connectors/source/HdfsFile.md
index ff113557c2..36df709042 100644
--- a/docs/zh/connectors/source/HdfsFile.md
+++ b/docs/zh/connectors/source/HdfsFile.md
@@ -82,7 +82,7 @@ import ChangeLog from '../changelog/connector-file-hadoop.md';
| compress_codec | string | 否 | none |
文件的压缩编解码器
|
| archive_compress_codec | string | 否 | none |
文件使用的归档压缩编解码器。支持:`none` `zip` `tar` `tar.gz` `gz`。 |
| encoding | string | 否 | UTF-8 |
|
-| null_format | string | 否 | - | 仅在
file_format_type 为 `text` 或 `csv` 时使用。null_format 定义哪些字符串可以表示为 null。例如:`\N`
|
+| null_format | string | 否 | - | 当
`file_format_type` 为 `text` 或 `csv` 时使用。指定表示 null 的字段值。 对于配置了
`schema.columns[].defaultValue` 的 CSV,如果字段值匹配
`null_format`,或文件表头缺少该列,则使用配置的默认值。空 CSV 字段仍为 null,除非它与 `null_format`
匹配;未配置默认值的列保持原有行为。 CSV 默认值在创建读取器时转换为列类型。无法转换的默认值会导致作业失败,即使没有字段需要使用该默认值。DATE 和
TIMESTAMP 默认值可以使用与文件数据不同的受支持格式。 例如:`\N` |
| excel_use_header_line | boolean | 否 | false | 使用 Excel
第一行作为表头按名称映射字段。仅在 file_format_type 为 `excel` 时使用。
|
| markdown_rag_metadata_enabled | boolean | 否 | false | 读取
markdown 文件时是否追加面向 RAG 的元数据列。仅在 file_format_type 为 `markdown` 时有效。
|
| pdf_rag_metadata_enabled | boolean | 否 | false | 读取 PDF
文件时是否追加面向 RAG 的元数据列。仅在 file_format_type 为 `pdf` 时有效。
|
diff --git a/docs/zh/connectors/source/LocalFile.md
b/docs/zh/connectors/source/LocalFile.md
index 18e14f9b9b..84c2c02d22 100644
--- a/docs/zh/connectors/source/LocalFile.md
+++ b/docs/zh/connectors/source/LocalFile.md
@@ -482,8 +482,9 @@ abc.*
### null_format [string]
-仅在 file_format_type 为 `text` 或 `csv` 时使用。
-null_format 定义哪些字符串可以表示为 null。
+当 `file_format_type` 为 `text` 或 `csv` 时使用。指定表示 null 的字段值。
+对于配置了 `schema.columns[].defaultValue` 的 CSV,如果字段值匹配
`null_format`,或文件表头缺少该列,则使用配置的默认值。空 CSV 字段仍为 null,除非它与 `null_format`
匹配;未配置默认值的列保持原有行为。
+CSV 默认值在创建读取器时进行校验。DATE 和 TIMESTAMP 默认值可以使用与文件数据不同的受支持格式。
例如:`\N`
diff --git a/docs/zh/connectors/source/ObsFile.md
b/docs/zh/connectors/source/ObsFile.md
index 4fc5a7dfd7..cc56deffae 100644
--- a/docs/zh/connectors/source/ObsFile.md
+++ b/docs/zh/connectors/source/ObsFile.md
@@ -91,7 +91,7 @@ import ChangeLog from '../changelog/connector-file-obs.md';
| compress_codec | string | 否 | none | 文件使用的压缩编解码器。 |
| archive_compress_codec | string | 否 | none | 文件使用的归档压缩编解码器。支持:`none` `zip`
`tar` `tar.gz` `gz`。 |
| encoding | string | 否 | UTF-8 | 文件编码,仅在 `file_format_type` 为
`json`、`text`、`csv` 或 `xml` 时使用。 |
-| null_format | string | 否 | - |
仅在file_format_type为text时使用。null_format用于定义哪些字符串可以表示为null。例如:`\N` |
+| null_format | string | 否 | - | 当 `file_format_type` 为 `text` 或 `csv`
时使用。指定表示 null 的字段值。 对于配置了 `schema.columns[].defaultValue` 的 CSV,如果字段值匹配
`null_format`,或文件表头缺少该列,则使用配置的默认值。空 CSV 字段仍为 null,除非它与 `null_format`
匹配;未配置默认值的列保持原有行为。 CSV 默认值在创建读取器时转换为列类型。无法转换的默认值会导致作业失败,即使没有字段需要使用该默认值。DATE 和
TIMESTAMP 默认值可以使用与文件数据不同的受支持格式。 例如:`\N` |
| binary_chunk_size | int | 否 | 1024 |
仅在file_format_type为binary时使用。读取二进制文件的块大小(以字节为单位)。默认为1024字节。较大的值可能会提高大文件的性能,但会使用更多内存。
|
| binary_complete_file_mode | boolean | 否 | false |
仅在file_format_type为binary时使用。是否将完整文件作为单个块读取,而不是分割成块。启用时,整个文件内容将一次性读入内存。默认为false。
|
| file_filter_pattern | string | 否 | - | 用于过滤文件的模式。 |
diff --git a/docs/zh/connectors/source/OssFile.md
b/docs/zh/connectors/source/OssFile.md
index 54ba631ea7..f7c759c544 100644
--- a/docs/zh/connectors/source/OssFile.md
+++ b/docs/zh/connectors/source/OssFile.md
@@ -212,7 +212,7 @@ schema {
| compress_codec | string | 否 | none |
文件使用的压缩编解码器。
|
| archive_compress_codec | string | 否 | none |
文件使用的归档压缩编解码器。支持:`none` `zip` `tar` `tar.gz` `gz`。
|
| encoding | string | 否 | UTF-8 | 文件编码,仅在
`file_format_type` 为 `json`、`text`、`csv` 或 `xml` 时使用。 |
-| null_format | string | 否 | - |
仅在file_format_type为text时使用。null_format用于定义哪些字符串可以表示为null。例如:`\N`
|
+| null_format | string | 否 | - | 当
`file_format_type` 为 `text` 或 `csv` 时使用。指定表示 null 的字段值。 对于配置了
`schema.columns[].defaultValue` 的 CSV,如果字段值匹配
`null_format`,或文件表头缺少该列,则使用配置的默认值。空 CSV 字段仍为 null,除非它与 `null_format`
匹配;未配置默认值的列保持原有行为。 CSV 默认值在创建读取器时转换为列类型。无法转换的默认值会导致作业失败,即使没有字段需要使用该默认值。DATE 和
TIMESTAMP 默认值可以使用与文件数据不同的受支持格式。 例如:`\N` |
| binary_chunk_size | int | 否 | 1024 |
仅在file_format_type为binary时使用。读取二进制文件的块大小(以字节为单位)。默认为1024字节。较大的值可能会提高大文件的性能,但会使用更多内存。
|
| binary_complete_file_mode | boolean | 否 | false |
仅在file_format_type为binary时使用。是否将完整文件作为单个块读取,而不是分割成块。启用时,整个文件内容将一次性读入内存。默认为false。
|
| discovery_mode | string | 否 | once | 文件发现模式,支持
`once`(默认)和 `continuous`。continuous 模式会定期扫描路径,目前需要同时配置 `sync_mode=update` 和
`file_format_type=binary`。 |
diff --git a/docs/zh/connectors/source/OssJindoFile.md
b/docs/zh/connectors/source/OssJindoFile.md
index b8584da979..23a8ff2199 100644
--- a/docs/zh/connectors/source/OssJindoFile.md
+++ b/docs/zh/connectors/source/OssJindoFile.md
@@ -86,7 +86,7 @@ import ChangeLog from
'../changelog/connector-file-oss-jindo.md';
| compress_codec | string | 否 | none |
读取文件时使用的压缩格式。 |
| archive_compress_codec | string | 否 | none |
读取归档文件时使用的归档压缩格式。 |
| encoding | string | 否 | UTF-8 | 读取
`json`、`text`、`csv` 或 `xml` 文件时使用的字符编码。 |
-| null_format | string | 否 | - | 仅用于
`text` 格式,指定应解析为 `null` 的字符串。 |
+| null_format | string | 否 | - | 当
`file_format_type` 为 `text` 或 `csv` 时使用。指定表示 null 的字段值。 对于配置了
`schema.columns[].defaultValue` 的 CSV,如果字段值匹配
`null_format`,或文件表头缺少该列,则使用配置的默认值。空 CSV 字段仍为 null,除非它与 `null_format`
匹配;未配置默认值的列保持原有行为。 CSV 默认值在创建读取器时转换为列类型。无法转换的默认值会导致作业失败,即使没有字段需要使用该默认值。DATE 和
TIMESTAMP 默认值可以使用与文件数据不同的受支持格式。 例如:`\N` |
| binary_chunk_size | int | 否 | 1024 | 仅在
file_format_type 为 binary 时使用。读取二进制文件的块大小(以字节为单位)。默认为 1024
字节。较大的值可能会提高大文件的性能,但会使用更多内存。 |
| binary_complete_file_mode | boolean | 否 | false | 仅在
file_format_type 为 binary 时使用。是否将完整文件作为单个块读取,而不是分割成块。 |
| common-options | | 否 | - |
Source 插件通用参数,详见 [Source Common
Options](../common-options/source-common-options.md)。 |
diff --git a/docs/zh/connectors/source/S3File.md
b/docs/zh/connectors/source/S3File.md
index 08209a1b19..3e2c5cd6de 100644
--- a/docs/zh/connectors/source/S3File.md
+++ b/docs/zh/connectors/source/S3File.md
@@ -253,7 +253,7 @@ schema {
| enable_file_split | boolean | 否 | false
| 开启大文件拆分以提升并行度。仅支持 `text`/`csv`/`json`/`parquet`
且非压缩格式(`compress_codec=none` 且 `archive_compress_codec=none`)。
|
| file_split_size | long | 否 | 134217728
| `enable_file_split=true`
时生效,单位字节。`text`/`csv`/`json` 按 `file_split_size` 拆分并对齐到下一个
`row_delimiter`;`parquet` 以 RowGroup 为拆分单位,不会切开 RowGroup。
|
| encoding | string | 否 | UTF-8
|
|
-| null_format | string | 否 | -
|
仅在file_format_type为text时使用。null_format用于定义哪些字符串可以表示为null。例如:`\N`
|
+| null_format | string | 否 | -
| 当 `file_format_type` 为 `text` 或 `csv` 时使用。指定表示
null 的字段值。 对于配置了 `schema.columns[].defaultValue` 的 CSV,如果字段值匹配
`null_format`,或文件表头缺少该列,则使用配置的默认值。空 CSV 字段仍为 null,除非它与 `null_format`
匹配;未配置默认值的列保持原有行为。 CSV 默认值在创建读取器时转换为列类型。无法转换的默认值会导致作业失败,即使没有字段需要使用该默认值。DATE 和
TIMESTAMP 默认值可以使用与文件数据不同的受支持格式。 例如:`\N` |
| binary_chunk_size | int | 否 | 1024
|
仅在file_format_type为binary时使用。读取二进制文件的块大小(以字节为单位)。默认为1024字节。较大的值可能会提高大文件的性能,但会使用更多内存。
|
| binary_complete_file_mode | boolean | 否 | false
|
仅在file_format_type为binary时使用。是否将完整文件作为单个块读取,而不是分割成块。启用时,整个文件内容将一次性读入内存。默认为false。
|
| discovery_mode | string | 否 | once
| 文件发现模式,支持 `once`(默认)和 `continuous`。continuous
模式会定期扫描路径并处理新增或变更对象,目前需要同时配置 `sync_mode=update` 和 `file_format_type=binary`。 |
diff --git a/docs/zh/connectors/source/SftpFile.md
b/docs/zh/connectors/source/SftpFile.md
index c94e04060f..7465c5ca06 100644
--- a/docs/zh/connectors/source/SftpFile.md
+++ b/docs/zh/connectors/source/SftpFile.md
@@ -109,7 +109,7 @@ import ChangeLog from '../changelog/connector-file-sftp.md';
| compress_codec | String | 否 | None |
文件的压缩编解码器,支持的详细信息如下所示:<br/> - txt: `lzo` `None` <br/> - json: `lzo` `None`
<br/> - csv: `lzo` `None` <br/> - orc: `lzo` `snappy` `lz4` `zlib` `None` <br/>
- parquet: `lzo` `snappy` `lz4` `gzip` `brotli` `zstd` `None` <br/>
提示:excel类型不支持任何压缩格式 |
| archive_compress_codec | string | 否 | none |
文件使用的归档压缩编解码器。支持:`none` `zip` `tar` `tar.gz` `gz`。 |
| encoding | string | 否 | UTF-8 | 文件编码,仅在
`file_format_type` 为 `json`、`text`、`csv` 或 `xml` 时使用。 |
-| null_format | string | 否 | - |
仅在file_format_type为text时使用。null_format用于定义哪些字符串可以表示为null。例如:`\N`
|
+| null_format | string | 否 | - | 当
`file_format_type` 为 `text` 或 `csv` 时使用。指定表示 null 的字段值。 对于配置了
`schema.columns[].defaultValue` 的 CSV,如果字段值匹配
`null_format`,或文件表头缺少该列,则使用配置的默认值。空 CSV 字段仍为 null,除非它与 `null_format`
匹配;未配置默认值的列保持原有行为。 CSV 默认值在创建读取器时转换为列类型。无法转换的默认值会导致作业失败,即使没有字段需要使用该默认值。DATE 和
TIMESTAMP 默认值可以使用与文件数据不同的受支持格式。 例如:`\N` |
| binary_chunk_size | int | 否 | 1024 |
仅在file_format_type为binary时使用。读取二进制文件的块大小(以字节为单位)。默认为1024字节。较大的值可能会提高大文件的性能,但会使用更多内存。
|
| binary_complete_file_mode | boolean | 否 | false |
仅在file_format_type为binary时使用。是否将完整文件作为单个块读取,而不是分割成块。启用时,整个文件内容将一次性读入内存。默认为false。
|
| discovery_mode | string | 否 | once |
文件发现模式,支持:`once`(默认)、`continuous`。continuous 模式下将周期性扫描并处理新/变更文件(无界)。当前实现中
continuous 需要配合 `sync_mode=update`(仅 binary)使用,以避免重复传输。
|
diff --git a/docs/zh/connectors/source/SmbFile.md
b/docs/zh/connectors/source/SmbFile.md
index 578efa01b7..fef2c6cc83 100644
--- a/docs/zh/connectors/source/SmbFile.md
+++ b/docs/zh/connectors/source/SmbFile.md
@@ -104,7 +104,7 @@ import ChangeLog from '../changelog/connector-file-smb.md';
| compress_codec | String | 否 | None |
文件的压缩编解码器 |
| archive_compress_codec | String | 否 | none |
文件使用的归档压缩编解码器。支持:`none` `zip` `tar` `tar.gz` `gz`。 |
| encoding | String | 否 | UTF-8 |
读取文件时使用的编码
|
-| null_format | String | 否 | - |
仅在 file_format_type 为 text 时使用。定义哪些字符串可以表示为 null,例如 `\N` |
+| null_format | String | 否 | - |
当 `file_format_type` 为 `text` 或 `csv` 时使用。指定表示 null 的字段值。 对于配置了
`schema.columns[].defaultValue` 的 CSV,如果字段值匹配
`null_format`,或文件表头缺少该列,则使用配置的默认值。空 CSV 字段仍为 null,除非它与 `null_format`
匹配;未配置默认值的列保持原有行为。 CSV 默认值在创建读取器时转换为列类型。无法转换的默认值会导致作业失败,即使没有字段需要使用该默认值。DATE 和
TIMESTAMP 默认值可以使用与文件数据不同的受支持格式。 例如:`\N` |
| binary_chunk_size | int | 否 | 1024 | 仅在 file_format_type 为 binary
时使用。读取二进制文件的块大小(以字节为单位)。较大的值可能会提高大文件的性能,但会使用更多内存。 |
| binary_complete_file_mode | boolean | 否 | false | 仅在 file_format_type 为
binary 时使用。是否将完整文件作为单个块读取,而不是分割成块。 |
| filename_extension | String | 否 | - |
文件扩展名过滤,用于过滤特定扩展名的文件。例如:`csv` `.txt` `json` `.xml` |
diff --git a/docs/zh/introduction/concepts/incompatible-changes.md
b/docs/zh/introduction/concepts/incompatible-changes.md
index 2c5f9190f6..9800b82878 100644
--- a/docs/zh/introduction/concepts/incompatible-changes.md
+++ b/docs/zh/introduction/concepts/incompatible-changes.md
@@ -393,6 +393,14 @@ Zeta SQL ARRAY 函数现在将 TINYINT 元素声明为 `ARRAY<TINYINT>`,与实
### 格式变更
+- **CSV Source 对缺失字段和已配置的 null 值应用默认值**:配置了 `schema.columns[].defaultValue` 时,
+ 如果文件表头缺少该列,或字段值与 `null_format` 匹配,读取结果现在使用默认值而不是 null。
+ 未配置默认值的作业保持不变;空字段仍为 null,除非它与 `null_format` 匹配。
+ 如果下游逻辑原本依赖“配置了默认值但仍输出 null”的行为,升级前请移除 Source Schema 中的该默认值。
+ 配置的默认值现在会在创建 CSV 读取器时转换为列类型。无法转换的默认值(例如 TIMESTAMP 列的
+ `CURRENT_TIMESTAMP`)现在会在创建读取器时导致作业失败,而不是被忽略,即使没有字段需要使用该默认值。
+ 默认值必须是字面量,不能是 SQL 表达式。升级前请修正或移除无效的默认值。
+
- **破坏性变更:JSON 数值字段的序列化改为按运行时实际类型处理**
-
**影响范围**:`seatunnel-formats/seatunnel-format-json`(`RowToJsonConverters`)--影响所有以
JSON 格式序列化行的连接器(例如 Kafka、RabbitMQ、Pulsar 及文件 JSON Sink)。
- **变更说明**:以前,目录 Schema
中声明为数值类型(`TINYINT`、`SMALLINT`、`INT`、`BIGINT`、`FLOAT`、`DOUBLE`、`DECIMAL`)的字段,序列化时会把运行时值强制转换为声明类型对应的
Java 类型(例如 `BIGINT` 直接 `(long) value`)。在多表作业(例如多表 CDC 作业写 JSON 到
RabbitMQ/Kafka)中,多张表共享同一份目录 Schema 但物理列类型不一致时,`String` 或 `BigDecimal` 运行时值会抛出原始
`ClassCastException`
并导致作业失败。现在数值字段按运行时实际类型序列化:任意数值包装类型(`Byte`、`Short`、`Integer`、`Long`、`Float`、`Double`、`BigInteger`、`BigDecimal`)输出为对应的
JSON 数字;可解析为数字的字符串会解析成 JSON 数字,无法解析的文本则输出为 JSON 字符串;声明为 `DECIMAL` 的字段遇到
`Float`/`Dou [...]
diff --git
a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/CsvReadStrategy.java
b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/CsvReadStrategy.java
index 6f52c45566..619c38d4e6 100644
---
a/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/CsvReadStrategy.java
+++
b/seatunnel-connectors-v2/connector-file/connector-file-base/src/main/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/CsvReadStrategy.java
@@ -292,6 +292,7 @@ public class CsvReadStrategy extends AbstractReadStrategy {
initFormatter();
CsvDeserializationSchema.Builder builder =
CsvDeserializationSchema.builder()
+ .setCatalogTable(catalogTable)
.delimiter(getDelimiter())
.csvLineProcessor(processor)
.nullFormat(
diff --git
a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/CsvReadStrategyTest.java
b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/CsvReadStrategyTest.java
index 965e6a714b..86ff64c1c8 100644
---
a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/CsvReadStrategyTest.java
+++
b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/java/org/apache/seatunnel/connectors/seatunnel/file/source/reader/CsvReadStrategyTest.java
@@ -17,11 +17,16 @@
package org.apache.seatunnel.connectors.seatunnel.file.source.reader;
+import org.apache.seatunnel.shade.com.typesafe.config.Config;
import org.apache.seatunnel.shade.com.typesafe.config.ConfigFactory;
+import org.apache.seatunnel.api.configuration.ReadonlyConfig;
import org.apache.seatunnel.api.source.Collector;
import org.apache.seatunnel.api.table.catalog.CatalogTable;
import org.apache.seatunnel.api.table.catalog.CatalogTableUtil;
+import org.apache.seatunnel.api.table.catalog.TableIdentifier;
+import org.apache.seatunnel.api.table.catalog.TablePath;
+import org.apache.seatunnel.api.table.catalog.schema.ReadonlyConfigParser;
import org.apache.seatunnel.api.table.type.BasicType;
import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
@@ -43,6 +48,8 @@ import java.io.IOException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.nio.file.Paths;
+import java.time.LocalDate;
+import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -201,6 +208,55 @@ public class CsvReadStrategyTest {
Assertions.assertArrayEquals(new Object[] {2, "Bob", 20},
rows.get(1).getFields());
}
+ @Test
+ public void testConfiguredDefaultsFromLocalCsvFile() throws Exception {
+ URL resource =
CsvReadStrategyTest.class.getResource("/csv/default-values.csv");
+ String path = Paths.get(resource.toURI()).toString();
+ Config config =
+ ConfigFactory.parseString(
+ "csv_use_header_line = true\n"
+ + "null_format = \"NULL\"\n"
+ + "schema { columns = ["
+ + "{ name = \"id\", type = \"int\", nullable =
false, defaultValue = 7 },"
+ + "{ name = \"name\", type = \"string\",
nullable = false, defaultValue = \"unknown\" },"
+ + "{ name = \"optional\", type = \"string\",
nullable = true },"
+ + "{ name = \"date\", type = \"date\",
defaultValue = \"2020-01-01\" },"
+ + "{ name = \"timestamp\", type =
\"timestamp\", defaultValue = \"2020-01-01 12:34:56\" }"
+ + "] }");
+ CatalogTable catalogTable =
+ CatalogTable.of(
+ TableIdentifier.of("test", TablePath.DEFAULT),
+ new
ReadonlyConfigParser().parse(ReadonlyConfig.fromConfig(config)),
+ new HashMap<>(),
+ Collections.emptyList(),
+ "test");
+ try (CsvReadStrategy csvReadStrategy = new CsvReadStrategy()) {
+ csvReadStrategy.init(new
LocalFileSystemConf.LocalConf(FS_DEFAULT_NAME_DEFAULT));
+ csvReadStrategy.getFileNamesByPath(path);
+ csvReadStrategy.setPluginConfig(config);
+ csvReadStrategy.setCatalogTable(catalogTable);
+ TestCollector collector = new TestCollector();
+ csvReadStrategy.read(path, "", collector);
+
+ Assertions.assertEquals(2, collector.getRows().size());
+ Assertions.assertEquals(7, collector.getRows().get(0).getField(0));
+ Assertions.assertEquals("unknown",
collector.getRows().get(0).getField(1));
+ Assertions.assertEquals("keep",
collector.getRows().get(0).getField(2));
+ Assertions.assertEquals(8, collector.getRows().get(1).getField(0));
+ Assertions.assertEquals("unknown",
collector.getRows().get(1).getField(1));
+ Assertions.assertNull(collector.getRows().get(1).getField(2));
+ Assertions.assertEquals(
+ LocalDate.of(2020, 1, 1),
collector.getRows().get(0).getField(3));
+ Assertions.assertEquals(
+ LocalDateTime.of(2024, 2, 3, 4, 5, 6),
collector.getRows().get(0).getField(4));
+ Assertions.assertEquals(
+ LocalDate.of(2024, 2, 3),
collector.getRows().get(1).getField(3));
+ Assertions.assertEquals(
+ LocalDateTime.of(2020, 1, 1, 12, 34, 56),
+ collector.getRows().get(1).getField(4));
+ }
+ }
+
@Test
public void testReadCsv() throws Exception {
URL resource = CsvReadStrategyTest.class.getResource("/test.csv");
diff --git
a/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/resources/csv/default-values.csv
b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/resources/csv/default-values.csv
new file mode 100644
index 0000000000..fc3fcc05a1
--- /dev/null
+++
b/seatunnel-connectors-v2/connector-file/connector-file-base/src/test/resources/csv/default-values.csv
@@ -0,0 +1,3 @@
+id,optional,date,timestamp
+NULL,keep,NULL,2024/02/03 04:05:06
+8,,2024/02/03,NULL
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileIT.java
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileIT.java
index 34ead22e85..97a7462fa0 100644
---
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileIT.java
+++
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/java/org/apache/seatunnel/e2e/connector/file/local/LocalFileIT.java
@@ -328,6 +328,10 @@ public class LocalFileIT extends TestSuiteBase {
"/csv/csv_with_header2.csv",
"/seatunnel/read/csv/header/csv_with_header2.csv",
container);
+ ContainerUtil.copyFileIntoContainers(
+ "/csv/default-values.csv",
+ "/seatunnel/read/csv/defaults/default-values.csv",
+ container);
Path csvWithPreamble = Files.createTempFile("csv-preamble-",
".csv");
try {
@@ -382,6 +386,7 @@ public class LocalFileIT extends TestSuiteBase {
helper.execute("/csv/local_csv_to_assert.conf");
helper.execute("/csv/local_csv_enable_split_to_assert.conf");
helper.execute("/csv/csv_with_header_to_assert.conf");
+ helper.execute("/csv/default_values_to_assert.conf");
helper.execute("/csv/breakline_csv_to_assert.conf");
}
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/csv/default-values.csv
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/csv/default-values.csv
new file mode 100644
index 0000000000..c09b0e288f
--- /dev/null
+++
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/csv/default-values.csv
@@ -0,0 +1,3 @@
+id,optional,date,timestamp
+NULL,keep,NULL,2020/01/01 12:34:56
+NULL,keep,2020/01/01,NULL
diff --git
a/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/csv/default_values_to_assert.conf
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/csv/default_values_to_assert.conf
new file mode 100644
index 0000000000..8c26597a25
--- /dev/null
+++
b/seatunnel-e2e/seatunnel-connector-v2-e2e/connector-file-local-e2e/src/test/resources/csv/default_values_to_assert.conf
@@ -0,0 +1,57 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+env {
+ parallelism = 1
+ job.mode = "BATCH"
+}
+
+source {
+ LocalFile {
+ path = "/seatunnel/read/csv/defaults"
+ file_format_type = csv
+ csv_use_header_line = true
+ null_format = "NULL"
+ schema {
+ columns = [
+ { name = "id", type = "int", nullable = false, defaultValue = 7 }
+ { name = "name", type = "string", nullable = false, defaultValue =
"unknown" }
+ { name = "optional", type = "string", nullable = true }
+ { name = "date", type = "date", defaultValue = "2020-01-01" }
+ { name = "timestamp", type = "timestamp", defaultValue = "2020-01-01
12:34:56" }
+ ]
+ }
+ }
+}
+
+sink {
+ Assert {
+ rules {
+ row_rules = [
+ { rule_type = MIN_ROW, rule_value = 2 }
+ { rule_type = MAX_ROW, rule_value = 2 }
+ ]
+ field_rules = [
+ { field_name = id, field_type = int, field_value = [{ equals_to = 7 }]
}
+ { field_name = name, field_type = string, field_value = [{ equals_to =
"unknown" }] }
+ { field_name = optional, field_type = string, field_value = [{
equals_to = "keep" }] }
+ { field_name = date, field_type = date, field_value = [{ rule_type =
NOT_NULL, equals_to = "2020-01-01" }] }
+ { field_name = timestamp, field_type = timestamp, field_value = [{
rule_type = NOT_NULL, equals_to = "2020-01-01T12:34:56" }] }
+ ]
+ }
+ }
+}
diff --git
a/seatunnel-formats/seatunnel-format-csv/src/main/java/org/apache/seatunnel/format/csv/CsvDeserializationSchema.java
b/seatunnel-formats/seatunnel-format-csv/src/main/java/org/apache/seatunnel/format/csv/CsvDeserializationSchema.java
index ba3b7be105..0bd12caaeb 100644
---
a/seatunnel-formats/seatunnel-format-csv/src/main/java/org/apache/seatunnel/format/csv/CsvDeserializationSchema.java
+++
b/seatunnel-formats/seatunnel-format-csv/src/main/java/org/apache/seatunnel/format/csv/CsvDeserializationSchema.java
@@ -20,6 +20,7 @@ package org.apache.seatunnel.format.csv;
import org.apache.seatunnel.shade.org.apache.commons.lang3.StringUtils;
import org.apache.seatunnel.api.table.catalog.CatalogTable;
+import org.apache.seatunnel.api.table.catalog.Column;
import org.apache.seatunnel.api.table.catalog.TablePath;
import org.apache.seatunnel.api.table.type.ArrayType;
import org.apache.seatunnel.api.table.type.MapType;
@@ -40,7 +41,9 @@ import
org.apache.seatunnel.format.csv.processor.DefaultCsvLineProcessor;
import lombok.NonNull;
import java.io.IOException;
+import java.io.ObjectInputStream;
import java.io.Serializable;
+import java.lang.reflect.Array;
import java.math.BigDecimal;
import java.nio.charset.StandardCharsets;
import java.time.LocalDate;
@@ -55,6 +58,7 @@ import java.time.temporal.ChronoField;
import java.time.temporal.TemporalAccessor;
import java.time.temporal.TemporalQueries;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
@@ -67,6 +71,7 @@ public class CsvDeserializationSchema implements Serializable
{
private final String nullFormat;
private final CsvLineProcessor processor;
private final CatalogTable catalogTable;
+ private final Object[] defaultValues;
@SuppressWarnings("MagicNumber")
public static final DateTimeFormatter TIME_FORMAT =
@@ -75,7 +80,8 @@ public class CsvDeserializationSchema implements Serializable
{
.appendFraction(ChronoField.NANO_OF_SECOND, 0, 9, true)
.toFormatter();
- public Map<String, DateTimeFormatter> fieldFormatterMap = new HashMap<>();
+ // Formatters are derived from file data and are not serializable.
+ public transient Map<String, DateTimeFormatter> fieldFormatterMap = new
HashMap<>();
private CsvDeserializationSchema(
@NonNull SeaTunnelRowType seaTunnelRowType,
@@ -90,6 +96,30 @@ public class CsvDeserializationSchema implements
Serializable {
this.nullFormat = nullFormat;
this.processor = processor;
this.catalogTable = catalogTable;
+ Map<String, Column> columnsByName = new HashMap<>();
+ if (catalogTable != null) {
+ for (Column column : catalogTable.getTableSchema().getColumns()) {
+ columnsByName.put(column.getName(), column);
+ }
+ }
+ this.defaultValues = new Object[seaTunnelRowType.getTotalFields()];
+ for (int i = 0; i < defaultValues.length; i++) {
+ String fieldName = seaTunnelRowType.getFieldName(i);
+ defaultValues[i] =
+ resolveDefaultValue(
+ columnsByName.get(fieldName),
+ seaTunnelRowType.getFieldType(i),
+ fieldName);
+ }
+ }
+
+ /**
+ * Restores the converted defaults, but rebuilds the transient,
non-serializable formatter
+ * cache. File-data formats are learned again from incoming values after
deserialization.
+ */
+ private void readObject(ObjectInputStream input) throws IOException,
ClassNotFoundException {
+ input.defaultReadObject();
+ fieldFormatterMap = new HashMap<>();
}
public static Builder builder() {
@@ -185,10 +215,11 @@ public class CsvDeserializationSchema implements
Serializable {
Object[] objects = new Object[seaTunnelRowType.getTotalFields()];
for (int i = 0; i < objects.length; i++) {
String fieldValue = splitsMap.get(i);
- if (StringUtils.isBlank(fieldValue)) {
+ if (fieldValue == null || StringUtils.equals(fieldValue,
nullFormat)) {
+ objects[i] = getDefaultValue(i);
continue;
}
- if (StringUtils.equals(fieldValue, nullFormat)) {
+ if (StringUtils.isBlank(fieldValue)) {
continue;
}
objects[i] =
@@ -207,6 +238,186 @@ public class CsvDeserializationSchema implements
Serializable {
return seaTunnelRow;
}
+ private Object getDefaultValue(int fieldIndex) {
+ return copyDefaultValue(defaultValues[fieldIndex]);
+ }
+
+ /** Resolves defaults once, independently of the format learned from file
data. */
+ private Object resolveDefaultValue(
+ Column column, SeaTunnelDataType<?> fieldType, String fieldName) {
+ if (column == null || column.getDefaultValue() == null) {
+ return null;
+ }
+ try {
+ return convertDefaultValue(column.getDefaultValue(), fieldType,
fieldName);
+ } catch (RuntimeException ignored) {
+ // Parser exceptions can include the configured value in their
messages.
+ throw new IllegalArgumentException(
+ String.format(
+ "Invalid defaultValue for CSV column [%s] of type
[%s]",
+ fieldName, fieldType));
+ }
+ }
+
+ /**
+ * Copies cached mutable defaults for each output row, including nested
bytes, arrays, map keys
+ * and values, and row fields. Immutable scalar and temporal values can be
shared.
+ */
+ private Object copyDefaultValue(Object value) {
+ if (value == null) {
+ return null;
+ }
+ if (value instanceof byte[]) {
+ return ((byte[]) value).clone();
+ }
+ if (value.getClass().isArray()) {
+ int length = Array.getLength(value);
+ Object copy =
Array.newInstance(value.getClass().getComponentType(), length);
+ for (int i = 0; i < length; i++) {
+ Array.set(copy, i, copyDefaultValue(Array.get(value, i)));
+ }
+ return copy;
+ }
+ if (value instanceof Map) {
+ Map<Object, Object> copy = new LinkedHashMap<>();
+ ((Map<?, ?>) value)
+ .forEach(
+ (key, mapValue) ->
+ copy.put(copyDefaultValue(key),
copyDefaultValue(mapValue)));
+ return copy;
+ }
+ if (value instanceof SeaTunnelRow) {
+ Object[] fields = ((SeaTunnelRow) value).getFields();
+ return new SeaTunnelRow((Object[]) copyDefaultValue(fields));
+ }
+ return value;
+ }
+
+ /**
+ * Converts a configured literal once to its declared type. Collections,
arrays, maps and rows
+ * are converted recursively; supported typed temporal and byte values
avoid string conversion.
+ * Text literals use default parsing without reading or populating the
file-data formatter
+ * cache. Conversion failures are reported by {@link #resolveDefaultValue}
with column context.
+ */
+ private Object convertDefaultValue(
+ Object value, SeaTunnelDataType<?> fieldType, String fieldName) {
+ if (value == null) {
+ return null;
+ }
+ switch (fieldType.getSqlType()) {
+ case STRING:
+ return value.toString();
+ case BOOLEAN:
+ if (!"true".equalsIgnoreCase(value.toString())
+ && !"false".equalsIgnoreCase(value.toString())) {
+ throw new IllegalArgumentException("Boolean default must
be true or false");
+ }
+ return Boolean.parseBoolean(value.toString());
+ case DATE:
+ if (value instanceof LocalDate) {
+ return value;
+ }
+ break;
+ case TIME:
+ if (value instanceof LocalTime) {
+ return value;
+ }
+ break;
+ case TIMESTAMP:
+ if (value instanceof LocalDateTime) {
+ return value;
+ }
+ break;
+ case TIMESTAMP_TZ:
+ if (value instanceof OffsetDateTime) {
+ return value;
+ }
+ break;
+ case BYTES:
+ if (value instanceof byte[]) {
+ return ((byte[]) value).clone();
+ }
+ if (value instanceof String) {
+ return ((String) value).getBytes(StandardCharsets.UTF_8);
+ }
+ break;
+ case ARRAY:
+ if (value instanceof Collection || value.getClass().isArray())
{
+ ArrayType<?, ?> arrayType = (ArrayType<?, ?>) fieldType;
+ SeaTunnelDataType<?> elementType =
arrayType.getElementType();
+ int length =
+ value instanceof Collection
+ ? ((Collection<?>) value).size()
+ : Array.getLength(value);
+ Object result =
Array.newInstance(elementType.getTypeClass(), length);
+ int index = 0;
+ if (value instanceof Collection) {
+ for (Object element : (Collection<?>) value) {
+ Array.set(
+ result,
+ index++,
+ convertDefaultValue(element, elementType,
fieldName));
+ }
+ } else {
+ for (int i = 0; i < length; i++) {
+ Array.set(
+ result,
+ i,
+ convertDefaultValue(
+ Array.get(value, i), elementType,
fieldName));
+ }
+ }
+ return result;
+ }
+ break;
+ case MAP:
+ if (value instanceof Map) {
+ MapType<?, ?> mapType = (MapType<?, ?>) fieldType;
+ Map<Object, Object> result = new LinkedHashMap<>();
+ ((Map<?, ?>) value)
+ .forEach(
+ (key, mapValue) ->
+ result.put(
+ convertDefaultValue(
+ key,
mapType.getKeyType(), fieldName),
+ convertDefaultValue(
+ mapValue,
+
mapType.getValueType(),
+ fieldName)));
+ return result;
+ }
+ break;
+ case ROW:
+ if (value instanceof Map || value instanceof SeaTunnelRow) {
+ SeaTunnelRowType rowType = (SeaTunnelRowType) fieldType;
+ Object[] fields = new Object[rowType.getTotalFields()];
+ for (int i = 0; i < fields.length; i++) {
+ Object fieldValue =
+ value instanceof Map
+ ? ((Map<?, ?>)
value).get(rowType.getFieldName(i))
+ : ((SeaTunnelRow) value).getField(i);
+ fields[i] =
+ convertDefaultValue(
+ fieldValue,
+ rowType.getFieldType(i),
+ fieldName + "." +
rowType.getFieldName(i));
+ }
+ return new SeaTunnelRow(fields);
+ }
+ break;
+ case NULL:
+ return null;
+ default:
+ break;
+ }
+ Object converted = convert(value.toString(), fieldType, 0, fieldName,
true);
+ if (converted == null) {
+ throw new IllegalArgumentException(
+ "Default value cannot be converted to CSV field type");
+ }
+ return converted;
+ }
+
public SeaTunnelDataType<SeaTunnelRow> getProducedType() {
return seaTunnelRowType;
}
@@ -230,6 +441,15 @@ public class CsvDeserializationSchema implements
Serializable {
private Object convert(
String field, SeaTunnelDataType<?> fieldType, int level, String
fieldName) {
+ return convert(field, fieldType, level, fieldName, false);
+ }
+
+ private Object convert(
+ String field,
+ SeaTunnelDataType<?> fieldType,
+ int level,
+ String fieldName,
+ boolean isDefaultValue) {
if (StringUtils.isBlank(field)) {
return null;
}
@@ -239,7 +459,8 @@ public class CsvDeserializationSchema implements
Serializable {
String[] elements = field.split(separators[level + 1]);
ArrayList<Object> objectArrayList = new ArrayList<>();
for (String element : elements) {
- objectArrayList.add(convert(element, elementType, level +
1, fieldName));
+ objectArrayList.add(
+ convert(element, elementType, level + 1,
fieldName, isDefaultValue));
}
switch (elementType.getSqlType()) {
case STRING:
@@ -283,11 +504,18 @@ public class CsvDeserializationSchema implements
Serializable {
for (String kv : kvs) {
String[] splits = kv.split(separators[level + 2]);
if (splits.length < 2) {
- objectMap.put(convert(splits[0], keyType, level + 1,
fieldName), null);
+ objectMap.put(
+ convert(splits[0], keyType, level + 1,
fieldName, isDefaultValue),
+ null);
} else {
objectMap.put(
- convert(splits[0], keyType, level + 1,
fieldName),
- convert(splits[1], valueType, level + 1,
fieldName));
+ convert(splits[0], keyType, level + 1,
fieldName, isDefaultValue),
+ convert(
+ splits[1],
+ valueType,
+ level + 1,
+ fieldName,
+ isDefaultValue));
}
}
return objectMap;
@@ -314,11 +542,13 @@ public class CsvDeserializationSchema implements
Serializable {
case BYTES:
return field.getBytes(StandardCharsets.UTF_8);
case DATE:
- return parseDate(field, fieldName);
+ return isDefaultValue ? DateUtils.parse(field) :
parseDate(field, fieldName);
case TIME:
return parseTime(field);
case TIMESTAMP:
- return parseTimestamp(field, fieldName);
+ return isDefaultValue
+ ? DateTimeUtils.parse(field)
+ : parseTimestamp(field, fieldName);
case TIMESTAMP_TZ:
return parseTimestampTz(field, fieldName);
case ROW:
@@ -332,7 +562,8 @@ public class CsvDeserializationSchema implements
Serializable {
splitsMap.get(i),
((SeaTunnelRowType)
fieldType).getFieldType(i),
level + 1,
- fieldName + "." + eleFieldNames[i]);
+ fieldName + "." + eleFieldNames[i],
+ isDefaultValue);
}
return new SeaTunnelRow(objects);
default:
diff --git
a/seatunnel-formats/seatunnel-format-csv/src/test/java/org/apache/seatunnel/format/csv/CsvTextFormatSchemaTest.java
b/seatunnel-formats/seatunnel-format-csv/src/test/java/org/apache/seatunnel/format/csv/CsvTextFormatSchemaTest.java
index 04f35ce554..4cbc47f7d8 100644
---
a/seatunnel-formats/seatunnel-format-csv/src/test/java/org/apache/seatunnel/format/csv/CsvTextFormatSchemaTest.java
+++
b/seatunnel-formats/seatunnel-format-csv/src/test/java/org/apache/seatunnel/format/csv/CsvTextFormatSchemaTest.java
@@ -17,11 +17,17 @@
package org.apache.seatunnel.format.csv;
+import org.apache.seatunnel.api.table.catalog.CatalogTable;
+import org.apache.seatunnel.api.table.catalog.PhysicalColumn;
+import org.apache.seatunnel.api.table.catalog.TableIdentifier;
+import org.apache.seatunnel.api.table.catalog.TablePath;
+import org.apache.seatunnel.api.table.catalog.TableSchema;
import org.apache.seatunnel.api.table.type.ArrayType;
import org.apache.seatunnel.api.table.type.BasicType;
import org.apache.seatunnel.api.table.type.DecimalType;
import org.apache.seatunnel.api.table.type.LocalTimeType;
import org.apache.seatunnel.api.table.type.MapType;
+import org.apache.seatunnel.api.table.type.PrimitiveByteArrayType;
import org.apache.seatunnel.api.table.type.SeaTunnelDataType;
import org.apache.seatunnel.api.table.type.SeaTunnelRow;
import org.apache.seatunnel.api.table.type.SeaTunnelRowType;
@@ -33,14 +39,23 @@ import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
import java.io.IOException;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.PrintWriter;
+import java.io.StringWriter;
import java.math.BigDecimal;
import java.nio.file.Path;
import java.time.LocalDate;
import java.time.LocalDateTime;
+import java.time.LocalTime;
import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.TimeZone;
@@ -299,6 +314,393 @@ public class CsvTextFormatSchemaTest {
}
}
+ @Test
+ public void testConfiguredDefaultsForNullMarkerAndMissingColumn() throws
Exception {
+ CatalogTable catalogTable =
+ CatalogTable.of(
+ TableIdentifier.of("test", TablePath.DEFAULT),
+ TableSchema.builder()
+ .column(
+ PhysicalColumn.of(
+ "id",
+ BasicType.INT_TYPE,
+ (Long) null,
+ false,
+ 7,
+ null))
+ .column(
+ PhysicalColumn.of(
+ "name",
+ BasicType.STRING_TYPE,
+ (Long) null,
+ false,
+ "unknown",
+ null))
+ .column(
+ PhysicalColumn.of(
+ "optional",
+ BasicType.STRING_TYPE,
+ (Long) null,
+ true,
+ null,
+ null))
+ .build(),
+ new HashMap<>(),
+ Collections.emptyList(),
+ "test");
+ CsvDeserializationSchema schema =
+ CsvDeserializationSchema.builder()
+ .seaTunnelRowType(catalogTable.getSeaTunnelRowType())
+ .setCatalogTable(catalogTable)
+ .nullFormat("NULL")
+ .build();
+
+ SeaTunnelRow row = schema.getSeaTunnelRow(Collections.singletonMap(0,
"NULL"));
+ Assertions.assertEquals(7, row.getField(0));
+ Assertions.assertEquals("unknown", row.getField(1));
+ Assertions.assertNull(row.getField(2));
+
+ Map<Integer, String> emptyValues = new HashMap<>();
+ emptyValues.put(0, "");
+ emptyValues.put(1, "");
+ SeaTunnelRow emptyRow = schema.getSeaTunnelRow(emptyValues);
+ Assertions.assertNull(emptyRow.getField(0));
+ Assertions.assertNull(emptyRow.getField(1));
+
+ ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+ try (ObjectOutputStream output = new ObjectOutputStream(bytes)) {
+ output.writeObject(schema);
+ }
+ CsvDeserializationSchema restored;
+ try (ObjectInputStream input =
+ new ObjectInputStream(new
ByteArrayInputStream(bytes.toByteArray()))) {
+ restored = (CsvDeserializationSchema) input.readObject();
+ }
+ Assertions.assertEquals(
+ 7, restored.getSeaTunnelRow(Collections.singletonMap(0,
"NULL")).getField(0));
+ }
+
+ @Test
+ public void testDateDefaultBeforeDataWithDifferentFormat() {
+
assertTemporalDefaultWithDifferentDataFormat(LocalTimeType.LOCAL_DATE_TYPE,
true);
+ }
+
+ @Test
+ public void testDateDefaultAfterDataWithDifferentFormat() {
+
assertTemporalDefaultWithDifferentDataFormat(LocalTimeType.LOCAL_DATE_TYPE,
false);
+ }
+
+ @Test
+ public void testTimestampDefaultBeforeDataWithDifferentFormat() {
+
assertTemporalDefaultWithDifferentDataFormat(LocalTimeType.LOCAL_DATE_TIME_TYPE,
true);
+ }
+
+ @Test
+ public void testTimestampDefaultAfterDataWithDifferentFormat() {
+
assertTemporalDefaultWithDifferentDataFormat(LocalTimeType.LOCAL_DATE_TIME_TYPE,
false);
+ }
+
+ private void assertTemporalDefaultWithDifferentDataFormat(
+ SeaTunnelDataType<?> type, boolean defaultFirst) {
+ boolean date = type.equals(LocalTimeType.LOCAL_DATE_TYPE);
+ CsvDeserializationSchema schema =
+ schemaWithDefault(type, date ? "2020-01-01" : "2020-01-01
12:34:56");
+ Object expectedDefault =
+ date ? LocalDate.of(2020, 1, 1) : LocalDateTime.of(2020, 1, 1,
12, 34, 56);
+ String data = date ? "2024/02/03" : "2024/02/03 04:05:06";
+ Object expectedData =
+ date ? LocalDate.of(2024, 2, 3) : LocalDateTime.of(2024, 2, 3,
4, 5, 6);
+ if (defaultFirst) {
+ assertEquals(
+ expectedDefault,
schema.getSeaTunnelRow(Collections.emptyMap()).getField(0));
+ }
+ assertEquals(
+ expectedData,
+ schema.getSeaTunnelRow(Collections.singletonMap(0,
data)).getField(0));
+ assertEquals(
+ expectedDefault,
+ schema.getSeaTunnelRow(Collections.singletonMap(0,
"NULL")).getField(0));
+ assertEquals(
+ expectedData,
+ schema.getSeaTunnelRow(Collections.singletonMap(0,
data)).getField(0));
+ }
+
+ @Test
+ public void testTemporalDefaultsSurviveSerializationAfterData() throws
Exception {
+ for (boolean date : new boolean[] {true, false}) {
+ CsvDeserializationSchema schema =
+ schemaWithDefault(
+ date
+ ? LocalTimeType.LOCAL_DATE_TYPE
+ : LocalTimeType.LOCAL_DATE_TIME_TYPE,
+ date ? "2020-01-01" : "2020-01-01 12:34:56");
+ Map<Integer, String> data =
+ Collections.singletonMap(0, date ? "2024/02/03" :
"2024/02/03 04:05:06");
+ Object expectedDefault =
+ date ? LocalDate.of(2020, 1, 1) : LocalDateTime.of(2020,
1, 1, 12, 34, 56);
+ assertEquals(
+ expectedDefault,
+
roundTrip(schema).getSeaTunnelRow(Collections.emptyMap()).getField(0));
+ SeaTunnelRow expected = schema.getSeaTunnelRow(data);
+ CsvDeserializationSchema restored = roundTrip(schema);
+ assertEquals(expected.getField(0),
restored.getSeaTunnelRow(data).getField(0));
+ assertEquals(
+ expectedDefault,
restored.getSeaTunnelRow(Collections.emptyMap()).getField(0));
+ assertEquals(
+ expected.getField(0),
roundTrip(restored).getSeaTunnelRow(data).getField(0));
+ }
+ }
+
+ @Test
+ public void testTypedTemporalDefaultsArePreserved() throws Exception {
+ SeaTunnelDataType<?>[] types = {
+ LocalTimeType.LOCAL_DATE_TYPE,
+ LocalTimeType.LOCAL_TIME_TYPE,
+ LocalTimeType.LOCAL_DATE_TIME_TYPE,
+ LocalTimeType.OFFSET_DATE_TIME_TYPE
+ };
+ Object[] values = {
+ LocalDate.of(2020, 1, 1),
+ LocalTime.of(12, 34),
+ LocalDateTime.of(2020, 1, 1, 12, 34),
+ OffsetDateTime.of(2020, 1, 1, 12, 34, 0, 0, ZoneOffset.ofHours(9))
+ };
+ for (int i = 0; i < types.length; i++) {
+ CsvDeserializationSchema schema = schemaWithDefault(types[i],
values[i]);
+ Assertions.assertSame(
+ values[i],
schema.getSeaTunnelRow(Collections.emptyMap()).getField(0));
+ assertEquals(
+ values[i],
+
roundTrip(schema).getSeaTunnelRow(Collections.emptyMap()).getField(0));
+ }
+ }
+
+ @Test
+ public void testStructuredTemporalDefaultsUseIndependentFormats() {
+ ArrayType<LocalDate[], LocalDate> dates =
+ new ArrayType<>(LocalDate[].class,
LocalTimeType.LOCAL_DATE_TYPE);
+ LocalDate[] expected = {LocalDate.of(2020, 1, 1), LocalDate.of(2020,
2, 3)};
+ for (Object value :
+ new Object[] {
+ Arrays.asList("2020-01-01", "2020/02/03"),
"2020-01-01\u00022020/02/03"
+ }) {
+ CsvDeserializationSchema schema = schemaWithDefault(dates, value);
+ Assertions.assertArrayEquals(
+ expected,
+ (LocalDate[])
schema.getSeaTunnelRow(Collections.emptyMap()).getField(0));
+ Assertions.assertArrayEquals(
+ new LocalDate[] {LocalDate.of(2024, 2, 3)},
+ (LocalDate[])
+ schema.getSeaTunnelRow(Collections.singletonMap(0,
"2024.02.03"))
+ .getField(0));
+ }
+ }
+
+ @Test
+ public void testTimestampTzDefaultDoesNotAffectDataParser() {
+ CsvDeserializationSchema schema =
+ schemaWithDefault(LocalTimeType.OFFSET_DATE_TIME_TYPE,
"2020-01-01T12:34:56+09:00");
+ OffsetDateTime expectedDefault =
OffsetDateTime.parse("2020-01-01T12:34:56+09:00");
+ assertEquals(expectedDefault,
schema.getSeaTunnelRow(Collections.emptyMap()).getField(0));
+ assertEquals(
+ OffsetDateTime.of(2024, 2, 3, 4, 5, 6, 0, ZoneOffset.UTC),
+ schema.getSeaTunnelRow(Collections.singletonMap(0, "2024/02/03
04:05:06"))
+ .getField(0));
+ assertEquals(expectedDefault,
schema.getSeaTunnelRow(Collections.emptyMap()).getField(0));
+ }
+
+ @Test
+ public void testInvalidTemporalDefaultDoesNotExposeValue() {
+ for (SeaTunnelDataType<?> type :
+ new SeaTunnelDataType<?>[] {
+ LocalTimeType.LOCAL_DATE_TYPE,
LocalTimeType.LOCAL_DATE_TIME_TYPE
+ }) {
+ String sensitiveDefault = "secret-invalid-temporal-default";
+ IllegalArgumentException exception =
+ Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () -> schemaWithDefault(type, sensitiveDefault));
+ StringWriter stackTrace = new StringWriter();
+ exception.printStackTrace(new PrintWriter(stackTrace));
+ Assertions.assertTrue(stackTrace.toString().contains("value"));
+
Assertions.assertTrue(stackTrace.toString().contains(type.toString()));
+
Assertions.assertFalse(stackTrace.toString().contains(sensitiveDefault));
+ }
+ }
+
+ private CsvDeserializationSchema schemaWithDefault(SeaTunnelDataType<?>
type, Object value) {
+ CatalogTable table =
+ CatalogTable.of(
+ TableIdentifier.of("test", TablePath.DEFAULT),
+ TableSchema.builder()
+ .column(
+ PhysicalColumn.of(
+ "value", type, (Long) null,
true, value, null))
+ .build(),
+ new HashMap<>(),
+ Collections.emptyList(),
+ "test");
+ return CsvDeserializationSchema.builder()
+ .seaTunnelRowType(table.getSeaTunnelRowType())
+ .setCatalogTable(table)
+ .nullFormat("NULL")
+ .build();
+ }
+
+ private CsvDeserializationSchema roundTrip(CsvDeserializationSchema
schema) throws Exception {
+ ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+ try (ObjectOutputStream output = new ObjectOutputStream(bytes)) {
+ output.writeObject(schema);
+ }
+ try (ObjectInputStream input =
+ new ObjectInputStream(new
ByteArrayInputStream(bytes.toByteArray()))) {
+ return (CsvDeserializationSchema) input.readObject();
+ }
+ }
+
+ @Test
+ public void testInvalidConfiguredDefaultIsReportedWithColumnContext() {
+ CatalogTable catalogTable =
+ CatalogTable.of(
+ TableIdentifier.of("test", TablePath.DEFAULT),
+ TableSchema.builder()
+ .column(
+ PhysicalColumn.of(
+ "enabled",
+ BasicType.BOOLEAN_TYPE,
+ (Long) null,
+ false,
+ "not-a-boolean",
+ null))
+ .build(),
+ new HashMap<>(),
+ Collections.emptyList(),
+ "test");
+ IllegalArgumentException exception =
+ Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () ->
+ CsvDeserializationSchema.builder()
+
.seaTunnelRowType(catalogTable.getSeaTunnelRowType())
+ .setCatalogTable(catalogTable)
+ .build());
+ Assertions.assertTrue(exception.getMessage().contains("enabled"));
+
Assertions.assertFalse(exception.getMessage().contains("not-a-boolean"));
+ }
+
+ @Test
+ public void testInvalidNumericDefaultDoesNotExposeValueInStackTrace() {
+ String sensitiveDefault = "secret-csv-default";
+ CatalogTable catalogTable =
+ CatalogTable.of(
+ TableIdentifier.of("test", TablePath.DEFAULT),
+ TableSchema.builder()
+ .column(
+ PhysicalColumn.of(
+ "count",
+ BasicType.INT_TYPE,
+ (Long) null,
+ false,
+ sensitiveDefault,
+ null))
+ .build(),
+ new HashMap<>(),
+ Collections.emptyList(),
+ "test");
+ IllegalArgumentException exception =
+ Assertions.assertThrows(
+ IllegalArgumentException.class,
+ () ->
+ CsvDeserializationSchema.builder()
+
.seaTunnelRowType(catalogTable.getSeaTunnelRowType())
+ .setCatalogTable(catalogTable)
+ .build());
+ StringWriter stackTrace = new StringWriter();
+ exception.printStackTrace(new PrintWriter(stackTrace));
+ Assertions.assertTrue(stackTrace.toString().contains("count"));
+ Assertions.assertTrue(stackTrace.toString().contains("INT"));
+
Assertions.assertFalse(stackTrace.toString().contains(sensitiveDefault));
+ }
+
+ @Test
+ public void testStructuredDefaultsAreIndependentBetweenRows() throws
Exception {
+ SeaTunnelRowType nestedRowType =
+ new SeaTunnelRowType(
+ new String[] {"numbers"},
+ new SeaTunnelDataType<?>[] {ArrayType.INT_ARRAY_TYPE});
+ Map<String, Object> mapDefault = new HashMap<>();
+ mapDefault.put("count", 3);
+ Map<String, Object> rowDefault = new HashMap<>();
+ rowDefault.put("numbers", Arrays.asList(4, 5));
+ CatalogTable catalogTable =
+ CatalogTable.of(
+ TableIdentifier.of("test", TablePath.DEFAULT),
+ TableSchema.builder()
+ .column(
+ PhysicalColumn.of(
+ "arrays",
+ new ArrayType<>(
+ Integer[][].class,
+
ArrayType.INT_ARRAY_TYPE),
+ (Long) null,
+ true,
+ Arrays.asList(
+ Arrays.asList(1, 2),
Arrays.asList(3)),
+ null))
+ .column(
+ PhysicalColumn.of(
+ "map",
+ new MapType<>(
+ BasicType.STRING_TYPE,
BasicType.INT_TYPE),
+ (Long) null,
+ true,
+ mapDefault,
+ null))
+ .column(
+ PhysicalColumn.of(
+ "row",
+ nestedRowType,
+ (Long) null,
+ true,
+ rowDefault,
+ null))
+ .column(
+ PhysicalColumn.of(
+ "bytes",
+
PrimitiveByteArrayType.INSTANCE,
+ (Long) null,
+ true,
+ new byte[] {1, 2},
+ null))
+ .build(),
+ new HashMap<>(),
+ Collections.emptyList(),
+ "test");
+ CsvDeserializationSchema schema =
+ CsvDeserializationSchema.builder()
+ .seaTunnelRowType(catalogTable.getSeaTunnelRowType())
+ .setCatalogTable(catalogTable)
+ .build();
+ mapDefault.clear();
+ rowDefault.clear();
+
+ SeaTunnelRow first = schema.getSeaTunnelRow(Collections.emptyMap());
+ ((Integer[][]) first.getField(0))[0][0] = 99;
+ ((Map<?, ?>) first.getField(1)).clear();
+ ((Integer[]) ((SeaTunnelRow) first.getField(2)).getField(0))[0] = 99;
+ ((byte[]) first.getField(3))[0] = 99;
+
+ for (CsvDeserializationSchema copy :
+ new CsvDeserializationSchema[] {schema, roundTrip(schema)}) {
+ SeaTunnelRow second = copy.getSeaTunnelRow(Collections.emptyMap());
+ Assertions.assertEquals(1, ((Integer[][])
second.getField(0))[0][0]);
+ Assertions.assertEquals(3, ((Map<?, ?>)
second.getField(1)).get("count"));
+ Assertions.assertEquals(
+ 4, ((Integer[]) ((SeaTunnelRow)
second.getField(2)).getField(0))[0]);
+ Assertions.assertArrayEquals(new byte[] {1, 2}, (byte[])
second.getField(3));
+ }
+ }
+
@Test
void testTimestampTzRoundTrip() throws IOException {
SeaTunnelRowType rowType =