This is an automated email from the ASF dual-hosted git repository.
jmclean pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 682878c180 [#7983] Fix Paimon type conversion to preserve nullability
for list elements and map values (#7995)
682878c180 is described below
commit 682878c1805f3c7d3672193ef91f047aa6d55fff
Author: Akshat Tiwari <[email protected]>
AuthorDate: Mon Aug 11 07:27:57 2025 +0530
[#7983] Fix Paimon type conversion to preserve nullability for list
elements and map values (#7995)
### What changes were proposed in this pull request?
This PR fixes the Paimon type conversion in `TypeUtils.java` to properly
preserve nullability information for list elements and map values when
converting from Gravitino types to Paimon types.
### Why are the changes needed?
The `GravitinoToPaimonTypeVisitor` was not checking the
`elementNullable()` and `valueNullable()` properties during type
conversion, causing:
- List elements to always be nullable regardless of the Gravitino type
specification
- Map values to always be nullable regardless of the Gravitino type
specification
This inconsistency could lead to data integrity issues and unexpected
behavior when working with Paimon catalogs.
### Does this PR introduce any user-facing change?
Yes, this change ensures that nullability constraints are properly
preserved during type conversion, maintaining data integrity and
consistency between Gravitino and Paimon type systems. Users will now
get the correct nullability behavior for list elements and map values.
### How was this patch tested?
Added comprehensive test coverage in `TestTypeUtils.java` with the new
`testToPaimonTypeNullability()` method that verifies:
- `Types.ListType.of(Types.IntegerType.get(), false)` converts to
`DataTypes.ARRAY(DataTypes.INT().notNull())`
- `Types.ListType.of(Types.IntegerType.get(), true)` converts to
`DataTypes.ARRAY(DataTypes.INT().nullable())`
- `Types.MapType.of(..., false)` converts to `DataTypes.MAP(...,
DataTypes.INT().notNull())`
- `Types.MapType.of(..., true)` converts to `DataTypes.MAP(...,
DataTypes.INT().nullable())`
All existing tests continue to pass, ensuring backward compatibility.
Fixes #7983
---
.../catalog/lakehouse/paimon/utils/TypeUtils.java | 11 +++++++++--
.../catalog/lakehouse/paimon/utils/TestTypeUtils.java | 16 ++++++++++++++++
2 files changed, 25 insertions(+), 2 deletions(-)
diff --git
a/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/utils/TypeUtils.java
b/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/utils/TypeUtils.java
index a9e7b2365a..0ee547f21e 100644
---
a/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/utils/TypeUtils.java
+++
b/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/utils/TypeUtils.java
@@ -258,10 +258,17 @@ public class TypeUtils {
return DataTypes.VARBINARY(VarBinaryType.MAX_LENGTH);
case LIST:
Types.ListType listType = (Types.ListType) type;
- return DataTypes.ARRAY(visit(listType.elementType()));
+ DataType elementType = visit(listType.elementType());
+ DataType elementTypeWithNullability =
+ listType.elementNullable() ? elementType.nullable() :
elementType.notNull();
+ return DataTypes.ARRAY(elementTypeWithNullability);
case MAP:
Types.MapType mapType = (Types.MapType) type;
- return DataTypes.MAP(visit(mapType.keyType()),
visit(mapType.valueType()));
+ DataType keyType = visit(mapType.keyType());
+ DataType valueType = visit(mapType.valueType());
+ DataType valueTypeWithNullability =
+ mapType.valueNullable() ? valueType.nullable() :
valueType.notNull();
+ return DataTypes.MAP(keyType, valueTypeWithNullability);
case STRUCT:
RowType.Builder builder = RowType.builder();
Arrays.stream(((Types.StructType) type).fields())
diff --git
a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/utils/TestTypeUtils.java
b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/utils/TestTypeUtils.java
index 019752b4a3..dc60037c57 100644
---
a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/utils/TestTypeUtils.java
+++
b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/utils/TestTypeUtils.java
@@ -112,6 +112,22 @@ public class TestTypeUtils {
.forEach(this::checkUnsupportedType);
}
+ @Test
+ void testToPaimonTypeNullability() {
+ assertEquals(
+ DataTypes.ARRAY(DataTypes.INT().notNull()),
+ toPaimonType(Types.ListType.of(Types.IntegerType.get(), false)));
+ assertEquals(
+ DataTypes.ARRAY(DataTypes.INT().nullable()),
+ toPaimonType(Types.ListType.of(Types.IntegerType.get(), true)));
+ assertEquals(
+ DataTypes.MAP(DataTypes.STRING(), DataTypes.INT().notNull()),
+ toPaimonType(Types.MapType.of(Types.StringType.get(),
Types.IntegerType.get(), false)));
+ assertEquals(
+ DataTypes.MAP(DataTypes.STRING(), DataTypes.INT().nullable()),
+ toPaimonType(Types.MapType.of(Types.StringType.get(),
Types.IntegerType.get(), true)));
+ }
+
private Type toGravitinoDataType(DataType dataType) {
switch (dataType.getTypeRoot()) {
case BOOLEAN:
