This is an automated email from the ASF dual-hosted git repository.
diqiu50 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 3b6fe775a6 [#13040] fix(postgresql): Report unconstrained NUMERIC as
an external type (#13042)
3b6fe775a6 is described below
commit 3b6fe775a65ffa486862c50e8d32877afd47ae83
Author: Yuhui <[email protected]>
AuthorDate: Thu Sep 10 09:02:27 2026 +0800
[#13040] fix(postgresql): Report unconstrained NUMERIC as an external type
(#13042)
### What changes were proposed in this pull request?
Report a PostgreSQL `NUMERIC` without precision and scale as the
external type `numeric`, and report array elements as nullable. Read
external types as string in the Trino, Spark and Flink connectors.
### Why are the changes needed?
An unconstrained `NUMERIC` holds values whose precision and scale vary
per row, so `Decimal(38, 18)` claims bounds the source never declared.
PostgreSQL array elements always accept NULL and cannot be declared
otherwise.
Once a common column type maps to an external type, a column carrying
one reaches the connectors, where an unmapped type fails the type
conversion of the whole table.
Fix: #13040
### Does this PR introduce _any_ user-facing change?
Yes. An unconstrained `NUMERIC` is reported as the external type
`numeric`, an array as a list with nullable elements, and a connector
reads an external type as a string. A `numeric(p, s)[]` column becomes a
list of the external type `numeric`, since array element metadata is not
carried through.
### How was this patch tested?
Unit tests in the PostgreSQL catalog and the three connectors, plus a
PostgreSQL integration test against a real container.
Co-authored-by: Claude Opus 5 <[email protected]>
---
.../converter/PostgreSqlTypeConverter.java | 18 ++++++-------
.../converter/TestPostgreSqlTypeConverter.java | 28 +++++++-------------
.../integration/test/CatalogPostgreSqlIT.java | 20 ++++++++++++++-
.../operation/TestPostgreSqlTableOperations.java | 2 +-
docs/jdbc-postgresql-catalog.md | 10 ++++++++
.../gravitino/flink/connector/utils/TypeUtils.java | 30 ++++++++++++++++++----
.../flink/connector/utils/TestTypeUtils.java | 27 +++++++++++++++++++
.../connector/jdbc/SparkJdbcTypeConverter.java | 12 +++++++++
.../connector/jdbc/TestSparkJdbcTypeConverter.java | 8 ++++++
.../postgresql/PostgreSQLDataTypeTransformer.java | 15 ++++++++---
.../TestPostgreSQLDataTypeTransformer.java | 11 ++++++++
11 files changed, 143 insertions(+), 38 deletions(-)
diff --git
a/catalogs/catalog-jdbc-postgresql/src/main/java/org/apache/gravitino/catalog/postgresql/converter/PostgreSqlTypeConverter.java
b/catalogs/catalog-jdbc-postgresql/src/main/java/org/apache/gravitino/catalog/postgresql/converter/PostgreSqlTypeConverter.java
index f5161b12f7..c87a63f86c 100644
---
a/catalogs/catalog-jdbc-postgresql/src/main/java/org/apache/gravitino/catalog/postgresql/converter/PostgreSqlTypeConverter.java
+++
b/catalogs/catalog-jdbc-postgresql/src/main/java/org/apache/gravitino/catalog/postgresql/converter/PostgreSqlTypeConverter.java
@@ -42,8 +42,6 @@ public class PostgreSqlTypeConverter extends
JdbcTypeConverter {
static final String UUID = "uuid";
@VisibleForTesting static final String JDBC_ARRAY_PREFIX = "_";
@VisibleForTesting static final String ARRAY_TOKEN = "[]";
- @VisibleForTesting static final int DEFAULT_NUMERIC_PRECISION = 38;
- @VisibleForTesting static final int DEFAULT_NUMERIC_SCALE = 18;
@Override
public Type toGravitino(JdbcTypeBean typeBean) {
@@ -81,11 +79,12 @@ public class PostgreSqlTypeConverter extends
JdbcTypeConverter {
case NUMERIC:
Integer columnSize = typeBean.getColumnSize();
Integer scale = typeBean.getScale();
+ // An unconstrained NUMERIC accepts values whose precision and scale
vary per row, up to
+ // 131072 digits before and 16383 after the decimal point, which no
DecimalType can
+ // describe, so it is reported as an external type. The driver reports
column size 0 for it,
+ // null is handled defensively.
if (columnSize == null || columnSize == 0) {
- // PostgreSQL unconstrained NUMERIC has no fixed precision/scale.
Gravitino DecimalType
- // cannot represent that exactly, so use the maximum supported
decimal as a compatibility
- // tradeoff for engines and clients that cannot consume ExternalType.
- return Types.DecimalType.of(DEFAULT_NUMERIC_PRECISION,
DEFAULT_NUMERIC_SCALE);
+ return Types.ExternalType.of(NUMERIC);
}
return Types.DecimalType.of(columnSize, scale == null ? 0 : scale);
case VARCHAR:
@@ -162,10 +161,11 @@ public class PostgreSqlTypeConverter extends
JdbcTypeConverter {
// the array size or number of dimensions in CREATE TABLE is simply
documentation; it does not
// affect run-time behavior.
// https://www.postgresql.org/docs/current/arrays.html#ARRAYS-DECLARATION
+ // PostgreSQL array elements always accept NULL and cannot be declared
otherwise, so
+ // elementNullable is ignored: a list declared with non-nullable elements is
created as an
+ // ordinary array whose elements accept NULL.
private String fromGravitinoArrayType(ListType listType) {
Type elementType = listType.elementType();
- Preconditions.checkArgument(
- !listType.elementNullable(), "PostgreSQL doesn't support element to
nullable");
Preconditions.checkArgument(
!(elementType instanceof ListType),
"PostgreSQL doesn't support multidimensional list internally, please
use one dimensional list");
@@ -176,6 +176,6 @@ public class PostgreSqlTypeConverter extends
JdbcTypeConverter {
private ListType toGravitinoArrayType(String typeName) {
String elementTypeName = typeName.substring(JDBC_ARRAY_PREFIX.length(),
typeName.length());
JdbcTypeBean bean = new JdbcTypeBean(elementTypeName);
- return ListType.of(toGravitino(bean), false);
+ return ListType.nullable(toGravitino(bean));
}
}
diff --git
a/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/converter/TestPostgreSqlTypeConverter.java
b/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/converter/TestPostgreSqlTypeConverter.java
index e4139441b7..23a65e19a0 100644
---
a/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/converter/TestPostgreSqlTypeConverter.java
+++
b/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/converter/TestPostgreSqlTypeConverter.java
@@ -27,8 +27,6 @@ import static
org.apache.gravitino.catalog.postgresql.converter.PostgreSqlTypeCo
import static
org.apache.gravitino.catalog.postgresql.converter.PostgreSqlTypeConverter.BOOL;
import static
org.apache.gravitino.catalog.postgresql.converter.PostgreSqlTypeConverter.BPCHAR;
import static
org.apache.gravitino.catalog.postgresql.converter.PostgreSqlTypeConverter.BYTEA;
-import static
org.apache.gravitino.catalog.postgresql.converter.PostgreSqlTypeConverter.DEFAULT_NUMERIC_PRECISION;
-import static
org.apache.gravitino.catalog.postgresql.converter.PostgreSqlTypeConverter.DEFAULT_NUMERIC_SCALE;
import static
org.apache.gravitino.catalog.postgresql.converter.PostgreSqlTypeConverter.FLOAT_4;
import static
org.apache.gravitino.catalog.postgresql.converter.PostgreSqlTypeConverter.FLOAT_8;
import static
org.apache.gravitino.catalog.postgresql.converter.PostgreSqlTypeConverter.INT_2;
@@ -71,16 +69,10 @@ public class TestPostgreSqlTypeConverter {
checkJdbcTypeToGravitinoType(Types.TimestampType.withoutTimeZone(3),
TIMESTAMP, 23, null, 3);
checkJdbcTypeToGravitinoType(Types.TimestampType.withoutTimeZone(6),
TIMESTAMP, 26, null, 6);
checkJdbcTypeToGravitinoType(Types.DecimalType.of(10, 2), NUMERIC, 10, 2,
0);
- // Unconstrained NUMERIC (no precision) returns columnSize=0 from JDBC
metadata;
- // mapped to Gravitino's maximum supported decimal as a compatibility
tradeoff.
- checkJdbcTypeToGravitinoType(
- Types.DecimalType.of(DEFAULT_NUMERIC_PRECISION,
DEFAULT_NUMERIC_SCALE), NUMERIC, 0, 0, 0);
- checkJdbcTypeToGravitinoType(
- Types.DecimalType.of(DEFAULT_NUMERIC_PRECISION, DEFAULT_NUMERIC_SCALE),
- NUMERIC,
- null,
- null,
- 0);
+ // Unconstrained NUMERIC (no precision) returns columnSize=0 from JDBC
metadata; its precision
+ // and scale vary per row, so it is reported as an external type instead
of a decimal.
+ checkJdbcTypeToGravitinoType(Types.ExternalType.of(NUMERIC), NUMERIC, 0,
0, 0);
+ checkJdbcTypeToGravitinoType(Types.ExternalType.of(NUMERIC), NUMERIC,
null, null, 0);
checkJdbcTypeToGravitinoType(Types.DecimalType.of(9, 0), NUMERIC, 9, 0, 0);
checkJdbcTypeToGravitinoType(Types.DecimalType.of(18, 0), NUMERIC, 18, 0,
0);
checkJdbcTypeToGravitinoType(Types.DecimalType.of(20, 0), NUMERIC, 20, 0,
0);
@@ -96,19 +88,17 @@ public class TestPostgreSqlTypeConverter {
@Test
public void testArrayType() {
Type elmentType = Types.IntegerType.get();
- Type list1 = Types.ListType.of(elmentType, false);
+ Type list1 = Types.ListType.of(elmentType, true);
checkGravitinoTypeToJdbcType(INT_4 + ARRAY_TOKEN, list1);
+ // PostgreSQL array elements are always nullable
checkJdbcTypeToGravitinoType(list1, JDBC_ARRAY_PREFIX + INT_4, null, null,
0);
- // not support element nullable
- Assertions.assertThrowsExactly(
- IllegalArgumentException.class,
- () ->
- checkGravitinoTypeToJdbcType(INT_4 + ARRAY_TOKEN,
Types.ListType.of(elmentType, true)));
+ // element nullability is not enforced, PostgreSQL cannot declare it
either way
+ checkGravitinoTypeToJdbcType(INT_4 + ARRAY_TOKEN,
Types.ListType.of(elmentType, false));
// not support multidimensional
- Type list2 = Types.ListType.of(list1, false);
+ Type list2 = Types.ListType.of(list1, true);
Assertions.assertThrowsExactly(
IllegalArgumentException.class,
() -> checkGravitinoTypeToJdbcType(INT_4 + ARRAY_TOKEN, list2));
diff --git
a/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java
b/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java
index 7bea7747eb..9441493386 100644
---
a/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java
+++
b/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/integration/test/CatalogPostgreSqlIT.java
@@ -278,7 +278,7 @@ public class CatalogPostgreSqlIT extends BaseIT {
@Test
void testCreateTableWithArrayType() {
String tableName =
GravitinoITUtils.genRandomName("postgresql_it_array_table");
- Column col = Column.of("array", Types.ListType.of(IntegerType.get(),
false), "col_4_comment");
+ Column col = Column.of("array", Types.ListType.of(IntegerType.get(),
true), "col_4_comment");
Column[] columns = new Column[] {col};
NameIdentifier tableIdentifier = NameIdentifier.of(schemaName, tableName);
@@ -1632,6 +1632,24 @@ public class CatalogPostgreSqlIT extends BaseIT {
Assertions.assertEquals(Types.ExternalType.of("bit"),
loadedTable.columns()[0].dataType());
}
+ @Test
+ void testUnconstrainedNumericAndArrayTypeConverter() {
+ String tableName =
GravitinoITUtils.genRandomName("test_numeric_array_type");
+ postgreSqlService.executeQuery(
+ String.format(
+ "CREATE TABLE %s.%s (numeric_col numeric, numeric_col_2
numeric(10,2), array_col integer[]);",
+ schemaName, tableName));
+ Table loadedTable =
+ catalog.asTableCatalog().loadTable(NameIdentifier.of(schemaName,
tableName));
+
+ // An unconstrained numeric holds values whose precision and scale vary
per row
+ Assertions.assertEquals(Types.ExternalType.of("numeric"),
loadedTable.columns()[0].dataType());
+ Assertions.assertEquals(Types.DecimalType.of(10, 2),
loadedTable.columns()[1].dataType());
+ // PostgreSQL array elements are always nullable
+ Assertions.assertEquals(
+ Types.ListType.of(Types.IntegerType.get(), true),
loadedTable.columns()[2].dataType());
+ }
+
@Test
void testOperationTableIndex() {
String tableName = GravitinoITUtils.genRandomName("test_add_index");
diff --git
a/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/operation/TestPostgreSqlTableOperations.java
b/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/operation/TestPostgreSqlTableOperations.java
index fb6b1a1fa8..9fb4aafa49 100644
---
a/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/operation/TestPostgreSqlTableOperations.java
+++
b/catalogs/catalog-jdbc-postgresql/src/test/java/org/apache/gravitino/catalog/postgresql/operation/TestPostgreSqlTableOperations.java
@@ -356,7 +356,7 @@ public class TestPostgreSqlTableOperations extends
TestPostgreSql {
columns.add(
JdbcColumn.builder()
.withName("col_16")
- .withType(Types.ListType.of(Types.IntegerType.get(), false))
+ .withType(Types.ListType.of(Types.IntegerType.get(), true))
.withNullable(true)
.build());
diff --git a/docs/jdbc-postgresql-catalog.md b/docs/jdbc-postgresql-catalog.md
index d61637b162..dcdf9428ec 100644
--- a/docs/jdbc-postgresql-catalog.md
+++ b/docs/jdbc-postgresql-catalog.md
@@ -118,6 +118,16 @@ Refer to [Manage Catalogs and
Schemas](./manage-catalogs-and-schemas.md#schema-o
:::info
PostgreSQL doesn't support Gravitino `Fixed` `Struct` `Map` `IntervalDay`
`IntervalYear` `Union` type.
Meanwhile, the data types other than listed above are mapped to Gravitino
**[External Type](./tables-and-views.md#external-type)** that represents an
unresolvable data type.
+
+An unconstrained `Numeric` column, that is one declared without precision and
scale, accepts values of up to
+131072 digits before and 16383 digits after the decimal point, and its
precision and scale vary per row.
+Gravitino `Decimal` caps precision at 38 and is fixed per column, so such a
column is mapped to the External
+Type `numeric` instead. A `Numeric(p, s)` column is mapped to `Decimal(p, s)`
and a `Numeric(p)` column to
+`Decimal(p, 0)` as usual.
+
+PostgreSQL array elements always accept NULL and cannot be declared otherwise,
so an `Array` column is always
+mapped to a `List` whose elements are nullable. A `List` created with
non-nullable elements is accepted and
+produces an ordinary array whose elements accept NULL.
:::
### Table Column Auto-Increment
diff --git
a/flink-connector/flink-common/src/main/java/org/apache/gravitino/flink/connector/utils/TypeUtils.java
b/flink-connector/flink-common/src/main/java/org/apache/gravitino/flink/connector/utils/TypeUtils.java
index 2b032dac7e..5dd84f2e7d 100644
---
a/flink-connector/flink-common/src/main/java/org/apache/gravitino/flink/connector/utils/TypeUtils.java
+++
b/flink-connector/flink-common/src/main/java/org/apache/gravitino/flink/connector/utils/TypeUtils.java
@@ -23,12 +23,14 @@ import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.flink.table.api.DataTypes;
+import org.apache.flink.table.api.ValidationException;
import org.apache.flink.table.types.DataType;
import org.apache.flink.table.types.logical.ArrayType;
import org.apache.flink.table.types.logical.BinaryType;
import org.apache.flink.table.types.logical.CharType;
import org.apache.flink.table.types.logical.DecimalType;
import org.apache.flink.table.types.logical.LogicalType;
+import org.apache.flink.table.types.logical.LogicalTypeRoot;
import org.apache.flink.table.types.logical.MapType;
import org.apache.flink.table.types.logical.MultisetType;
import org.apache.flink.table.types.logical.RowType;
@@ -36,9 +38,13 @@ import
org.apache.flink.table.types.logical.utils.LogicalTypeParser;
import org.apache.flink.table.types.utils.TypeConversions;
import org.apache.gravitino.rel.types.Type;
import org.apache.gravitino.rel.types.Types;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TypeUtils {
+ private static final Logger LOG = LoggerFactory.getLogger(TypeUtils.class);
+
// Flink supports time/timestamp precision from 0 to 9 (nanosecond
precision).
// @see
//
https://nightlies.apache.org/flink/flink-docs-release-2.1/docs/dev/table/types/#date-and-time
@@ -246,11 +252,25 @@ public class TypeUtils {
case EXTERNAL:
Types.ExternalType externalType = (Types.ExternalType) gravitinoType;
String catalogString = externalType.catalogString();
- // Parse the external catalog type string back to Flink LogicalType.
- // This is used to restore types like MULTISET that Gravitino doesn't
natively support.
- LogicalType parsedType =
- LogicalTypeParser.parse(catalogString,
TypeUtils.class.getClassLoader());
- return TypeConversions.fromLogicalToDataType(parsedType);
+ // MULTISET is the only Flink type carried as an external type,
written here by
+ // toGravitinoType and by the Paimon catalog. Every other catalog
string is a type name of
+ // some other data source, and some of those parse as a Flink type by
accident, a
+ // PostgreSQL numeric parses as DECIMAL(10, 0). Only a parsed MULTISET
is therefore
+ // restored, and any other type is read as a string to keep the rest
of the table usable.
+ try {
+ LogicalType parsedType =
+ LogicalTypeParser.parse(catalogString,
TypeUtils.class.getClassLoader());
+ if (parsedType.getTypeRoot() == LogicalTypeRoot.MULTISET) {
+ return TypeConversions.fromLogicalToDataType(parsedType);
+ }
+ LOG.warn("External type {} is not a Flink type, reading it as a
string.", catalogString);
+ } catch (ValidationException e) {
+ LOG.warn(
+ "External type {} cannot be parsed as a Flink type, reading it
as a string.",
+ catalogString,
+ e);
+ }
+ return DataTypes.STRING();
default:
throw new UnsupportedOperationException("Not support " +
gravitinoType.toString());
}
diff --git
a/flink-connector/flink-common/src/test/java/org/apache/gravitino/flink/connector/utils/TestTypeUtils.java
b/flink-connector/flink-common/src/test/java/org/apache/gravitino/flink/connector/utils/TestTypeUtils.java
index 9d0c71ec84..ce64056c5b 100644
---
a/flink-connector/flink-common/src/test/java/org/apache/gravitino/flink/connector/utils/TestTypeUtils.java
+++
b/flink-connector/flink-common/src/test/java/org/apache/gravitino/flink/connector/utils/TestTypeUtils.java
@@ -285,4 +285,31 @@ public class TestTypeUtils {
DataTypes.MULTISET(DataTypes.BIGINT()),
TypeUtils.toFlinkType(Types.ExternalType.of("MULTISET<BIGINT>")));
}
+
+ @Test
+ public void testExternalTypeFromOtherCatalogConversion() {
+ // A type name of another data source that parses as a Flink type by
accident: a PostgreSQL
+ // numeric parses as DECIMAL(10, 0)
+ Assertions.assertEquals(
+ DataTypes.STRING(),
TypeUtils.toFlinkType(Types.ExternalType.of("numeric")));
+
+ // A type name that is no Flink type at all
+ Assertions.assertEquals(
+ DataTypes.STRING(),
TypeUtils.toFlinkType(Types.ExternalType.of("money")));
+ Assertions.assertEquals(
+ DataTypes.STRING(),
TypeUtils.toFlinkType(Types.ExternalType.of("json")));
+ }
+
+ @Test
+ public void testMultisetSpellingsConversion() {
+ // The Paimon catalog writes the external type with asSQLString, which
spells the element type
+ // STRING rather than VARCHAR(2147483647)
+ Assertions.assertEquals(
+ DataTypes.MULTISET(DataTypes.STRING()),
+ TypeUtils.toFlinkType(Types.ExternalType.of("MULTISET<STRING>")));
+
+ Assertions.assertEquals(
+ DataTypes.MULTISET(DataTypes.BIGINT()),
+ TypeUtils.toFlinkType(Types.ExternalType.of("multiset<bigint>")));
+ }
}
diff --git
a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/jdbc/SparkJdbcTypeConverter.java
b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/jdbc/SparkJdbcTypeConverter.java
index e7dccbf1c8..e8b6433a47 100644
---
a/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/jdbc/SparkJdbcTypeConverter.java
+++
b/spark-connector/spark-common/src/main/java/org/apache/gravitino/spark/connector/jdbc/SparkJdbcTypeConverter.java
@@ -24,14 +24,26 @@ import org.apache.gravitino.rel.types.Types;
import org.apache.gravitino.spark.connector.SparkTypeConverter;
import org.apache.spark.sql.types.DataType;
import org.apache.spark.sql.types.DataTypes;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class SparkJdbcTypeConverter extends SparkTypeConverter {
+ private static final Logger LOG =
LoggerFactory.getLogger(SparkJdbcTypeConverter.class);
+
@Override
public DataType toSparkType(Type gravitinoType) {
if (gravitinoType instanceof Types.VarCharType) {
// Spark's JDBC dialects reject VarcharType, so widen it to StringType.
return DataTypes.StringType;
+ } else if (gravitinoType instanceof Types.ExternalType) {
+ // An external type carries a source type that Gravitino cannot
represent, such as an
+ // unconstrained PostgreSQL numeric. Reading it as a string keeps an
unmapped type from
+ // failing the type conversion of the whole table.
+ LOG.warn(
+ "Reading type {} as a string, Gravitino cannot represent it",
+ ((Types.ExternalType) gravitinoType).catalogString());
+ return DataTypes.StringType;
} else if (gravitinoType instanceof Types.TimestampType) {
// Both timestamp flavors map to TimestampType, never TimestampNTZType:
the MySQL dialect only
// pushes TimestampType literals down correctly, and NTZ literals
produce invalid SQL.
diff --git
a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/jdbc/TestSparkJdbcTypeConverter.java
b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/jdbc/TestSparkJdbcTypeConverter.java
index 00fd1c7784..044af357d8 100644
---
a/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/jdbc/TestSparkJdbcTypeConverter.java
+++
b/spark-connector/spark-common/src/test/java/org/apache/gravitino/spark/connector/jdbc/TestSparkJdbcTypeConverter.java
@@ -44,4 +44,12 @@ public class TestSparkJdbcTypeConverter {
Assertions.assertEquals(
DataTypes.StringType,
sparkJdbcTypeConverter.toSparkType(Types.VarCharType.of(10)));
}
+
+ @Test
+ void testConvertExternalTypeToSparkString() {
+ Assertions.assertEquals(
+ DataTypes.StringType,
sparkJdbcTypeConverter.toSparkType(Types.ExternalType.of("numeric")));
+ Assertions.assertEquals(
+ DataTypes.StringType,
sparkJdbcTypeConverter.toSparkType(Types.ExternalType.of("json")));
+ }
}
diff --git
a/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/jdbc/postgresql/PostgreSQLDataTypeTransformer.java
b/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/jdbc/postgresql/PostgreSQLDataTypeTransformer.java
index cd04413e32..636544f0ba 100644
---
a/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/jdbc/postgresql/PostgreSQLDataTypeTransformer.java
+++
b/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/jdbc/postgresql/PostgreSQLDataTypeTransformer.java
@@ -19,6 +19,7 @@
package org.apache.gravitino.trino.connector.catalog.jdbc.postgresql;
+import io.airlift.log.Logger;
import io.trino.spi.TrinoException;
import io.trino.spi.type.CharType;
import io.trino.spi.type.TimeType;
@@ -32,6 +33,8 @@ import
org.apache.gravitino.trino.connector.util.GeneralDataTypeTransformer;
/** Type transformer between PostgreSQL and Trino */
public class PostgreSQLDataTypeTransformer extends GeneralDataTypeTransformer {
+ private static final Logger LOG =
Logger.get(PostgreSQLDataTypeTransformer.class);
+
@SuppressWarnings("UnusedVariable")
private static final int POSTGRESQL_CHAR_LENGTH_LIMIT = 10485760;
// 1 GB, please refer to
@@ -82,6 +85,15 @@ public class PostgreSQLDataTypeTransformer extends
GeneralDataTypeTransformer {
}
// When precision is not set, the default precision is 3 (milliseconds
precision)
return TimeType.TIME_MILLIS;
+ } else if (Name.EXTERNAL == type.name()) {
+ // An external type carries a PostgreSQL type that Gravitino cannot
represent, such as an
+ // unconstrained numeric or money. Reading it as varchar is equivalent
to configuring
+ // unsupported_type_handling=CONVERT_TO_VARCHAR, whose Trino default is
IGNORE, and keeps an
+ // unmapped type from failing the type conversion of the whole table.
+ LOG.warn(
+ "Reading PostgreSQL type %s as varchar, Gravitino cannot represent
it",
+ ((Types.ExternalType) type).catalogString());
+ return io.trino.spi.type.VarcharType.createUnboundedVarcharType();
}
return super.getTrinoType(type);
@@ -113,9 +125,6 @@ public class PostgreSQLDataTypeTransformer extends
GeneralDataTypeTransformer {
}
return Types.VarCharType.of(varcharType.getLength().get());
- } else if (typeClass == io.trino.spi.type.ArrayType.class) {
- return Types.ListType.of(
- getGravitinoType(((io.trino.spi.type.ArrayType)
type).getElementType()), false);
}
return super.getGravitinoType(type);
diff --git
a/trino-connector/trino-connector/src/test/java/org/apache/gravitino/trino/connector/catalog/jdbc/postgresql/TestPostgreSQLDataTypeTransformer.java
b/trino-connector/trino-connector/src/test/java/org/apache/gravitino/trino/connector/catalog/jdbc/postgresql/TestPostgreSQLDataTypeTransformer.java
index 831fce11f7..7e72fc4fa2 100644
---
a/trino-connector/trino-connector/src/test/java/org/apache/gravitino/trino/connector/catalog/jdbc/postgresql/TestPostgreSQLDataTypeTransformer.java
+++
b/trino-connector/trino-connector/src/test/java/org/apache/gravitino/trino/connector/catalog/jdbc/postgresql/TestPostgreSQLDataTypeTransformer.java
@@ -58,6 +58,17 @@ public class TestPostgreSQLDataTypeTransformer {
Types.StringType.get());
}
+ @Test
+ public void testGravitinoExternalTypeToTrinoType() {
+ GeneralDataTypeTransformer generalDataTypeTransformer = new
PostgreSQLDataTypeTransformer();
+ Assertions.assertEquals(
+
generalDataTypeTransformer.getTrinoType(Types.ExternalType.of("numeric")),
+ io.trino.spi.type.VarcharType.createUnboundedVarcharType());
+ Assertions.assertEquals(
+
generalDataTypeTransformer.getTrinoType(Types.ExternalType.of("money")),
+ io.trino.spi.type.VarcharType.createUnboundedVarcharType());
+ }
+
@Test
public void testGravitinoCharToTrinoType() {
GeneralDataTypeTransformer generalDataTypeTransformer = new
PostgreSQLDataTypeTransformer();
