This is an automated email from the ASF dual-hosted git repository.
yuqi1129 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 278102990e [#13547] fix(mysql): Handle decimal precision beyond
Gravitino limit (#13548)
278102990e is described below
commit 278102990eb60e48f46d3cc9db8786ddafa37a6e
Author: jarred0214 <[email protected]>
AuthorDate: Sat Oct 10 09:06:58 2026 +0800
[#13547] fix(mysql): Handle decimal precision beyond Gravitino limit
(#13548)
### What changes were proposed in this pull request?
Update the MySQL JDBC type converter to handle MySQL `DECIMAL` columns
whose precision exceeds Gravitino's maximum decimal precision of 38.
- Keep mapping `DECIMAL` columns with precision up to 38 to
`DecimalType`.
- Map higher-precision `DECIMAL` columns to `ExternalType`.
- Preserve the native precision, scale, and unsigned attribute, for
example:
- `decimal(65,30)`
- `decimal(65,30) unsigned`
- Add tests for precision 38, precision 39, precision 65, and unsigned
decimals.
### Why are the changes needed?
MySQL supports `DECIMAL` precision up to 65, while Gravitino
`DecimalType` only supports precision up to 38.
The existing converter directly calls `Types.DecimalType.of` with the
JDBC precision and scale. Loading a table containing `DECIMAL(65,30)`
therefore throws:
```text
Decimal precision must be in range[1, 38]: precision: 65
```
The table cannot be loaded, and operations that require table loading,
such as granting table privileges, fail as a consequence.
Fix: #13547
### Does this PR introduce _any_ user-facing change?
Yes. MySQL tables containing `DECIMAL` columns with precision above 38
can now be loaded. These columns are exposed as `ExternalType` values
that preserve their native MySQL declaration. `DECIMAL` columns with
precision up to 38 are unchanged.
### How was this patch tested?
- Added unit coverage to `TestMysqlTypeConverter`.
- Ran:
```shell
./gradlew :catalogs:catalog-jdbc-mysql:test -PskipITs
```
The MySQL JDBC catalog module tests passed.
---
.../mysql/converter/MysqlTypeConverter.java | 21 +++++++++++++++-
.../mysql/converter/TestMysqlTypeConverter.java | 12 +++++++++
.../mysql/integration/test/CatalogMysqlIT.java | 29 ++++++++++++++++++++++
3 files changed, 61 insertions(+), 1 deletion(-)
diff --git
a/catalogs/catalog-jdbc-mysql/src/main/java/org/apache/gravitino/catalog/mysql/converter/MysqlTypeConverter.java
b/catalogs/catalog-jdbc-mysql/src/main/java/org/apache/gravitino/catalog/mysql/converter/MysqlTypeConverter.java
index 317263690d..46c8c016f4 100644
---
a/catalogs/catalog-jdbc-mysql/src/main/java/org/apache/gravitino/catalog/mysql/converter/MysqlTypeConverter.java
+++
b/catalogs/catalog-jdbc-mysql/src/main/java/org/apache/gravitino/catalog/mysql/converter/MysqlTypeConverter.java
@@ -26,6 +26,8 @@ import org.apache.gravitino.rel.types.Types;
/** Type converter for MySQL. */
public class MysqlTypeConverter extends JdbcTypeConverter {
+ // Keep this limit in sync with Types.DecimalType.checkPrecisionScale.
+ static final int MAX_DECIMAL_PRECISION = 38;
static final String BIT = "bit";
static final String TINYINT = "tinyint";
static final String TINYINT_UNSIGNED = "tinyint unsigned";
@@ -91,7 +93,7 @@ public class MysqlTypeConverter extends JdbcTypeConverter {
.orElseGet(Types.TimestampType::withoutTimeZone);
case DECIMAL_UNSIGNED:
case DECIMAL:
- return Types.DecimalType.of(typeBean.getColumnSize(),
typeBean.getScale());
+ return toGravitinoDecimal(typeBean);
case VARCHAR:
return Types.VarCharType.of(typeBean.getColumnSize());
case CHAR:
@@ -166,4 +168,21 @@ public class MysqlTypeConverter extends JdbcTypeConverter {
throw new IllegalArgumentException(
String.format("Couldn't convert Gravitino type %s to MySQL type",
type.simpleString()));
}
+
+ private static Type toGravitinoDecimal(JdbcTypeBean typeBean) {
+ int precision = typeBean.getColumnSize();
+ int scale = typeBean.getScale();
+ if (precision <= MAX_DECIMAL_PRECISION) {
+ // Preserve the existing mapping for signed and unsigned decimals within
the core limit.
+ // DecimalType does not represent unsignedness.
+ return Types.DecimalType.of(precision, scale);
+ }
+
+ // MySQL supports precision up to 65, while Gravitino DecimalType is
limited to 38.
+ // Preserve larger decimals as ExternalType so loading the table does not
fail.
+ String unsignedSuffix =
+ DECIMAL_UNSIGNED.equalsIgnoreCase(typeBean.getTypeName()) ? "
unsigned" : "";
+ return Types.ExternalType.of(
+ String.format("%s(%d,%d)%s", DECIMAL, precision, scale,
unsignedSuffix));
+ }
}
diff --git
a/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/converter/TestMysqlTypeConverter.java
b/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/converter/TestMysqlTypeConverter.java
index 85ccc85ed9..b1851853fe 100644
---
a/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/converter/TestMysqlTypeConverter.java
+++
b/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/converter/TestMysqlTypeConverter.java
@@ -74,6 +74,14 @@ public class TestMysqlTypeConverter {
checkJdbcTypeToGravitinoType(Types.TimestampType.withTimeZone(9),
TIMESTAMP, 29, null, 9);
checkJdbcTypeToGravitinoType(Types.DecimalType.of(10, 2), DECIMAL, 10, 2,
0);
checkJdbcTypeToGravitinoType(Types.DecimalType.of(10, 2),
DECIMAL_UNSIGNED, 10, 2, 0);
+ checkJdbcTypeToGravitinoType(Types.DecimalType.of(38, 30), DECIMAL, 38,
30, 0);
+ checkJdbcTypeToGravitinoType(Types.DecimalType.of(38, 30),
DECIMAL_UNSIGNED, 38, 30, 0);
+ checkJdbcTypeToGravitinoType(
+ Types.ExternalType.of("decimal(39,30) unsigned"), DECIMAL_UNSIGNED,
39, 30, 0);
+ checkJdbcTypeToGravitinoType(Types.ExternalType.of("decimal(39,30)"),
DECIMAL, 39, 30, 0);
+ checkJdbcTypeToGravitinoType(Types.ExternalType.of("decimal(65,30)"),
DECIMAL, 65, 30, 0);
+ checkJdbcTypeToGravitinoType(
+ Types.ExternalType.of("decimal(65,30) unsigned"), DECIMAL_UNSIGNED,
65, 30, 0);
checkJdbcTypeToGravitinoType(Types.VarCharType.of(20), VARCHAR, 20, null,
0);
checkJdbcTypeToGravitinoType(Types.FixedCharType.of(20), CHAR, 20, null,
0);
checkJdbcTypeToGravitinoType(Types.StringType.get(), TEXT, null, null, 0);
@@ -94,6 +102,10 @@ public class TestMysqlTypeConverter {
checkGravitinoTypeToJdbcType(TIMESTAMP,
Types.TimestampType.withTimeZone());
checkGravitinoTypeToJdbcType(DATETIME,
Types.TimestampType.withoutTimeZone());
checkGravitinoTypeToJdbcType(DECIMAL + "(10,2)", Types.DecimalType.of(10,
2));
+ checkGravitinoTypeToJdbcType("decimal(39,30)",
Types.ExternalType.of("decimal(39,30)"));
+ checkGravitinoTypeToJdbcType("decimal(65,30)",
Types.ExternalType.of("decimal(65,30)"));
+ checkGravitinoTypeToJdbcType(
+ "decimal(65,30) unsigned", Types.ExternalType.of("decimal(65,30)
unsigned"));
checkGravitinoTypeToJdbcType(VARCHAR + "(20)", Types.VarCharType.of(20));
checkGravitinoTypeToJdbcType(CHAR + "(20)", Types.FixedCharType.of(20));
checkGravitinoTypeToJdbcType(TEXT, Types.StringType.get());
diff --git
a/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java
b/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java
index 7d23739c5e..b0a8e08139 100644
---
a/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java
+++
b/catalogs/catalog-jdbc-mysql/src/test/java/org/apache/gravitino/catalog/mysql/integration/test/CatalogMysqlIT.java
@@ -679,6 +679,35 @@ public class CatalogMysqlIT extends BaseIT {
}
}
+ @Test
+ void testLoadHighPrecisionDecimalTable() {
+ mysqlService.executeQuery(
+ String.format(
+ "CREATE TABLE %s.%s ("
+ + "decimal_38 DECIMAL(38,30), "
+ + "decimal_39 DECIMAL(39,30), "
+ + "decimal_65 DECIMAL(65,30), "
+ + "decimal_65_unsigned DECIMAL(65,30) UNSIGNED, "
+ + "decimal_10_unsigned DECIMAL(10,2) UNSIGNED, "
+ + "decimal_38_unsigned DECIMAL(38,30) UNSIGNED, "
+ + "decimal_39_unsigned DECIMAL(39,30) UNSIGNED)",
+ schemaName, tableName));
+
+ Table loadedTable =
+ catalog.asTableCatalog().loadTable(NameIdentifier.of(schemaName,
tableName));
+ Column[] columns = loadedTable.columns();
+ Assertions.assertEquals(7, columns.length);
+ Assertions.assertEquals(Types.DecimalType.of(38, 30),
columns[0].dataType());
+ Assertions.assertEquals(Types.ExternalType.of("decimal(39,30)"),
columns[1].dataType());
+ Assertions.assertEquals(Types.ExternalType.of("decimal(65,30)"),
columns[2].dataType());
+ Assertions.assertEquals(
+ Types.ExternalType.of("decimal(65,30) unsigned"),
columns[3].dataType());
+ Assertions.assertEquals(Types.DecimalType.of(10, 2),
columns[4].dataType());
+ Assertions.assertEquals(Types.DecimalType.of(38, 30),
columns[5].dataType());
+ Assertions.assertEquals(
+ Types.ExternalType.of("decimal(39,30) unsigned"),
columns[6].dataType());
+ }
+
@Test
void testColumnTypeConverter() {
// test convert from MySQL to Gravitino
