This is an automated email from the ASF dual-hosted git repository.
fanng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 519754ca0d [#4755] improvement (trino-connector): Trino connector
support the format table properties of Iceberg catalog (#7612)
519754ca0d is described below
commit 519754ca0ddc140e4349ac6610476f746ed4dc98
Author: Yuhui <[email protected]>
AuthorDate: Tue Jul 15 09:46:45 2025 +0800
[#4755] improvement (trino-connector): Trino connector support the format
table properties of Iceberg catalog (#7612)
### What changes were proposed in this pull request?
Trino connector support the format table properties of Iceberg catalog
### Why are the changes needed?
Fix: #4755
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
IT
---
.../lakehouse-iceberg/00000_create_table.sql | 1 +
.../lakehouse-iceberg/00000_create_table.txt | 12 ++++++++++++
.../lakehouse-iceberg/00002_alter_table.txt | 12 ++++++++++++
.../00005_partition_sort_order.txt | 4 ++++
.../testsets/lakehouse-iceberg/00006_datatype.txt | 4 ++++
.../testsets/lakehouse-iceberg/00007_varchar.txt | 2 ++
.../catalog/iceberg/IcebergMetadataAdapter.java | 21 +++++++++++++++++++++
.../catalog/iceberg/IcebergPropertyMeta.java | 18 +++++++++++++++++-
.../iceberg/IcebergTablePropertyConverter.java | 5 +++++
9 files changed, 78 insertions(+), 1 deletion(-)
diff --git
a/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00000_create_table.sql
b/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00000_create_table.sql
index 7a9bbb6087..75dbd1e816 100644
---
a/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00000_create_table.sql
+++
b/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00000_create_table.sql
@@ -11,6 +11,7 @@ CREATE TABLE gt_db2.tb02 (
name varchar,
salary int
) with (
+ format = 'ORC',
partitioning = ARRAY['name'],
sorted_by = ARRAY['salary']
);
diff --git
a/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00000_create_table.txt
b/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00000_create_table.txt
index 57a6313ccf..b1760888f3 100644
---
a/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00000_create_table.txt
+++
b/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00000_create_table.txt
@@ -8,6 +8,8 @@ CREATE TABLE
)
COMMENT ''
WITH (
+ format = 'PARQUET',
+ format_version = '2',
location = 'hdfs://%/user/iceberg/warehouse/TrinoQueryIT/gt_db2%/tb01'
)"
@@ -19,6 +21,8 @@ CREATE TABLE
)
COMMENT ''
WITH (
+ format = 'ORC',
+ format_version = '2',
location = 'hdfs://%/user/iceberg/warehouse/TrinoQueryIT/gt_db2%/tb02',
partitioning = ARRAY['name'],
sorted_by = ARRAY['salary']
@@ -34,6 +38,8 @@ CREATE TABLE
)
COMMENT ''
WITH (
+ format = 'PARQUET',
+ format_version = '2',
location = 'hdfs://%/user/iceberg/warehouse/TrinoQueryIT/gt_db2%/tb03',
partitioning = ARRAY['name'],
sorted_by = ARRAY['name']
@@ -47,6 +53,8 @@ CREATE TABLE
)
COMMENT ''
WITH (
+ format = 'PARQUET',
+ format_version = '2',
location = 'hdfs://%/user/iceberg/warehouse/TrinoQueryIT/gt_db2%/tb04',
sorted_by = ARRAY['name']
)"
@@ -59,6 +67,8 @@ CREATE TABLE
)
COMMENT ''
WITH (
+ format = 'PARQUET',
+ format_version = '2',
location = 'hdfs://%/user/iceberg/warehouse/TrinoQueryIT/gt_db2%/tb05',
partitioning = ARRAY['name']
)"
@@ -71,6 +81,8 @@ CREATE TABLE
)
COMMENT ''
WITH (
+ format = 'PARQUET',
+ format_version = '2',
location = 'hdfs://%/user/iceberg/warehouse/TrinoQueryIT/%/gt_db2/tb06'
)"
diff --git
a/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00002_alter_table.txt
b/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00002_alter_table.txt
index b404f03e5d..e19dfa9a58 100644
---
a/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00002_alter_table.txt
+++
b/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00002_alter_table.txt
@@ -18,6 +18,8 @@ DROP COLUMN
)
COMMENT ''
WITH (
+ format = 'PARQUET',
+ format_version = '2',
location = 'hdfs://%/user/iceberg/warehouse/TrinoQueryIT/gt_db2%/tb01'
)"
@@ -29,6 +31,8 @@ RENAME COLUMN
)
COMMENT ''
WITH (
+ format = 'PARQUET',
+ format_version = '2',
location = 'hdfs://%/user/iceberg/warehouse/TrinoQueryIT/gt_db2%/tb01'
)"
@@ -40,6 +44,8 @@ SET COLUMN TYPE
)
COMMENT ''
WITH (
+ format = 'PARQUET',
+ format_version = '2',
location = 'hdfs://%/user/iceberg/warehouse/TrinoQueryIT/gt_db2%/tb01'
)"
@@ -51,6 +57,8 @@ COMMENT
)
COMMENT 'test table comments'
WITH (
+ format = 'PARQUET',
+ format_version = '2',
location = 'hdfs://%/user/iceberg/warehouse/TrinoQueryIT/gt_db2%/tb01'
)"
@@ -62,6 +70,8 @@ COMMENT
)
COMMENT 'test table comments'
WITH (
+ format = 'PARQUET',
+ format_version = '2',
location = 'hdfs://%/user/iceberg/warehouse/TrinoQueryIT/gt_db2%/tb01'
)"
@@ -74,6 +84,8 @@ ADD COLUMN
)
COMMENT 'test table comments'
WITH (
+ format = 'PARQUET',
+ format_version = '2',
location = 'hdfs://%/user/iceberg/warehouse/TrinoQueryIT/gt_db2%/tb01'
)"
diff --git
a/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00005_partition_sort_order.txt
b/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00005_partition_sort_order.txt
index 429ae871cc..6bbe53cb0e 100644
---
a/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00005_partition_sort_order.txt
+++
b/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00005_partition_sort_order.txt
@@ -24,6 +24,8 @@ CREATE TABLE
)
COMMENT ''
WITH (
+ format = 'PARQUET',
+ format_version = '2',
location = 'hdfs://%/user/iceberg/warehouse/TrinoQueryIT/gt_db2%/lineitem',
partitioning = ARRAY['year(commitdate)'],
sorted_by = ARRAY['partkey','extendedprice DESC']
@@ -59,6 +61,8 @@ CREATE TABLE
)
COMMENT ''
WITH (
+ format = 'PARQUET',
+ format_version = '2',
location = 'hdfs://%/user/iceberg/warehouse/TrinoQueryIT/gt_db2%/tb01',
partitioning = ARRAY['day(commitdate)','month(shipdate)','bucket(partkey,
2)','truncate(shipinstruct, 2)'],
sorted_by = ARRAY['partkey ASC NULLS LAST','extendedprice DESC NULLS FIRST']
diff --git
a/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00006_datatype.txt
b/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00006_datatype.txt
index 79c5177f34..0e703c8028 100644
---
a/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00006_datatype.txt
+++
b/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00006_datatype.txt
@@ -21,6 +21,8 @@ CREATE TABLE
)
COMMENT ''
WITH (
+ format = 'PARQUET',
+ format_version = '2',
location = 'hdfs://%/user/iceberg/warehouse/TrinoQueryIT/gt_db2%/tb01'
)"
@@ -51,6 +53,8 @@ CREATE TABLE
)
COMMENT ''
WITH (
+ format = 'PARQUET',
+ format_version = '2',
location = 'hdfs://%/user/iceberg/warehouse/TrinoQueryIT/gt_db2%/tb02'
)"
diff --git
a/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00007_varchar.txt
b/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00007_varchar.txt
index 2ab524c8dd..54f2ccf7cf 100644
---
a/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00007_varchar.txt
+++
b/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00007_varchar.txt
@@ -16,6 +16,8 @@ CREATE TABLE
)
COMMENT ''
WITH (
+ format = 'PARQUET',
+ format_version = '2',
location = 'hdfs://%/user/iceberg/warehouse/TrinoQueryIT/gt_db2%/tb04'
)"
diff --git
a/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergMetadataAdapter.java
b/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergMetadataAdapter.java
index 075fc7d3e0..20a9111af1 100644
---
a/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergMetadataAdapter.java
+++
b/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergMetadataAdapter.java
@@ -18,6 +18,13 @@
*/
package org.apache.gravitino.trino.connector.catalog.iceberg;
+import static
org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants.FORMAT;
+import static
org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants.FORMAT_VERSION;
+import static
org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants.PROVIDER;
+import static
org.apache.gravitino.trino.connector.catalog.iceberg.IcebergPropertyMeta.ICEBERG_FORMAT_PROPERTY;
+import static
org.apache.gravitino.trino.connector.catalog.iceberg.IcebergPropertyMeta.ICEBERG_FORMAT_VERSION_PROPERTY;
+import static
org.apache.gravitino.trino.connector.catalog.iceberg.IcebergTablePropertyConverter.convertTableFormatToTrino;
+
import com.google.common.collect.ImmutableSet;
import io.trino.spi.connector.ColumnMetadata;
import io.trino.spi.connector.ConnectorTableMetadata;
@@ -110,6 +117,14 @@ public class IcebergMetadataAdapter extends
CatalogConnectorMetadataAdapter {
toGravitinoTableProperties(
removeKeys(tableMetadata.getProperties(),
ICEBERG_PROPERTIES_TO_REMOVE));
+ if (propertyMap.containsKey(ICEBERG_FORMAT_PROPERTY)) {
+ String format = propertyMap.get(ICEBERG_FORMAT_PROPERTY).toString();
+ properties.put(PROVIDER, format);
+ if (propertyMap.containsKey((ICEBERG_FORMAT_VERSION_PROPERTY))) {
+ properties.put(FORMAT_VERSION,
propertyMap.get(ICEBERG_FORMAT_VERSION_PROPERTY).toString());
+ }
+ }
+
List<GravitinoColumn> columns = new ArrayList<>();
for (int i = 0; i < tableMetadata.getColumns().size(); i++) {
ColumnMetadata column = tableMetadata.getColumns().get(i);
@@ -150,6 +165,12 @@ public class IcebergMetadataAdapter extends
CatalogConnectorMetadataAdapter {
Map<String, Object> properties =
toTrinoTableProperties(gravitinoTable.getProperties());
+ properties.put(
+ ICEBERG_FORMAT_PROPERTY,
+ convertTableFormatToTrino(gravitinoTable.getProperties().get(FORMAT)));
+ properties.put(
+ ICEBERG_FORMAT_VERSION_PROPERTY,
gravitinoTable.getProperties().get(FORMAT_VERSION));
+
if (ArrayUtils.isNotEmpty(gravitinoTable.getPartitioning())) {
properties.put(
IcebergPropertyMeta.ICEBERG_PARTITIONING_PROPERTY,
diff --git
a/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergPropertyMeta.java
b/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergPropertyMeta.java
index 8c60c10830..94f42cea3e 100644
---
a/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergPropertyMeta.java
+++
b/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergPropertyMeta.java
@@ -44,6 +44,15 @@ public class IcebergPropertyMeta implements HasPropertyMeta {
/** Property key for table location configuration. */
public static final String ICEBERG_LOCATION_PROPERTY = "location";
+ /** Property key for table format configuration. */
+ public static final String ICEBERG_FORMAT_PROPERTY = "format";
+
+ /** Property key for table format version configuration. */
+ public static final String ICEBERG_FORMAT_VERSION_PROPERTY =
"format_version";
+
+ private static final String DEFAULT_ICEBERG_FORMAT = "PARQUET";
+ private static final String DEFAULT_ICEBERG_FORMAT_VERSION = "2";
+
// Value is whether this property is reserved and cannot be used by users
// TODO (yuqi) add more properties
/** Map of table property metadata to their reservation status. */
@@ -72,7 +81,14 @@ public class IcebergPropertyMeta implements HasPropertyMeta {
value -> (List<?>) value,
value -> value),
PropertyMetadata.stringProperty(
- ICEBERG_LOCATION_PROPERTY, "Location for table storage", null,
false));
+ ICEBERG_LOCATION_PROPERTY, "Location for table storage", null,
false),
+ PropertyMetadata.stringProperty(
+ ICEBERG_FORMAT_PROPERTY, "Table format", DEFAULT_ICEBERG_FORMAT,
false),
+ PropertyMetadata.stringProperty(
+ ICEBERG_FORMAT_VERSION_PROPERTY,
+ "Format version",
+ DEFAULT_ICEBERG_FORMAT_VERSION,
+ false));
// TODO (yuqi) add more properties
/** Map of schema property metadata to their reservation status. */
diff --git
a/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergTablePropertyConverter.java
b/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergTablePropertyConverter.java
index 267a007b21..51dbf04abe 100644
---
a/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergTablePropertyConverter.java
+++
b/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergTablePropertyConverter.java
@@ -47,4 +47,9 @@ public class IcebergTablePropertyConverter extends
PropertyConverter {
// no matter what the mapping is, the properties will be empty.
return TRINO_KEY_TO_GRAVITINO_KEY;
}
+
+ public static String convertTableFormatToTrino(String format) {
+ format = format.replace("iceberg/", "");
+ return format.toUpperCase();
+ }
}
