This is an automated email from the ASF dual-hosted git repository. yuqi4733 pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/gravitino.git
commit c3ce4e4ee2abb67b5df0afa5f856fe1a3bfcc2de Author: Yuhui <[email protected]> AuthorDate: Fri Feb 27 21:46:57 2026 +0800 [#9964] feat (trino-connector): Add the version segment module to support Trino 469-472 (#9965) Add the version segment module for 469-472 to support Trino 469 through Trino 472 Fix: #9964 NO UT and IT Fix Fix --- .github/workflows/build.yml | 2 +- .github/workflows/frontend-integration-test.yml | 12 +- AGENTS.md | 8 + build.gradle.kts | 52 +---- .../integration/test/CatalogOceanBaseIT.java | 2 - .../operation/TestOceanBaseDatabaseOperations.java | 2 - .../operation/TestOceanBaseTableOperations.java | 2 - dev/docker/trino/Dockerfile | 40 +--- dev/docker/trino/trino-dependency.sh | 4 +- docs/trino-connector/catalog-hive.md | 2 +- docs/trino-connector/catalog-iceberg.md | 2 +- docs/trino-connector/configuration.md | 2 +- docs/trino-connector/development.md | 252 +++++++++++++-------- docs/trino-connector/installation.md | 76 +++++-- docs/trino-connector/requirements.md | 7 +- docs/trino-connector/trino-connector.md | 3 +- .../docker-script/docker-compose.yaml | 8 +- .../test/container/TrinoITContainers.java | 2 +- settings.gradle.kts | 1 + trino-connector/integration-test/build.gradle.kts | 1 + ...leanup.sql => catalog_iceberg_hive_cleanup.sql} | 0 ...repare.sql => catalog_iceberg_hive_prepare.sql} | 0 .../{ => ignored}/00011_table_statistics.sql | 0 .../{ => ignored}/00011_table_statistics.txt | 0 .../catalog_iceberg_mysql_cleanup.sql | 0 .../catalog_iceberg_mysql_prepare.sql | 0 .../trino-connector-469-472/build.gradle.kts | 169 ++++++++++++++ .../trino/connector/GravitinoConnector469.java | 67 ++++++ .../connector/GravitinoConnectorFactory469.java | 62 +++++ .../trino/connector/GravitinoMetadata469.java | 105 +++++++++ .../GravitinoNodePartitioningProvider469.java | 32 +++ .../trino/connector/GravitinoPlugin469.java | 38 ++++ .../trino/connector/GravitinoSplitManager469.java | 61 +++++ .../connector/GravitinoSystemConnector469.java | 82 +++++++ .../META-INF/services/io.trino.spi.Plugin | 19 ++ .../src/test/java/TestGravitinoConnector469.java | 71 ++++++ trino-connector/trino-connector/build.gradle.kts | 1 + .../catalog/hive/HiveConnectorAdapter.java | 1 + .../iceberg/IcebergCatalogPropertyConverter.java | 1 + .../catalog/iceberg/IcebergMetadataAdapter.java | 14 +- .../catalog/iceberg/IcebergPropertyMeta.java | 4 +- 41 files changed, 972 insertions(+), 235 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d6fc247115..cd6ca287dc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,7 +15,7 @@ concurrency: # A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: changes: - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 diff --git a/.github/workflows/frontend-integration-test.yml b/.github/workflows/frontend-integration-test.yml index e636df7c89..ec8b03c58c 100644 --- a/.github/workflows/frontend-integration-test.yml +++ b/.github/workflows/frontend-integration-test.yml @@ -37,7 +37,7 @@ jobs: - scripts/** - server/** - server-common/** - - web/** + - web-v2/** - build.gradle.kts - gradle.properties - gradlew @@ -49,7 +49,7 @@ jobs: needs: changes if: needs.changes.outputs.source_changes == 'true' runs-on: ubuntu-latest - timeout-minutes: 60 + timeout-minutes: 90 strategy: matrix: # Integration test for AMD64 architecture @@ -75,7 +75,7 @@ jobs: - name: Package Gravitino run: | - ./gradlew compileDistribution compileTrinoConnector -x test + GRAVITINO_USE_WEB_V2=true ./gradlew compileDistribution compileTrinoConnector -x test - name: Free up disk space run: | @@ -84,8 +84,8 @@ jobs: - name: Frontend Integration Test id: integrationTest run: | - ./gradlew -PskipTests -PtestMode=embedded -PskipDockerTests=false :web:integration-test:test - ./gradlew -PskipTests -PtestMode=deploy -PskipDockerTests=false :web:integration-test:test + GRAVITINO_USE_WEB_V2=true ./gradlew -PskipTests -PtestMode=embedded -PskipDockerTests=false :web-v2:integration-test:test + GRAVITINO_USE_WEB_V2=true ./gradlew -PskipTests -PtestMode=deploy -PskipDockerTests=false :web-v2:integration-test:test - name: Upload integrate tests reports uses: actions/upload-artifact@v4 @@ -99,4 +99,4 @@ jobs: distribution/package/logs/gravitino-server.log catalogs/**/*.log catalogs/**/*.tar - web/integration-test/build/*.log + web-v2/integration-test/build/*.log \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md index adf11b3d74..cf0c820047 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -57,3 +57,11 @@ - **Unit Tests**: `./gradlew test -PskipITs -PskipDockerTests=false` - **Integration Tests**: `./gradlew test -PskipTests -PskipDockerTests=false` - **OpenAPI Docs Validation**: `./gradlew :docs:build` — Run this after any changes to `docs/open-api/*.yaml` to validate OpenAPI specification correctness. + +## Claude Memory Usage +- Before starting any task, use mcp-search to check if similar work has been done before. + When encountering unfamiliar code or configuration, search memory for prior context. +- When hitting a problem, search memory first for known solutions before debugging from scratch. +- After completing a task, save key findings and solutions to claude-mem for future reference. +- Use multiple keyword combinations when searching (e.g., module name + issue type, class name + error). + diff --git a/build.gradle.kts b/build.gradle.kts index 8b9d6d95a1..19029deffe 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -903,25 +903,16 @@ tasks { } val compileTrinoConnector by registering { - dependsOn("trino-connector:trino-connector:copyLibs") + dependsOn("trino-connector:trino-connector-469-472:copyLibs") group = "gravitino distribution" - outputs.dir(projectDir.dir("distribution/${rootProject.name}-trino-connector")) - doLast { - copy { - from(projectDir.dir("licenses")) { into("${rootProject.name}-trino-connector/licenses") } - from(projectDir.file("LICENSE.trino")) { into("${rootProject.name}-trino-connector") } - from(projectDir.file("NOTICE.trino")) { into("${rootProject.name}-trino-connector") } - from(projectDir.file("README.md")) { into("${rootProject.name}-trino-connector") } - into(outputDir) - rename { fileName -> - fileName.replace(".trino", "") - } - } - } } val assembleDistribution by registering(Tar::class) { - dependsOn("assembleTrinoConnector", "assembleIcebergRESTServer", "assembleLanceRESTServer", "assembleDistributionAll") + dependsOn( + ":trino-connector:trino-connector-469-472:assembleTrinoConnector", + "assembleIcebergRESTServer", + "assembleLanceRESTServer" + ) group = "gravitino distribution" finalizedBy("checksumDistribution") into("${rootProject.name}-$version-bin") @@ -958,17 +949,6 @@ tasks { } } - val assembleTrinoConnector by registering(Tar::class) { - dependsOn("compileTrinoConnector") - group = "gravitino distribution" - finalizedBy("checksumTrinoConnector") - into("${rootProject.name}-trino-connector-$version") - from(compileTrinoConnector.map { it.outputs.files.single() }) - compression = Compression.GZIP - archiveFileName.set("${rootProject.name}-trino-connector-$version.tar.gz") - destinationDirectory.set(projectDir.dir("distribution")) - } - val assembleLanceRESTServer by registering(Tar::class) { dependsOn("compileLanceRESTServer") group = "gravitino distribution" @@ -1025,7 +1005,7 @@ tasks { register("checksumDistribution") { group = "gravitino distribution" - dependsOn(assembleDistribution, "checksumTrinoConnector", "checksumIcebergRESTServerDistribution", "checksumLanceRESTServerDistribution") + dependsOn(assembleDistribution, "checksumIcebergRESTServerDistribution", "checksumLanceRESTServerDistribution") val archiveFile = assembleDistribution.flatMap { it.archiveFile } val checksumFile = archiveFile.map { archive -> archive.asFile.let { it.resolveSibling("${it.name}.sha256") } @@ -1039,22 +1019,6 @@ tasks { } } - register("checksumTrinoConnector") { - group = "gravitino distribution" - dependsOn(assembleTrinoConnector) - val archiveFile = assembleTrinoConnector.flatMap { it.archiveFile } - val checksumFile = archiveFile.map { archive -> - archive.asFile.let { it.resolveSibling("${it.name}.sha256") } - } - inputs.file(archiveFile) - outputs.file(checksumFile) - doLast { - checksumFile.get().writeText( - serviceOf<ChecksumService>().sha256(archiveFile.get().asFile).toString() - ) - } - } - val cleanDistribution by registering(Delete::class) { group = "gravitino distribution" delete(outputDir) @@ -1073,9 +1037,9 @@ tasks { !it.name.startsWith("optimizer") && !it.name.startsWith("spark") && !it.name.startsWith("hive-metastore") && + !it.name.startsWith("trino-connector") && it.name != "hadoop-common" && it.name != "integration-test" && - it.name != "trino-connector" && it.parent?.name != "bundles" && it.parent?.name != "maintenance" && it.name != "mcp-server" diff --git a/catalogs-contrib/catalog-jdbc-oceanbase/src/test/java/org/apache/gravitino/catalog/oceanbase/integration/test/CatalogOceanBaseIT.java b/catalogs-contrib/catalog-jdbc-oceanbase/src/test/java/org/apache/gravitino/catalog/oceanbase/integration/test/CatalogOceanBaseIT.java index b0cb06bb82..260d851ac7 100644 --- a/catalogs-contrib/catalog-jdbc-oceanbase/src/test/java/org/apache/gravitino/catalog/oceanbase/integration/test/CatalogOceanBaseIT.java +++ b/catalogs-contrib/catalog-jdbc-oceanbase/src/test/java/org/apache/gravitino/catalog/oceanbase/integration/test/CatalogOceanBaseIT.java @@ -75,7 +75,6 @@ import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; @@ -85,7 +84,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; @Tag("gravitino-docker-test") -@Disabled @TestInstance(Lifecycle.PER_CLASS) public class CatalogOceanBaseIT extends BaseIT { private static final Logger LOG = LoggerFactory.getLogger(CatalogOceanBaseIT.class); diff --git a/catalogs-contrib/catalog-jdbc-oceanbase/src/test/java/org/apache/gravitino/catalog/oceanbase/operation/TestOceanBaseDatabaseOperations.java b/catalogs-contrib/catalog-jdbc-oceanbase/src/test/java/org/apache/gravitino/catalog/oceanbase/operation/TestOceanBaseDatabaseOperations.java index 2564b39a6a..df27c6b70b 100644 --- a/catalogs-contrib/catalog-jdbc-oceanbase/src/test/java/org/apache/gravitino/catalog/oceanbase/operation/TestOceanBaseDatabaseOperations.java +++ b/catalogs-contrib/catalog-jdbc-oceanbase/src/test/java/org/apache/gravitino/catalog/oceanbase/operation/TestOceanBaseDatabaseOperations.java @@ -23,12 +23,10 @@ import java.util.List; import java.util.Map; import org.apache.gravitino.utils.RandomNameUtils; import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; @Tag("gravitino-docker-test") -@Disabled public class TestOceanBaseDatabaseOperations extends TestOceanBase { @Test diff --git a/catalogs-contrib/catalog-jdbc-oceanbase/src/test/java/org/apache/gravitino/catalog/oceanbase/operation/TestOceanBaseTableOperations.java b/catalogs-contrib/catalog-jdbc-oceanbase/src/test/java/org/apache/gravitino/catalog/oceanbase/operation/TestOceanBaseTableOperations.java index ee56cf1a16..c26d25b608 100644 --- a/catalogs-contrib/catalog-jdbc-oceanbase/src/test/java/org/apache/gravitino/catalog/oceanbase/operation/TestOceanBaseTableOperations.java +++ b/catalogs-contrib/catalog-jdbc-oceanbase/src/test/java/org/apache/gravitino/catalog/oceanbase/operation/TestOceanBaseTableOperations.java @@ -43,12 +43,10 @@ import org.apache.gravitino.rel.types.Types; import org.apache.gravitino.utils.RandomNameUtils; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; @Tag("gravitino-docker-test") -@Disabled public class TestOceanBaseTableOperations extends TestOceanBase { private static final Type VARCHAR = Types.VarCharType.of(255); private static final Type INT = Types.IntegerType.get(); diff --git a/dev/docker/trino/Dockerfile b/dev/docker/trino/Dockerfile index 8d4e4ff3ca..7684283779 100644 --- a/dev/docker/trino/Dockerfile +++ b/dev/docker/trino/Dockerfile @@ -16,49 +16,11 @@ # specific language governing permissions and limitations # under the License. # -FROM trinodb/trino:435 +FROM trinodb/trino:469 LABEL maintainer="[email protected]" USER root -# Only mysql, hudi, iceberg, mariadb, jmx, memory, tpch, tpcds, postgresql, hive plugin are kept -RUN rm -rf /usr/lib/trino/plugin/accumulo \ - && rm -rf /usr/lib/trino/plugin/blackhole \ - && rm -rf /usr/lib/trino/plugin/delta-lake \ - && rm -rf /usr/lib/trino/plugin/example-http \ - && rm -rf /usr/lib/trino/plugin/geospatial \ - && rm -rf /usr/lib/trino/plugin/kafka \ - && rm -rf /usr/lib/trino/plugin/local-file \ - && rm -rf /usr/lib/trino/plugin/ml \ - && rm -rf /usr/lib/trino/plugin/mysql-event-listener \ - && rm -rf /usr/lib/trino/plugin/phoenix5 \ - && rm -rf /usr/lib/trino/plugin/prometheus \ - && rm -rf /usr/lib/trino/plugin/redshift \ - && rm -rf /usr/lib/trino/plugin/singlestore \ - && rm -rf /usr/lib/trino/plugin/thrift \ - && rm -rf /usr/lib/trino/plugin/atop \ - && rm -rf /usr/lib/trino/plugin/cassandra \ - && rm -rf /usr/lib/trino/plugin/druid \ - && rm -rf /usr/lib/trino/plugin/exchange-filesystem \ - && rm -rf /usr/lib/trino/plugin/google-sheets \ - && rm -rf /usr/lib/trino/plugin/http-event-listener \ - && rm -rf /usr/lib/trino/plugin/ignite \ - && rm -rf /usr/lib/trino/plugin/kinesis \ - && rm -rf /usr/lib/trino/plugin/mongodb \ - && rm -rf /usr/lib/trino/plugin/oracle \ - && rm -rf /usr/lib/trino/plugin/pinot \ - && rm -rf /usr/lib/trino/plugin/raptor-legacy \ - && rm -rf /usr/lib/trino/plugin/resource-group-managers \ - && rm -rf /usr/lib/trino/plugin/sqlserver \ - && rm -rf /usr/lib/trino/plugin/bigquery \ - && rm -rf /usr/lib/trino/plugin/clickhouse \ - && rm -rf /usr/lib/trino/plugin/elasticsearch \ - && rm -rf /usr/lib/trino/plugin/exchange-hdfs \ - && rm -rf /usr/lib/trino/plugin/kudu \ - && rm -rf /usr/lib/trino/plugin/password-authenticators \ - && rm -rf /usr/lib/trino/plugin/redis \ - && rm -rf /usr/lib/trino/plugin/session-property-managers \ - && rm -rf /usr/lib/trino/plugin/teradata-functions COPY --chown=trino:trino packages/trino/conf /etc/trino COPY --chown=trino:trino packages/gravitino-trino-connector/mysql-connector-java-8.0.27.jar /usr/lib/trino/plugin/iceberg/ COPY --chown=trino:trino packages/gravitino-trino-connector/postgresql-42.7.0.jar /usr/lib/trino/plugin/iceberg/ diff --git a/dev/docker/trino/trino-dependency.sh b/dev/docker/trino/trino-dependency.sh index c07128a8c7..09b89e89e8 100755 --- a/dev/docker/trino/trino-dependency.sh +++ b/dev/docker/trino/trino-dependency.sh @@ -28,8 +28,8 @@ rm -rf "${trino_dir}/packages" mkdir -p "${trino_dir}/packages" cd ${gravitino_home} -${gravitino_home}/gradlew clean assembleTrinoConnector -x test -cp -r "${gravitino_home}/distribution/gravitino-trino-connector" "${trino_dir}/packages/gravitino-trino-connector" +${gravitino_home}/gradlew :trino-connector:trino-connector-469-472:assembleTrinoConnector -x test +cp -r "${gravitino_home}/distribution/gravitino-trino-connector-469-472" "${trino_dir}/packages/gravitino-trino-connector" MYSQL_VERSION="8.0.27" PG_VERSION="42.7.0" diff --git a/docs/trino-connector/catalog-hive.md b/docs/trino-connector/catalog-hive.md index d6d9313e0a..87b391dedf 100644 --- a/docs/trino-connector/catalog-hive.md +++ b/docs/trino-connector/catalog-hive.md @@ -374,7 +374,7 @@ replacing hdfs_user with the appropriate username: When using AWS S3 within the Hive catalog, users need to configure the Trino Hive connector's AWS S3-related properties in the catalog's properteis. Please refer to the documentation -of [Hive connector with Amazon S3](https://trino.io/docs/435/connector/hive-s3.html). +of [Hive connector with Amazon S3](https://trino.io/docs/current/connector/hive-s3.html). To create a Hive catalog with AWS S3 configuration in the Trino CLI, use the following command: diff --git a/docs/trino-connector/catalog-iceberg.md b/docs/trino-connector/catalog-iceberg.md index 8fa63bfb6f..20cd0b08f9 100644 --- a/docs/trino-connector/catalog-iceberg.md +++ b/docs/trino-connector/catalog-iceberg.md @@ -281,7 +281,7 @@ replacing hdfs_user with the appropriate username: When using AWS S3 within the Iceberg catalog, users need to configure the Trino Iceberg connector's AWS S3-related properties in the catalog's properties. Please refer to the documentation -of [Hive connector with Amazon S3](https://trino.io/docs/435/connector/hive-s3.html). +of [Hive connector with Amazon S3](https://trino.io/docs/current/connector/hive-s3.html). These configurations must use the `trino.bypass.` prefix in the Iceberg catalog's attributes to be effective. To create an Iceberg catalog with AWS S3 configuration in the Trino CLI, use the following command: diff --git a/docs/trino-connector/configuration.md b/docs/trino-connector/configuration.md index 6d407a84e9..2e300eee85 100644 --- a/docs/trino-connector/configuration.md +++ b/docs/trino-connector/configuration.md @@ -13,7 +13,7 @@ license: "This software is licensed under the Apache License version 2." | trino.jdbc.user | string | admin | The jdbc user name of current Trino. | NO | 0.5.1 | | trino.jdbc.password | string | (none) | The jdbc password of current Trino. | NO | 0.5.1 | | gravitino.metadata.refresh-interval-seconds | integer | 10 | The `gravitino.metadata.refresh-interval-seconds` defines the interval in seconds to refresh metadata from Gravitino server, the default value is 10 seconds. | No | 0.9.0 | -| gravitino.trino.skip-version-validation | boolean | false | The `gravitino.trino.skip-version-validation` defines whether skip Trino version validation or not. Note that Gravitino only supports Trino which version between 435 and 439, other versions of Trino have not undergone thorough testing, so there may be compatiablity problem if true. | No | 1.0.0 | +| gravitino.trino.skip-version-validation | boolean | false | The `gravitino.trino.skip-version-validation` defines whether to skip Trino version validation. Gravitino supports Trino versions between 435 and 462. If this option is `true`, unsupported Trino versions can still be used, but compatibility is not guaranteed. | No | 1.0.0 | | gravitino.client. | string | (none) | The configuration key prefix for the Gravitino client config. | No | 1.0.0 | | gravitino.trino.skip-catalog-patterns | string | (none) | The `gravitino.trino.skip-catalog-patterns` defines a comma-separated list of catalog name regex patterns that should be excluded from loading. For example, `test_.*, .*_tmp` excludes all catalogs starting with `test_` or ending with `_tmp`. | No | 1.2.0 | diff --git a/docs/trino-connector/development.md b/docs/trino-connector/development.md index 6b1fb79ffe..b507999747 100644 --- a/docs/trino-connector/development.md +++ b/docs/trino-connector/development.md @@ -1,18 +1,43 @@ --- title: "Apache Gravitino Trino connector development" slug: /trino-connector/development -keyword: gravitino connector development +keyword: gravitino connector development license: "This software is licensed under the Apache License version 2." --- -This document is to guide users through the development of the Apache Gravitino Trino connector for Trino locally. +This document guides you through developing the Apache Gravitino Trino connector locally. + +## Multi-version architecture + +The Gravitino Trino connector supports multiple Trino versions (see [Requirements](requirements.md)). The source code is organized into a shared base module and several version-segment modules: + +```text +trino-connector/ +├── trino-connector/ # Shared base source code +│ └── src/main/java/ # Common implementation used by all versions +├── trino-connector-435-439/ # Version-specific adapters for Trino 435-439 +│ └── src/main/java/ +├── trino-connector-440-445/ # Version-specific adapters for Trino 440-445 +│ └── src/main/java/ +├── trino-connector-446-451/ # Version-specific adapters for Trino 446-451 +│ └── src/main/java/ +├── trino-connector-452-468/ # Version-specific adapters for Trino 452-468 +│ └── src/main/java/ +├── trino-connector-469-472/ # Version-specific adapters for Trino 469-472 +│ └── src/main/java/ +└── integration-test/ # Integration tests +``` + +Each version-segment module includes the shared base source via Gradle `sourceSets` and adds version-specific adapter classes (e.g., `GravitinoConnector469.java`, `GravitinoPlugin469.java`) to handle Trino SPI differences across versions. + +When developing against a specific Trino version in the Trino project, you need to include **both** the shared base source and the matching version-segment source as source directories in the Maven `pom.xml`. ## Prerequisites -Before you start developing the Gravitino Trino connector, you need to have the following prerequisites: +Before you start, ensure the following: -1. You need to start the Gravitino server locally, for more information, please refer to the [start Gravitino server](../how-to-install.md) -2. Create a catalog in the Gravitino server, for more information, please refer to the [Gravitino metadata management](../manage-relational-metadata-using-gravitino.md). Assuming we have just created a MySQL catalog using the following command: +1. Start the Gravitino server locally. For more information, refer to [How to install](../how-to-install.md). +2. Create a catalog in the Gravitino server. For more information, refer to [Gravitino metadata management](../manage-relational-metadata-using-gravitino.md). For example, create a MySQL catalog: ```curl curl -X POST -H "Content-Type: application/json" -d '{"name":"test","comment":"comment","properties":{}}' http://localhost:8090/api/metalakes @@ -26,29 +51,31 @@ curl -X POST -H "Content-Type: application/json" -d '{"name":"mysql_catalog3","t ``` :::note -Please change the above `localhost`, `port` and the names of metalake and catalogs accordingly. +Change `localhost`, `port`, and the names of metalake and catalogs to match your environment. ::: - ## Development environment -To develop the Gravitino Trino connector locally, you need to do the following steps: - ### IDEA -1. Clone the Trino repository from the [GitHub](https://github.com/trinodb/trino) repository. The released version Trino-435 is the least version that Gravitino supports. -2. Open the Trino project in your IDEA. -3. Create a new module for the Gravitino Trino connector in the Trino project as the following picture (we will use the name `trino-gravitino` as the module name in the following steps).  -4. Add a soft link to the Gravitino Trino connector module in the Trino project. Assuming the src java main directory of the Gravitino Trino connector in project Gravitino is `gravitino/path/to/gravitino-trino-connector/src/main/java`, and the src java main directory of trino-gravitino in the Trino project is `trino/path/to/trino-gravitino/src/main/java`, you can use the following command to create a soft link: +1. Clone the Trino repository from [GitHub](https://github.com/trinodb/trino). This document uses Trino `469` by default. Check out the tag matching your target Trino version (e.g., `git checkout 469`). +2. Open the Trino project in IDEA. +3. Create a new module named `trino-gravitino` in the Trino project as shown below: +  -```shell -ln -s gravitino/path/to/trino-connector/src/main/java trino/path/to/trino-gravitino/src/main/java -``` -then you can see the `gravitino-trino-connecor` source files and directories in the `trino-gravitino` module as follows: +4. Identify which version-segment module matches your Trino version: + + | Trino Version | Version-Segment Module | + |---------------|------------------------| + | 435-439 | `trino-connector-435-439` | + | 440-445 | `trino-connector-440-445` | + | 446-451 | `trino-connector-446-451` | + | 452-468 | `trino-connector-452-468` | + | 469-472 | `trino-connector-469-472` | - +5. Add `<module>plugin/trino-gravitino</module>` to `trino/pom.xml` and create the `pom.xml` for the `trino-gravitino` module. The example below uses Trino `469`. Ensure the `trino-root` version matches the Trino version you are developing against. -5. Add `<module>plugin/trino-gravitino</module>` to `trino/pom.xml` and change the `pom.xml` file in the `trino-gravitino` module accordingly. This is an example content of the `pom.xml` file in the `trino-gravitino` module. Ensure that the version of trino-root is identical to the version of trino. + The `<build>` section uses `build-helper-maven-plugin` to add the Gravitino source directories directly. Replace `/path/to/gravitino` with the absolute path of your local Gravitino project root, and change `trino-connector-469-472` to the version-segment module from step 4. ```xml <?xml version="1.0" encoding="UTF-8"?> @@ -57,7 +84,7 @@ then you can see the `gravitino-trino-connecor` source files and directories in <parent> <groupId>io.trino</groupId> <artifactId>trino-root</artifactId> - <version>435</version> + <version>469</version> <relativePath>../../pom.xml</relativePath> </parent> @@ -72,9 +99,8 @@ then you can see the `gravitino-trino-connecor` source files and directories in <dependencies> <!-- - You can switch to the snapshot version as you like, for example, - if you want to use the jar of latest main branch, - you can execute the following command to install Gravitino `client-java-runtime` jar locally. + You can use the snapshot version. For example, to use the jar from + the latest main branch, run the following command in the Gravitino project: ./gradlew publishToMavenLocal --> <dependency> @@ -190,99 +216,125 @@ then you can see the `gravitino-trino-connecor` source files and directories in </dependencies> + <build> + <plugins> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>build-helper-maven-plugin</artifactId> + <executions> + <execution> + <id>add-source</id> + <phase>generate-sources</phase> + <goals> + <goal>add-source</goal> + </goals> + <configuration> + <sources> + <!-- Shared base source --> + <source>/path/to/gravitino/trino-connector/trino-connector/src/main/java</source> + <!-- Version-segment source (change to match your Trino version) --> + <source>/path/to/gravitino/trino-connector/trino-connector-469-472/src/main/java</source> + </sources> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </project> ``` -6. Try to compile module `trino-gravitino` to see if there are any errors. +6. Try to compile the `trino-gravitino` module to check for errors: + ```shell -# build the whole trino project +# Build the whole Trino project ./mvnw -pl '!core/trino-server-rpm' package -DskipTests -Dair.check.skip-all=true - -# build the trino-gravitino module if we change the code in the trino-gravitino module -./mvnw clean -pl 'plugin/trino-gravitino' package -DskipTests -Dcheckstyle.skip -Dair.check.skip-checkstyle=true -DskipTests -Dair.check.skip-all=true +# Build only the trino-gravitino module +./mvnw clean -pl 'plugin/trino-gravitino' package -DskipTests -Dair.check.skip-all=true ``` + :::note -If a compile error occurs due to `The following artifacts could not be resolved: org.apache.gravitino:xxx:jar`, which can be resolved by executing `./gradlew publishToMavenLocal` in gravitino beforehand. +If a compile error occurs due to `The following artifacts could not be resolved: org.apache.gravitino:xxx:jar`, run `./gradlew publishToMavenLocal` in the Gravitino project first. ::: -7. Set up the configuration for the Gravitino Trino connector in the Trino project. You can do as the following picture shows: - +7. Set up the configuration for the Gravitino Trino connector in the Trino project as shown below: +  -The corresponding configuration files are here: + The corresponding configuration files: -- Gravitino properties file: `gravitino.properties` -```properties -# the connector name is always 'gravitino' -connector.name=gravitino + - Gravitino properties file: `gravitino.properties` -# uri of the gravitino server, you need to change it according to your environment -gravitino.uri=http://localhost:8090 + ```properties + connector.name=gravitino + gravitino.uri=http://localhost:8090 + gravitino.metalake=test + ``` -# The name of the metalake to which the connector is connected, you need to change it according to your environment -gravitino.metalake=test + - Trino configuration file: `config.properties` -``` -- Trino configuration file: `config.properties` -```properties -# -# WARNING -# ^^^^^^^ -# This configuration file is for development only and should NOT be used -# in production. For example configuration, see the Trino documentation. -# sample nodeId to provide consistency across test runs -node.id=ffffffff-ffff-ffff-ffff-ffffffffffff -node.environment=test -node.internal-address=localhost -experimental.concurrent-startup=true - -# Default port is 8080, We change it to 8180 -http-server.http.port=8180 - -discovery.uri=http://localhost:8180 - -exchange.http-client.max-connections=1000 -exchange.http-client.max-connections-per-server=1000 -exchange.http-client.connect-timeout=1m -exchange.http-client.idle-timeout=1m - -scheduler.http-client.max-connections=1000 -scheduler.http-client.max-connections-per-server=1000 -scheduler.http-client.connect-timeout=1m -scheduler.http-client.idle-timeout=1m - -query.client.timeout=5m -query.min-expire-age=30m - -# We removed several catalogs that won't be used in Gravitino -plugin.bundles=\ - ../../plugin/trino-iceberg/pom.xml,\ - ../../plugin/trino-hive/pom.xml,\ - ../../plugin/trino-local-file/pom.xml, \ - ../../plugin/trino-mysql/pom.xml,\ - ../../plugin/trino-postgresql/pom.xml, \ - ../../plugin/trino-exchange-filesystem/pom.xml, \ - ../../plugin/trino-gravitino/pom.xml - -node-scheduler.include-coordinator=true - -# Note: The Gravitino Trino connector olny supports with The dynamic catalog manager -catalog.management=dynamic -``` + ```properties + # + # WARNING + # ^^^^^^^ + # This configuration file is for development only and should NOT be used + # in production. For example configuration, see the Trino documentation. + # + node.id=ffffffff-ffff-ffff-ffff-ffffffffffff + node.environment=test + node.internal-address=localhost + experimental.concurrent-startup=true -:::note -Remove the file `/etc/catalogs/xxx.properties` if the corresponding `plugin/trino-xxx/pom.xml` is not recorded in the `/etc/config.properties`. For the hive plugin, please use `plugin/trino-hive/pom.xml` after release version 435. Others should use `plugin/trino-hive-hadoop2/pom.xml`. -::: + # Default port is 8080, change it to 8180 to avoid conflicts + http-server.http.port=8180 + + discovery.uri=http://localhost:8180 + + exchange.http-client.max-connections=1000 + exchange.http-client.max-connections-per-server=1000 + exchange.http-client.connect-timeout=1m + exchange.http-client.idle-timeout=1m + + scheduler.http-client.max-connections=1000 + scheduler.http-client.max-connections-per-server=1000 + scheduler.http-client.connect-timeout=1m + scheduler.http-client.idle-timeout=1m + + query.client.timeout=5m + query.min-expire-age=30m + + plugin.bundles=\ + ../../plugin/trino-iceberg/pom.xml,\ + ../../plugin/trino-hive/pom.xml,\ + ../../plugin/trino-local-file/pom.xml,\ + ../../plugin/trino-mysql/pom.xml,\ + ../../plugin/trino-postgresql/pom.xml,\ + ../../plugin/trino-exchange-filesystem/pom.xml,\ + ../../plugin/trino-gravitino/pom.xml + + node-scheduler.include-coordinator=true + + # The Gravitino Trino connector only supports the dynamic catalog manager + catalog.management=dynamic + ``` + + :::note + Remove `/etc/catalogs/xxx.properties` if the corresponding `plugin/trino-xxx/pom.xml` is not listed in `plugin.bundles`. For the Hive plugin, use `plugin/trino-hive/pom.xml` for Trino 435 and later; for earlier versions, use `plugin/trino-hive-hadoop2/pom.xml`. + ::: 8. Start the Trino server and connect to the Gravitino server. - -9. If `DevelopmentServer` has started successfully, you can connect to the Trino server using the `trino-cli` and run the following command to see if the Gravitino Trino connector is available: -```shell -java -jar trino-cli-429-executable.jar --server localhost:8180 -``` -:::note -The `trino-cli-429-executable.jar` is the Trino CLI jar file, you can download it from the [Trino release page](https://trino.io/docs/current/client/cli.html). **Users can use the version of the Trino CLI jar file according to the version of the Trino server.** -::: +  + +9. Once `DevelopmentServer` starts successfully, connect to the Trino server using the Trino CLI: + + ```shell + java -jar trino-cli-*-executable.jar --server localhost:8180 + ``` + + :::note + Download the `trino-cli` jar from the [Trino release page](https://trino.io/docs/469/client/cli.html). Use the CLI version that matches your Trino server version. + ::: -10. If nothing goes wrong, you can start developing the Gravitino Trino connector in the Gravitino project and debug it in the Trino project. - +10. You can now develop the Gravitino Trino connector in the Gravitino project and debug it in the Trino project. +  diff --git a/docs/trino-connector/installation.md b/docs/trino-connector/installation.md index ef2dce574a..cfd7077663 100644 --- a/docs/trino-connector/installation.md +++ b/docs/trino-connector/installation.md @@ -8,10 +8,58 @@ license: "This software is licensed under the Apache License version 2." To install the Apache Gravitino Trino connector, you should first deploy the Trino environment, and then install the Gravitino Trino connector plugin into Trino. Please refer to the [Deploying Trino documentation](https://trino.io/docs/current/installation/deployment.html) and do the following steps: -1. [Download](https://github.com/apache/gravitino/releases) the Gravitino Trino connector tarball and unpack it. - The tarball contains a single top-level directory `gravitino-trino-connector-<version>`, - which called the connector directory. -2. Copy the connector directory to the Trino's plugin directory. +The examples in this document use Trino `469` by default. + +## Get the connector package + +Gravitino provides different Trino connector packages for different Trino version segments. +You need to choose the package by your Trino server version first. + +| Trino server version | Connector package segment | +|----------------------|---------------------------| +| 435-439 | `trino-connector-435-439` | +| 440-445 | `trino-connector-440-445` | +| 446-451 | `trino-connector-446-451` | +| 452-468 | `trino-connector-452-468` | +| 469-472 | `trino-connector-469-472` | + +For Trino `469`, choose the `trino-connector-469-472` package. + +The release package naming format is: + +`gravitino-trino-connector-<segment>-<version>.tar.gz` + +For example: + +`gravitino-trino-connector-469-472-<version>.tar.gz` + +You can get the package in the following ways: + +1. Download a released package from [GitHub Releases](https://github.com/apache/gravitino/releases). For Trino `469`, download the `469-472` package: + +```shell +cd /tmp +wget https://github.com/apache/gravitino/releases/download/<version>/gravitino-trino-connector-469-472-<version>.tar.gz +tar -zxvf gravitino-trino-connector-469-472-<version>.tar.gz +``` + +2. Build from source in the Gravitino project. For Trino `469`, build module `trino-connector-469-472`: + +```shell +cd <gravitino-source-root> +./gradlew :trino-connector:trino-connector-469-472:assembleTrinoConnector +cd distribution +tar -zxvf gravitino-trino-connector-469-472-<version>.tar.gz +``` + +After unpacking, you can see the connector directory: + +`gravitino-trino-connector-469-472-<version>` + +## Install the connector package + +1. Download and unpack the correct Gravitino Trino connector tarball for your Trino version. +2. Rename the unpacked connector directory to `gravitino`, and then copy it to the Trino plugin directory. Normally, the directory location is `Trino-server-<version>/plugin`, and the directory contains other catalogs used by Trino. 3. Add Trino JVM arguments `-Dlog4j.configurationFile=file:////etc/trino/log4j2.properties` to enable logging for the Gravitino Trino connector. 4. Update Trino coordinator configuration. @@ -25,11 +73,6 @@ catalog.management=dynamic discovery.uri=http://0.0.0.0:8080 ``` -Alternatively, -you can build the Gravitino Trino connector package from the sources -and obtain the `gravitino-trino-connector-<version>.tar.gz` file in the `$PROJECT/distribution` directory. -Please refer to the [Gravitino Development documentation](../how-to-build.md) - ## Example You can install the Gravitino Trino connector in Trino office docker images step by step. @@ -40,7 +83,7 @@ Use the docker command to create a container from the `trinodb/trino` image. Ass Run it in the background, and map the default Trino port, which is 8080, from inside the container to port 8080 on your machine. ```shell -docker run --name trino-gravitino -d -p 8080:8080 trinodb/trino:435 +docker run --name trino-gravitino -d -p 8080:8080 trinodb/trino:469 ``` Run `docker ps` to check whether the container is running. @@ -48,20 +91,21 @@ Run `docker ps` to check whether the container is running. ### Installing the Apache Gravitino Trino connector -Download the Gravitino Trino connector tarball and unpack it. +Download the Gravitino Trino connector tarball for Trino `469` and unpack it. ```shell cd /tmp -wget https://github.com/apache/gravitino/releases/gravitino-trino-connector-<version>.tar.gz -tar -zxvf gravitino-trino-connector-<version>.tar.gz +wget https://github.com/apache/gravitino/releases/download/<version>/gravitino-trino-connector-469-472-<version>.tar.gz +tar -zxvf gravitino-trino-connector-469-472-<version>.tar.gz ``` -You can see the connector directory `gravitino-trino-connector-<version>` after unpacking. +You can see the connector directory `gravitino-trino-connector-469-472-<version>` after unpacking. -Copy the connector directory to the Trino container's plugin directory. +Rename the connector directory to `gravitino`, then copy it to the Trino container's plugin directory. ```shell -docker cp /tmp/gravitino-trino-connector-<version> trino-gravitino:/lib/trino/plugin +mv /tmp/gravitino-trino-connector-469-472-<version> /tmp/gravitino +docker cp /tmp/gravitino trino-gravitino:/lib/trino/plugin ``` Check the plugin directory in the container. diff --git a/docs/trino-connector/requirements.md b/docs/trino-connector/requirements.md index 5c1d7c7541..21a3d12997 100644 --- a/docs/trino-connector/requirements.md +++ b/docs/trino-connector/requirements.md @@ -7,9 +7,10 @@ license: "This software is licensed under the Apache License version 2." To install and deploy the Apache Gravitino Trino connector, The following environmental setup is necessary: -- Trino server version should be between Trino-server-435 and Trino-server-439. - If you use other versions of Trino, you can set the `gravitino.trino.skip-version-validation` to `true`. - Other versions of Trino have not undergone thorough testing. +- Trino server version should be between Trino-server-435 and Trino-server-472. + The examples in this document use Trino `469` by default. +- If you use an unsupported Trino version, you can set `gravitino.trino.skip-version-validation` to `true`. + Unsupported versions have not undergone thorough testing. - Ensure that all nodes running Trino can access the Gravitino server's port, which defaults to 8090. - Ensure that all nodes running Trino can access the real catalogs resources, such as Hive, Iceberg, MySQL, PostgreSQL, etc. - Ensure that you have installed the following connectors in Trino: Hive, Iceberg, MySQL, PostgreSQL. diff --git a/docs/trino-connector/trino-connector.md b/docs/trino-connector/trino-connector.md index 6f22fe7b4f..8033436190 100644 --- a/docs/trino-connector/trino-connector.md +++ b/docs/trino-connector/trino-connector.md @@ -12,6 +12,8 @@ The Gravitino Trino connector uses the [Trino dynamic catalog managed mechanism] When the Gravitino Trino connector retrieves catalogs from the Gravitino server, it generates a `CREATE CATALOG` statement and executes the statement on the current Trino server to register the catalogs with Trino +The connector supports multiple Trino versions. For the supported version ranges, see [Requirements](requirements.md). The examples in this document set Trino `469` as the default. + :::note Once metadata such as catalogs are changed in Gravitino, Trino can update itself through Gravitino, this process usually takes about 3~10 seconds. @@ -29,4 +31,3 @@ Usage in queries is as follows: SELECT * from catalog.dbname.tablename ``` - diff --git a/integration-test-common/docker-script/docker-compose.yaml b/integration-test-common/docker-script/docker-compose.yaml index 37cc54cbb9..ddbf976a2e 100644 --- a/integration-test-common/docker-script/docker-compose.yaml +++ b/integration-test-common/docker-script/docker-compose.yaml @@ -79,7 +79,7 @@ services: retries: 5 trino: - image: trinodb/trino:${TRINO_VERSION:-452} + image: trinodb/trino:${TRINO_VERSION:-469} networks: - trino-net container_name: trino-ci-trino @@ -94,7 +94,7 @@ services: entrypoint: /bin/bash /tmp/trino/init.sh volumes: - ./init/trino:/tmp/trino - - ${GRAVITINO_TRINO_CONNECTOR_DIR:-../../trino-connector/trino-connector-452-468/build/libs}:/usr/lib/trino/plugin/gravitino + - ${GRAVITINO_TRINO_CONNECTOR_DIR:-../../trino-connector/trino-connector-469-472/build/libs}:/usr/lib/trino/plugin/gravitino extra_hosts: - "host.docker.internal:host-gateway" healthcheck: @@ -113,7 +113,7 @@ services: condition: service_healthy trino-worker: - image: trinodb/trino:${TRINO_VERSION:-452} + image: trinodb/trino:${TRINO_VERSION:-469} networks: - trino-net deploy: @@ -128,7 +128,7 @@ services: entrypoint: /bin/bash /tmp/trino/init.sh volumes: - ./init/trino:/tmp/trino - - ${GRAVITINO_TRINO_CONNECTOR_DIR:-../../trino-connector/trino-connector-452-468/build/libs}:/usr/lib/trino/plugin/gravitino + - ${GRAVITINO_TRINO_CONNECTOR_DIR:-../../trino-connector/trino-connector-469-472/build/libs}:/usr/lib/trino/plugin/gravitino extra_hosts: - "host.docker.internal:host-gateway" depends_on: diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TrinoITContainers.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TrinoITContainers.java index fb808c3f59..a02e12de6a 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TrinoITContainers.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TrinoITContainers.java @@ -72,7 +72,7 @@ public class TrinoITContainers implements AutoCloseable { } if (trinoConnectorDir != null) { File dir = new File(trinoConnectorDir); - if (!dir.exists() || dir.list().length == 0) { + if (!dir.exists() || !dir.isDirectory() || dir.list().length == 0) { throw new Exception( "Gravitino trino connector directory %s does not exist or is empty" .formatted(trinoConnectorDir)); diff --git a/settings.gradle.kts b/settings.gradle.kts index 71a7bc776e..8ec52fc206 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -73,6 +73,7 @@ include( "trino-connector:trino-connector-440-445", "trino-connector:trino-connector-446-451", "trino-connector:trino-connector-452-468", + "trino-connector:trino-connector-469-472", "trino-connector:integration-test" ) include("spark-connector:spark-common") diff --git a/trino-connector/integration-test/build.gradle.kts b/trino-connector/integration-test/build.gradle.kts index fa0e8cd36b..7ec53676fd 100644 --- a/trino-connector/integration-test/build.gradle.kts +++ b/trino-connector/integration-test/build.gradle.kts @@ -75,6 +75,7 @@ tasks.register("setupDependencies") { dependsOn(":trino-connector:trino-connector-440-445:copyLibs") dependsOn(":trino-connector:trino-connector-446-451:copyLibs") dependsOn(":trino-connector:trino-connector-452-468:copyLibs") + dependsOn(":trino-connector:trino-connector-469-472:copyLibs") dependsOn(":catalogs:catalog-lakehouse-iceberg:jar", ":catalogs:catalog-lakehouse-iceberg:runtimeJars") dependsOn(":catalogs:catalog-jdbc-mysql:jar", ":catalogs:catalog-jdbc-mysql:runtimeJars") dependsOn(":catalogs:catalog-jdbc-postgresql:jar", ":catalogs:catalog-jdbc-postgresql:runtimeJars") diff --git a/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/catalog_iceberg_cleanup.sql b/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/catalog_iceberg_hive_cleanup.sql similarity index 100% rename from trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/catalog_iceberg_cleanup.sql rename to trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/catalog_iceberg_hive_cleanup.sql diff --git a/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/catalog_iceberg_prepare.sql b/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/catalog_iceberg_hive_prepare.sql similarity index 100% rename from trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/catalog_iceberg_prepare.sql rename to trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/catalog_iceberg_hive_prepare.sql diff --git a/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00011_table_statistics.sql b/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/ignored/00011_table_statistics.sql similarity index 100% rename from trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00011_table_statistics.sql rename to trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/ignored/00011_table_statistics.sql diff --git a/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00011_table_statistics.txt b/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/ignored/00011_table_statistics.txt similarity index 100% rename from trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/00011_table_statistics.txt rename to trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/ignored/00011_table_statistics.txt diff --git a/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/catalog_iceberg_mysql_cleanup.sql b/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/ignored/catalog_iceberg_mysql_cleanup.sql similarity index 100% rename from trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/catalog_iceberg_mysql_cleanup.sql rename to trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/ignored/catalog_iceberg_mysql_cleanup.sql diff --git a/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/catalog_iceberg_mysql_prepare.sql b/trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/ignored/catalog_iceberg_mysql_prepare.sql similarity index 100% rename from trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/catalog_iceberg_mysql_prepare.sql rename to trino-connector/integration-test/src/test/resources/trino-ci-testset/testsets/lakehouse-iceberg/ignored/catalog_iceberg_mysql_prepare.sql diff --git a/trino-connector/trino-connector-469-472/build.gradle.kts b/trino-connector/trino-connector-469-472/build.gradle.kts new file mode 100644 index 0000000000..ef3857a23c --- /dev/null +++ b/trino-connector/trino-connector-469-472/build.gradle.kts @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import com.diffplug.gradle.spotless.SpotlessExtension +import net.ltgt.gradle.errorprone.errorprone +import org.gradle.internal.hash.ChecksumService +import org.gradle.kotlin.dsl.support.serviceOf + +plugins { + `java-library` + `maven-publish` +} + +// This module supports Trino versions 469-472 +val minTrinoVersion = 469 +val maxTrinoVersion = 472 +val otelSemconvVersion = "1.32.0" + +val trinoVersion = providers.gradleProperty("trinoVersion") + .map { it.trim().toInt() } + .orElse(minTrinoVersion) + .get() + +// Validate version range +check(trinoVersion in minTrinoVersion..maxTrinoVersion) { + "Module ${project.path} supports Trino versions $minTrinoVersion-$maxTrinoVersion, " + + "but trinoVersion=$trinoVersion was specified. " + + "Please set '-PtrinoVersion=$minTrinoVersion' (or any version in the supported range)." +} + +java { + toolchain.languageVersion.set(JavaLanguageVersion.of(24)) +} + +dependencies { + implementation(project(":catalogs:catalog-common")) + implementation(project(":clients:client-java-runtime", configuration = "shadow")) + + compileOnly(project(":api")) + compileOnly(project(":clients:client-java")) + compileOnly(project(":common")) + + implementation(libs.airlift.json) + implementation(libs.bundles.log4j) + implementation(libs.commons.collections4) + implementation(libs.commons.lang3) + implementation("io.trino:trino-jdbc:$trinoVersion") + runtimeOnly("io.opentelemetry.semconv:opentelemetry-semconv:$otelSemconvVersion") + compileOnly(libs.airlift.resolver) + compileOnly("io.trino:trino-spi:$trinoVersion") { + exclude("org.apache.logging.log4j") + } + testImplementation(libs.awaitility) + testImplementation(libs.mockito.core) + testImplementation(libs.mysql.driver) + testImplementation("io.trino:trino-memory:$trinoVersion") { + exclude("org.antlr") + exclude("org.apache.logging.log4j") + } + testImplementation("io.trino:trino-testing:$trinoVersion") { + exclude("org.apache.logging.log4j") + } + testRuntimeOnly(libs.junit.jupiter.engine) +} + +sourceSets { + main { + java.srcDirs("../trino-connector/src/main/java") + } + test { + java.srcDirs("../trino-connector/src/test/java") + resources.srcDirs("../trino-connector/src/test/resources") + } +} + +plugins.withId("com.diffplug.spotless") { + configure<SpotlessExtension> { + java { + // Keep Spotless within this module to avoid cross-project target errors. + target(project.fileTree("src") { include("**/*.java") }) + } + } +} + +tasks.withType<JavaCompile>().configureEach { + // Error Prone is incompatible with the JDK 24 toolchain required by this Trino range. + options.errorprone.isEnabled.set(false) + options.release.set(17) +} + +tasks.withType<Test>().configureEach { + extensions + .findByType(org.gradle.testing.jacoco.plugins.JacocoTaskExtension::class.java) + ?.isEnabled = false +} + +tasks { + val copyRuntimeLibs by registering(Copy::class) { + dependsOn("jar") + from({ configurations.runtimeClasspath.get().filter(File::isFile) }) + into(layout.buildDirectory.dir("libs")) + } + + val distributionDir = rootProject.layout.projectDirectory.dir("distribution/${rootProject.name}-${project.name}") + + val copyLibs by registering(Copy::class) { + dependsOn(copyRuntimeLibs, "build") + from(layout.buildDirectory.dir("libs")) + from(rootProject.layout.projectDirectory.dir("licenses")) { + into("licenses") + } + from(rootProject.file("LICENSE.trino")) + from(rootProject.file("NOTICE.trino")) + from(rootProject.file("README.md")) + into(distributionDir) + rename { fileName -> + fileName.replace(".trino", "") + } + outputs.dir(distributionDir) + } + + val assembleTrinoConnector by registering(Tar::class) { + dependsOn(copyLibs) + group = "gravitino distribution" + finalizedBy("checksumTrinoConnector") + val archiveBase = "${rootProject.name}-${project.name}-$version" + into(archiveBase) + from(distributionDir) + compression = Compression.GZIP + archiveFileName.set("$archiveBase.tar.gz") + destinationDirectory.set(rootProject.layout.projectDirectory.dir("distribution")) + } + + val checksumTrinoConnector by registering { + group = "gravitino distribution" + dependsOn(assembleTrinoConnector) + val archiveFile = assembleTrinoConnector.flatMap { it.archiveFile } + val checksumFile = archiveFile.map { archive -> + archive.asFile.let { it.resolveSibling("${it.name}.sha256") } + } + inputs.file(archiveFile) + outputs.file(checksumFile) + doLast { + checksumFile.get().writeText( + serviceOf<ChecksumService>().sha256(archiveFile.get().asFile).toString() + ) + } + } + + named("build") { + finalizedBy(copyRuntimeLibs) + } +} diff --git a/trino-connector/trino-connector-469-472/src/main/java/org/apache/gravitino/trino/connector/GravitinoConnector469.java b/trino-connector/trino-connector-469-472/src/main/java/org/apache/gravitino/trino/connector/GravitinoConnector469.java new file mode 100644 index 0000000000..eba983b738 --- /dev/null +++ b/trino-connector/trino-connector-469-472/src/main/java/org/apache/gravitino/trino/connector/GravitinoConnector469.java @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.trino.connector; + +import io.trino.spi.connector.Connector; +import io.trino.spi.connector.ConnectorMetadata; +import io.trino.spi.connector.ConnectorNodePartitioningProvider; +import io.trino.spi.connector.ConnectorPageSourceProviderFactory; +import io.trino.spi.connector.ConnectorSplitManager; +import org.apache.gravitino.trino.connector.catalog.CatalogConnectorContext; +import org.apache.gravitino.trino.connector.catalog.CatalogConnectorMetadata; +import org.apache.gravitino.trino.connector.catalog.CatalogConnectorMetadataAdapter; + +public class GravitinoConnector469 extends GravitinoConnector { + + public GravitinoConnector469(CatalogConnectorContext connectorContext) { + super(connectorContext); + } + + @Override + protected GravitinoMetadata createGravitinoMetadata( + CatalogConnectorMetadata catalogConnectorMetadata, + CatalogConnectorMetadataAdapter metadataAdapter, + ConnectorMetadata internalMetadata) { + return new GravitinoMetadata469(catalogConnectorMetadata, metadataAdapter, internalMetadata); + } + + @Override + public ConnectorSplitManager getSplitManager() { + ConnectorSplitManager splitManager = + catalogConnectorContext.getInternalConnector().getSplitManager(); + return new GravitinoSplitManager469(splitManager); + } + + @Override + public ConnectorNodePartitioningProvider getNodePartitioningProvider() { + ConnectorNodePartitioningProvider nodePartitioningProvider = + catalogConnectorContext.getInternalConnector().getNodePartitioningProvider(); + return new GravitinoNodePartitioningProvider469(nodePartitioningProvider); + } + + @Override + public ConnectorPageSourceProviderFactory getPageSourceProviderFactory() { + Connector internalConnector = catalogConnectorContext.getInternalConnector(); + ConnectorPageSourceProviderFactory internalConnectorPageSourceProviderFactory = + internalConnector.getPageSourceProviderFactory(); + return () -> + new GravitinoDataSourceProvider( + internalConnectorPageSourceProviderFactory.createPageSourceProvider()); + } +} diff --git a/trino-connector/trino-connector-469-472/src/main/java/org/apache/gravitino/trino/connector/GravitinoConnectorFactory469.java b/trino-connector/trino-connector-469-472/src/main/java/org/apache/gravitino/trino/connector/GravitinoConnectorFactory469.java new file mode 100644 index 0000000000..b365c81412 --- /dev/null +++ b/trino-connector/trino-connector-469-472/src/main/java/org/apache/gravitino/trino/connector/GravitinoConnectorFactory469.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.trino.connector; + +import org.apache.gravitino.client.GravitinoAdminClient; +import org.apache.gravitino.trino.connector.catalog.CatalogConnectorContext; +import org.apache.gravitino.trino.connector.system.GravitinoSystemConnector; +import org.apache.gravitino.trino.connector.system.storedprocedure.GravitinoStoredProcedureFactory; + +public class GravitinoConnectorFactory469 extends GravitinoConnectorFactory { + + public GravitinoConnectorFactory469(GravitinoAdminClient client) { + super(client); + } + + @Override + protected int getMinSupportTrinoSpiVersion() { + return 469; + } + + @Override + protected int getMaxSupportTrinoSpiVersion() { + return 472; + } + + @Override + protected String getTrinoCatalogName(String metalake, String catalog) { + return "\"" + metalake + "." + catalog + "\""; + } + + @Override + protected boolean supportCatalogNameWithMetalake() { + return false; + } + + @Override + protected GravitinoConnector createConnector(CatalogConnectorContext connectorContext) { + return new GravitinoConnector469(connectorContext); + } + + @Override + protected GravitinoSystemConnector createSystemConnector( + GravitinoStoredProcedureFactory storedProcedureFactory) { + return new GravitinoSystemConnector469(storedProcedureFactory); + } +} diff --git a/trino-connector/trino-connector-469-472/src/main/java/org/apache/gravitino/trino/connector/GravitinoMetadata469.java b/trino-connector/trino-connector-469-472/src/main/java/org/apache/gravitino/trino/connector/GravitinoMetadata469.java new file mode 100644 index 0000000000..27307f2036 --- /dev/null +++ b/trino-connector/trino-connector-469-472/src/main/java/org/apache/gravitino/trino/connector/GravitinoMetadata469.java @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.trino.connector; + +import io.airlift.slice.Slice; +import io.trino.spi.connector.ColumnHandle; +import io.trino.spi.connector.ColumnMetadata; +import io.trino.spi.connector.ColumnPosition; +import io.trino.spi.connector.ConnectorInsertTableHandle; +import io.trino.spi.connector.ConnectorMergeTableHandle; +import io.trino.spi.connector.ConnectorOutputMetadata; +import io.trino.spi.connector.ConnectorSession; +import io.trino.spi.connector.ConnectorTableHandle; +import io.trino.spi.connector.RetryMode; +import io.trino.spi.connector.SchemaTableName; +import io.trino.spi.statistics.ComputedStatistics; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import org.apache.gravitino.trino.connector.catalog.CatalogConnectorMetadata; +import org.apache.gravitino.trino.connector.catalog.CatalogConnectorMetadataAdapter; +import org.apache.gravitino.trino.connector.metadata.GravitinoColumn; + +public class GravitinoMetadata469 extends GravitinoMetadata { + + public GravitinoMetadata469( + CatalogConnectorMetadata catalogConnectorMetadata, + CatalogConnectorMetadataAdapter metadataAdapter, + io.trino.spi.connector.ConnectorMetadata internalMetadata) { + super(catalogConnectorMetadata, metadataAdapter, internalMetadata); + } + + @Override + public void addColumn( + ConnectorSession session, + ConnectorTableHandle tableHandle, + ColumnMetadata column, + ColumnPosition position) { + GravitinoColumn gravitinoColumn = metadataAdapter.createColumn(column); + catalogConnectorMetadata.addColumn(getTableName(tableHandle), gravitinoColumn); + } + + @Override + public Optional<ConnectorOutputMetadata> finishInsert( + ConnectorSession session, + ConnectorInsertTableHandle insertHandle, + List<ConnectorTableHandle> sourceTableHandles, + Collection<Slice> fragments, + Collection<ComputedStatistics> computedStatistics) { + return internalMetadata.finishInsert( + session, + GravitinoHandle.unWrap(insertHandle), + sourceTableHandles.stream().map(GravitinoHandle::unWrap).collect(Collectors.toList()), + fragments, + computedStatistics); + } + + @Override + public ConnectorMergeTableHandle beginMerge( + ConnectorSession session, + ConnectorTableHandle tableHandle, + Map<Integer, Collection<ColumnHandle>> updateCaseColumns, + RetryMode retryMode) { + ConnectorMergeTableHandle connectorMergeTableHandle = + internalMetadata.beginMerge( + session, GravitinoHandle.unWrap(tableHandle), updateCaseColumns, retryMode); + SchemaTableName tableName = getTableName(tableHandle); + + return new GravitinoMergeTableHandle( + tableName.getSchemaName(), tableName.getTableName(), connectorMergeTableHandle); + } + + @Override + public void finishMerge( + ConnectorSession session, + ConnectorMergeTableHandle mergeTableHandle, + List<ConnectorTableHandle> sourceTableHandles, + Collection<Slice> fragments, + Collection<ComputedStatistics> computedStatistics) { + internalMetadata.finishMerge( + session, + GravitinoHandle.unWrap(mergeTableHandle), + sourceTableHandles.stream().map(GravitinoHandle::unWrap).collect(Collectors.toList()), + fragments, + computedStatistics); + } +} diff --git a/trino-connector/trino-connector-469-472/src/main/java/org/apache/gravitino/trino/connector/GravitinoNodePartitioningProvider469.java b/trino-connector/trino-connector-469-472/src/main/java/org/apache/gravitino/trino/connector/GravitinoNodePartitioningProvider469.java new file mode 100644 index 0000000000..1af2fe7f02 --- /dev/null +++ b/trino-connector/trino-connector-469-472/src/main/java/org/apache/gravitino/trino/connector/GravitinoNodePartitioningProvider469.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.trino.connector; + +import io.trino.spi.connector.ConnectorNodePartitioningProvider; + +/** + * Trino 469+ NodePartitioningProvider implementation with the new split bucket function signature. + */ +public class GravitinoNodePartitioningProvider469 extends GravitinoNodePartitioningProvider { + + public GravitinoNodePartitioningProvider469( + ConnectorNodePartitioningProvider nodePartitioningProvider) { + super(nodePartitioningProvider); + } +} diff --git a/trino-connector/trino-connector-469-472/src/main/java/org/apache/gravitino/trino/connector/GravitinoPlugin469.java b/trino-connector/trino-connector-469-472/src/main/java/org/apache/gravitino/trino/connector/GravitinoPlugin469.java new file mode 100644 index 0000000000..5cc6460a0e --- /dev/null +++ b/trino-connector/trino-connector-469-472/src/main/java/org/apache/gravitino/trino/connector/GravitinoPlugin469.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.trino.connector; + +import org.apache.gravitino.client.GravitinoAdminClient; + +/** Trino plugin endpoint, using java spi mechanism */ +public class GravitinoPlugin469 extends GravitinoPlugin { + + public GravitinoPlugin469() { + super(); + } + + public GravitinoPlugin469(GravitinoAdminClient client) { + super(client); + } + + @Override + protected GravitinoConnectorFactory createConnectorFactory(GravitinoAdminClient client) { + return new GravitinoConnectorFactory469(client); + } +} diff --git a/trino-connector/trino-connector-469-472/src/main/java/org/apache/gravitino/trino/connector/GravitinoSplitManager469.java b/trino-connector/trino-connector-469-472/src/main/java/org/apache/gravitino/trino/connector/GravitinoSplitManager469.java new file mode 100644 index 0000000000..af660c5a6d --- /dev/null +++ b/trino-connector/trino-connector-469-472/src/main/java/org/apache/gravitino/trino/connector/GravitinoSplitManager469.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.trino.connector; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.trino.spi.connector.ConnectorSplit; +import io.trino.spi.connector.ConnectorSplitManager; +import io.trino.spi.connector.ConnectorSplitSource; + +public class GravitinoSplitManager469 extends GravitinoSplitManager { + + public GravitinoSplitManager469(ConnectorSplitManager internalSplitManager) { + super(internalSplitManager); + } + + @Override + protected ConnectorSplitSource createSplitSource(ConnectorSplitSource splits) { + return new GravitinoSplitSource469(splits); + } + + static class GravitinoSplitSource469 extends GravitinoSplitSource { + + GravitinoSplitSource469(ConnectorSplitSource connectorSplitSource) { + super(connectorSplitSource); + } + + @Override + protected ConnectorSplit createSplit(ConnectorSplit split) { + return new GravitinoSplit469(split); + } + } + + public static class GravitinoSplit469 extends GravitinoSplit { + + @JsonCreator + public GravitinoSplit469(@JsonProperty(HANDLE_STRING) String handleString) { + super(handleString); + } + + public GravitinoSplit469(ConnectorSplit split) { + super(split); + } + } +} diff --git a/trino-connector/trino-connector-469-472/src/main/java/org/apache/gravitino/trino/connector/GravitinoSystemConnector469.java b/trino-connector/trino-connector-469-472/src/main/java/org/apache/gravitino/trino/connector/GravitinoSystemConnector469.java new file mode 100644 index 0000000000..b2d5483b27 --- /dev/null +++ b/trino-connector/trino-connector-469-472/src/main/java/org/apache/gravitino/trino/connector/GravitinoSystemConnector469.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.trino.connector; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.trino.spi.Page; +import io.trino.spi.connector.ConnectorPageSource; +import io.trino.spi.connector.ConnectorPageSourceProvider; +import io.trino.spi.connector.ConnectorSplit; +import io.trino.spi.connector.ConnectorSplitManager; +import io.trino.spi.connector.SchemaTableName; +import org.apache.gravitino.trino.connector.system.GravitinoSystemConnector; +import org.apache.gravitino.trino.connector.system.storedprocedure.GravitinoStoredProcedureFactory; + +public class GravitinoSystemConnector469 extends GravitinoSystemConnector { + + public GravitinoSystemConnector469( + GravitinoStoredProcedureFactory gravitinoStoredProcedureFactory) { + super(gravitinoStoredProcedureFactory); + } + + @Override + protected ConnectorSplitManager createSplitManager() { + return new GravitinoSplitManager469(); + } + + @Override + protected ConnectorPageSourceProvider createPageSourceProvider() { + return new DatasourceProvider469(); + } + + static class DatasourceProvider469 extends DatasourceProvider { + + @Override + protected ConnectorPageSource createPageSource(Page page) { + return new SystemTablePageSource469(page); + } + } + + static class GravitinoSplitManager469 extends SplitManager { + + protected ConnectorSplit createSplit(SchemaTableName tableName) { + return new Split469(tableName); + } + } + + static class SystemTablePageSource469 extends SystemTablePageSource { + + public SystemTablePageSource469(Page page) { + super(page); + } + + public Page getNextPage() { + return nextPage(); + } + } + + public static class Split469 extends Split { + + @JsonCreator + public Split469(@JsonProperty("tableName") SchemaTableName tableName) { + super(tableName); + } + } +} diff --git a/trino-connector/trino-connector-469-472/src/main/resources/META-INF/services/io.trino.spi.Plugin b/trino-connector/trino-connector-469-472/src/main/resources/META-INF/services/io.trino.spi.Plugin new file mode 100644 index 0000000000..dcd20705ab --- /dev/null +++ b/trino-connector/trino-connector-469-472/src/main/resources/META-INF/services/io.trino.spi.Plugin @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +org.apache.gravitino.trino.connector.GravitinoPlugin469 \ No newline at end of file diff --git a/trino-connector/trino-connector-469-472/src/test/java/TestGravitinoConnector469.java b/trino-connector/trino-connector-469-472/src/test/java/TestGravitinoConnector469.java new file mode 100644 index 0000000000..c3e49e2bba --- /dev/null +++ b/trino-connector/trino-connector-469-472/src/test/java/TestGravitinoConnector469.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import static io.trino.testing.TestingSession.testSessionBuilder; + +import io.trino.Session; +import io.trino.testing.DistributedQueryRunner; +import org.apache.gravitino.client.GravitinoAdminClient; +import org.apache.gravitino.trino.connector.GravitinoPlugin; +import org.apache.gravitino.trino.connector.GravitinoPlugin469; +import org.apache.gravitino.trino.connector.TestGravitinoConnector; +import org.apache.gravitino.trino.connector.TestGravitinoConnectorWithMetalakeCatalogName; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Nested; + +public class TestGravitinoConnector469 { + @Nested + class SingleMetalake extends TestGravitinoConnector { + @Override + protected GravitinoPlugin createGravitinoPlugin(GravitinoAdminClient client) { + return new GravitinoPlugin469(client); + } + + @Override + protected DistributedQueryRunner createTrinoQueryRunner() throws Exception { + Session session = testSessionBuilder().setCatalog("gravitino").build(); + return DistributedQueryRunner.builder(session).setWorkerCount(1).build(); + } + } + + @Nested + @Disabled + class MultiMetalake extends TestGravitinoConnectorWithMetalakeCatalogName { + @Override + protected GravitinoPlugin createGravitinoPlugin(GravitinoAdminClient client) { + return new GravitinoPlugin469(client); + } + + @Override + protected DistributedQueryRunner createTrinoQueryRunner() throws Exception { + Session session = testSessionBuilder().setCatalog("gravitino").build(); + return DistributedQueryRunner.builder(session).setWorkerCount(1).build(); + } + + @Override + protected String getTrinoCliCatalogName(String metalake, String catalog) { + return metalake + "." + catalog; + } + + @Override + protected String getTrinoSqlCatalogName(String metalakeName, String catalogName) { + return "\"" + metalakeName + "." + catalogName + "\""; + } + } +} diff --git a/trino-connector/trino-connector/build.gradle.kts b/trino-connector/trino-connector/build.gradle.kts index 9e734d46c5..69ed4e7593 100644 --- a/trino-connector/trino-connector/build.gradle.kts +++ b/trino-connector/trino-connector/build.gradle.kts @@ -33,6 +33,7 @@ val trinoVersion = trinoVersionProperty.map { it.trim().toInt() }.get() dependencies { implementation(project(":catalogs:catalog-common")) implementation(project(":clients:client-java-runtime", configuration = "shadow")) + implementation(libs.airlift.json) implementation(libs.bundles.log4j) implementation(libs.commons.collections4) diff --git a/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/hive/HiveConnectorAdapter.java b/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/hive/HiveConnectorAdapter.java index ce2ef4363b..e097b2145e 100644 --- a/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/hive/HiveConnectorAdapter.java +++ b/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/hive/HiveConnectorAdapter.java @@ -56,6 +56,7 @@ public class HiveConnectorAdapter implements CatalogConnectorAdapter { config.putAll(trinoProperty); config.put("hive.metastore.uri", metastoreUri); config.put("hive.security", "allow-all"); + config.put("fs.hadoop.enabled", "true"); return config; } diff --git a/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergCatalogPropertyConverter.java b/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergCatalogPropertyConverter.java index 02f43c88dd..b0b220c934 100644 --- a/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergCatalogPropertyConverter.java +++ b/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergCatalogPropertyConverter.java @@ -68,6 +68,7 @@ public class IcebergCatalogPropertyConverter extends CatalogPropertyConverter { // The order of put operations determines the priority of parameters. config.putAll(super.gravitinoToEngineProperties(properties)); config.putAll(stringStringMap); + config.put("fs.hadoop.enabled", "true"); return config; } diff --git a/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergMetadataAdapter.java b/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergMetadataAdapter.java index 20a9111af1..b0d8e8c072 100644 --- a/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergMetadataAdapter.java +++ b/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergMetadataAdapter.java @@ -21,6 +21,7 @@ package org.apache.gravitino.trino.connector.catalog.iceberg; import static org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants.FORMAT; import static org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants.FORMAT_VERSION; import static org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants.PROVIDER; +import static org.apache.gravitino.trino.connector.catalog.iceberg.IcebergPropertyMeta.DEFAULT_ICEBERG_FORMAT_VERSION; import static org.apache.gravitino.trino.connector.catalog.iceberg.IcebergPropertyMeta.ICEBERG_FORMAT_PROPERTY; import static org.apache.gravitino.trino.connector.catalog.iceberg.IcebergPropertyMeta.ICEBERG_FORMAT_VERSION_PROPERTY; import static org.apache.gravitino.trino.connector.catalog.iceberg.IcebergTablePropertyConverter.convertTableFormatToTrino; @@ -117,13 +118,12 @@ public class IcebergMetadataAdapter extends CatalogConnectorMetadataAdapter { toGravitinoTableProperties( removeKeys(tableMetadata.getProperties(), ICEBERG_PROPERTIES_TO_REMOVE)); - if (propertyMap.containsKey(ICEBERG_FORMAT_PROPERTY)) { - String format = propertyMap.get(ICEBERG_FORMAT_PROPERTY).toString(); - properties.put(PROVIDER, format); - if (propertyMap.containsKey((ICEBERG_FORMAT_VERSION_PROPERTY))) { - properties.put(FORMAT_VERSION, propertyMap.get(ICEBERG_FORMAT_VERSION_PROPERTY).toString()); - } - } + properties.put(PROVIDER, propertyMap.get(ICEBERG_FORMAT_PROPERTY).toString()); + properties.put( + FORMAT_VERSION, + propertyMap + .getOrDefault(ICEBERG_FORMAT_VERSION_PROPERTY, DEFAULT_ICEBERG_FORMAT_VERSION) + .toString()); List<GravitinoColumn> columns = new ArrayList<>(); for (int i = 0; i < tableMetadata.getColumns().size(); i++) { diff --git a/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergPropertyMeta.java b/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergPropertyMeta.java index 94f42cea3e..212d6fdf40 100644 --- a/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergPropertyMeta.java +++ b/trino-connector/trino-connector/src/main/java/org/apache/gravitino/trino/connector/catalog/iceberg/IcebergPropertyMeta.java @@ -50,8 +50,8 @@ public class IcebergPropertyMeta implements HasPropertyMeta { /** Property key for table format version configuration. */ public static final String ICEBERG_FORMAT_VERSION_PROPERTY = "format_version"; - private static final String DEFAULT_ICEBERG_FORMAT = "PARQUET"; - private static final String DEFAULT_ICEBERG_FORMAT_VERSION = "2"; + public static final String DEFAULT_ICEBERG_FORMAT = "PARQUET"; + public static final String DEFAULT_ICEBERG_FORMAT_VERSION = "2"; // Value is whether this property is reserved and cannot be used by users // TODO (yuqi) add more properties
