This is an automated email from the ASF dual-hosted git repository. yuqi4733 pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/gravitino.git
commit 2ebe07166ce443ed132cd0e767bf6a9b15dbd730 Author: Yuhui <[email protected]> AuthorDate: Thu Feb 5 14:25:03 2026 +0800 [#9719] feat (trino-connector): Add the version segment module to support Trino-435 to Trino–439 (#9861) Add the version segment module for 435–439 to support Trino 345 through Trino 439 Fix: #9719 NO UT and IT --- .github/workflows/frontend-integration-test.yml | 9 +- .github/workflows/trino-integration-test.yml | 14 +- gradle.properties | 5 + .../docker-script/docker-compose.yaml | 4 +- .../docker-script/init/trino/init.sh | 2 +- integration-test-common/docker-script/shutdown.sh | 1 + .../test/container/TrinoITContainers.java | 8 ++ settings.gradle.kts | 6 +- .../connector/integration/test/TrinoQueryIT.java | 10 -- .../integration/test/TrinoQueryITBase.java | 2 +- .../{run_test.sh => run_trino_test_with_env.sh} | 0 .../{trino_test.sh => trino_integration_test.sh} | 0 .../trino-connector-435-439/build.gradle.kts | 159 +++++++++++++++++++++ .../trino/connector/GravitinoConnector435.java | 55 +++++++ .../connector/GravitinoConnectorFactory435.java | 52 +++++++ .../trino/connector/GravitinoMetadata435.java | 83 +++++++++++ .../GravitinoNodePartitioningProvider435.java | 30 ++++ .../trino/connector/GravitinoPlugin435.java | 38 +++++ .../trino/connector/GravitinoSplitManager435.java | 66 +++++++++ .../connector/GravitinoSystemConnector435.java | 87 +++++++++++ .../META-INF/services/io.trino.spi.Plugin | 12 +- .../src/test/java/TestGravitinoConnector435.java | 59 ++++++++ trino-connector/trino-connector/build.gradle.kts | 6 +- .../trino/connector/GravitinoMockServer.java | 7 + 24 files changed, 677 insertions(+), 38 deletions(-) diff --git a/.github/workflows/frontend-integration-test.yml b/.github/workflows/frontend-integration-test.yml index 71f788371c..e636df7c89 100644 --- a/.github/workflows/frontend-integration-test.yml +++ b/.github/workflows/frontend-integration-test.yml @@ -1,8 +1,11 @@ name: Frontend Integration Test # Controls when the workflow will run -# Disable it temporarily. -on: {} +on: + push: + branches: [ "main", "branch-*" ] + pull_request: + branches: [ "main", "branch-*" ] concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -72,7 +75,7 @@ jobs: - name: Package Gravitino run: | - ./gradlew compileDistribution -x test + ./gradlew compileDistribution compileTrinoConnector -x test - name: Free up disk space run: | diff --git a/.github/workflows/trino-integration-test.yml b/.github/workflows/trino-integration-test.yml index a8097e0337..ae9c0d5ec4 100644 --- a/.github/workflows/trino-integration-test.yml +++ b/.github/workflows/trino-integration-test.yml @@ -1,8 +1,11 @@ name: Trino Integration Test # Controls when the workflow will run -# Disable it temporarily. -on: {} +on: + push: + branches: [ "main", "branch-*" ] + pull_request: + branches: [ "main", "branch-*" ] concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -71,9 +74,9 @@ jobs: run: | dev/ci/check_commands.sh - - name: Package Gravitino + - name: Build Gravitino run: | - ./gradlew compileDistribution compileTrinoConnector -x test + ./gradlew build -x test - name: Free up disk space run: | @@ -82,8 +85,7 @@ jobs: - name: Trino Integration Test id: integrationTest run: | - ./gradlew -PskipTests -PtestMode=embedded -PskipDockerTests=false :trino-connector:integration-test:test - ./gradlew -PskipTests -PtestMode=deploy -PskipDockerTests=false :trino-connector:integration-test:test + trino-connector/integration-test/trino-test-tools/trino_integration_test.sh # Disable the Trino cascading query integration test, because the connector jars are private now. #trino-connector/integration-test/trino-test-tools/run_test.sh diff --git a/gradle.properties b/gradle.properties index 98e15862e2..45123f6aa7 100644 --- a/gradle.properties +++ b/gradle.properties @@ -21,6 +21,8 @@ org.gradle.parallel=true # Build cache can be disabled with --no-build-cache option org.gradle.caching=true org.gradle.jvmargs=-Xmx4g +org.gradle.java.installations.auto-download=true +org.gradle.java.installations.auto-detect=true # version that is going to be updated automatically by releases version = 1.2.0-SNAPSHOT @@ -40,3 +42,6 @@ skipDockerTests = true # enableFuse is used to enable the fuse module in the build. enableFuse = false + +# The minimum supported Trino version. +minSupportedTrinoVersion= 435 diff --git a/integration-test-common/docker-script/docker-compose.yaml b/integration-test-common/docker-script/docker-compose.yaml index 7b3d46b89e..2641e53992 100644 --- a/integration-test-common/docker-script/docker-compose.yaml +++ b/integration-test-common/docker-script/docker-compose.yaml @@ -94,7 +94,7 @@ services: entrypoint: /bin/bash /tmp/trino/init.sh volumes: - ./init/trino:/tmp/trino - - ${GRAVITINO_TRINO_CONNECTOR_DIR:-../../trino-connector/trino-connector/build/libs}:/usr/lib/trino/plugin/gravitino + - ${GRAVITINO_TRINO_CONNECTOR_DIR:-../../trino-connector/trino-connector-435-439/build/libs}:/usr/lib/trino/plugin/gravitino extra_hosts: - "host.docker.internal:host-gateway" healthcheck: @@ -128,7 +128,7 @@ services: entrypoint: /bin/bash /tmp/trino/init.sh volumes: - ./init/trino:/tmp/trino - - ${GRAVITINO_TRINO_CONNECTOR_DIR:-../../trino-connector/trino-connector/build/libs}:/usr/lib/trino/plugin/gravitino + - ${GRAVITINO_TRINO_CONNECTOR_DIR:-../../trino-connector/trino-connector-435-439/build/libs}:/usr/lib/trino/plugin/gravitino extra_hosts: - "host.docker.internal:host-gateway" depends_on: diff --git a/integration-test-common/docker-script/init/trino/init.sh b/integration-test-common/docker-script/init/trino/init.sh index 385c06e8f5..051d56ee6c 100644 --- a/integration-test-common/docker-script/init/trino/init.sh +++ b/integration-test-common/docker-script/init/trino/init.sh @@ -60,7 +60,7 @@ fi # Container start up if [[ "${TRINO_ROLE}" == "coordinator" ]]; then - nohup /usr/lib/trino/bin/run-trino & + nohup /usr/lib/trino/bin/run-trino > /tmp/trino.out 2>&1 & counter=0 while [ $counter -le 300 ]; do diff --git a/integration-test-common/docker-script/shutdown.sh b/integration-test-common/docker-script/shutdown.sh index f2a60d3d9d..90288834ff 100755 --- a/integration-test-common/docker-script/shutdown.sh +++ b/integration-test-common/docker-script/shutdown.sh @@ -26,4 +26,5 @@ if [ -d $LOG_DIR ]; then docker cp trino-ci-hive:/tmp/root $LOG_DIR/hive fi +export GRAVITINO_TRINO_CONNECTOR_DIR=/dev/null docker compose down diff --git a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TrinoITContainers.java b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TrinoITContainers.java index b26984c6d2..d5b2636d6c 100644 --- a/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TrinoITContainers.java +++ b/integration-test-common/src/test/java/org/apache/gravitino/integration/test/container/TrinoITContainers.java @@ -19,6 +19,9 @@ package org.apache.gravitino.integration.test.container; import com.google.common.collect.ImmutableSet; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.HashMap; import java.util.Map; import org.apache.gravitino.integration.test.util.CommandExecutor; @@ -70,6 +73,10 @@ public class TrinoITContainers implements AutoCloseable { env.put("TRINO_VERSION", String.valueOf(trinoVersion)); } if (trinoConnectorDir != null) { + Path path = Paths.get(trinoConnectorDir); + if (!Files.exists(path)) { + throw new Exception("Provided GRAVITINO_TRINO_CONNECTOR_DIR '" + path + "' does not exist"); + } env.put("GRAVITINO_TRINO_CONNECTOR_DIR", trinoConnectorDir); } env.put("GRAVITINO_SERVER_PORT", String.valueOf(gravitinoServerPort)); @@ -79,6 +86,7 @@ public class TrinoITContainers implements AutoCloseable { env.put("GRAVITINO_LOG_PATH", System.getProperty("gravitino.log.path")); } + LOG.info("Launching containers with env: {}", env); String command = ITUtils.joinPath(dockerComposeDir, "launch.sh"); Object output = CommandExecutor.executeCommandLocalHost( diff --git a/settings.gradle.kts b/settings.gradle.kts index 6e4a70a48b..bd9ce95d9f 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -67,7 +67,11 @@ include("iceberg:iceberg-rest-server") include("lance:lance-common") include("lance:lance-rest-server") include("authorizations:authorization-ranger", "authorizations:authorization-common", "authorizations:authorization-chain") -include("trino-connector:trino-connector", "trino-connector:integration-test") +include( + "trino-connector:trino-connector", + "trino-connector:trino-connector-435-439", + "trino-connector:integration-test" +) include("spark-connector:spark-common") if (scalaVersion == "2.12") { // flink only support scala 2.12 diff --git a/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryIT.java b/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryIT.java index 811e8229f7..ef06725de9 100644 --- a/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryIT.java +++ b/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryIT.java @@ -42,16 +42,9 @@ import java.util.regex.Pattern; import org.apache.gravitino.integration.test.util.ITUtils; import org.apache.logging.log4j.util.Strings; import org.awaitility.Awaitility; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Tag; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -@Tag("gravitino-docker-test") -@TestInstance(TestInstance.Lifecycle.PER_CLASS) public class TrinoQueryIT extends TrinoQueryITBase { private static final Logger LOG = LoggerFactory.getLogger(TrinoQueryIT.class); @@ -82,7 +75,6 @@ public class TrinoQueryIT extends TrinoQueryITBase { testsetsDir = ITUtils.joinPath(testsetsDir, "testsets"); } - @BeforeAll public void setup() throws Exception { trinoQueryITBase = new TrinoQueryITBase(trinoWorkerNum, trinoVersion, trinoConnectorDir); trinoQueryITBase.setup(); @@ -123,7 +115,6 @@ public class TrinoQueryIT extends TrinoQueryITBase { } } - @AfterAll public static void cleanup() { TrinoQueryITBase.cleanup(); } @@ -341,7 +332,6 @@ public class TrinoQueryIT extends TrinoQueryITBase { return match; } - @Test public void testSql() throws Exception { ExecutorService executor = Executors.newFixedThreadPool(testParallelism); CompletionService<Integer> completionService = new ExecutorCompletionService<>(executor); diff --git a/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryITBase.java b/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryITBase.java index f2197c0e57..456e24d4c4 100644 --- a/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryITBase.java +++ b/trino-connector/integration-test/src/test/java/org/apache/gravitino/trino/connector/integration/test/TrinoQueryITBase.java @@ -150,7 +150,7 @@ public class TrinoQueryITBase { try { if (autoStart) { if (trinoITContainers != null) trinoITContainers.shutdown(); - baseIT.stopIntegrationTest(); + if (baseIT != null) baseIT.stopIntegrationTest(); } } catch (Exception e) { LOG.error("Error in cleanup", e); diff --git a/trino-connector/integration-test/trino-test-tools/run_test.sh b/trino-connector/integration-test/trino-test-tools/run_trino_test_with_env.sh similarity index 100% rename from trino-connector/integration-test/trino-test-tools/run_test.sh rename to trino-connector/integration-test/trino-test-tools/run_trino_test_with_env.sh diff --git a/trino-connector/integration-test/trino-test-tools/trino_test.sh b/trino-connector/integration-test/trino-test-tools/trino_integration_test.sh similarity index 100% rename from trino-connector/integration-test/trino-test-tools/trino_test.sh rename to trino-connector/integration-test/trino-test-tools/trino_integration_test.sh diff --git a/trino-connector/trino-connector-435-439/build.gradle.kts b/trino-connector/trino-connector-435-439/build.gradle.kts new file mode 100644 index 0000000000..a9602647f8 --- /dev/null +++ b/trino-connector/trino-connector-435-439/build.gradle.kts @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import com.diffplug.gradle.spotless.SpotlessExtension +import net.ltgt.gradle.errorprone.errorprone +import org.gradle.internal.hash.ChecksumService +import org.gradle.kotlin.dsl.support.serviceOf + +plugins { + `java-library` + `maven-publish` +} + +val supportedTrinoVersion = 435 +val minSupportedTrinoVersionProperty = providers.gradleProperty("minSupportedTrinoVersion").map { it.trim().toInt() } +val trinoVersionProvider = + providers.gradleProperty("trinoVersion").map { it.trim().toInt() }.orElse(minSupportedTrinoVersionProperty) + +val resolvedTrinoVersion = trinoVersionProvider.get() +check(resolvedTrinoVersion >= supportedTrinoVersion) { + "Failed to build the module ${project.path} of trinoVersion=$resolvedTrinoVersion. " + + "This module requires trinoVersion >= $supportedTrinoVersion. " + + "Please set '-PtrinoVersion' in the command or set config the 'minSupportedTrinoVersion' of gradle.properties." +} +val trinoVersion = resolvedTrinoVersion + +java { + toolchain.languageVersion.set(JavaLanguageVersion.of(24)) +} + +dependencies { + implementation(project(":catalogs:catalog-common")) + implementation(project(":clients:client-java-runtime", configuration = "shadow")) + implementation(libs.airlift.json) + implementation(libs.bundles.log4j) + implementation(libs.commons.collections4) + implementation(libs.commons.lang3) + implementation("io.trino:trino-jdbc:$trinoVersion") + compileOnly(libs.airlift.resolver) + compileOnly("io.trino:trino-spi:$trinoVersion") { + exclude("org.apache.logging.log4j") + } + testImplementation(libs.awaitility) + testImplementation(libs.mockito.core) + testImplementation(libs.mysql.driver) + testImplementation("io.trino:trino-memory:$trinoVersion") { + exclude("org.antlr") + exclude("org.apache.logging.log4j") + } + testImplementation("io.trino:trino-testing:$trinoVersion") { + exclude("org.apache.logging.log4j") + } + testRuntimeOnly(libs.junit.jupiter.engine) +} + +sourceSets { + main { + java.srcDirs("../trino-connector/src/main/java") + } + test { + java.srcDirs("../trino-connector/src/test/java") + resources.srcDirs("../trino-connector/src/test/resources") + } +} + +plugins.withId("com.diffplug.spotless") { + configure<SpotlessExtension> { + java { + // Keep Spotless within this module to avoid cross-project target errors. + target(project.fileTree("src") { include("**/*.java") }) + } + } +} + +tasks.withType<JavaCompile>().configureEach { + // Error Prone is incompatible with the JDK 24 toolchain required by this Trino range. + options.errorprone.isEnabled.set(false) + options.release.set(17) +} + +tasks.withType<Test>().configureEach { + extensions + .findByType(org.gradle.testing.jacoco.plugins.JacocoTaskExtension::class.java) + ?.isEnabled = false +} + +tasks { + val copyRuntimeLibs by registering(Copy::class) { + dependsOn("jar") + from({ configurations.runtimeClasspath.get().filter(File::isFile) }) + into(layout.buildDirectory.dir("libs")) + } + + val distributionDir = rootProject.layout.projectDirectory.dir("distribution/${rootProject.name}-${project.name}") + + val copyLibs by registering(Copy::class) { + dependsOn(copyRuntimeLibs, "build") + from(layout.buildDirectory.dir("libs")) + from(rootProject.layout.projectDirectory.dir("licenses")) { + into("licenses") + } + from(rootProject.file("LICENSE.trino")) + from(rootProject.file("NOTICE.trino")) + from(rootProject.file("README.md")) + into(distributionDir) + rename { fileName -> + fileName.replace(".trino", "") + } + outputs.dir(distributionDir) + } + + val assembleTrinoConnector by registering(Tar::class) { + dependsOn(copyLibs) + group = "gravitino distribution" + finalizedBy("checksumTrinoConnector") + val archiveBase = "${rootProject.name}-${project.name}-$version" + into(archiveBase) + from(distributionDir) + compression = Compression.GZIP + archiveFileName.set("$archiveBase.tar.gz") + destinationDirectory.set(rootProject.layout.projectDirectory.dir("distribution")) + } + + val checksumTrinoConnector by registering { + group = "gravitino distribution" + dependsOn(assembleTrinoConnector) + val archiveFile = assembleTrinoConnector.flatMap { it.archiveFile } + val checksumFile = archiveFile.map { archive -> + archive.asFile.let { it.resolveSibling("${it.name}.sha256") } + } + inputs.file(archiveFile) + outputs.file(checksumFile) + doLast { + checksumFile.get().writeText( + serviceOf<ChecksumService>().sha256(archiveFile.get().asFile).toString() + ) + } + } + + named("build") { + finalizedBy(copyRuntimeLibs) + } +} diff --git a/trino-connector/trino-connector-435-439/src/main/java/org/apache/gravitino/trino/connector/GravitinoConnector435.java b/trino-connector/trino-connector-435-439/src/main/java/org/apache/gravitino/trino/connector/GravitinoConnector435.java new file mode 100644 index 0000000000..9d6dc932d4 --- /dev/null +++ b/trino-connector/trino-connector-435-439/src/main/java/org/apache/gravitino/trino/connector/GravitinoConnector435.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.trino.connector; + +import io.trino.spi.connector.ConnectorMetadata; +import io.trino.spi.connector.ConnectorNodePartitioningProvider; +import io.trino.spi.connector.ConnectorSplitManager; +import org.apache.gravitino.trino.connector.catalog.CatalogConnectorContext; +import org.apache.gravitino.trino.connector.catalog.CatalogConnectorMetadata; +import org.apache.gravitino.trino.connector.catalog.CatalogConnectorMetadataAdapter; + +public class GravitinoConnector435 extends GravitinoConnector { + + public GravitinoConnector435(CatalogConnectorContext connectorContext) { + super(connectorContext); + } + + @Override + protected GravitinoMetadata createGravitinoMetadata( + CatalogConnectorMetadata catalogConnectorMetadata, + CatalogConnectorMetadataAdapter metadataAdapter, + ConnectorMetadata internalMetadata) { + return new GravitinoMetadata435(catalogConnectorMetadata, metadataAdapter, internalMetadata); + } + + @Override + public ConnectorSplitManager getSplitManager() { + ConnectorSplitManager splitManager = + catalogConnectorContext.getInternalConnector().getSplitManager(); + return new GravitinoSplitManager435(splitManager); + } + + @Override + public ConnectorNodePartitioningProvider getNodePartitioningProvider() { + ConnectorNodePartitioningProvider nodePartitioningProvider = + catalogConnectorContext.getInternalConnector().getNodePartitioningProvider(); + return new GravitinoNodePartitioningProvider435(nodePartitioningProvider); + } +} diff --git a/trino-connector/trino-connector-435-439/src/main/java/org/apache/gravitino/trino/connector/GravitinoConnectorFactory435.java b/trino-connector/trino-connector-435-439/src/main/java/org/apache/gravitino/trino/connector/GravitinoConnectorFactory435.java new file mode 100644 index 0000000000..2ac78816ce --- /dev/null +++ b/trino-connector/trino-connector-435-439/src/main/java/org/apache/gravitino/trino/connector/GravitinoConnectorFactory435.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.trino.connector; + +import org.apache.gravitino.client.GravitinoAdminClient; +import org.apache.gravitino.trino.connector.catalog.CatalogConnectorContext; +import org.apache.gravitino.trino.connector.system.GravitinoSystemConnector; +import org.apache.gravitino.trino.connector.system.storedprocedure.GravitinoStoredProcedureFactory; + +public class GravitinoConnectorFactory435 extends GravitinoConnectorFactory { + + public GravitinoConnectorFactory435(GravitinoAdminClient client) { + super(client); + } + + @Override + protected int getMinSupportTrinoSpiVersion() { + return 435; + } + + @Override + protected int getMaxSupportTrinoSpiVersion() { + return 439; + } + + @Override + protected GravitinoConnector createConnector(CatalogConnectorContext connectorContext) { + return new GravitinoConnector435(connectorContext); + } + + @Override + protected GravitinoSystemConnector createSystemConnector( + GravitinoStoredProcedureFactory storedProcedureFactory) { + return new GravitinoSystemConnector435(storedProcedureFactory); + } +} diff --git a/trino-connector/trino-connector-435-439/src/main/java/org/apache/gravitino/trino/connector/GravitinoMetadata435.java b/trino-connector/trino-connector-435-439/src/main/java/org/apache/gravitino/trino/connector/GravitinoMetadata435.java new file mode 100644 index 0000000000..919d53f4a6 --- /dev/null +++ b/trino-connector/trino-connector-435-439/src/main/java/org/apache/gravitino/trino/connector/GravitinoMetadata435.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.trino.connector; + +import io.airlift.slice.Slice; +import io.trino.spi.connector.ColumnMetadata; +import io.trino.spi.connector.ConnectorInsertTableHandle; +import io.trino.spi.connector.ConnectorMergeTableHandle; +import io.trino.spi.connector.ConnectorOutputMetadata; +import io.trino.spi.connector.ConnectorSession; +import io.trino.spi.connector.ConnectorTableHandle; +import io.trino.spi.connector.RetryMode; +import io.trino.spi.connector.SchemaTableName; +import io.trino.spi.statistics.ComputedStatistics; +import java.util.Collection; +import java.util.Optional; +import org.apache.gravitino.trino.connector.catalog.CatalogConnectorMetadata; +import org.apache.gravitino.trino.connector.catalog.CatalogConnectorMetadataAdapter; +import org.apache.gravitino.trino.connector.metadata.GravitinoColumn; + +public class GravitinoMetadata435 extends GravitinoMetadata { + + public GravitinoMetadata435( + CatalogConnectorMetadata catalogConnectorMetadata, + CatalogConnectorMetadataAdapter metadataAdapter, + io.trino.spi.connector.ConnectorMetadata internalMetadata) { + super(catalogConnectorMetadata, metadataAdapter, internalMetadata); + } + + @Override + public void addColumn( + ConnectorSession session, ConnectorTableHandle tableHandle, ColumnMetadata column) { + GravitinoColumn gravitinoColumn = metadataAdapter.createColumn(column); + catalogConnectorMetadata.addColumn(getTableName(tableHandle), gravitinoColumn); + } + + @Override + public Optional<ConnectorOutputMetadata> finishInsert( + ConnectorSession session, + ConnectorInsertTableHandle insertHandle, + Collection<Slice> fragments, + Collection<ComputedStatistics> computedStatistics) { + return internalMetadata.finishInsert( + session, GravitinoHandle.unWrap(insertHandle), fragments, computedStatistics); + } + + @Override + public ConnectorMergeTableHandle beginMerge( + ConnectorSession session, ConnectorTableHandle tableHandle, RetryMode retryMode) { + ConnectorMergeTableHandle connectorMergeTableHandle = + internalMetadata.beginMerge(session, GravitinoHandle.unWrap(tableHandle), retryMode); + SchemaTableName tableName = getTableName(tableHandle); + + return new GravitinoMergeTableHandle( + tableName.getSchemaName(), tableName.getTableName(), connectorMergeTableHandle); + } + + @Override + public void finishMerge( + ConnectorSession session, + ConnectorMergeTableHandle mergeTableHandle, + Collection<Slice> fragments, + Collection<ComputedStatistics> computedStatistics) { + internalMetadata.finishMerge( + session, GravitinoHandle.unWrap(mergeTableHandle), fragments, computedStatistics); + } +} diff --git a/trino-connector/trino-connector-435-439/src/main/java/org/apache/gravitino/trino/connector/GravitinoNodePartitioningProvider435.java b/trino-connector/trino-connector-435-439/src/main/java/org/apache/gravitino/trino/connector/GravitinoNodePartitioningProvider435.java new file mode 100644 index 0000000000..e45753da54 --- /dev/null +++ b/trino-connector/trino-connector-435-439/src/main/java/org/apache/gravitino/trino/connector/GravitinoNodePartitioningProvider435.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.trino.connector; + +import io.trino.spi.connector.ConnectorNodePartitioningProvider; + +/** Trino NodePartitioningProvider implementation with the new split bucket function signature. */ +public class GravitinoNodePartitioningProvider435 extends GravitinoNodePartitioningProvider { + + public GravitinoNodePartitioningProvider435( + ConnectorNodePartitioningProvider nodePartitioningProvider) { + super(nodePartitioningProvider); + } +} diff --git a/trino-connector/trino-connector-435-439/src/main/java/org/apache/gravitino/trino/connector/GravitinoPlugin435.java b/trino-connector/trino-connector-435-439/src/main/java/org/apache/gravitino/trino/connector/GravitinoPlugin435.java new file mode 100644 index 0000000000..9a0e8f328a --- /dev/null +++ b/trino-connector/trino-connector-435-439/src/main/java/org/apache/gravitino/trino/connector/GravitinoPlugin435.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.trino.connector; + +import org.apache.gravitino.client.GravitinoAdminClient; + +/** Trino plugin endpoint, using java spi mechanism */ +public class GravitinoPlugin435 extends GravitinoPlugin { + + public GravitinoPlugin435() { + super(); + } + + public GravitinoPlugin435(GravitinoAdminClient client) { + super(client); + } + + @Override + protected GravitinoConnectorFactory createConnectorFactory(GravitinoAdminClient client) { + return new GravitinoConnectorFactory435(client); + } +} diff --git a/trino-connector/trino-connector-435-439/src/main/java/org/apache/gravitino/trino/connector/GravitinoSplitManager435.java b/trino-connector/trino-connector-435-439/src/main/java/org/apache/gravitino/trino/connector/GravitinoSplitManager435.java new file mode 100644 index 0000000000..306feea732 --- /dev/null +++ b/trino-connector/trino-connector-435-439/src/main/java/org/apache/gravitino/trino/connector/GravitinoSplitManager435.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.trino.connector; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.trino.spi.connector.ConnectorSplit; +import io.trino.spi.connector.ConnectorSplitManager; +import io.trino.spi.connector.ConnectorSplitSource; + +public class GravitinoSplitManager435 extends GravitinoSplitManager { + + public GravitinoSplitManager435(ConnectorSplitManager internalSplitManager) { + super(internalSplitManager); + } + + @Override + protected ConnectorSplitSource createSplitSource(ConnectorSplitSource splits) { + return new GravitinoSplitSource435(splits); + } + + static class GravitinoSplitSource435 extends GravitinoSplitSource { + + GravitinoSplitSource435(ConnectorSplitSource connectorSplitSource) { + super(connectorSplitSource); + } + + @Override + protected ConnectorSplit createSplit(ConnectorSplit split) { + return new GravitinoSplit435(split); + } + } + + public static class GravitinoSplit435 extends GravitinoSplit { + + @JsonCreator + public GravitinoSplit435(@JsonProperty(HANDLE_STRING) String handleString) { + super(handleString); + } + + public GravitinoSplit435(ConnectorSplit split) { + super(split); + } + + @Override + public Object getInfo() { + return getInternalHandle().getInfo(); + } + } +} diff --git a/trino-connector/trino-connector-435-439/src/main/java/org/apache/gravitino/trino/connector/GravitinoSystemConnector435.java b/trino-connector/trino-connector-435-439/src/main/java/org/apache/gravitino/trino/connector/GravitinoSystemConnector435.java new file mode 100644 index 0000000000..e3c3074367 --- /dev/null +++ b/trino-connector/trino-connector-435-439/src/main/java/org/apache/gravitino/trino/connector/GravitinoSystemConnector435.java @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.gravitino.trino.connector; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.trino.spi.Page; +import io.trino.spi.connector.ConnectorPageSource; +import io.trino.spi.connector.ConnectorPageSourceProvider; +import io.trino.spi.connector.ConnectorSplit; +import io.trino.spi.connector.ConnectorSplitManager; +import io.trino.spi.connector.SchemaTableName; +import org.apache.gravitino.trino.connector.system.GravitinoSystemConnector; +import org.apache.gravitino.trino.connector.system.storedprocedure.GravitinoStoredProcedureFactory; + +public class GravitinoSystemConnector435 extends GravitinoSystemConnector { + + public GravitinoSystemConnector435( + GravitinoStoredProcedureFactory gravitinoStoredProcedureFactory) { + super(gravitinoStoredProcedureFactory); + } + + @Override + protected ConnectorSplitManager createSplitManager() { + return new GravitinoSplitManager435(); + } + + @Override + protected ConnectorPageSourceProvider createPageSourceProvider() { + return new DatasourceProvider435(); + } + + static class DatasourceProvider435 extends DatasourceProvider { + + @Override + protected ConnectorPageSource createPageSource(Page page) { + return new SystemTablePageSource435(page); + } + } + + static class GravitinoSplitManager435 extends SplitManager { + + protected ConnectorSplit createSplit(SchemaTableName tableName) { + return new Split435(tableName); + } + } + + static class SystemTablePageSource435 extends SystemTablePageSource { + + public SystemTablePageSource435(Page page) { + super(page); + } + + public Page getNextPage() { + return nextPage(); + } + } + + public static class Split435 extends Split { + + @JsonCreator + public Split435(@JsonProperty("tableName") SchemaTableName tableName) { + super(tableName); + } + + @Override + public Object getInfo() { + return this; + } + } +} diff --git a/integration-test-common/docker-script/shutdown.sh b/trino-connector/trino-connector-435-439/src/main/resources/META-INF/services/io.trino.spi.Plugin old mode 100755 new mode 100644 similarity index 76% copy from integration-test-common/docker-script/shutdown.sh copy to trino-connector/trino-connector-435-439/src/main/resources/META-INF/services/io.trino.spi.Plugin index f2a60d3d9d..aafac3a96c --- a/integration-test-common/docker-script/shutdown.sh +++ b/trino-connector/trino-connector-435-439/src/main/resources/META-INF/services/io.trino.spi.Plugin @@ -1,4 +1,3 @@ -#!/bin/bash # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -16,14 +15,5 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. - # -cd "$(dirname "$0")" - -LOG_DIR=../build/trino-ci-container-log -if [ -d $LOG_DIR ]; then - docker cp trino-ci-hive:/usr/local/hadoop/logs $LOG_DIR/hdfs - docker cp trino-ci-hive:/tmp/root $LOG_DIR/hive -fi - -docker compose down +org.apache.gravitino.trino.connector.GravitinoPlugin435 \ No newline at end of file diff --git a/trino-connector/trino-connector-435-439/src/test/java/TestGravitinoConnector435.java b/trino-connector/trino-connector-435-439/src/test/java/TestGravitinoConnector435.java new file mode 100644 index 0000000000..eb767c7dda --- /dev/null +++ b/trino-connector/trino-connector-435-439/src/test/java/TestGravitinoConnector435.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import static io.trino.testing.TestingSession.testSessionBuilder; + +import io.trino.Session; +import io.trino.testing.DistributedQueryRunner; +import org.apache.gravitino.client.GravitinoAdminClient; +import org.apache.gravitino.trino.connector.GravitinoPlugin; +import org.apache.gravitino.trino.connector.GravitinoPlugin435; +import org.apache.gravitino.trino.connector.TestGravitinoConnector; +import org.apache.gravitino.trino.connector.TestGravitinoConnectorWithMetalakeCatalogName; +import org.junit.jupiter.api.Nested; + +public class TestGravitinoConnector435 { + @Nested + class SingleMetalake extends TestGravitinoConnector { + @Override + protected GravitinoPlugin createGravitinoPlugin(GravitinoAdminClient client) { + return new GravitinoPlugin435(client); + } + + @Override + protected DistributedQueryRunner createTrinoQueryRunner() throws Exception { + Session session = testSessionBuilder().setCatalog("gravitino").build(); + return DistributedQueryRunner.builder(session).setNodeCount(1).build(); + } + } + + @Nested + class MultiMetalake extends TestGravitinoConnectorWithMetalakeCatalogName { + @Override + protected GravitinoPlugin createGravitinoPlugin(GravitinoAdminClient client) { + return new GravitinoPlugin435(client); + } + + @Override + protected DistributedQueryRunner createTrinoQueryRunner() throws Exception { + Session session = testSessionBuilder().setCatalog("gravitino").build(); + return DistributedQueryRunner.builder(session).setNodeCount(1).build(); + } + } +} diff --git a/trino-connector/trino-connector/build.gradle.kts b/trino-connector/trino-connector/build.gradle.kts index 39c7a078c0..9e734d46c5 100644 --- a/trino-connector/trino-connector/build.gradle.kts +++ b/trino-connector/trino-connector/build.gradle.kts @@ -26,9 +26,9 @@ repositories { mavenCentral() } -val trinoVersionProvider = - providers.gradleProperty("trinoVersion").map { it.toInt() }.orElse(435) -val trinoVersion = trinoVersionProvider.get() +val minSupportedTrinoVersionProperty = providers.gradleProperty("minSupportedTrinoVersion") +val trinoVersionProperty = providers.gradleProperty("trinoVersion").orElse(minSupportedTrinoVersionProperty) +val trinoVersion = trinoVersionProperty.map { it.trim().toInt() }.get() dependencies { implementation(project(":catalogs:catalog-common")) diff --git a/trino-connector/trino-connector/src/test/java/org/apache/gravitino/trino/connector/GravitinoMockServer.java b/trino-connector/trino-connector/src/test/java/org/apache/gravitino/trino/connector/GravitinoMockServer.java index 2f8097f88b..d8f5e0d35b 100644 --- a/trino-connector/trino-connector/src/test/java/org/apache/gravitino/trino/connector/GravitinoMockServer.java +++ b/trino-connector/trino-connector/src/test/java/org/apache/gravitino/trino/connector/GravitinoMockServer.java @@ -30,6 +30,7 @@ import static org.mockito.Mockito.when; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import io.trino.plugin.memory.MemoryConnector; +import io.trino.spi.TrinoException; import io.trino.spi.connector.ColumnHandle; import io.trino.spi.connector.ColumnMetadata; import io.trino.spi.connector.ConnectorMetadata; @@ -632,6 +633,9 @@ public class GravitinoMockServer implements AutoCloseable { ColumnMetadata.class) .invoke(metadata, null, tableHandle, columnMetadata); } catch (ReflectiveOperationException e) { + if (e.getCause() instanceof TrinoException) { + throw new RuntimeException(e.getCause().getMessage(), e); + } throw new RuntimeException("Failed to invoke legacy addColumn by reflection", e); } } else { @@ -650,6 +654,9 @@ public class GravitinoMockServer implements AutoCloseable { columnPositionClass) .invoke(metadata, null, tableHandle, columnMetadata, position); } catch (Exception e) { + if (e.getCause() instanceof TrinoException) { + throw new RuntimeException(e.getCause().getMessage(), e); + } throw new RuntimeException("Failed to invoke addColumn with ColumnPosition", e); } }
