This is an automated email from the ASF dual-hosted git repository.
yuqi1129 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 5a9ee9aba8 [#12682] improve(paimon): document optional Hadoop S3A
support for Paimon catalog (#12683)
5a9ee9aba8 is described below
commit 5a9ee9aba8fb15ebf6a913de58bdb6ec3f1a2895
Author: Qi Yu <[email protected]>
AuthorDate: Thu Sep 10 22:33:20 2026 +0800
[#12682] improve(paimon): document optional Hadoop S3A support for Paimon
catalog (#12683)
### What changes were proposed in this pull request?
Document Hadoop S3A as an optional filesystem for Paimon catalogs,
including the required dependencies and configuration.
Add `hadoop-aws` to the test runtime classpath only, and retain unit and
LocalStack integration coverage for an `s3a://` Paimon filesystem
catalog.
### Why are the changes needed?
Paimon native S3 support uses `s3://` with `paimon-s3`. Hadoop S3A is an
optional Hadoop-compatible filesystem and requires version-aligned
`hadoop-aws` and `aws-java-sdk-bundle` JARs in the Paimon catalog
classpath.
Packaging these dependencies by default would increase the Paimon
catalog package by about 298 MB. Users who need `s3a://` can install
them explicitly instead.
Fix: #12682
### Does this PR introduce _any_ user-facing change?
Yes. The Paimon catalog documentation now explains how to enable and
configure optional Hadoop S3A support.
No API or property-key changes are introduced, and S3A dependencies are
not included in the production runtime classpath.
### How was this patch tested?
- `./gradlew --no-daemon :catalogs:catalog-lakehouse-paimon:test --tests
org.apache.gravitino.catalog.lakehouse.paimon.TestPaimonCatalog.testS3AFileSystemIsAvailable
--tests
org.apache.gravitino.catalog.lakehouse.paimon.integration.test.CatalogPaimonS3AIT.testAccessS3AFileSystem
-PskipDockerTests=false --rerun-tasks`
- Verified that `runtimeClasspath` does not contain `hadoop-aws`.
- Verified that `testRuntimeClasspath` contains `hadoop-aws:3.3.6` and
`aws-java-sdk-bundle:1.12.367`.
---
catalogs/catalog-lakehouse-paimon/build.gradle.kts | 47 ++++++++
.../lakehouse/paimon/TestPaimonCatalog.java | 10 ++
.../integration/test/CatalogPaimonS3AIT.java | 125 +++++++++++++++++++++
docs/lakehouse-paimon-catalog.md | 39 ++++++-
4 files changed, 220 insertions(+), 1 deletion(-)
diff --git a/catalogs/catalog-lakehouse-paimon/build.gradle.kts
b/catalogs/catalog-lakehouse-paimon/build.gradle.kts
index 0ba959ba90..7b34b2812f 100644
--- a/catalogs/catalog-lakehouse-paimon/build.gradle.kts
+++ b/catalogs/catalog-lakehouse-paimon/build.gradle.kts
@@ -28,6 +28,14 @@ val scalaVersion: String =
project.properties["scalaVersion"] as? String ?: extr
val sparkVersion: String = libs.versions.spark35.get()
val sparkMajorVersion: String = sparkVersion.substringBeforeLast(".")
val paimonVersion: String = libs.versions.paimon.get()
+val s3aTestRuntime by configurations.creating {
+ isCanBeConsumed = false
+ isCanBeResolved = true
+}
+
+configurations.testRuntimeOnly {
+ extendsFrom(s3aTestRuntime)
+}
dependencies {
compileOnly(project(":api"))
@@ -210,9 +218,43 @@ dependencies {
testImplementation(libs.testcontainers.localstack)
testImplementation(libs.testcontainers.mysql)
+ // Keep optional Hadoop S3A dependencies available only to tests.
+ s3aTestRuntime(libs.hadoop3.aws)
testRuntimeOnly(libs.junit.jupiter.engine)
}
+val s3aTestLibDirectory =
+ "$rootDir/distribution/package-all/catalogs/lakehouse-paimon/libs"
+val copyS3ATestDependencies =
+ tasks.register("copyS3ATestDependencies") {
+ inputs.files(s3aTestRuntime)
+ outputs.files(
+ provider {
+ s3aTestRuntime.files.filter {
+ it.name.startsWith("hadoop-aws-") ||
it.name.startsWith("aws-java-sdk-bundle-")
+ }.map { file("$s3aTestLibDirectory/${it.name}") }
+ }
+ )
+ // Copy tasks targeting the distribution depend on
cleanDistributionPackage. Test setup
+ // must preserve the existing deployment, so copy the optional jars in a
regular task.
+ doLast {
+ copy {
+ from(s3aTestRuntime) {
+ include("hadoop-aws-*.jar", "aws-java-sdk-bundle-*.jar")
+ }
+ into(s3aTestLibDirectory)
+ }
+ }
+ }
+val cleanS3ATestDependencies =
+ tasks.register<Delete>("cleanS3ATestDependencies") {
+ delete(
+ fileTree(s3aTestLibDirectory) {
+ include("hadoop-aws-*.jar", "aws-java-sdk-bundle-*.jar")
+ }
+ )
+ }
+
tasks {
register("runtimeJars", Copy::class) {
from(configurations.runtimeClasspath)
@@ -266,6 +308,11 @@ tasks.test {
} else {
dependsOn(tasks.jar)
}
+
+ if (project.properties["testMode"] == "deploy") {
+ dependsOn(copyS3ATestDependencies)
+ finalizedBy(cleanS3ATestDependencies)
+ }
}
tasks.getByName("generateMetadataFileForMavenJavaPublication") {
diff --git
a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/TestPaimonCatalog.java
b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/TestPaimonCatalog.java
index 93510faf29..998d1d9386 100644
---
a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/TestPaimonCatalog.java
+++
b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/TestPaimonCatalog.java
@@ -48,6 +48,8 @@ import org.apache.gravitino.rel.ViewCatalog;
import org.apache.gravitino.storage.AzureProperties;
import org.apache.gravitino.storage.OSSProperties;
import org.apache.gravitino.storage.S3Properties;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
@@ -146,6 +148,14 @@ public class TestPaimonCatalog {
ImmutableMap.of()));
}
+ @Test
+ void testS3AFileSystemIsAvailable() throws IOException {
+ Class<? extends FileSystem> fileSystemClass =
+ FileSystem.getFileSystemClass("s3a", new Configuration());
+
+ Assertions.assertEquals("org.apache.hadoop.fs.s3a.S3AFileSystem",
fileSystemClass.getName());
+ }
+
@Test
void testAsViewCatalog() {
PaimonCatalog paimonCatalog = newPaimonCatalog("catalog_view");
diff --git
a/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonS3AIT.java
b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonS3AIT.java
new file mode 100644
index 0000000000..969c4e7e63
--- /dev/null
+++
b/catalogs/catalog-lakehouse-paimon/src/test/java/org/apache/gravitino/catalog/lakehouse/paimon/integration/test/CatalogPaimonS3AIT.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.gravitino.catalog.lakehouse.paimon.integration.test;
+
+import static org.apache.gravitino.connector.BaseCatalog.CATALOG_BYPASS_PREFIX;
+
+import com.google.common.collect.Maps;
+import java.util.Collections;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import org.apache.gravitino.Catalog;
+import org.apache.gravitino.Schema;
+import
org.apache.gravitino.catalog.lakehouse.paimon.PaimonCatalogPropertiesMetadata;
+import org.apache.gravitino.client.GravitinoMetalake;
+import
org.apache.gravitino.integration.test.container.GravitinoLocalStackContainer;
+import org.apache.gravitino.integration.test.util.BaseIT;
+import org.apache.gravitino.integration.test.util.GravitinoITUtils;
+import org.awaitility.Awaitility;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+import org.testcontainers.containers.Container;
+
+/** Integration test for accessing a Paimon filesystem catalog through Hadoop
S3A. */
+@Tag("gravitino-docker-test")
+public class CatalogPaimonS3AIT extends BaseIT {
+
+ private static final String PROVIDER = "lakehouse-paimon";
+ private static final String S3A_PROPERTY_PREFIX = CATALOG_BYPASS_PREFIX +
"hadoop.fs.s3a.";
+ private static final String ACCESS_KEY = "test";
+ private static final String SECRET_KEY = "test";
+
+ private final String bucketName = "paimon-s3a-" +
UUID.randomUUID().toString().replace("-", "");
+ private final String metalakeName =
GravitinoITUtils.genRandomName("paimon_s3a_metalake");
+ private final String catalogName =
GravitinoITUtils.genRandomName("paimon_s3a_catalog");
+ private final String schemaName =
GravitinoITUtils.genRandomName("paimon_s3a_schema");
+
+ private GravitinoMetalake metalake;
+ private GravitinoLocalStackContainer localStackContainer;
+
+ @BeforeAll
+ void setUp() {
+ containerSuite.startLocalStackContainer();
+ localStackContainer = containerSuite.getLocalStackContainer();
+
+ Awaitility.await()
+ .atMost(60, TimeUnit.SECONDS)
+ .pollInterval(1, TimeUnit.SECONDS)
+ .until(
+ () -> {
+ try {
+ Container.ExecResult result =
+ localStackContainer.executeInContainer(
+ "awslocal", "s3", "mb", "s3://" + bucketName);
+ return result.getExitCode() == 0;
+ } catch (Exception e) {
+ return false;
+ }
+ });
+
+ metalake = client.createMetalake(metalakeName, "Paimon S3A metalake",
Collections.emptyMap());
+ }
+
+ @AfterAll
+ void tearDown() {
+ if (metalake != null && metalake.catalogExists(catalogName)) {
+ metalake.disableCatalog(catalogName);
+ metalake.dropCatalog(catalogName, true);
+ }
+ if (client != null && client.metalakeExists(metalakeName)) {
+ client.disableMetalake(metalakeName);
+ client.dropMetalake(metalakeName, true);
+ }
+ }
+
+ @Test
+ void testAccessS3AFileSystem() {
+ Map<String, String> catalogProperties = Maps.newHashMap();
+
catalogProperties.put(PaimonCatalogPropertiesMetadata.GRAVITINO_CATALOG_BACKEND,
"filesystem");
+ catalogProperties.put(
+ PaimonCatalogPropertiesMetadata.WAREHOUSE, "s3a://" + bucketName +
"/warehouse");
+ catalogProperties.put(S3A_PROPERTY_PREFIX + "access.key", ACCESS_KEY);
+ catalogProperties.put(S3A_PROPERTY_PREFIX + "secret.key", SECRET_KEY);
+ catalogProperties.put(
+ S3A_PROPERTY_PREFIX + "endpoint",
+ String.format(
+ "http://%s:%d";,
+ localStackContainer.getContainer().getHost(),
+
localStackContainer.getMappedPort(GravitinoLocalStackContainer.PORT)));
+ catalogProperties.put(S3A_PROPERTY_PREFIX + "path.style.access", "true");
+ catalogProperties.put(S3A_PROPERTY_PREFIX + "connection.ssl.enabled",
"false");
+
+ Catalog catalog =
+ metalake.createCatalog(
+ catalogName,
+ Catalog.Type.RELATIONAL,
+ PROVIDER,
+ "Paimon S3A catalog",
+ catalogProperties);
+ Schema schema =
+ catalog.asSchemas().createSchema(schemaName, "Paimon S3A schema",
Collections.emptyMap());
+
+ Assertions.assertEquals(schemaName, schema.name());
+ Assertions.assertEquals(schemaName,
catalog.asSchemas().loadSchema(schemaName).name());
+ }
+}
diff --git a/docs/lakehouse-paimon-catalog.md b/docs/lakehouse-paimon-catalog.md
index dc861dd6b2..2ebd10fe3e 100644
--- a/docs/lakehouse-paimon-catalog.md
+++ b/docs/lakehouse-paimon-catalog.md
@@ -56,7 +56,7 @@ Builds with Apache Paimon `1.2`.
| `dlf-token-loader` | The token loader for
Aliyun DLF.
| (none)
| No
[...]
:::note
-- If you want to use the `oss` or `s3` warehouse, you need to place related
jars in the `catalogs/lakehouse-paimon/lib` directory, more information can be
found in the [Paimon
S3](https://paimon.apache.org/docs/1.2/maintenance/filesystems/#s3).
+- If you want to use the `oss` or `s3` warehouse, you need to place related
jars in the `catalogs/lakehouse-paimon/libs` directory, more information can be
found in the [Paimon
S3](https://paimon.apache.org/docs/1.2/maintenance/filesystems/#s3).
- If you use an S3 warehouse, also download
[`gravitino-aws-<version>.jar`](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-aws)
and place it in the `catalogs/lakehouse-paimon/libs` directory to enable
credential vending. For OSS, use
[`gravitino-aliyun-<version>.jar`](https://mvnrepository.com/artifact/org.apache.gravitino/gravitino-aliyun)
instead.
- If you want to use REST backend, Gravitino Paimon catalog supports Aliyun
DLF (Data Lake Formation) as the REST catalog service. You need to configure
the DLF-related properties eg:
```
@@ -80,6 +80,43 @@ connect to Aliyun DLF, more information can be found in the
[Paimon REST Catalog
Any properties not defined by Gravitino with `gravitino.bypass.` prefix will
pass to Paimon catalog properties and HDFS configuration. For example, if
specify `gravitino.bypass.table.type`, `table.type` will pass to Paimon catalog
properties.
+#### Hadoop S3A (optional)
+
+Paimon's native S3 filesystem uses an `s3://` warehouse and the `paimon-s3`
dependency. As an
+alternative, you can use Hadoop S3A with an `s3a://` warehouse. Gravitino does
not package the
+optional S3A dependencies by default. To enable it, add the following JARs to
+`catalogs/lakehouse-paimon/libs` and restart Gravitino:
+
+- `hadoop-aws-<hadoop-version>.jar`, with the same version as the Hadoop
libraries packaged by
+ Gravitino.
+- The `aws-java-sdk-bundle` version required by that `hadoop-aws` release.
+
+The following example configures static credentials and a custom HTTP S3
endpoint:
+
+```json
+{
+ "name": "paimon_s3a",
+ "type": "RELATIONAL",
+ "provider": "lakehouse-paimon",
+ "properties": {
+ "catalog-backend": "filesystem",
+ "warehouse": "s3a://bucket/warehouse",
+ "gravitino.bypass.hadoop.fs.s3a.access.key": "<access-key-id>",
+ "gravitino.bypass.hadoop.fs.s3a.secret.key": "<secret-access-key>",
+ "gravitino.bypass.hadoop.fs.s3a.endpoint": "http://s3.example.com";,
+ "gravitino.bypass.hadoop.fs.s3a.path.style.access": "true",
+ "gravitino.bypass.hadoop.fs.s3a.connection.ssl.enabled": "false"
+ }
+}
+```
+
+The endpoint-related properties are only needed for a custom endpoint. Omit
+`gravitino.bypass.hadoop.fs.s3a.connection.ssl.enabled` when the endpoint uses
HTTPS, and enable
+path-style access only when the S3-compatible service requires it. Hadoop S3A
automatically
+discovers `S3AFileSystem` and uses its default credential provider chain, so
`fs.s3a.impl` and
+`fs.s3a.aws.credentials.provider` do not need to be configured for this
example. You can pass other
+Hadoop S3A properties by prefixing them with `gravitino.bypass.hadoop.`.
+
#### JDBC Backend
If you are using JDBC backend, you must specify the properties like
`jdbc-user`, `jdbc-password` and `jdbc-driver`.
