This is an automated email from the ASF dual-hosted git repository.
jshao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new f7d84b36c4 [#9791] improvement(catalog-lakehouse-paimon): Support
paimon rest backend (#9751)
f7d84b36c4 is described below
commit f7d84b36c4e857d226f42f4295fb1b46ce0c86e9
Author: kevin <[email protected]>
AuthorDate: Fri Jan 30 01:48:11 2026 +0800
[#9791] improvement(catalog-lakehouse-paimon): Support paimon rest backend
(#9751)
### What changes were proposed in this pull request?
Add `REST` enum value to `PaimonCatalogBackend` to support Paimon REST
catalog backend.
This enables Gravitino to connect to Paimon REST-compatible services
(e.g., Alibaba Cloud DLF) via the `token-provider`, `dlf-access-key-id`,
`dlf-access-key-secret`.
### Why are the changes needed?
Paimon natively supports `rest` as a metastore type, but Gravitino's
`PaimonCatalogBackend` enum only includes `FILESYSTEM`, `JDBC`, and
`HIVE`. This prevents users from using Paimon REST catalog services
through Gravitino.
With this change, users can:
- Connect to Paimon REST catalog services
- Use `dlf-token-provider`,`dlf-access-key-id`, `dlf-access-key-secret`
to access Aliyun DLF Paimon REST server
Example configuration:
```json
{
"catalog-backend": "rest",
"uri": "http://example.com";,
"warehouse": "my_catalog",
"token-provider": "dlf",
"dlf-access-key-id": "xxx",
"dlf-access-key-secret": "xxx"
}
---------
Co-authored-by: QingweiYang <[email protected]>
---
.../catalog/lakehouse/paimon/PaimonConstants.java | 18 +++++
.../lakehouse/paimon/PaimonCatalogBackend.java | 3 +-
.../paimon/PaimonCatalogPropertiesMetadata.java | 94 +++++++++++++++++++---
docs/lakehouse-paimon-catalog.md | 33 ++++++--
4 files changed, 128 insertions(+), 20 deletions(-)
diff --git
a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonConstants.java
b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonConstants.java
index 04e8988f49..9c2f472f39 100644
---
a/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonConstants.java
+++
b/catalogs/catalog-common/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonConstants.java
@@ -35,6 +35,10 @@ public class PaimonConstants {
public static final String GRAVITINO_JDBC_DRIVER = "jdbc-driver";
+ public static final String GRAVITINO_TOKEN_PROVIDER = "token-provider";
+ public static final String PAIMON_TOKEN_PROVIDER = "token.provider";
+ public static final String TOKEN = "token";
+
// S3 properties needed by Paimon
public static final String S3_ENDPOINT = "s3.endpoint";
public static final String S3_ACCESS_KEY = "s3.access-key";
@@ -45,6 +49,20 @@ public class PaimonConstants {
public static final String OSS_ACCESS_KEY = "fs.oss.accessKeyId";
public static final String OSS_SECRET_KEY = "fs.oss.accessKeySecret";
+ // DLF (Data Lake Formation) related properties
+ public static final String GRAVITINO_DLF_ACCESS_KEY_ID = "dlf-access-key-id";
+ public static final String GRAVITINO_DLF_ACCESS_KEY_SECRET =
"dlf-access-key-secret";
+ public static final String GRAVITINO_DLF_SECURITY_TOKEN =
"dlf-security-token";
+ public static final String GRAVITINO_DLF_TOKEN_PATH = "dlf-token-path";
+ public static final String GRAVITINO_DLF_TOKEN_LOADER = "dlf-token-loader";
+
+ // DLF related properties - Paimon backend keys
+ public static final String PAIMON_DLF_ACCESS_KEY_ID = "dlf.access-key-id";
+ public static final String PAIMON_DLF_ACCESS_KEY_SECRET =
"dlf.access-key-secret";
+ public static final String PAIMON_DLF_SECURITY_TOKEN = "dlf.security-token";
+ public static final String PAIMON_DLF_TOKEN_PATH = "dlf.token-path";
+ public static final String PAIMON_DLF_TOKEN_LOADER = "dlf.token-loader";
+
// Iceberg Table properties constants
public static final String COMMENT = "comment";
public static final String OWNER = "owner";
diff --git
a/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonCatalogBackend.java
b/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonCatalogBackend.java
index 7371c5be36..c8914c0a86 100644
---
a/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonCatalogBackend.java
+++
b/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonCatalogBackend.java
@@ -22,5 +22,6 @@ package org.apache.gravitino.catalog.lakehouse.paimon;
public enum PaimonCatalogBackend {
FILESYSTEM,
JDBC,
- HIVE
+ HIVE,
+ REST
}
diff --git
a/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonCatalogPropertiesMetadata.java
b/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonCatalogPropertiesMetadata.java
index 4c9dcb07a8..b82061ac36 100644
---
a/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonCatalogPropertiesMetadata.java
+++
b/catalogs/catalog-lakehouse-paimon/src/main/java/org/apache/gravitino/catalog/lakehouse/paimon/PaimonCatalogPropertiesMetadata.java
@@ -63,19 +63,26 @@ public class PaimonCatalogPropertiesMetadata extends
BaseCatalogPropertiesMetada
public static final String S3_SECRET_KEY = PaimonConstants.S3_SECRET_KEY;
public static final Map<String, String> GRAVITINO_CONFIG_TO_PAIMON =
- ImmutableMap.of(
- GRAVITINO_CATALOG_BACKEND,
- PAIMON_METASTORE,
- WAREHOUSE,
- WAREHOUSE,
- URI,
- URI,
- GRAVITINO_JDBC_USER,
- PAIMON_JDBC_USER,
- GRAVITINO_JDBC_PASSWORD,
- PAIMON_JDBC_PASSWORD,
- GRAVITINO_JDBC_DRIVER,
- GRAVITINO_JDBC_DRIVER);
+ ImmutableMap.<String, String>builder()
+ .put(GRAVITINO_CATALOG_BACKEND, PAIMON_METASTORE)
+ .put(WAREHOUSE, WAREHOUSE)
+ .put(URI, URI)
+ .put(GRAVITINO_JDBC_USER, PAIMON_JDBC_USER)
+ .put(GRAVITINO_JDBC_PASSWORD, PAIMON_JDBC_PASSWORD)
+ .put(GRAVITINO_JDBC_DRIVER, GRAVITINO_JDBC_DRIVER)
+ .put(PaimonConstants.GRAVITINO_TOKEN_PROVIDER,
PaimonConstants.PAIMON_TOKEN_PROVIDER)
+ .put(PaimonConstants.TOKEN, PaimonConstants.TOKEN)
+ .put(
+ PaimonConstants.GRAVITINO_DLF_ACCESS_KEY_ID,
PaimonConstants.PAIMON_DLF_ACCESS_KEY_ID)
+ .put(
+ PaimonConstants.GRAVITINO_DLF_ACCESS_KEY_SECRET,
+ PaimonConstants.PAIMON_DLF_ACCESS_KEY_SECRET)
+ .put(
+ PaimonConstants.GRAVITINO_DLF_SECURITY_TOKEN,
+ PaimonConstants.PAIMON_DLF_SECURITY_TOKEN)
+ .put(PaimonConstants.GRAVITINO_DLF_TOKEN_PATH,
PaimonConstants.PAIMON_DLF_TOKEN_PATH)
+ .put(PaimonConstants.GRAVITINO_DLF_TOKEN_LOADER,
PaimonConstants.PAIMON_DLF_TOKEN_LOADER)
+ .build();
private static final Map<String, PropertyEntry<?>> PROPERTIES_METADATA;
public static final Map<String, String> KERBEROS_CONFIGURATION =
ImmutableMap.of(
@@ -102,6 +109,66 @@ public class PaimonCatalogPropertiesMetadata extends
BaseCatalogPropertiesMetada
OSSProperties.GRAVITINO_OSS_ACCESS_KEY_SECRET,
PaimonOSSFileSystemConfig.OSS_SECRET_KEY,
OSSProperties.GRAVITINO_OSS_ENDPOINT,
PaimonOSSFileSystemConfig.OSS_ENDPOINT);
+ public static final Map<String, PropertyEntry<?>> REST_PROPERTY_ENTRIES =
+ new ImmutableMap.Builder<String, PropertyEntry<?>>()
+ .put(
+ PaimonConstants.GRAVITINO_TOKEN_PROVIDER,
+ stringOptionalPropertyEntry(
+ PaimonConstants.GRAVITINO_TOKEN_PROVIDER,
+ "The token provider type for Paimon",
+ false /* immutable */,
+ null /* defaultValue */,
+ false /* hidden */))
+ .put(
+ PaimonConstants.TOKEN,
+ stringOptionalPropertyEntry(
+ PaimonConstants.TOKEN,
+ "The bear token for REST catalog authentication",
+ false /* immutable */,
+ null /* defaultValue */,
+ false /* hidden */))
+ .put(
+ PaimonConstants.GRAVITINO_DLF_ACCESS_KEY_ID,
+ stringOptionalPropertyEntry(
+ PaimonConstants.GRAVITINO_DLF_ACCESS_KEY_ID,
+ "The access key ID for Aliyun DLF",
+ false /* immutable */,
+ null /* defaultValue */,
+ false /* hidden */))
+ .put(
+ PaimonConstants.GRAVITINO_DLF_ACCESS_KEY_SECRET,
+ stringOptionalPropertyEntry(
+ PaimonConstants.GRAVITINO_DLF_ACCESS_KEY_SECRET,
+ "The access key secret for Aliyun DLF",
+ false /* immutable */,
+ null /* defaultValue */,
+ false /* hidden */))
+ .put(
+ PaimonConstants.GRAVITINO_DLF_SECURITY_TOKEN,
+ stringOptionalPropertyEntry(
+ PaimonConstants.GRAVITINO_DLF_SECURITY_TOKEN,
+ "The security token for Aliyun DLF",
+ false /* immutable */,
+ null /* defaultValue */,
+ false /* hidden */))
+ .put(
+ PaimonConstants.GRAVITINO_DLF_TOKEN_PATH,
+ stringOptionalPropertyEntry(
+ PaimonConstants.GRAVITINO_DLF_TOKEN_PATH,
+ "The token path for Aliyun DLF",
+ false /* immutable */,
+ null /* defaultValue */,
+ false /* hidden */))
+ .put(
+ PaimonConstants.GRAVITINO_DLF_TOKEN_LOADER,
+ stringOptionalPropertyEntry(
+ PaimonConstants.GRAVITINO_DLF_TOKEN_LOADER,
+ "The token loader for Aliyun DLF",
+ false /* immutable */,
+ null /* defaultValue */,
+ false /* hidden */))
+ .build();
+
static {
List<PropertyEntry<?>> propertyEntries =
ImmutableList.of(
@@ -149,6 +216,7 @@ public class PaimonCatalogPropertiesMetadata extends
BaseCatalogPropertiesMetada
result.putAll(AuthenticationConfig.AUTHENTICATION_PROPERTY_ENTRIES);
result.putAll(PaimonS3FileSystemConfig.S3_FILESYSTEM_PROPERTY_ENTRIES);
result.putAll(PaimonOSSFileSystemConfig.OSS_FILESYSTEM_PROPERTY_ENTRIES);
+ result.putAll(REST_PROPERTY_ENTRIES);
PROPERTIES_METADATA = ImmutableMap.copyOf(result);
}
diff --git a/docs/lakehouse-paimon-catalog.md b/docs/lakehouse-paimon-catalog.md
index 20945fe44f..d1da340c16 100644
--- a/docs/lakehouse-paimon-catalog.md
+++ b/docs/lakehouse-paimon-catalog.md
@@ -31,7 +31,7 @@ Builds with Apache Paimon `1.2`.
| Property name | Description
| Default value
| Required
[...]
|----------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------
[...]
-| `catalog-backend` | Catalog backend of
Gravitino Paimon catalog. Supports `filesystem`, `jdbc` and `hive`.
| (none)
| Yes
[...]
+| `catalog-backend` | Catalog backend of
Gravitino Paimon catalog. Supports `filesystem`, `jdbc`, `hive` and `rest`.
| (none)
| Yes
[...]
| `uri` | The URI configuration
of the Paimon catalog. `thrift://127.0.0.1:9083` or
`jdbc:postgresql://127.0.0.1:5432/db_name` or
`jdbc:mysql://127.0.0.1:3306/metastore_db`. It is optional for
`FilesystemCatalog`. | (none)
| required if the value of `catalog-backend` is not
`filesystem`.
[...]
| `warehouse` | Warehouse directory of
catalog. `file:///user/hive/warehouse-paimon/` for local fs,
`hdfs://namespace/hdfs/path` for HDFS , `s3://{bucket-name}/path/` for S3 or
`oss://{bucket-name}/path` for Aliyun OSS | (none)
| Yes
[...]
| `catalog-backend-name` | The catalog name passed
to underlying Paimon catalog backend.
| The property value of `catalog-backend`, like `jdbc` for
JDBC catalog backend. | No
[...]
@@ -47,13 +47,34 @@ Builds with Apache Paimon `1.2`.
| `s3-endpoint` | The endpoint of the AWS
S3.
| (none)
| required if the value of `warehouse` is a S3 path
[...]
| `s3-access-key-id` | The access key of the
AWS S3.
| (none)
| required if the value of `warehouse` is a S3 path
[...]
| `s3-secret-access-key` | The secret key of the
AWS S3.
| (none)
| required if the value of `warehouse` is a S3 path
[...]
+| `token-provider` | The token provider type
for Paimon catalog backend.
| Token provider could be `bear` or `dlf`.
| required if the value of `catalog-backend` is `rest`.
[...]
+| `token` | The bear token for
Paimon REST catalog authentication.
| (none)
| required if the value of `token-provider` is
`bear`.
[...]
+| `dlf-access-key-id` | The access key ID for
Aliyun DLF (Data Lake Formation).
| (none)
| required if the value of `catalog-backend` is `rest`
and accessing Aliyun DLF Paimon REST server.
[...]
+| `dlf-access-key-secret` | The access key secret
for Aliyun DLF.
| (none)
| required if the value of `catalog-backend` is `rest`
and accessing Aliyun DLF Paimon REST server.
[...]
+| `dlf-security-token` | The security token for
Aliyun DLF.
| (none)
| No
[...]
+| `dlf-token-path` | The token path for
Aliyun DLF.
| (none)
| No
[...]
+| `dlf-token-loader` | The token loader for
Aliyun DLF.
| (none)
| No
[...]
:::note
-If you want to use the `oss` or `s3` warehouse, you need to place related jars
in the `catalogs/lakehouse-paimon/lib` directory, more information can be found
in the [Paimon S3](https://paimon.apache.org/docs/master/filesystems/s3/).
-:::
-
-:::note
-The hive backend does not support the kerberos authentication now.
+- If you want to use the `oss` or `s3` warehouse, you need to place related
jars in the `catalogs/lakehouse-paimon/lib` directory, more information can be
found in the [Paimon
S3](https://paimon.apache.org/docs/1.2/project/download/#filesystem-jars).
+- If you want to use REST backend, Gravitino Paimon catalog supports Aliyun
DLF (Data Lake Formation) as the REST catalog service. You need to configure
the DLF-related properties eg:
+```
+{
+ "name": "dlf_paimon",
+ "type": "RELATIONAL",
+ "provider": "lakehouse-paimon",
+ "properties": {
+ "catalog-backend": "rest",
+ "uri": "<catalog server url>",
+ "warehouse": "gravitino",
+ "token-provider": "dlf",
+ "dlf-access-key-id": "<access-key-id>",
+ "dlf-access-key-secret": "<access-key-secret>"
+ }
+}
+```
+connect to Aliyun DLF, more information can be found in the [Paimon REST
Catalog](https://paimon.apache.org/docs/master/concepts/rest/overview/).
+- The hive backend does not support the kerberos authentication now.
:::
Any properties not defined by Gravitino with `gravitino.bypass.` prefix will
pass to Paimon catalog properties and HDFS configuration. For example, if
specify `gravitino.bypass.table.type`, `table.type` will pass to Paimon catalog
properties.
