This is an automated email from the ASF dual-hosted git repository.
fanng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new d35e5f526 [#5894] feat(iceberg): support Azure account key credential
(#5938)
d35e5f526 is described below
commit d35e5f526317933a17b9c4ba75607384c0384c5f
Author: JUN <oren....@gmail.com>
AuthorDate: Sun Dec 22 11:32:16 2024 +0800
[#5894] feat(iceberg): support Azure account key credential (#5938)
### What changes were proposed in this pull request?
Support Azure account key credential
### Why are the changes needed?
Fix: #5894
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Unit Test IcebergRESTADLSAccountKeyIT at iceberg 1.6.0
---
.../credential/AzureAccountKeyCredential.java | 109 +++++++++++++++++++++
.../org.apache.gravitino.credential.Credential | 1 +
.../abs/credential/ADLSTokenProvider.java | 14 +--
.../abs/credential/AzureAccountKeyProvider.java | 54 ++++++++++
....apache.gravitino.credential.CredentialProvider | 3 +-
.../gravitino/credential/CredentialConstants.java | 2 +
.../credential/CredentialPropertyUtils.java | 31 ++++--
.../credential/TestCredentialFactory.java | 27 +++++
...ntialConfig.java => AzureCredentialConfig.java} | 6 +-
docs/iceberg-rest-service.md | 43 ++++----
...RESTADLSIT.java => IcebergRESTADLSTokenIT.java} | 13 +--
...LSIT.java => IcebergRESTAzureAccountKeyIT.java} | 98 +-----------------
12 files changed, 262 insertions(+), 139 deletions(-)
diff --git
a/api/src/main/java/org/apache/gravitino/credential/AzureAccountKeyCredential.java
b/api/src/main/java/org/apache/gravitino/credential/AzureAccountKeyCredential.java
new file mode 100644
index 000000000..be24d7cda
--- /dev/null
+++
b/api/src/main/java/org/apache/gravitino/credential/AzureAccountKeyCredential.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.gravitino.credential;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import java.util.Map;
+import org.apache.commons.lang3.StringUtils;
+
+/** Azure account key credential. */
+public class AzureAccountKeyCredential implements Credential {
+
+ /** Azure account key credential type. */
+ public static final String AZURE_ACCOUNT_KEY_CREDENTIAL_TYPE =
"azure-account-key";
+ /** Azure storage account name */
+ public static final String GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME =
"azure-storage-account-name";
+ /** Azure storage account key */
+ public static final String GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY =
"azure-storage-account-key";
+
+ private String accountName;
+ private String accountKey;
+
+ /**
+ * Constructs an instance of {@link AzureAccountKeyCredential}.
+ *
+ * @param accountName The Azure account name.
+ * @param accountKey The Azure account key.
+ */
+ public AzureAccountKeyCredential(String accountName, String accountKey) {
+ validate(accountName, accountKey);
+ this.accountName = accountName;
+ this.accountKey = accountKey;
+ }
+
+ /**
+ * This is the constructor that is used by credential factory to create an
instance of credential
+ * according to the credential information.
+ */
+ public AzureAccountKeyCredential() {}
+
+ @Override
+ public String credentialType() {
+ return AZURE_ACCOUNT_KEY_CREDENTIAL_TYPE;
+ }
+
+ @Override
+ public long expireTimeInMs() {
+ return 0;
+ }
+
+ @Override
+ public Map<String, String> credentialInfo() {
+ return (new ImmutableMap.Builder<String, String>())
+ .put(GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME, accountName)
+ .put(GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY, accountKey)
+ .build();
+ }
+
+ @Override
+ public void initialize(Map<String, String> credentialInfo, long
expireTimeInMS) {
+ String accountName =
credentialInfo.get(GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME);
+ String accountKey =
credentialInfo.get(GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY);
+ validate(accountName, accountKey);
+ this.accountName = accountName;
+ this.accountKey = accountKey;
+ }
+
+ /**
+ * Get Azure account name
+ *
+ * @return The Azure account name
+ */
+ public String accountName() {
+ return accountName;
+ }
+
+ /**
+ * Get Azure account key
+ *
+ * @return The Azure account key
+ */
+ public String accountKey() {
+ return accountKey;
+ }
+
+ private void validate(String accountName, String accountKey) {
+ Preconditions.checkArgument(
+ StringUtils.isNotBlank(accountName), "Azure account name should not be
empty.");
+ Preconditions.checkArgument(
+ StringUtils.isNotBlank(accountKey), "Azure account key should not be
empty.");
+ }
+}
diff --git
a/api/src/main/resources/META-INF/services/org.apache.gravitino.credential.Credential
b/api/src/main/resources/META-INF/services/org.apache.gravitino.credential.Credential
index f130b4b64..6071cb916 100644
---
a/api/src/main/resources/META-INF/services/org.apache.gravitino.credential.Credential
+++
b/api/src/main/resources/META-INF/services/org.apache.gravitino.credential.Credential
@@ -23,3 +23,4 @@ org.apache.gravitino.credential.GCSTokenCredential
org.apache.gravitino.credential.OSSTokenCredential
org.apache.gravitino.credential.OSSSecretKeyCredential
org.apache.gravitino.credential.ADLSTokenCredential
+org.apache.gravitino.credential.AzureAccountKeyCredential
diff --git
a/bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/credential/ADLSTokenProvider.java
b/bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/credential/ADLSTokenProvider.java
index e2ee3ed82..c2b684acb 100644
---
a/bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/credential/ADLSTokenProvider.java
+++
b/bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/credential/ADLSTokenProvider.java
@@ -38,7 +38,7 @@ import org.apache.gravitino.credential.CredentialConstants;
import org.apache.gravitino.credential.CredentialContext;
import org.apache.gravitino.credential.CredentialProvider;
import org.apache.gravitino.credential.PathBasedCredentialContext;
-import org.apache.gravitino.credential.config.ADLSCredentialConfig;
+import org.apache.gravitino.credential.config.AzureCredentialConfig;
/** Generates ADLS token to access ADLS data. */
public class ADLSTokenProvider implements CredentialProvider {
@@ -51,14 +51,14 @@ public class ADLSTokenProvider implements
CredentialProvider {
@Override
public void initialize(Map<String, String> properties) {
- ADLSCredentialConfig adlsCredentialConfig = new
ADLSCredentialConfig(properties);
- this.storageAccountName = adlsCredentialConfig.storageAccountName();
- this.tenantId = adlsCredentialConfig.tenantId();
- this.clientId = adlsCredentialConfig.clientId();
- this.clientSecret = adlsCredentialConfig.clientSecret();
+ AzureCredentialConfig azureCredentialConfig = new
AzureCredentialConfig(properties);
+ this.storageAccountName = azureCredentialConfig.storageAccountName();
+ this.tenantId = azureCredentialConfig.tenantId();
+ this.clientId = azureCredentialConfig.clientId();
+ this.clientSecret = azureCredentialConfig.clientSecret();
this.endpoint =
String.format("https://%s.%s";, storageAccountName,
ADLSTokenCredential.ADLS_DOMAIN);
- this.tokenExpireSecs = adlsCredentialConfig.tokenExpireInSecs();
+ this.tokenExpireSecs = azureCredentialConfig.adlsTokenExpireInSecs();
}
@Override
diff --git
a/bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/credential/AzureAccountKeyProvider.java
b/bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/credential/AzureAccountKeyProvider.java
new file mode 100644
index 000000000..726c4f2d9
--- /dev/null
+++
b/bundles/azure-bundle/src/main/java/org/apache/gravitino/abs/credential/AzureAccountKeyProvider.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.gravitino.abs.credential;
+
+import java.util.Map;
+import org.apache.gravitino.credential.AzureAccountKeyCredential;
+import org.apache.gravitino.credential.Credential;
+import org.apache.gravitino.credential.CredentialConstants;
+import org.apache.gravitino.credential.CredentialContext;
+import org.apache.gravitino.credential.CredentialProvider;
+import org.apache.gravitino.credential.config.AzureCredentialConfig;
+
+/** Generates Azure account key to access data. */
+public class AzureAccountKeyProvider implements CredentialProvider {
+ private String accountName;
+ private String accountKey;
+
+ @Override
+ public void initialize(Map<String, String> properties) {
+ AzureCredentialConfig azureCredentialConfig = new
AzureCredentialConfig(properties);
+ this.accountName = azureCredentialConfig.storageAccountName();
+ this.accountKey = azureCredentialConfig.storageAccountKey();
+ }
+
+ @Override
+ public void close() {}
+
+ @Override
+ public String credentialType() {
+ return CredentialConstants.AZURE_ACCOUNT_KEY_CREDENTIAL_PROVIDER_TYPE;
+ }
+
+ @Override
+ public Credential getCredential(CredentialContext context) {
+ return new AzureAccountKeyCredential(accountName, accountKey);
+ }
+}
diff --git
a/bundles/azure-bundle/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider
b/bundles/azure-bundle/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider
index fb53efffa..4c7e7982c 100644
---
a/bundles/azure-bundle/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider
+++
b/bundles/azure-bundle/src/main/resources/META-INF/services/org.apache.gravitino.credential.CredentialProvider
@@ -16,4 +16,5 @@
# specific language governing permissions and limitations
# under the License.
#
-org.apache.gravitino.abs.credential.ADLSTokenProvider
\ No newline at end of file
+org.apache.gravitino.abs.credential.ADLSTokenProvider
+org.apache.gravitino.abs.credential.AzureAccountKeyProvider
\ No newline at end of file
diff --git
a/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java
b/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java
index 7dd74d084..29f9241c8 100644
---
a/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java
+++
b/catalogs/catalog-common/src/main/java/org/apache/gravitino/credential/CredentialConstants.java
@@ -32,5 +32,7 @@ public class CredentialConstants {
public static final String ADLS_TOKEN_CREDENTIAL_PROVIDER_TYPE =
"adls-token";
public static final String ADLS_TOKEN_EXPIRE_IN_SECS =
"adls-token-expire-in-secs";
+ public static final String AZURE_ACCOUNT_KEY_CREDENTIAL_PROVIDER_TYPE =
"azure-account-key";
+
private CredentialConstants() {}
}
diff --git
a/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java
b/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java
index e1803a6dd..d7a3caf06 100644
---
a/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java
+++
b/common/src/main/java/org/apache/gravitino/credential/CredentialPropertyUtils.java
@@ -33,12 +33,19 @@ public class CredentialPropertyUtils {
@VisibleForTesting static final String ICEBERG_S3_SECRET_ACCESS_KEY =
"s3.secret-access-key";
@VisibleForTesting static final String ICEBERG_S3_TOKEN = "s3.session-token";
@VisibleForTesting static final String ICEBERG_GCS_TOKEN =
"gcs.oauth2.token";
- @VisibleForTesting static final String ICEBERG_ADLS_TOKEN = "adls.sas-token";
@VisibleForTesting static final String ICEBERG_OSS_ACCESS_KEY_ID =
"client.access-key-id";
@VisibleForTesting static final String ICEBERG_OSS_ACCESS_KEY_SECRET =
"client.access-key-secret";
@VisibleForTesting static final String ICEBERG_OSS_SECURITY_TOKEN =
"client.security-token";
+ @VisibleForTesting static final String ICEBERG_ADLS_TOKEN = "adls.sas-token";
+
+ @VisibleForTesting
+ static final String ICEBERG_ADLS_ACCOUNT_NAME =
"adls.auth.shared-key.account.name";
+
+ @VisibleForTesting
+ static final String ICEBERG_ADLS_ACCOUNT_KEY =
"adls.auth.shared-key.account.key";
+
private static Map<String, String> icebergCredentialPropertyMap =
ImmutableMap.of(
GCSTokenCredential.GCS_TOKEN_NAME,
@@ -54,7 +61,11 @@ public class CredentialPropertyUtils {
OSSTokenCredential.GRAVITINO_OSS_SESSION_ACCESS_KEY_ID,
ICEBERG_OSS_ACCESS_KEY_ID,
OSSTokenCredential.GRAVITINO_OSS_SESSION_SECRET_ACCESS_KEY,
- ICEBERG_OSS_ACCESS_KEY_SECRET);
+ ICEBERG_OSS_ACCESS_KEY_SECRET,
+ AzureAccountKeyCredential.GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME,
+ ICEBERG_ADLS_ACCOUNT_NAME,
+ AzureAccountKeyCredential.GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY,
+ ICEBERG_ADLS_ACCOUNT_KEY);
/**
* Transforms a specific credential into a map of Iceberg properties.
@@ -63,6 +74,14 @@ public class CredentialPropertyUtils {
* @return a map of Iceberg properties derived from the credential
*/
public static Map<String, String> toIcebergProperties(Credential credential)
{
+ if (credential instanceof S3TokenCredential
+ || credential instanceof S3SecretKeyCredential
+ || credential instanceof OSSTokenCredential
+ || credential instanceof OSSSecretKeyCredential
+ || credential instanceof AzureAccountKeyCredential) {
+ return transformProperties(credential.credentialInfo(),
icebergCredentialPropertyMap);
+ }
+
if (credential instanceof GCSTokenCredential) {
Map<String, String> icebergGCSCredentialProperties =
transformProperties(credential.credentialInfo(),
icebergCredentialPropertyMap);
@@ -70,12 +89,7 @@ public class CredentialPropertyUtils {
"gcs.oauth2.token-expires-at",
String.valueOf(credential.expireTimeInMs()));
return icebergGCSCredentialProperties;
}
- if (credential instanceof S3TokenCredential || credential instanceof
S3SecretKeyCredential) {
- return transformProperties(credential.credentialInfo(),
icebergCredentialPropertyMap);
- }
- if (credential instanceof OSSTokenCredential || credential instanceof
OSSSecretKeyCredential) {
- return transformProperties(credential.credentialInfo(),
icebergCredentialPropertyMap);
- }
+
if (credential instanceof ADLSTokenCredential) {
ADLSTokenCredential adlsCredential = (ADLSTokenCredential) credential;
String sasTokenKey =
@@ -87,6 +101,7 @@ public class CredentialPropertyUtils {
icebergADLSCredentialProperties.put(sasTokenKey,
adlsCredential.sasToken());
return icebergADLSCredentialProperties;
}
+
return credential.toProperties();
}
diff --git
a/common/src/test/java/org/apache/gravitino/credential/TestCredentialFactory.java
b/common/src/test/java/org/apache/gravitino/credential/TestCredentialFactory.java
index 75a669e38..6291b8857 100644
---
a/common/src/test/java/org/apache/gravitino/credential/TestCredentialFactory.java
+++
b/common/src/test/java/org/apache/gravitino/credential/TestCredentialFactory.java
@@ -165,4 +165,31 @@ public class TestCredentialFactory {
Assertions.assertEquals(sasToken, adlsTokenCredential.sasToken());
Assertions.assertEquals(expireTime, adlsTokenCredential.expireTimeInMs());
}
+
+ @Test
+ void testAzureAccountKeyCredential() {
+ String storageAccountName = "storage-account-name";
+ String storageAccountKey = "storage-account-key";
+
+ Map<String, String> azureAccountKeyCredentialInfo =
+ ImmutableMap.of(
+ AzureAccountKeyCredential.GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME,
+ storageAccountName,
+ AzureAccountKeyCredential.GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY,
+ storageAccountKey);
+ long expireTime = 0;
+ Credential credential =
+ CredentialFactory.create(
+ AzureAccountKeyCredential.AZURE_ACCOUNT_KEY_CREDENTIAL_TYPE,
+ azureAccountKeyCredentialInfo,
+ expireTime);
+ Assertions.assertEquals(
+ AzureAccountKeyCredential.AZURE_ACCOUNT_KEY_CREDENTIAL_TYPE,
credential.credentialType());
+ Assertions.assertInstanceOf(AzureAccountKeyCredential.class, credential);
+
+ AzureAccountKeyCredential azureAccountKeyCredential =
(AzureAccountKeyCredential) credential;
+ Assertions.assertEquals(storageAccountName,
azureAccountKeyCredential.accountName());
+ Assertions.assertEquals(storageAccountKey,
azureAccountKeyCredential.accountKey());
+ Assertions.assertEquals(expireTime,
azureAccountKeyCredential.expireTimeInMs());
+ }
}
diff --git
a/core/src/main/java/org/apache/gravitino/credential/config/ADLSCredentialConfig.java
b/core/src/main/java/org/apache/gravitino/credential/config/AzureCredentialConfig.java
similarity index 96%
rename from
core/src/main/java/org/apache/gravitino/credential/config/ADLSCredentialConfig.java
rename to
core/src/main/java/org/apache/gravitino/credential/config/AzureCredentialConfig.java
index e9d368e67..155cc6806 100644
---
a/core/src/main/java/org/apache/gravitino/credential/config/ADLSCredentialConfig.java
+++
b/core/src/main/java/org/apache/gravitino/credential/config/AzureCredentialConfig.java
@@ -29,7 +29,7 @@ import org.apache.gravitino.config.ConfigEntry;
import org.apache.gravitino.credential.CredentialConstants;
import org.apache.gravitino.storage.AzureProperties;
-public class ADLSCredentialConfig extends Config {
+public class AzureCredentialConfig extends Config {
public static final ConfigEntry<String> AZURE_STORAGE_ACCOUNT_NAME =
new ConfigBuilder(AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME)
@@ -79,7 +79,7 @@ public class ADLSCredentialConfig extends Config {
.intConf()
.createWithDefault(3600);
- public ADLSCredentialConfig(Map<String, String> properties) {
+ public AzureCredentialConfig(Map<String, String> properties) {
super(false);
loadFromMap(properties, k -> true);
}
@@ -110,7 +110,7 @@ public class ADLSCredentialConfig extends Config {
}
@NotNull
- public Integer tokenExpireInSecs() {
+ public Integer adlsTokenExpireInSecs() {
return this.get(ADLS_TOKEN_EXPIRE_IN_SECS);
}
}
diff --git a/docs/iceberg-rest-service.md b/docs/iceberg-rest-service.md
index 8d9d49745..f31aa1368 100644
--- a/docs/iceberg-rest-service.md
+++ b/docs/iceberg-rest-service.md
@@ -106,18 +106,18 @@ The detailed configuration items are as follows:
Gravitino Iceberg REST service supports using static S3 secret key or
generating temporary token to access S3 data.
-| Configuration item | Description
| Default value | Required
| Since Version |
-|---------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|----------------------------------------------------|------------------|
-| `gravitino.iceberg-rest.io-impl` | The IO implementation
for `FileIO` in Iceberg, use `org.apache.iceberg.aws.s3.S3FileIO` for S3.
| (none) | No
| 0.6.0-incubating |
-| `gravitino.iceberg-rest.credential-provider-type` | Supports `s3-token` and
`s3-secret-key` for S3. `s3-token` generates a temporary token according to the
query data path while `s3-secret-key` using the s3 secret access key to access
S3 data. | (none) | No
| 0.7.0-incubating |
-| `gravitino.iceberg-rest.s3-access-key-id` | The static access key ID
used to access S3 data.
| (none) | No
| 0.6.0-incubating |
-| `gravitino.iceberg-rest.s3-secret-access-key` | The static secret access
key used to access S3 data.
| (none) | No
| 0.6.0-incubating |
-| `gravitino.iceberg-rest.s3-endpoint` | An alternative endpoint
of the S3 service, This could be used for S3FileIO with any s3-compatible
object storage service that has a different endpoint, or access a private S3
endpoint in a virtual private cloud. | (none) | No
| 0.6.0-incubating |
-| `gravitino.iceberg-rest.s3-region` | The region of the S3
service, like `us-west-2`.
| (none) | No
| 0.6.0-incubating |
-| `gravitino.iceberg-rest.s3-role-arn` | The ARN of the role to
access the S3 data.
| (none) | Yes, when
`credential-provider-type` is `s3-token` | 0.7.0-incubating |
-| `gravitino.iceberg-rest.s3-external-id` | The S3 external id to
generate token, only used when `credential-provider-type` is `s3-token`.
| (none) | No
| 0.7.0-incubating |
-| `gravitino.iceberg-rest.s3-token-expire-in-secs` | The S3 session token
expire time in secs, it couldn't exceed the max session time of the assumed
role, only used when `credential-provider-type` is `s3-token`.
| 3600 | No
| 0.7.0-incubating |
-| `gravitino.iceberg-rest.s3-token-service-endpoint` | An alternative
endpoint of the S3 token service, This could be used with s3-compatible object
storage service like MINIO that has a different STS endpoint. | (none) |
No | 0.8.0-incubating |
+| Configuration item | Description
| Default value | Required
| Since Version |
+|----------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|----------------------------------------------------|------------------|
+| `gravitino.iceberg-rest.io-impl` | The IO implementation
for `FileIO` in Iceberg, use `org.apache.iceberg.aws.s3.S3FileIO` for S3.
| (none) | No
| 0.6.0-incubating |
+| `gravitino.iceberg-rest.credential-provider-type` | Supports `s3-token` and
`s3-secret-key` for S3. `s3-token` generates a temporary token according to the
query data path while `s3-secret-key` using the s3 secret access key to access
S3 data. | (none) | No
| 0.7.0-incubating |
+| `gravitino.iceberg-rest.s3-access-key-id` | The static access key
ID used to access S3 data.
| (none) | No
| 0.6.0-incubating |
+| `gravitino.iceberg-rest.s3-secret-access-key` | The static secret
access key used to access S3 data.
| (none) | No
| 0.6.0-incubating |
+| `gravitino.iceberg-rest.s3-endpoint` | An alternative endpoint
of the S3 service, This could be used for S3FileIO with any s3-compatible
object storage service that has a different endpoint, or access a private S3
endpoint in a virtual private cloud. | (none) | No
| 0.6.0-incubating |
+| `gravitino.iceberg-rest.s3-region` | The region of the S3
service, like `us-west-2`.
| (none) | No
| 0.6.0-incubating |
+| `gravitino.iceberg-rest.s3-role-arn` | The ARN of the role to
access the S3 data.
| (none) | Yes, when
`credential-provider-type` is `s3-token` | 0.7.0-incubating |
+| `gravitino.iceberg-rest.s3-external-id` | The S3 external id to
generate token, only used when `credential-provider-type` is `s3-token`.
| (none) | No
| 0.7.0-incubating |
+| `gravitino.iceberg-rest.s3-token-expire-in-secs` | The S3 session token
expire time in secs, it couldn't exceed the max session time of the assumed
role, only used when `credential-provider-type` is `s3-token`.
| 3600 | No
| 0.7.0-incubating |
+| `gravitino.iceberg-rest.s3-token-service-endpoint` | An alternative endpoint
of the S3 token service, This could be used with s3-compatible object storage
service like MINIO that has a different STS endpoint.
| (none) | No
| 0.8.0-incubating |
For other Iceberg s3 properties not managed by Gravitino like `s3.sse.type`,
you could config it directly by `gravitino.iceberg-rest.s3.sse.type`.
@@ -175,15 +175,16 @@ Please set `gravitino.iceberg-rest.warehouse` to
`gs://{bucket_name}/${prefix_na
Gravitino Iceberg REST service supports generating SAS token to access ADLS
data.
-| Configuration item | Description
| Default value | Required | Since Version |
-|-----------------------------------------------------|-----------------------------------------------------------------------------------------------------------|---------------|----------|------------------|
-| `gravitino.iceberg-rest.io-impl` | The IO implementation
for `FileIO` in Iceberg, use `org.apache.iceberg.azure.adlsv2.ADLSFileIO` for
ADLS. | (none) | Yes | 0.8.0-incubating |
-| `gravitino.iceberg-rest.credential-provider-type` | Supports `adls-token`,
generates a temporary token according to the query data path.
| (none) | Yes | 0.8.0-incubating |
-| `gravitino.iceberg-rest.azure-storage-account-name` | The static storage
account name used to access ADLS data.
| (none) | Yes | 0.8.0-incubating |
-| `gravitino.iceberg-rest.azure-storage-account-key` | The static storage
account key used to access ADLS data.
| (none) | Yes | 0.8.0-incubating |
-| `gravitino.iceberg-rest.azure-tenant-id` | Azure Active Directory
(AAD) tenant ID.
| (none) | Yes | 0.8.0-incubating |
-| `gravitino.iceberg-rest.azure-client-id` | Azure Active Directory
(AAD) client ID used for authentication.
| (none) | Yes | 0.8.0-incubating |
-| `gravitino.iceberg-rest.azure-client-secret` | Azure Active Directory
(AAD) client secret used for authentication.
| (none) | Yes | 0.8.0-incubating |
+| Configuration item | Description
| Default value | Required | Since Version |
+|-----------------------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------|----------|------------------|
+| `gravitino.iceberg-rest.io-impl` | The IO implementation
for `FileIO` in Iceberg, use `org.apache.iceberg.azure.adlsv2.ADLSFileIO` for
ADLS.
| (none) | Yes | 0.8.0-incubating |
+| `gravitino.iceberg-rest.credential-provider-type` | Supports `adls-token`
and `azure-account-key`. `adls-token` generates a temporary token according to
the query data path while `azure-account-key` uses a storage account key to
access ADLS data. | (none) | Yes | 0.8.0-incubating |
+| `gravitino.iceberg-rest.azure-storage-account-name` | The static storage
account name used to access ADLS data.
| (none) | Yes | 0.8.0-incubating |
+| `gravitino.iceberg-rest.azure-storage-account-key` | The static storage
account key used to access ADLS data.
| (none) | Yes | 0.8.0-incubating |
+| `gravitino.iceberg-rest.azure-tenant-id` | Azure Active Directory
(AAD) tenant ID, only used when `credential-provider-type` is `adls-token`.
| (none) | Yes | 0.8.0-incubating |
+| `gravitino.iceberg-rest.azure-client-id` | Azure Active Directory
(AAD) client ID used for authentication, only used when
`credential-provider-type` is `adls-token`.
| (none) | Yes |
0.8.0-incubating |
+| `gravitino.iceberg-rest.azure-client-secret` | Azure Active Directory
(AAD) client secret used for authentication, only used when
`credential-provider-type` is `adls-token`.
| (none) | Yes | 0.8.0-incubating |
+| `gravitino.iceberg-rest.adls-token-expire-in-secs` | The ADLS SAS token
expire time in secs, only used when `credential-provider-type` is
`adls-token`.
| 3600 | No | 0.8.0-incubating |
For other Iceberg ADLS properties not managed by Gravitino like
`adls.read.block-size-bytes`, you could config it directly by
`gravitino.iceberg-rest.adls.read.block-size-bytes`.
diff --git
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSIT.java
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSTokenIT.java
similarity index 92%
copy from
iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSIT.java
copy to
iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSTokenIT.java
index 570298d05..b16d504e1 100644
---
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSIT.java
+++
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSTokenIT.java
@@ -36,7 +36,7 @@ import
org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
@SuppressWarnings("FormatStringAnnotation")
@EnabledIfEnvironmentVariable(named = "GRAVITINO_TEST_CLOUD_IT", matches =
"true")
-public class IcebergRESTADLSIT extends IcebergRESTJdbcCatalogIT {
+public class IcebergRESTADLSTokenIT extends IcebergRESTJdbcCatalogIT {
private String storageAccountName;
private String storageAccountKey;
@@ -49,13 +49,14 @@ public class IcebergRESTADLSIT extends
IcebergRESTJdbcCatalogIT {
void initEnv() {
this.storageAccountName =
System.getenv()
- .getOrDefault("GRAVITINO_ADLS_STORAGE_ACCOUNT_NAME",
"{STORAGE_ACCOUNT_NAME}");
+ .getOrDefault("GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME",
"{STORAGE_ACCOUNT_NAME}");
this.storageAccountKey =
- System.getenv().getOrDefault("GRAVITINO_ADLS_STORAGE_ACCOUNT_KEY",
"{STORAGE_ACCOUNT_KEY}");
- this.tenantId = System.getenv().getOrDefault("GRAVITINO_ADLS_TENANT_ID",
"{TENANT_ID}");
- this.clientId = System.getenv().getOrDefault("GRAVITINO_ADLS_CLIENT_ID",
"{CLIENT_ID}");
+ System.getenv()
+ .getOrDefault("GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY",
"{STORAGE_ACCOUNT_KEY}");
+ this.tenantId = System.getenv().getOrDefault("GRAVITINO_AZURE_TENANT_ID",
"{TENANT_ID}");
+ this.clientId = System.getenv().getOrDefault("GRAVITINO_AZURE_CLIENT_ID",
"{CLIENT_ID}");
this.clientSecret =
- System.getenv().getOrDefault("GRAVITINO_ADLS_CLIENT_SECRET",
"{CLIENT_SECRET}");
+ System.getenv().getOrDefault("GRAVITINO_AZURE_CLIENT_SECRET",
"{CLIENT_SECRET}");
this.warehousePath =
String.format(
"abfss://%s@%s.dfs.core.windows.net/data/test",
diff --git
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSIT.java
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTAzureAccountKeyIT.java
similarity index 53%
rename from
iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSIT.java
rename to
iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTAzureAccountKeyIT.java
index 570298d05..42709162a 100644
---
a/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTADLSIT.java
+++
b/iceberg/iceberg-rest-server/src/test/java/org/apache/gravitino/iceberg/integration/test/IcebergRESTAzureAccountKeyIT.java
@@ -22,7 +22,6 @@ package org.apache.gravitino.iceberg.integration.test;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
-import org.apache.gravitino.abs.credential.ADLSLocationUtils;
import org.apache.gravitino.catalog.lakehouse.iceberg.IcebergConstants;
import org.apache.gravitino.credential.CredentialConstants;
import org.apache.gravitino.iceberg.common.IcebergConfig;
@@ -30,32 +29,24 @@ import org.apache.gravitino.integration.test.util.BaseIT;
import org.apache.gravitino.integration.test.util.DownloaderUtils;
import org.apache.gravitino.integration.test.util.ITUtils;
import org.apache.gravitino.storage.AzureProperties;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable;
@SuppressWarnings("FormatStringAnnotation")
@EnabledIfEnvironmentVariable(named = "GRAVITINO_TEST_CLOUD_IT", matches =
"true")
-public class IcebergRESTADLSIT extends IcebergRESTJdbcCatalogIT {
+public class IcebergRESTAzureAccountKeyIT extends IcebergRESTJdbcCatalogIT {
private String storageAccountName;
private String storageAccountKey;
- private String tenantId;
- private String clientId;
- private String clientSecret;
private String warehousePath;
@Override
void initEnv() {
this.storageAccountName =
System.getenv()
- .getOrDefault("GRAVITINO_ADLS_STORAGE_ACCOUNT_NAME",
"{STORAGE_ACCOUNT_NAME}");
+ .getOrDefault("GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME",
"{STORAGE_ACCOUNT_NAME}");
this.storageAccountKey =
- System.getenv().getOrDefault("GRAVITINO_ADLS_STORAGE_ACCOUNT_KEY",
"{STORAGE_ACCOUNT_KEY}");
- this.tenantId = System.getenv().getOrDefault("GRAVITINO_ADLS_TENANT_ID",
"{TENANT_ID}");
- this.clientId = System.getenv().getOrDefault("GRAVITINO_ADLS_CLIENT_ID",
"{CLIENT_ID}");
- this.clientSecret =
- System.getenv().getOrDefault("GRAVITINO_ADLS_CLIENT_SECRET",
"{CLIENT_SECRET}");
+ System.getenv()
+ .getOrDefault("GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY",
"{STORAGE_ACCOUNT_KEY}");
this.warehousePath =
String.format(
"abfss://%s@%s.dfs.core.windows.net/data/test",
@@ -91,20 +82,13 @@ public class IcebergRESTADLSIT extends
IcebergRESTJdbcCatalogIT {
configMap.put(
IcebergConfig.ICEBERG_CONFIG_PREFIX +
CredentialConstants.CREDENTIAL_PROVIDER_TYPE,
- CredentialConstants.ADLS_TOKEN_CREDENTIAL_PROVIDER_TYPE);
+ CredentialConstants.AZURE_ACCOUNT_KEY_CREDENTIAL_PROVIDER_TYPE);
configMap.put(
IcebergConfig.ICEBERG_CONFIG_PREFIX +
AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME,
storageAccountName);
configMap.put(
IcebergConfig.ICEBERG_CONFIG_PREFIX +
AzureProperties.GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY,
storageAccountKey);
- configMap.put(
- IcebergConfig.ICEBERG_CONFIG_PREFIX +
AzureProperties.GRAVITINO_AZURE_TENANT_ID, tenantId);
- configMap.put(
- IcebergConfig.ICEBERG_CONFIG_PREFIX +
AzureProperties.GRAVITINO_AZURE_CLIENT_ID, clientId);
- configMap.put(
- IcebergConfig.ICEBERG_CONFIG_PREFIX +
AzureProperties.GRAVITINO_AZURE_CLIENT_SECRET,
- clientSecret);
configMap.put(
IcebergConfig.ICEBERG_CONFIG_PREFIX + IcebergConstants.IO_IMPL,
@@ -130,76 +114,4 @@ public class IcebergRESTADLSIT extends
IcebergRESTJdbcCatalogIT {
String targetDir = String.format("%s/iceberg-rest-server/libs/",
gravitinoHome);
BaseIT.copyBundleJarsToDirectory("azure-bundle", targetDir);
}
-
- @Test
- public void testParseLocationValidInput() {
- String location =
"abfss://contai...@account.dfs.core.windows.net/data/test/path";
-
- ADLSLocationUtils.ADLSLocationParts parts =
ADLSLocationUtils.parseLocation(location);
-
- Assertions.assertEquals("container", parts.getContainer(), "Container name
should match");
- Assertions.assertEquals("account", parts.getAccountName(), "Account name
should match");
- Assertions.assertEquals("/data/test/path", parts.getPath(), "Path should
match");
- }
-
- @Test
- public void testParseLocationInvalidInput() {
- String location = "abfss://container/invalid/location";
-
- Exception exception =
- Assertions.assertThrows(
- IllegalArgumentException.class,
- () -> {
- ADLSLocationUtils.parseLocation(location);
- });
-
- Assertions.assertTrue(
- exception.getMessage().contains("Invalid location"),
- "Exception message should indicate invalid location");
- }
-
- @Test
- public void testTrimSlashesNullInput() {
- Assertions.assertNull(ADLSLocationUtils.trimSlashes(null), "Null input
should return null");
- }
-
- @Test
- public void testTrimSlashesEmptyInput() {
- Assertions.assertEquals(
- "", ADLSLocationUtils.trimSlashes(""), "Empty input should return
empty string");
- }
-
- @Test
- public void testTrimSlashesNoSlashes() {
- String input = "data/test/path";
- Assertions.assertEquals(
- "data/test/path",
- ADLSLocationUtils.trimSlashes(input),
- "Input without slashes should remain unchanged");
- }
-
- @Test
- public void testTrimSlashesLeadingAndTrailingSlashes() {
- String input = "/data/test/path/";
- Assertions.assertEquals(
- "data/test/path",
- ADLSLocationUtils.trimSlashes(input),
- "Leading and trailing slashes should be trimmed");
- }
-
- @Test
- public void testTrimSlashesMultipleLeadingAndTrailingSlashes() {
- String input = "///data/test/path///";
- Assertions.assertEquals(
- "data/test/path",
- ADLSLocationUtils.trimSlashes(input),
- "Multiple leading and trailing slashes should be trimmed");
- }
-
- @Test
- public void testTrimSlashesOnlySlashes() {
- String input = "////";
- Assertions.assertEquals(
- "", ADLSLocationUtils.trimSlashes(input), "Only slashes should result
in an empty string");
- }
}
