This is an automated email from the ASF dual-hosted git repository.
liuxun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new c98c15197 [#5623] feat(python): supports credential API in python
client (#5777)
c98c15197 is described below
commit c98c151972790793d5f5411052b10f381d75e95d
Author: FANNG <xiaoj...@datastrato.com>
AuthorDate: Thu Dec 12 21:48:07 2024 +0800
[#5623] feat(python): supports credential API in python client (#5777)
### What changes were proposed in this pull request?
supports credential API in python client
```python
catalog = gravitino_client.load_catalog(catalog_name)
catalog.as_fileset_catalog().support_credentials().get_credentials()
fileset = catalog.as_fileset_catalog().load_fileset(
NameIdentifier.of("schema", "fileset")
)
credentials = fileset.support_credentials().get_credentials()
```
### Why are the changes needed?
Fix: #5623
### Does this PR introduce _any_ user-facing change?
no
### How was this patch tested?
1. add UT
2. setup a Gravitino server which returns credential, test with python
client
---
.../gravitino/api/credential/__init__.py | 16 +++
.../gravitino/api/credential/credential.py | 51 ++++++++++
.../api/credential/gcs_token_credential.py | 75 ++++++++++++++
.../api/credential/oss_token_credential.py | 105 ++++++++++++++++++++
.../api/credential/s3_secret_key_credential.py | 91 +++++++++++++++++
.../api/credential/s3_token_credential.py | 110 +++++++++++++++++++++
.../api/credential/supports_credentials.py | 73 ++++++++++++++
.../client-python/gravitino/api/metadata_object.py | 56 +++++++++++
.../gravitino/catalog/base_schema_catalog.py | 13 +++
.../gravitino/catalog/fileset_catalog.py | 13 ++-
.../gravitino/client/generic_fileset.py | 75 ++++++++++++++
.../metadata_object_credential_operations.py | 74 ++++++++++++++
.../gravitino/client/metadata_object_impl.py | 35 +++++++
.../client-python/gravitino/dto/credential_dto.py | 42 ++++++++
.../gravitino/dto/responses/credential_response.py | 44 +++++++++
clients/client-python/gravitino/exceptions/base.py | 8 ++
.../handlers/credential_error_handler.py | 45 +++++++++
.../gravitino/utils/credential_factory.py | 39 ++++++++
.../client-python/gravitino/utils/precondition.py | 48 +++++++++
clients/client-python/tests/unittests/mock_base.py | 4 +
.../tests/unittests/test_credential_api.py | 105 ++++++++++++++++++++
.../tests/unittests/test_credential_factory.py | 101 +++++++++++++++++++
.../tests/unittests/test_error_handler.py | 23 +++++
.../tests/unittests/test_precondition.py | 46 +++++++++
.../tests/unittests/test_responses.py | 35 +++++++
25 files changed, 1325 insertions(+), 2 deletions(-)
diff --git a/clients/client-python/gravitino/api/credential/__init__.py
b/clients/client-python/gravitino/api/credential/__init__.py
new file mode 100644
index 000000000..325597ecf
--- /dev/null
+++ b/clients/client-python/gravitino/api/credential/__init__.py
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
diff --git a/clients/client-python/gravitino/api/credential/credential.py
b/clients/client-python/gravitino/api/credential/credential.py
new file mode 100644
index 000000000..37b97694a
--- /dev/null
+++ b/clients/client-python/gravitino/api/credential/credential.py
@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from abc import ABC, abstractmethod
+from typing import Dict
+
+
+class Credential(ABC):
+ """Represents the credential in Gravitino."""
+
+ @abstractmethod
+ def credential_type(self) -> str:
+ """The type of the credential.
+
+ Returns:
+ the type of the credential.
+ """
+ pass
+
+ @abstractmethod
+ def expire_time_in_ms(self) -> int:
+ """Returns the expiration time of the credential in milliseconds since
+ the epoch, 0 means it will never expire.
+
+ Returns:
+ The expiration time of the credential.
+ """
+ pass
+
+ @abstractmethod
+ def credential_info(self) -> Dict[str, str]:
+ """The credential information.
+
+ Returns:
+ The credential information.
+ """
+ pass
diff --git
a/clients/client-python/gravitino/api/credential/gcs_token_credential.py
b/clients/client-python/gravitino/api/credential/gcs_token_credential.py
new file mode 100644
index 000000000..1362383f0
--- /dev/null
+++ b/clients/client-python/gravitino/api/credential/gcs_token_credential.py
@@ -0,0 +1,75 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from abc import ABC
+from typing import Dict
+
+from gravitino.api.credential.credential import Credential
+from gravitino.utils.precondition import Precondition
+
+
+class GCSTokenCredential(Credential, ABC):
+ """Represents the GCS token credential."""
+
+ GCS_TOKEN_CREDENTIAL_TYPE: str = "gcs-token"
+ _GCS_TOKEN_NAME: str = "token"
+
+ _expire_time_in_ms: int = 0
+
+ def __init__(self, credential_info: Dict[str, str], expire_time_in_ms:
int):
+ self._token = credential_info[self._GCS_TOKEN_NAME]
+ self._expire_time_in_ms = expire_time_in_ms
+ Precondition.check_string_not_empty(
+ self._token, "GCS token should not be empty"
+ )
+ Precondition.check_argument(
+ self._expire_time_in_ms > 0,
+ "The expiration time of GCS token credential should be greater
than 0",
+ )
+
+ def credential_type(self) -> str:
+ """The type of the credential.
+
+ Returns:
+ the type of the credential.
+ """
+ return self.GCS_TOKEN_CREDENTIAL_TYPE
+
+ def expire_time_in_ms(self) -> int:
+ """Returns the expiration time of the credential in milliseconds since
+ the epoch, 0 means it will never expire.
+
+ Returns:
+ The expiration time of the credential.
+ """
+ return self._expire_time_in_ms
+
+ def credential_info(self) -> Dict[str, str]:
+ """The credential information.
+
+ Returns:
+ The credential information.
+ """
+ return {self._GCS_TOKEN_NAME: self._token}
+
+ def token(self) -> str:
+ """The GCS token.
+
+ Returns:
+ The GCS token.
+ """
+ return self._token
diff --git
a/clients/client-python/gravitino/api/credential/oss_token_credential.py
b/clients/client-python/gravitino/api/credential/oss_token_credential.py
new file mode 100644
index 000000000..70dad14a1
--- /dev/null
+++ b/clients/client-python/gravitino/api/credential/oss_token_credential.py
@@ -0,0 +1,105 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from abc import ABC
+from typing import Dict
+
+from gravitino.api.credential.credential import Credential
+from gravitino.utils.precondition import Precondition
+
+
+class OSSTokenCredential(Credential, ABC):
+ """Represents OSS token credential."""
+
+ OSS_TOKEN_CREDENTIAL_TYPE: str = "oss-token"
+ _GRAVITINO_OSS_SESSION_ACCESS_KEY_ID: str = "oss-access-key-id"
+ _GRAVITINO_OSS_SESSION_SECRET_ACCESS_KEY: str = "oss-secret-access-key"
+ _GRAVITINO_OSS_TOKEN: str = "oss-security-token"
+
+ def __init__(self, credential_info: Dict[str, str], expire_time_in_ms:
int):
+ self._access_key_id =
credential_info[self._GRAVITINO_OSS_SESSION_ACCESS_KEY_ID]
+ self._secret_access_key = credential_info[
+ self._GRAVITINO_OSS_SESSION_SECRET_ACCESS_KEY
+ ]
+ self._security_token = credential_info[self._GRAVITINO_OSS_TOKEN]
+ self._expire_time_in_ms = expire_time_in_ms
+ Precondition.check_string_not_empty(
+ self._access_key_id, "The OSS access key ID should not be empty"
+ )
+ Precondition.check_string_not_empty(
+ self._secret_access_key, "The OSS secret access key should not be
empty"
+ )
+ Precondition.check_string_not_empty(
+ self._security_token, "The OSS security token should not be empty"
+ )
+ Precondition.check_argument(
+ self._expire_time_in_ms > 0,
+ "The expiration time of OSS token credential should be greater
than 0",
+ )
+
+ def credential_type(self) -> str:
+ """The type of the credential.
+
+ Returns:
+ the type of the credential.
+ """
+ return self.OSS_TOKEN_CREDENTIAL_TYPE
+
+ def expire_time_in_ms(self) -> int:
+ """Returns the expiration time of the credential in milliseconds since
+ the epoch, 0 means it will never expire.
+
+ Returns:
+ The expiration time of the credential.
+ """
+ return self._expire_time_in_ms
+
+ def credential_info(self) -> Dict[str, str]:
+ """The credential information.
+
+ Returns:
+ The credential information.
+ """
+ return {
+ self._GRAVITINO_OSS_TOKEN: self._security_token,
+ self._GRAVITINO_OSS_SESSION_ACCESS_KEY_ID: self._access_key_id,
+ self._GRAVITINO_OSS_SESSION_SECRET_ACCESS_KEY:
self._secret_access_key,
+ }
+
+ def access_key_id(self) -> str:
+ """The OSS access key id.
+
+ Returns:
+ The OSS access key id.
+ """
+ return self._access_key_id
+
+ def secret_access_key(self) -> str:
+ """The OSS secret access key.
+
+ Returns:
+ The OSS secret access key.
+ """
+ return self._secret_access_key
+
+ def security_token(self) -> str:
+ """The OSS security token.
+
+ Returns:
+ The OSS security token.
+ """
+ return self._security_token
diff --git
a/clients/client-python/gravitino/api/credential/s3_secret_key_credential.py
b/clients/client-python/gravitino/api/credential/s3_secret_key_credential.py
new file mode 100644
index 000000000..735c41e2e
--- /dev/null
+++ b/clients/client-python/gravitino/api/credential/s3_secret_key_credential.py
@@ -0,0 +1,91 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from abc import ABC
+from typing import Dict
+
+from gravitino.api.credential.credential import Credential
+from gravitino.utils.precondition import Precondition
+
+
+class S3SecretKeyCredential(Credential, ABC):
+ """Represents S3 secret key credential."""
+
+ S3_SECRET_KEY_CREDENTIAL_TYPE: str = "s3-secret-key"
+ _GRAVITINO_S3_STATIC_ACCESS_KEY_ID: str = "s3-access-key-id"
+ _GRAVITINO_S3_STATIC_SECRET_ACCESS_KEY: str = "s3-secret-access-key"
+
+ def __init__(self, credential_info: Dict[str, str], expire_time: int):
+ self._access_key_id =
credential_info[self._GRAVITINO_S3_STATIC_ACCESS_KEY_ID]
+ self._secret_access_key = credential_info[
+ self._GRAVITINO_S3_STATIC_SECRET_ACCESS_KEY
+ ]
+ Precondition.check_string_not_empty(
+ self._access_key_id, "S3 access key id should not be empty"
+ )
+ Precondition.check_string_not_empty(
+ self._secret_access_key, "S3 secret access key should not be empty"
+ )
+ Precondition.check_argument(
+ expire_time == 0,
+ "The expiration time of S3 secret key credential should be 0",
+ )
+
+ def credential_type(self) -> str:
+ """Returns the expiration time of the credential in milliseconds since
+ the epoch, 0 means it will never expire.
+
+ Returns:
+ The expiration time of the credential.
+ """
+ return self.S3_SECRET_KEY_CREDENTIAL_TYPE
+
+ def expire_time_in_ms(self) -> int:
+ """Returns the expiration time of the credential in milliseconds since
+ the epoch, 0 means it will never expire.
+
+ Returns:
+ The expiration time of the credential.
+ """
+ return 0
+
+ def credential_info(self) -> Dict[str, str]:
+ """The credential information.
+
+ Returns:
+ The credential information.
+ """
+ return {
+ self._GRAVITINO_S3_STATIC_SECRET_ACCESS_KEY:
self._secret_access_key,
+ self._GRAVITINO_S3_STATIC_ACCESS_KEY_ID: self._access_key_id,
+ }
+
+ def access_key_id(self) -> str:
+ """The S3 access key id.
+
+ Returns:
+ The S3 access key id.
+ """
+ return self._access_key_id
+
+ def secret_access_key(self) -> str:
+ """The S3 secret access key.
+
+ Returns:
+ The S3 secret access key.
+ """
+ return self._secret_access_key
diff --git
a/clients/client-python/gravitino/api/credential/s3_token_credential.py
b/clients/client-python/gravitino/api/credential/s3_token_credential.py
new file mode 100644
index 000000000..c72d9f02a
--- /dev/null
+++ b/clients/client-python/gravitino/api/credential/s3_token_credential.py
@@ -0,0 +1,110 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from abc import ABC
+from typing import Dict
+
+from gravitino.api.credential.credential import Credential
+from gravitino.utils.precondition import Precondition
+
+
+class S3TokenCredential(Credential, ABC):
+ """Represents the S3 token credential."""
+
+ S3_TOKEN_CREDENTIAL_TYPE: str = "s3-token"
+ _GRAVITINO_S3_SESSION_ACCESS_KEY_ID: str = "s3-access-key-id"
+ _GRAVITINO_S3_SESSION_SECRET_ACCESS_KEY: str = "s3-secret-access-key"
+ _GRAVITINO_S3_TOKEN: str = "s3-session-token"
+
+ _expire_time_in_ms: int = 0
+ _access_key_id: str = None
+ _secret_access_key: str = None
+ _session_token: str = None
+
+ def __init__(self, credential_info: Dict[str, str], expire_time_in_ms:
int):
+ self._access_key_id =
credential_info[self._GRAVITINO_S3_SESSION_ACCESS_KEY_ID]
+ self._secret_access_key = credential_info[
+ self._GRAVITINO_S3_SESSION_SECRET_ACCESS_KEY
+ ]
+ self._session_token = credential_info[self._GRAVITINO_S3_TOKEN]
+ self._expire_time_in_ms = expire_time_in_ms
+ Precondition.check_string_not_empty(
+ self._access_key_id, "The S3 access key ID should not be empty"
+ )
+ Precondition.check_string_not_empty(
+ self._secret_access_key, "The S3 secret access key should not be
empty"
+ )
+ Precondition.check_string_not_empty(
+ self._session_token, "The S3 session token should not be empty"
+ )
+ Precondition.check_argument(
+ self._expire_time_in_ms > 0,
+ "The expiration time of S3 token credential should be greater than
0",
+ )
+
+ def credential_type(self) -> str:
+ """The type of the credential.
+
+ Returns:
+ the type of the credential.
+ """
+ return self.S3_TOKEN_CREDENTIAL_TYPE
+
+ def expire_time_in_ms(self) -> int:
+ """Returns the expiration time of the credential in milliseconds since
+ the epoch, 0 means it will never expire.
+
+ Returns:
+ The expiration time of the credential.
+ """
+ return self._expire_time_in_ms
+
+ def credential_info(self) -> Dict[str, str]:
+ """The credential information.
+
+ Returns:
+ The credential information.
+ """
+ return {
+ self._GRAVITINO_S3_TOKEN: self._session_token,
+ self._GRAVITINO_S3_SESSION_ACCESS_KEY_ID: self._access_key_id,
+ self._GRAVITINO_S3_SESSION_SECRET_ACCESS_KEY:
self._secret_access_key,
+ }
+
+ def access_key_id(self) -> str:
+ """The S3 access key id.
+
+ Returns:
+ The S3 access key id.
+ """
+ return self._access_key_id
+
+ def secret_access_key(self) -> str:
+ """The S3 secret access key.
+
+ Returns:
+ The S3 secret access key.
+ """
+ return self._secret_access_key
+
+ def session_token(self) -> str:
+ """The S3 session token.
+
+ Returns:
+ The S3 session token.
+ """
+ return self._session_token
diff --git
a/clients/client-python/gravitino/api/credential/supports_credentials.py
b/clients/client-python/gravitino/api/credential/supports_credentials.py
new file mode 100644
index 000000000..cf4856667
--- /dev/null
+++ b/clients/client-python/gravitino/api/credential/supports_credentials.py
@@ -0,0 +1,73 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from abc import ABC, abstractmethod
+from typing import List
+from gravitino.api.credential.credential import Credential
+from gravitino.exceptions.base import (
+ NoSuchCredentialException,
+ IllegalStateException,
+)
+
+
+class SupportsCredentials(ABC):
+ """Represents interface to get credentials."""
+
+ @abstractmethod
+ def get_credentials(self) -> List[Credential]:
+ """Retrieves a List of Credential objects.
+
+ Returns:
+ A List of Credential objects. In most cases the array only contains
+ one credential. If the object like Fileset contains multiple locations
+ for different storages like HDFS, S3, the array will contain multiple
+ credentials. The array could be empty if you request a credential for
+ a catalog but the credential provider couldn't generate the credential
+ for the catalog, like S3 token credential provider only generate
+ credential for the specific object like Fileset,Table. There will be at
+ most one credential for one credential type.
+ """
+ pass
+
+ def get_credential(self, credential_type: str) -> Credential:
+ """Retrieves Credential object based on the specified credential type.
+
+ Args:
+ credential_type: The type of the credential like s3-token,
+ s3-secret-key which are defined in the specific credentials.
+ Returns:
+ An Credential object with the specified credential type.
+ Raises:
+ NoSuchCredentialException If the specific credential cannot be
found.
+ IllegalStateException if multiple credential can be found.
+ """
+
+ credentials = self.get_credentials()
+ matched_credentials = [
+ credential
+ for credential in credentials
+ if credential.credential_type == credential_type
+ ]
+ if len(matched_credentials) == 0:
+ raise NoSuchCredentialException(
+ f"No credential found for the credential type:
{credential_type}"
+ )
+ if len(matched_credentials) > 1:
+ raise IllegalStateException(
+ f"Multiple credentials found for the credential type:
{credential_type}"
+ )
+ return matched_credentials[0]
diff --git a/clients/client-python/gravitino/api/metadata_object.py
b/clients/client-python/gravitino/api/metadata_object.py
new file mode 100644
index 000000000..f0429893e
--- /dev/null
+++ b/clients/client-python/gravitino/api/metadata_object.py
@@ -0,0 +1,56 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from abc import ABC, abstractmethod
+from enum import Enum
+
+
+class MetadataObject(ABC):
+ """The MetadataObject is the basic unit of the Gravitino system. It
+ represents the metadata object in the Apache Gravitino system. The object
+ can be a metalake, catalog, schema, table, topic, etc.
+ """
+
+ class Type(Enum):
+ """The type of object in the Gravitino system. Every type will map one
+ kind of the entity of the underlying system."""
+
+ CATALOG = "catalog"
+ """"Metadata Type for catalog."""
+
+ FILESET = "fileset"
+ """Metadata Type for Fileset System (including HDFS, S3, etc.), like
path/to/file"""
+
+ @abstractmethod
+ def type(self) -> Type:
+ """
+ The type of the object.
+
+ Returns:
+ The type of the object.
+ """
+ pass
+
+ @abstractmethod
+ def name(self) -> str:
+ """
+ The name of the object.
+
+ Returns:
+ The name of the object.
+ """
+ pass
diff --git a/clients/client-python/gravitino/catalog/base_schema_catalog.py
b/clients/client-python/gravitino/catalog/base_schema_catalog.py
index a04e7698d..6e5d212a2 100644
--- a/clients/client-python/gravitino/catalog/base_schema_catalog.py
+++ b/clients/client-python/gravitino/catalog/base_schema_catalog.py
@@ -19,9 +19,14 @@ import logging
from typing import Dict, List
from gravitino.api.catalog import Catalog
+from gravitino.api.metadata_object import MetadataObject
from gravitino.api.schema import Schema
from gravitino.api.schema_change import SchemaChange
from gravitino.api.supports_schemas import SupportsSchemas
+from gravitino.client.metadata_object_credential_operations import (
+ MetadataObjectCredentialOperations,
+)
+from gravitino.client.metadata_object_impl import MetadataObjectImpl
from gravitino.dto.audit_dto import AuditDTO
from gravitino.dto.catalog_dto import CatalogDTO
from gravitino.dto.requests.schema_create_request import SchemaCreateRequest
@@ -52,6 +57,9 @@ class BaseSchemaCatalog(CatalogDTO, SupportsSchemas):
# The namespace of current catalog, which is the metalake name.
_catalog_namespace: Namespace
+ # The metadata object credential operations
+ _object_credential_operations: MetadataObjectCredentialOperations
+
def __init__(
self,
catalog_namespace: Namespace,
@@ -74,6 +82,11 @@ class BaseSchemaCatalog(CatalogDTO, SupportsSchemas):
self.rest_client = rest_client
self._catalog_namespace = catalog_namespace
+ metadata_object = MetadataObjectImpl([name],
MetadataObject.Type.CATALOG)
+ self._object_credential_operations =
MetadataObjectCredentialOperations(
+ catalog_namespace.level(0), metadata_object, rest_client
+ )
+
self.validate()
def as_schemas(self):
diff --git a/clients/client-python/gravitino/catalog/fileset_catalog.py
b/clients/client-python/gravitino/catalog/fileset_catalog.py
index ffa252e62..f7ad2aebd 100644
--- a/clients/client-python/gravitino/catalog/fileset_catalog.py
+++ b/clients/client-python/gravitino/catalog/fileset_catalog.py
@@ -19,10 +19,13 @@ import logging
from typing import List, Dict
from gravitino.api.catalog import Catalog
+from gravitino.api.credential.supports_credentials import SupportsCredentials
+from gravitino.api.credential.credential import Credential
from gravitino.api.fileset import Fileset
from gravitino.api.fileset_change import FilesetChange
from gravitino.audit.caller_context import CallerContextHolder, CallerContext
from gravitino.catalog.base_schema_catalog import BaseSchemaCatalog
+from gravitino.client.generic_fileset import GenericFileset
from gravitino.dto.audit_dto import AuditDTO
from gravitino.dto.requests.fileset_create_request import FilesetCreateRequest
from gravitino.dto.requests.fileset_update_request import FilesetUpdateRequest
@@ -40,7 +43,7 @@ from gravitino.exceptions.handlers.fileset_error_handler
import FILESET_ERROR_HA
logger = logging.getLogger(__name__)
-class FilesetCatalog(BaseSchemaCatalog):
+class FilesetCatalog(BaseSchemaCatalog, SupportsCredentials):
"""
Fileset catalog is a catalog implementation that supports fileset like
metadata operations, for
example, schemas and filesets list, creation, update and deletion. A
Fileset catalog is under the metalake.
@@ -124,7 +127,7 @@ class FilesetCatalog(BaseSchemaCatalog):
fileset_resp = FilesetResponse.from_json(resp.body, infer_missing=True)
fileset_resp.validate()
- return fileset_resp.fileset()
+ return GenericFileset(fileset_resp.fileset(), self.rest_client,
full_namespace)
def create_fileset(
self,
@@ -321,3 +324,9 @@ class FilesetCatalog(BaseSchemaCatalog):
if isinstance(change, FilesetChange.RemoveComment):
return FilesetUpdateRequest.UpdateFilesetCommentRequest(None)
raise ValueError(f"Unknown change type: {type(change).__name__}")
+
+ def support_credentials(self) -> SupportsCredentials:
+ return self
+
+ def get_credentials(self) -> List[Credential]:
+ return self._object_credential_operations.get_credentials()
diff --git a/clients/client-python/gravitino/client/generic_fileset.py
b/clients/client-python/gravitino/client/generic_fileset.py
new file mode 100644
index 000000000..3b7aa5326
--- /dev/null
+++ b/clients/client-python/gravitino/client/generic_fileset.py
@@ -0,0 +1,75 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from typing import Optional, Dict, List
+
+from gravitino.api.fileset import Fileset
+from gravitino.api.metadata_object import MetadataObject
+from gravitino.api.credential.supports_credentials import SupportsCredentials
+from gravitino.api.credential.credential import Credential
+from gravitino.client.metadata_object_credential_operations import (
+ MetadataObjectCredentialOperations,
+)
+from gravitino.client.metadata_object_impl import MetadataObjectImpl
+from gravitino.dto.audit_dto import AuditDTO
+from gravitino.dto.fileset_dto import FilesetDTO
+from gravitino.namespace import Namespace
+from gravitino.utils import HTTPClient
+
+
+class GenericFileset(Fileset, SupportsCredentials):
+
+ _fileset: FilesetDTO
+ """The fileset data transfer object"""
+
+ _object_credential_operations: MetadataObjectCredentialOperations
+ """The metadata object credential operations"""
+
+ def __init__(
+ self, fileset: FilesetDTO, rest_client: HTTPClient, full_namespace:
Namespace
+ ):
+ self._fileset = fileset
+ metadata_object = MetadataObjectImpl(
+ [full_namespace.level(1), full_namespace.level(2), fileset.name()],
+ MetadataObject.Type.FILESET,
+ )
+ self._object_credential_operations =
MetadataObjectCredentialOperations(
+ full_namespace.level(0), metadata_object, rest_client
+ )
+
+ def name(self) -> str:
+ return self._fileset.name()
+
+ def type(self) -> Fileset.Type:
+ return self._fileset.type()
+
+ def storage_location(self) -> str:
+ return self._fileset.storage_location()
+
+ def comment(self) -> Optional[str]:
+ return self._fileset.comment()
+
+ def properties(self) -> Dict[str, str]:
+ return self._fileset.properties()
+
+ def audit_info(self) -> AuditDTO:
+ return self._fileset.audit_info()
+
+ def support_credentials(self) -> SupportsCredentials:
+ return self
+
+ def get_credentials(self) -> List[Credential]:
+ return self._object_credential_operations.get_credentials()
diff --git
a/clients/client-python/gravitino/client/metadata_object_credential_operations.py
b/clients/client-python/gravitino/client/metadata_object_credential_operations.py
new file mode 100644
index 000000000..93d538cfa
--- /dev/null
+++
b/clients/client-python/gravitino/client/metadata_object_credential_operations.py
@@ -0,0 +1,74 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import logging
+from typing import List
+from gravitino.api.credential.supports_credentials import SupportsCredentials
+from gravitino.api.credential.credential import Credential
+from gravitino.api.metadata_object import MetadataObject
+from gravitino.dto.credential_dto import CredentialDTO
+from gravitino.dto.responses.credential_response import CredentialResponse
+from gravitino.exceptions.handlers.credential_error_handler import (
+ CREDENTIAL_ERROR_HANDLER,
+)
+from gravitino.utils import HTTPClient
+from gravitino.utils.credential_factory import CredentialFactory
+
+logger = logging.getLogger(__name__)
+
+
+class MetadataObjectCredentialOperations(SupportsCredentials):
+ _rest_client: HTTPClient
+ """The REST client to communicate with the REST server"""
+
+ _request_path: str
+ """The REST API path to do credential operations"""
+
+ def __init__(
+ self,
+ metalake_name: str,
+ metadata_object: MetadataObject,
+ rest_client: HTTPClient,
+ ):
+ self._rest_client = rest_client
+ metadata_object_type = metadata_object.type().value
+ metadata_object_name = metadata_object.name()
+ self._request_path = (
+ f"api/metalakes/{metalake_name}objects/{metadata_object_type}/"
+ f"{metadata_object_name}/credentials"
+ )
+
+ def get_credentials(self) -> List[Credential]:
+ resp = self._rest_client.get(
+ self._request_path,
+ error_handler=CREDENTIAL_ERROR_HANDLER,
+ )
+
+ credential_resp = CredentialResponse.from_json(resp.body,
infer_missing=True)
+ credential_resp.validate()
+ credential_dtos = credential_resp.credentials()
+ return self.to_credentials(credential_dtos)
+
+ def to_credentials(self, credentials: List[CredentialDTO]) ->
List[Credential]:
+ return [self.to_credential(credential) for credential in credentials]
+
+ def to_credential(self, credential_dto: CredentialDTO) -> Credential:
+ return CredentialFactory.create(
+ credential_dto.credential_type(),
+ credential_dto.credential_info(),
+ credential_dto.expire_time_in_ms(),
+ )
diff --git a/clients/client-python/gravitino/client/metadata_object_impl.py
b/clients/client-python/gravitino/client/metadata_object_impl.py
new file mode 100644
index 000000000..af16b71c4
--- /dev/null
+++ b/clients/client-python/gravitino/client/metadata_object_impl.py
@@ -0,0 +1,35 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import List, ClassVar
+
+from gravitino.api.metadata_object import MetadataObject
+
+
+class MetadataObjectImpl(MetadataObject):
+
+ _DOT: ClassVar[str] = "."
+
+ def __init__(self, names: List[str], metadata_object_type:
MetadataObject.Type):
+ self._name = self._DOT.join(names)
+ self._metadata_object_type = metadata_object_type
+
+ def type(self) -> MetadataObject.Type:
+ return self._metadata_object_type
+
+ def name(self) -> str:
+ return self._name
diff --git a/clients/client-python/gravitino/dto/credential_dto.py
b/clients/client-python/gravitino/dto/credential_dto.py
new file mode 100644
index 000000000..518c0460c
--- /dev/null
+++ b/clients/client-python/gravitino/dto/credential_dto.py
@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from dataclasses import dataclass, field
+from typing import Dict
+
+from dataclasses_json import config, DataClassJsonMixin
+
+from gravitino.api.credential.credential import Credential
+
+
+@dataclass
+class CredentialDTO(Credential, DataClassJsonMixin):
+ """Represents a Credential DTO (Data Transfer Object)."""
+
+ _credential_type: str = field(metadata=config(field_name="credentialType"))
+ _expire_time_in_ms: int =
field(metadata=config(field_name="expireTimeInMs"))
+ _credential_info: Dict[str, str] = field(
+ metadata=config(field_name="credentialInfo")
+ )
+
+ def credential_type(self) -> str:
+ return self._credential_type
+
+ def expire_time_in_ms(self) -> int:
+ return self._expire_time_in_ms
+
+ def credential_info(self) -> Dict[str, str]:
+ return self._credential_info
diff --git
a/clients/client-python/gravitino/dto/responses/credential_response.py
b/clients/client-python/gravitino/dto/responses/credential_response.py
new file mode 100644
index 000000000..1883c7580
--- /dev/null
+++ b/clients/client-python/gravitino/dto/responses/credential_response.py
@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import List
+from dataclasses import dataclass, field
+from dataclasses_json import config
+
+from gravitino.dto.credential_dto import CredentialDTO
+from gravitino.dto.responses.base_response import BaseResponse
+from gravitino.exceptions.base import IllegalArgumentException
+
+
+@dataclass
+class CredentialResponse(BaseResponse):
+ """Response for credential response."""
+
+ _credentials: List[CredentialDTO] =
field(metadata=config(field_name="credentials"))
+
+ def credentials(self) -> List[CredentialDTO]:
+ return self._credentials
+
+ def validate(self):
+ """Validates the response data.
+
+ Raises:
+ IllegalArgumentException if credentials are None.
+ """
+ if self._credentials is None:
+ raise IllegalArgumentException("credentials should be set")
+ super().validate()
diff --git a/clients/client-python/gravitino/exceptions/base.py
b/clients/client-python/gravitino/exceptions/base.py
index cd71de236..9091116dd 100644
--- a/clients/client-python/gravitino/exceptions/base.py
+++ b/clients/client-python/gravitino/exceptions/base.py
@@ -61,6 +61,10 @@ class NoSuchFilesetException(NotFoundException):
"""Exception thrown when a file with specified name is not existed."""
+class NoSuchCredentialException(NotFoundException):
+ """Exception thrown when a credential with specified credential type is
not existed."""
+
+
class NoSuchMetalakeException(NotFoundException):
"""An exception thrown when a metalake is not found."""
@@ -135,3 +139,7 @@ class UnauthorizedException(GravitinoRuntimeException):
class BadRequestException(GravitinoRuntimeException):
"""An exception thrown when the request is invalid."""
+
+
+class IllegalStateException(GravitinoRuntimeException):
+ """An exception thrown when the state is invalid."""
diff --git
a/clients/client-python/gravitino/exceptions/handlers/credential_error_handler.py
b/clients/client-python/gravitino/exceptions/handlers/credential_error_handler.py
new file mode 100644
index 000000000..542fb27cf
--- /dev/null
+++
b/clients/client-python/gravitino/exceptions/handlers/credential_error_handler.py
@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from gravitino.constants.error import ErrorConstants
+from gravitino.dto.responses.error_response import ErrorResponse
+from gravitino.exceptions.handlers.rest_error_handler import RestErrorHandler
+from gravitino.exceptions.base import (
+ CatalogNotInUseException,
+ NoSuchCredentialException,
+)
+
+
+class CredentialErrorHandler(RestErrorHandler):
+
+ def handle(self, error_response: ErrorResponse):
+
+ error_message = error_response.format_error_message()
+ code = error_response.code()
+ exception_type = error_response.type()
+
+ if code == ErrorConstants.NOT_FOUND_CODE:
+ if exception_type == NoSuchCredentialException.__name__:
+ raise NoSuchCredentialException(error_message)
+
+ if code == ErrorConstants.NOT_IN_USE_CODE:
+ raise CatalogNotInUseException(error_message)
+
+ super().handle(error_response)
+
+
+CREDENTIAL_ERROR_HANDLER = CredentialErrorHandler()
diff --git a/clients/client-python/gravitino/utils/credential_factory.py
b/clients/client-python/gravitino/utils/credential_factory.py
new file mode 100644
index 000000000..2dfbf619b
--- /dev/null
+++ b/clients/client-python/gravitino/utils/credential_factory.py
@@ -0,0 +1,39 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import Dict
+from gravitino.api.credential.credential import Credential
+from gravitino.api.credential.gcs_token_credential import GCSTokenCredential
+from gravitino.api.credential.oss_token_credential import OSSTokenCredential
+from gravitino.api.credential.s3_secret_key_credential import
S3SecretKeyCredential
+from gravitino.api.credential.s3_token_credential import S3TokenCredential
+
+
+class CredentialFactory:
+ @staticmethod
+ def create(
+ credential_type: str, credential_info: Dict[str, str],
expire_time_in_ms: int
+ ) -> Credential:
+ if credential_type == S3TokenCredential.S3_TOKEN_CREDENTIAL_TYPE:
+ return S3TokenCredential(credential_info, expire_time_in_ms)
+ if credential_type ==
S3SecretKeyCredential.S3_SECRET_KEY_CREDENTIAL_TYPE:
+ return S3SecretKeyCredential(credential_info, expire_time_in_ms)
+ if credential_type == GCSTokenCredential.GCS_TOKEN_CREDENTIAL_TYPE:
+ return GCSTokenCredential(credential_info, expire_time_in_ms)
+ if credential_type == OSSTokenCredential.OSS_TOKEN_CREDENTIAL_TYPE:
+ return OSSTokenCredential(credential_info, expire_time_in_ms)
+ raise NotImplementedError(f"Credential type {credential_type} is not
supported")
diff --git a/clients/client-python/gravitino/utils/precondition.py
b/clients/client-python/gravitino/utils/precondition.py
new file mode 100644
index 000000000..da3490555
--- /dev/null
+++ b/clients/client-python/gravitino/utils/precondition.py
@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from gravitino.exceptions.base import IllegalArgumentException
+
+
+class Precondition:
+ @staticmethod
+ def check_argument(expression_result: bool, error_message: str):
+ """Ensures the truth of an expression involving one or more parameters
+ to the calling method.
+
+ Args:
+ expression_result: A boolean expression.
+ error_message: The error message to use if the check fails.
+ Raises:
+ IllegalArgumentException – if expression is false
+ """
+ if not expression_result:
+ raise IllegalArgumentException(error_message)
+
+ @staticmethod
+ def check_string_not_empty(check_string: str, error_message: str):
+ """Ensures the string is not empty.
+
+ Args:
+ check_string: The string to check.
+ error_message: The error message to use if the check fails.
+ Raises:
+ IllegalArgumentException – if the check fails.
+ """
+ Precondition.check_argument(
+ check_string is not None and check_string.strip() != "",
error_message
+ )
diff --git a/clients/client-python/tests/unittests/mock_base.py
b/clients/client-python/tests/unittests/mock_base.py
index 9fd60a702..16a3d03c3 100644
--- a/clients/client-python/tests/unittests/mock_base.py
+++ b/clients/client-python/tests/unittests/mock_base.py
@@ -93,6 +93,10 @@ def mock_data(cls):
"gravitino.client.gravitino_metalake.GravitinoMetalake.load_catalog",
return_value=mock_load_fileset_catalog(),
)
+ @patch(
+ "gravitino.catalog.fileset_catalog.FilesetCatalog.load_fileset",
+ return_value=mock_load_fileset("fileset", ""),
+ )
@patch(
"gravitino.client.gravitino_client_base.GravitinoClientBase.check_version",
return_value=True,
diff --git a/clients/client-python/tests/unittests/test_credential_api.py
b/clients/client-python/tests/unittests/test_credential_api.py
new file mode 100644
index 000000000..2811a226f
--- /dev/null
+++ b/clients/client-python/tests/unittests/test_credential_api.py
@@ -0,0 +1,105 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from typing import List
+import json
+import unittest
+from http.client import HTTPResponse
+from unittest.mock import patch, Mock
+
+from gravitino import GravitinoClient, NameIdentifier
+from gravitino.api.credential.credential import Credential
+from gravitino.api.credential.s3_token_credential import S3TokenCredential
+from gravitino.client.generic_fileset import GenericFileset
+from gravitino.namespace import Namespace
+from gravitino.utils import Response, HTTPClient
+from tests.unittests import mock_base
+
+
+@mock_base.mock_data
+class TestCredentialApi(unittest.TestCase):
+
+ def test_get_credentials(self, *mock_method):
+ json_str = self._get_s3_token_str()
+ mock_resp = self._get_mock_http_resp(json_str)
+
+ metalake_name: str = "metalake_demo"
+ catalog_name: str = "fileset_catalog"
+ gravitino_client = GravitinoClient(
+ uri="http://localhost:8090";, metalake_name=metalake_name
+ )
+ catalog = gravitino_client.load_catalog(catalog_name)
+
+ with patch(
+ "gravitino.utils.http_client.HTTPClient.get",
+ return_value=mock_resp,
+ ):
+ credentials = (
+
catalog.as_fileset_catalog().support_credentials().get_credentials()
+ )
+ self._check_credential(credentials)
+
+ fileset_dto = catalog.as_fileset_catalog().load_fileset(
+ NameIdentifier.of("schema", "fileset")
+ )
+ fileset = GenericFileset(
+ fileset_dto,
+ HTTPClient("http://localhost:8090";),
+ Namespace.of(metalake_name, catalog_name, "schema"),
+ )
+ with patch(
+ "gravitino.utils.http_client.HTTPClient.get",
+ return_value=mock_resp,
+ ):
+ credentials = fileset.support_credentials().get_credentials()
+ self._check_credential(credentials)
+
+ def _get_mock_http_resp(self, json_str: str):
+ mock_http_resp = Mock(HTTPResponse)
+ mock_http_resp.getcode.return_value = 200
+ mock_http_resp.read.return_value = json_str
+ mock_http_resp.info.return_value = None
+ mock_http_resp.url = None
+ mock_resp = Response(mock_http_resp)
+ return mock_resp
+
+ def _get_s3_token_str(self):
+ json_data = {
+ "code": 0,
+ "credentials": [
+ {
+ "credentialType": "s3-token",
+ "expireTimeInMs": 1000,
+ "credentialInfo": {
+ "s3-access-key-id": "access_id",
+ "s3-secret-access-key": "secret_key",
+ "s3-session-token": "token",
+ },
+ }
+ ],
+ }
+ return json.dumps(json_data)
+
+ def _check_credential(self, credentials: List[Credential]):
+ self.assertEqual(1, len(credentials))
+ s3_credential: S3TokenCredential = credentials[0]
+ self.assertEqual(
+ S3TokenCredential.S3_TOKEN_CREDENTIAL_TYPE,
s3_credential.credential_type()
+ )
+ self.assertEqual("access_id", s3_credential.access_key_id())
+ self.assertEqual("secret_key", s3_credential.secret_access_key())
+ self.assertEqual("token", s3_credential.session_token())
+ self.assertEqual(1000, s3_credential.expire_time_in_ms())
diff --git a/clients/client-python/tests/unittests/test_credential_factory.py
b/clients/client-python/tests/unittests/test_credential_factory.py
new file mode 100644
index 000000000..0a7e78251
--- /dev/null
+++ b/clients/client-python/tests/unittests/test_credential_factory.py
@@ -0,0 +1,101 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# pylint: disable=protected-access,too-many-lines,too-many-locals
+
+import unittest
+
+from gravitino.api.credential.gcs_token_credential import GCSTokenCredential
+from gravitino.api.credential.oss_token_credential import OSSTokenCredential
+from gravitino.api.credential.s3_secret_key_credential import
S3SecretKeyCredential
+from gravitino.api.credential.s3_token_credential import S3TokenCredential
+from gravitino.utils.credential_factory import CredentialFactory
+
+
+class TestCredentialFactory(unittest.TestCase):
+
+ def test_s3_token_credential(self):
+ s3_credential_info = {
+ S3TokenCredential._GRAVITINO_S3_SESSION_ACCESS_KEY_ID:
"access_key",
+ S3TokenCredential._GRAVITINO_S3_SESSION_SECRET_ACCESS_KEY:
"secret_key",
+ S3TokenCredential._GRAVITINO_S3_TOKEN: "session_token",
+ }
+ s3_credential = S3TokenCredential(s3_credential_info, 1000)
+ credential_info = s3_credential.credential_info()
+ expire_time = s3_credential.expire_time_in_ms()
+
+ check_credential = CredentialFactory.create(
+ s3_credential.S3_TOKEN_CREDENTIAL_TYPE, credential_info,
expire_time
+ )
+ self.assertEqual("access_key", check_credential.access_key_id())
+ self.assertEqual("secret_key", check_credential.secret_access_key())
+ self.assertEqual("session_token", check_credential.session_token())
+ self.assertEqual(1000, check_credential.expire_time_in_ms())
+
+ def test_s3_secret_key_credential(self):
+ s3_credential_info = {
+ S3SecretKeyCredential._GRAVITINO_S3_STATIC_ACCESS_KEY_ID:
"access_key",
+ S3SecretKeyCredential._GRAVITINO_S3_STATIC_SECRET_ACCESS_KEY:
"secret_key",
+ }
+ s3_credential = S3SecretKeyCredential(s3_credential_info, 0)
+ credential_info = s3_credential.credential_info()
+ expire_time = s3_credential.expire_time_in_ms()
+
+ check_credential = CredentialFactory.create(
+ s3_credential.S3_SECRET_KEY_CREDENTIAL_TYPE, credential_info,
expire_time
+ )
+ self.assertEqual("access_key", check_credential.access_key_id())
+ self.assertEqual("secret_key", check_credential.secret_access_key())
+ self.assertEqual(0, check_credential.expire_time_in_ms())
+
+ def test_gcs_token_credential(self):
+ credential_info = {GCSTokenCredential._GCS_TOKEN_NAME: "token"}
+ credential = GCSTokenCredential(credential_info, 1000)
+ credential_info = credential.credential_info()
+ expire_time = credential.expire_time_in_ms()
+
+ check_credential = CredentialFactory.create(
+ credential.credential_type(), credential_info, expire_time
+ )
+ self.assertEqual(
+ GCSTokenCredential.GCS_TOKEN_CREDENTIAL_TYPE,
+ check_credential.credential_type(),
+ )
+ self.assertEqual("token", check_credential.token())
+ self.assertEqual(1000, check_credential.expire_time_in_ms())
+
+ def test_oss_token_credential(self):
+ credential_info = {
+ OSSTokenCredential._GRAVITINO_OSS_TOKEN: "token",
+ OSSTokenCredential._GRAVITINO_OSS_SESSION_ACCESS_KEY_ID:
"access_id",
+ OSSTokenCredential._GRAVITINO_OSS_SESSION_SECRET_ACCESS_KEY:
"secret_key",
+ }
+ credential = OSSTokenCredential(credential_info, 1000)
+ credential_info = credential.credential_info()
+ expire_time = credential.expire_time_in_ms()
+
+ check_credential = CredentialFactory.create(
+ credential.credential_type(), credential_info, expire_time
+ )
+ self.assertEqual(
+ OSSTokenCredential.OSS_TOKEN_CREDENTIAL_TYPE,
+ check_credential.credential_type(),
+ )
+ self.assertEqual("token", check_credential.security_token())
+ self.assertEqual("access_id", check_credential.access_key_id())
+ self.assertEqual("secret_key", check_credential.secret_access_key())
+ self.assertEqual(1000, check_credential.expire_time_in_ms())
diff --git a/clients/client-python/tests/unittests/test_error_handler.py
b/clients/client-python/tests/unittests/test_error_handler.py
index 4b9cbf1ca..a402ae111 100644
--- a/clients/client-python/tests/unittests/test_error_handler.py
+++ b/clients/client-python/tests/unittests/test_error_handler.py
@@ -35,6 +35,10 @@ from gravitino.exceptions.base import (
UnsupportedOperationException,
ConnectionFailedException,
CatalogAlreadyExistsException,
+ NoSuchCredentialException,
+)
+from gravitino.exceptions.handlers.credential_error_handler import (
+ CREDENTIAL_ERROR_HANDLER,
)
from gravitino.exceptions.handlers.rest_error_handler import REST_ERROR_HANDLER
@@ -127,6 +131,25 @@ class TestErrorHandler(unittest.TestCase):
ErrorResponse.generate_error_response(Exception, "mock error")
)
+ def test_credential_error_handler(self):
+
+ with self.assertRaises(NoSuchCredentialException):
+ CREDENTIAL_ERROR_HANDLER.handle(
+ ErrorResponse.generate_error_response(
+ NoSuchCredentialException, "mock error"
+ )
+ )
+
+ with self.assertRaises(InternalError):
+ CREDENTIAL_ERROR_HANDLER.handle(
+ ErrorResponse.generate_error_response(InternalError, "mock
error")
+ )
+
+ with self.assertRaises(RESTException):
+ CREDENTIAL_ERROR_HANDLER.handle(
+ ErrorResponse.generate_error_response(Exception, "mock error")
+ )
+
def test_metalake_error_handler(self):
with self.assertRaises(NoSuchMetalakeException):
diff --git a/clients/client-python/tests/unittests/test_precondition.py
b/clients/client-python/tests/unittests/test_precondition.py
new file mode 100644
index 000000000..78a246597
--- /dev/null
+++ b/clients/client-python/tests/unittests/test_precondition.py
@@ -0,0 +1,46 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# pylint: disable=protected-access,too-many-lines,too-many-locals
+
+import unittest
+
+from gravitino.exceptions.base import IllegalArgumentException
+from gravitino.utils.precondition import Precondition
+
+
+class TestPrecondition(unittest.TestCase):
+
+ def test_check_argument(self):
+ with self.assertRaises(IllegalArgumentException):
+ Precondition.check_argument(False, "error")
+ try:
+ Precondition.check_argument(True, "error")
+ except IllegalArgumentException:
+ self.fail("should not raise IllegalArgumentException")
+
+ def test_check_string_empty(self):
+ with self.assertRaises(IllegalArgumentException):
+ Precondition.check_string_not_empty("", "empty")
+ with self.assertRaises(IllegalArgumentException):
+ Precondition.check_string_not_empty(" ", "empty")
+ with self.assertRaises(IllegalArgumentException):
+ Precondition.check_string_not_empty(None, "empty")
+ try:
+ Precondition.check_string_not_empty("test", "empty")
+ except IllegalArgumentException:
+ self.fail("should not raised an exception")
diff --git a/clients/client-python/tests/unittests/test_responses.py
b/clients/client-python/tests/unittests/test_responses.py
index 19d403ad3..da8340bdf 100644
--- a/clients/client-python/tests/unittests/test_responses.py
+++ b/clients/client-python/tests/unittests/test_responses.py
@@ -17,6 +17,7 @@
import json
import unittest
+from gravitino.dto.responses.credential_response import CredentialResponse
from gravitino.dto.responses.file_location_response import FileLocationResponse
from gravitino.exceptions.base import IllegalArgumentException
@@ -39,3 +40,37 @@ class TestResponses(unittest.TestCase):
)
with self.assertRaises(IllegalArgumentException):
file_location_resp.validate()
+
+ def test_credential_response(self):
+ json_data = {"code": 0, "credentials": []}
+ json_str = json.dumps(json_data)
+ credential_resp: CredentialResponse =
CredentialResponse.from_json(json_str)
+ self.assertEqual(0, len(credential_resp.credentials()))
+ credential_resp.validate()
+
+ json_data = {
+ "code": 0,
+ "credentials": [
+ {
+ "credentialType": "s3-token",
+ "expireTimeInMs": 1000,
+ "credentialInfo": {
+ "s3-access-key-id": "access-id",
+ "s3-secret-access-key": "secret-key",
+ "s3-session-token": "token",
+ },
+ }
+ ],
+ }
+ json_str = json.dumps(json_data)
+ credential_resp: CredentialResponse =
CredentialResponse.from_json(json_str)
+ credential_resp.validate()
+ self.assertEqual(1, len(credential_resp.credentials()))
+ credential = credential_resp.credentials()[0]
+ self.assertEqual("s3-token", credential.credential_type())
+ self.assertEqual(1000, credential.expire_time_in_ms())
+ self.assertEqual("access-id",
credential.credential_info()["s3-access-key-id"])
+ self.assertEqual(
+ "secret-key", credential.credential_info()["s3-secret-access-key"]
+ )
+ self.assertEqual("token",
credential.credential_info()["s3-session-token"])
