This is an automated email from the ASF dual-hosted git repository.
liuxun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gravitino.git
The following commit(s) were added to refs/heads/main by this push:
new 689c0e3030 [#3253] improvement(python): add git pre-commit hook to
client-python (#6929)
689c0e3030 is described below
commit 689c0e303053e10da299c336f99f775d46d00ef4
Author: Yunchi Pang <[email protected]>
AuthorDate: Sun May 11 09:08:58 2025 -0700
[#3253] improvement(python): add git pre-commit hook to client-python
(#6929)
### What changes were proposed in this pull request?
Add git pre-commit hook to client-python to run linter and formatter
before `git commit`.
### Why are the changes needed?
Fix: #3253
### Does this PR introduce _any_ user-facing change?
### How was this patch tested?
pre-commit hook was tested manually to format a incorrectly formatted
`test.py`:
<img width="788" alt="Screenshot 2025-04-14 at 4 43 39 PM"
src="https://github.com/user-attachments/assets/a28c6021-83d8-4503-a21f-b56370c997df";
/>
---
.../utils/__init__.py => .pre-commit-config.yaml} | 9 +-
clients/client-python/docs/source/conf.py | 20 ++--
clients/client-python/gravitino/__init__.py | 14 +++
clients/client-python/gravitino/api/catalog.py | 8 +-
.../api/expressions/partitions/partitions.py | 2 +-
.../gravitino/api/expressions/sorts/sort_orders.py | 1 -
.../auth/default_oauth2_token_provider.py | 2 -
.../gravitino/client/fileset_catalog.py | 1 -
.../gravitino/client/generic_fileset.py | 1 -
.../gravitino/client/generic_model.py | 1 -
.../gravitino/client/generic_model_version.py | 1 -
.../gravitino/client/metadata_object_impl.py | 1 -
.../requests/oauth2_client_credential_request.py | 1 -
.../dto/responses/oauth2_token_response.py | 1 -
.../exceptions/handlers/catalog_error_handler.py | 2 -
.../handlers/credential_error_handler.py | 2 -
.../exceptions/handlers/fileset_error_handler.py | 2 -
.../exceptions/handlers/metalake_error_handler.py | 2 -
.../exceptions/handlers/model_error_handler.py | 1 -
.../exceptions/handlers/oauth_error_handler.py | 2 -
.../exceptions/handlers/rest_error_handler.py | 1 -
.../exceptions/handlers/schema_error_handler.py | 2 -
clients/client-python/gravitino/rest/rest_utils.py | 1 -
clients/client-python/gravitino/utils/__init__.py | 2 +
.../client-python/gravitino/utils/http_client.py | 1 -
.../utils/__init__.py => pyproject.toml} | 9 +-
clients/client-python/requirements.txt | 5 +-
.../tests/integration/auth/test_auth_common.py | 1 -
.../tests/integration/auth/test_oauth2_client.py | 4 -
.../integration/auth/test_simple_auth_client.py | 1 -
.../tests/integration/base_hadoop_env.py | 1 -
.../tests/integration/containers/hdfs_container.py | 1 -
.../integration/containers/oauth2_container.py | 1 -
.../tests/integration/test_fileset_catalog.py | 1 -
.../tests/integration/test_gvfs_with_hdfs.py | 5 +-
.../tests/unittests/audit/test_caller_context.py | 1 -
.../unittests/auth/test_oauth2_token_provider.py | 5 -
.../unittests/auth/test_simple_auth_provider.py | 1 -
clients/client-python/tests/unittests/mock_base.py | 1 -
.../tests/unittests/rel/test_distributions.py | 1 -
.../tests/unittests/rel/test_types.py | 1 -
.../tests/unittests/test_credential_api.py | 1 -
.../tests/unittests/test_credential_factory.py | 1 -
.../tests/unittests/test_error_handler.py | 7 --
.../tests/unittests/test_fileset_catalog_api.py | 1 -
.../tests/unittests/test_precondition.py | 1 -
dev/docker/iceberg-rest-server/rewrite_config.py | 114 +++++++++++----------
47 files changed, 110 insertions(+), 135 deletions(-)
diff --git a/clients/client-python/gravitino/utils/__init__.py
b/clients/client-python/.pre-commit-config.yaml
similarity index 79%
copy from clients/client-python/gravitino/utils/__init__.py
copy to clients/client-python/.pre-commit-config.yaml
index ee8b21b902..3efc883501 100644
--- a/clients/client-python/gravitino/utils/__init__.py
+++ b/clients/client-python/.pre-commit-config.yaml
@@ -15,4 +15,11 @@
# specific language governing permissions and limitations
# under the License.
-from gravitino.utils.http_client import Response, HTTPClient, unpack
+repos:
+ - repo: https://github.com/astral-sh/ruff-pre-commit
+ rev: v0.3.0
+ hooks:
+ - id: ruff # runs the linter
+ - id: ruff-format # runs the formatter
+
+default_stages: [pre-commit]
\ No newline at end of file
diff --git a/clients/client-python/docs/source/conf.py
b/clients/client-python/docs/source/conf.py
index ff06cc2f23..7141577709 100644
--- a/clients/client-python/docs/source/conf.py
+++ b/clients/client-python/docs/source/conf.py
@@ -26,11 +26,11 @@ import os
import sys
import re
-sys.path.insert(0, os.path.abspath('../..'))
-
from gravitino.constants.version import SETUP_FILE
from gravitino.exceptions.base import GravitinoRuntimeException
+sys.path.insert(0, os.path.abspath("../.."))
+
VERSION_PATTERN = r"version\s*=\s*['\"]([^'\"]+)['\"]"
AUTHOR_PATTERN = r"author\s*=\s*['\"]([^'\"]+)['\"]"
@@ -43,15 +43,15 @@ with open(SETUP_FILE, "r", encoding="utf-8") as f:
project_version = version_group.group(1)
else:
raise GravitinoRuntimeException("Can't find valid author info in
setup.py")
-
+
author_group = re.search(AUTHOR_PATTERN, setup_content)
if author_group is not None:
project_author = author_group.group(1)
else:
raise GravitinoRuntimeException("Can't find valid author info in
setup.py")
-project = 'Apache Gravitino Python Client'
-copyright = '2024, Apache Software Foundation'
+project = "Apache Gravitino Python Client"
+copyright = "2024, Apache Software Foundation"
author = project_author
release = project_version
@@ -60,15 +60,15 @@ release = project_version
extensions = [
- 'sphinx.ext.autodoc',
- 'sphinx.ext.autosummary',
+ "sphinx.ext.autodoc",
+ "sphinx.ext.autosummary",
]
-templates_path = ['_templates']
+templates_path = ["_templates"]
exclude_patterns = []
# -- Options for HTML output -------------------------------------------------
#
https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
-html_theme = 'furo'
-html_static_path = ['_static']
+html_theme = "furo"
+html_static_path = ["_static"]
diff --git a/clients/client-python/gravitino/__init__.py
b/clients/client-python/gravitino/__init__.py
index c9f4aac347..63ccaacf73 100644
--- a/clients/client-python/gravitino/__init__.py
+++ b/clients/client-python/gravitino/__init__.py
@@ -26,3 +26,17 @@ from gravitino.client.gravitino_admin_client import
GravitinoAdminClient
from gravitino.client.gravitino_metalake import GravitinoMetalake
from gravitino.name_identifier import NameIdentifier
from gravitino.filesystem import gvfs
+
+__all__ = [
+ "Catalog",
+ "Schema",
+ "Fileset",
+ "FilesetChange",
+ "MetalakeChange",
+ "SchemaChange",
+ "GravitinoClient",
+ "GravitinoAdminClient",
+ "GravitinoMetalake",
+ "NameIdentifier",
+ "gvfs",
+]
diff --git a/clients/client-python/gravitino/api/catalog.py
b/clients/client-python/gravitino/api/catalog.py
index babf0421b8..8a28dedb3f 100644
--- a/clients/client-python/gravitino/api/catalog.py
+++ b/clients/client-python/gravitino/api/catalog.py
@@ -147,7 +147,7 @@ class Catalog(Auditable):
"Catalog does not support schema operations"
)
- def as_table_catalog(self) -> "TableCatalog":
+ def as_table_catalog(self) -> "TableCatalog": # noqa: F821
"""
Raises:
UnsupportedOperationException if the catalog does not support
table operations.
@@ -157,7 +157,7 @@ class Catalog(Auditable):
"""
raise UnsupportedOperationException("Catalog does not support table
operations")
- def as_fileset_catalog(self) -> "FilesetCatalog":
+ def as_fileset_catalog(self) -> "FilesetCatalog": # noqa: F821
"""
Raises:
UnsupportedOperationException if the catalog does not support
fileset operations.
@@ -169,7 +169,7 @@ class Catalog(Auditable):
"Catalog does not support fileset operations"
)
- def as_topic_catalog(self) -> "TopicCatalog":
+ def as_topic_catalog(self) -> "TopicCatalog": # noqa: F821
"""
Returns:
the {@link TopicCatalog} if the catalog supports topic operations.
@@ -179,7 +179,7 @@ class Catalog(Auditable):
"""
raise UnsupportedOperationException("Catalog does not support topic
operations")
- def as_model_catalog(self) -> "ModelCatalog":
+ def as_model_catalog(self) -> "ModelCatalog": # noqa: F821
"""
Returns:
the {@link ModelCatalog} if the catalog supports model operations.
diff --git
a/clients/client-python/gravitino/api/expressions/partitions/partitions.py
b/clients/client-python/gravitino/api/expressions/partitions/partitions.py
index 6cb4b4a472..c116b3bdcb 100644
--- a/clients/client-python/gravitino/api/expressions/partitions/partitions.py
+++ b/clients/client-python/gravitino/api/expressions/partitions/partitions.py
@@ -177,7 +177,7 @@ class ListPartitionImpl(ListPartition):
return hash(
(
self._name,
- tuple(tuple(l) for l in self._lists),
+ tuple(tuple(sublist) for sublist in self._lists),
frozenset(self._properties.items()),
)
)
diff --git
a/clients/client-python/gravitino/api/expressions/sorts/sort_orders.py
b/clients/client-python/gravitino/api/expressions/sorts/sort_orders.py
index 9deaa4bacd..4d01cb1421 100644
--- a/clients/client-python/gravitino/api/expressions/sorts/sort_orders.py
+++ b/clients/client-python/gravitino/api/expressions/sorts/sort_orders.py
@@ -23,7 +23,6 @@ from gravitino.api.expressions.sorts.sort_order import
SortOrder
class SortImpl(SortOrder):
-
def __init__(
self,
expression: Expression,
diff --git
a/clients/client-python/gravitino/auth/default_oauth2_token_provider.py
b/clients/client-python/gravitino/auth/default_oauth2_token_provider.py
index 127123c539..8b3fa4937d 100644
--- a/clients/client-python/gravitino/auth/default_oauth2_token_provider.py
+++ b/clients/client-python/gravitino/auth/default_oauth2_token_provider.py
@@ -72,7 +72,6 @@ class DefaultOAuth2TokenProvider(OAuth2TokenProvider):
raise IllegalArgumentException("OAuth2TokenProvider must set path")
def _get_access_token(self) -> Optional[str]:
-
expires = self._expires_at_millis()
if expires is None:
@@ -102,7 +101,6 @@ class DefaultOAuth2TokenProvider(OAuth2TokenProvider):
return client_id, client_secret
def _fetch_token(self) -> str:
-
client_id, client_secret = self._parse_credential()
client_credential_request = OAuth2ClientCredentialRequest(
diff --git a/clients/client-python/gravitino/client/fileset_catalog.py
b/clients/client-python/gravitino/client/fileset_catalog.py
index d5bf996be4..27931f7d70 100644
--- a/clients/client-python/gravitino/client/fileset_catalog.py
+++ b/clients/client-python/gravitino/client/fileset_catalog.py
@@ -61,7 +61,6 @@ class FilesetCatalog(BaseSchemaCatalog, SupportsCredentials):
audit: AuditDTO = None,
rest_client: HTTPClient = None,
):
-
super().__init__(
namespace,
name,
diff --git a/clients/client-python/gravitino/client/generic_fileset.py
b/clients/client-python/gravitino/client/generic_fileset.py
index cb3728c187..ccadb26335 100644
--- a/clients/client-python/gravitino/client/generic_fileset.py
+++ b/clients/client-python/gravitino/client/generic_fileset.py
@@ -31,7 +31,6 @@ from gravitino.utils import HTTPClient
class GenericFileset(Fileset, SupportsCredentials):
-
_fileset: FilesetDTO
"""The fileset data transfer object"""
diff --git a/clients/client-python/gravitino/client/generic_model.py
b/clients/client-python/gravitino/client/generic_model.py
index 68c479232f..882383a119 100644
--- a/clients/client-python/gravitino/client/generic_model.py
+++ b/clients/client-python/gravitino/client/generic_model.py
@@ -22,7 +22,6 @@ from gravitino.dto.model_dto import ModelDTO
class GenericModel(Model):
-
_model_dto: ModelDTO
"""The model DTO object."""
diff --git a/clients/client-python/gravitino/client/generic_model_version.py
b/clients/client-python/gravitino/client/generic_model_version.py
index cbeab85ac4..3cd5003977 100644
--- a/clients/client-python/gravitino/client/generic_model_version.py
+++ b/clients/client-python/gravitino/client/generic_model_version.py
@@ -22,7 +22,6 @@ from gravitino.dto.model_version_dto import ModelVersionDTO
class GenericModelVersion(ModelVersion):
-
_model_version_dto: ModelVersionDTO
"""The model version DTO object."""
diff --git a/clients/client-python/gravitino/client/metadata_object_impl.py
b/clients/client-python/gravitino/client/metadata_object_impl.py
index af16b71c4f..8186307552 100644
--- a/clients/client-python/gravitino/client/metadata_object_impl.py
+++ b/clients/client-python/gravitino/client/metadata_object_impl.py
@@ -21,7 +21,6 @@ from gravitino.api.metadata_object import MetadataObject
class MetadataObjectImpl(MetadataObject):
-
_DOT: ClassVar[str] = "."
def __init__(self, names: List[str], metadata_object_type:
MetadataObject.Type):
diff --git
a/clients/client-python/gravitino/dto/requests/oauth2_client_credential_request.py
b/clients/client-python/gravitino/dto/requests/oauth2_client_credential_request.py
index cf10502a76..8f41d517d4 100644
---
a/clients/client-python/gravitino/dto/requests/oauth2_client_credential_request.py
+++
b/clients/client-python/gravitino/dto/requests/oauth2_client_credential_request.py
@@ -21,7 +21,6 @@ from dataclasses import dataclass
@dataclass
class OAuth2ClientCredentialRequest:
-
grant_type: str
client_id: Optional[str]
client_secret: str
diff --git
a/clients/client-python/gravitino/dto/responses/oauth2_token_response.py
b/clients/client-python/gravitino/dto/responses/oauth2_token_response.py
index 55113a4904..9d0cee9178 100644
--- a/clients/client-python/gravitino/dto/responses/oauth2_token_response.py
+++ b/clients/client-python/gravitino/dto/responses/oauth2_token_response.py
@@ -26,7 +26,6 @@ from gravitino.exceptions.base import IllegalArgumentException
@dataclass
class OAuth2TokenResponse(BaseResponse):
-
_access_token: str = field(metadata=config(field_name="access_token"))
_issue_token_type: Optional[str] = field(
metadata=config(field_name="issued_token_type")
diff --git
a/clients/client-python/gravitino/exceptions/handlers/catalog_error_handler.py
b/clients/client-python/gravitino/exceptions/handlers/catalog_error_handler.py
index 7835f96773..2a921fd268 100644
---
a/clients/client-python/gravitino/exceptions/handlers/catalog_error_handler.py
+++
b/clients/client-python/gravitino/exceptions/handlers/catalog_error_handler.py
@@ -29,9 +29,7 @@ from gravitino.exceptions.base import (
class CatalogErrorHandler(RestErrorHandler):
-
def handle(self, error_response: ErrorResponse):
-
error_message = error_response.format_error_message()
code = error_response.code()
exception_type = error_response.type()
diff --git
a/clients/client-python/gravitino/exceptions/handlers/credential_error_handler.py
b/clients/client-python/gravitino/exceptions/handlers/credential_error_handler.py
index 542fb27cf7..3706163fb3 100644
---
a/clients/client-python/gravitino/exceptions/handlers/credential_error_handler.py
+++
b/clients/client-python/gravitino/exceptions/handlers/credential_error_handler.py
@@ -25,9 +25,7 @@ from gravitino.exceptions.base import (
class CredentialErrorHandler(RestErrorHandler):
-
def handle(self, error_response: ErrorResponse):
-
error_message = error_response.format_error_message()
code = error_response.code()
exception_type = error_response.type()
diff --git
a/clients/client-python/gravitino/exceptions/handlers/fileset_error_handler.py
b/clients/client-python/gravitino/exceptions/handlers/fileset_error_handler.py
index 569ce1832b..f11bda7ba2 100644
---
a/clients/client-python/gravitino/exceptions/handlers/fileset_error_handler.py
+++
b/clients/client-python/gravitino/exceptions/handlers/fileset_error_handler.py
@@ -27,9 +27,7 @@ from gravitino.exceptions.base import (
class FilesetErrorHandler(RestErrorHandler):
-
def handle(self, error_response: ErrorResponse):
-
error_message = error_response.format_error_message()
code = error_response.code()
exception_type = error_response.type()
diff --git
a/clients/client-python/gravitino/exceptions/handlers/metalake_error_handler.py
b/clients/client-python/gravitino/exceptions/handlers/metalake_error_handler.py
index 60a7dd0ac3..86f7404a08 100644
---
a/clients/client-python/gravitino/exceptions/handlers/metalake_error_handler.py
+++
b/clients/client-python/gravitino/exceptions/handlers/metalake_error_handler.py
@@ -27,9 +27,7 @@ from gravitino.exceptions.base import (
class MetalakeErrorHandler(RestErrorHandler):
-
def handle(self, error_response: ErrorResponse):
-
error_message = error_response.format_error_message()
code = error_response.code()
diff --git
a/clients/client-python/gravitino/exceptions/handlers/model_error_handler.py
b/clients/client-python/gravitino/exceptions/handlers/model_error_handler.py
index 9f5e97260e..eec52d19d5 100644
--- a/clients/client-python/gravitino/exceptions/handlers/model_error_handler.py
+++ b/clients/client-python/gravitino/exceptions/handlers/model_error_handler.py
@@ -32,7 +32,6 @@ from gravitino.exceptions.handlers.rest_error_handler import
RestErrorHandler
class ModelErrorHandler(RestErrorHandler):
-
def handle(self, error_response: ErrorResponse):
error_message = error_response.format_error_message()
code = error_response.code()
diff --git
a/clients/client-python/gravitino/exceptions/handlers/oauth_error_handler.py
b/clients/client-python/gravitino/exceptions/handlers/oauth_error_handler.py
index 7f4d0f3684..cce8145e71 100644
--- a/clients/client-python/gravitino/exceptions/handlers/oauth_error_handler.py
+++ b/clients/client-python/gravitino/exceptions/handlers/oauth_error_handler.py
@@ -28,9 +28,7 @@ INVALID_SCOPE_ERROR = "invalid_scope"
class OAuthErrorHandler(RestErrorHandler):
-
def handle(self, error_response: OAuth2ErrorResponse):
-
error_message = error_response.message()
exception_type = error_response.type()
diff --git
a/clients/client-python/gravitino/exceptions/handlers/rest_error_handler.py
b/clients/client-python/gravitino/exceptions/handlers/rest_error_handler.py
index ccfcbb6287..578410cea2 100644
--- a/clients/client-python/gravitino/exceptions/handlers/rest_error_handler.py
+++ b/clients/client-python/gravitino/exceptions/handlers/rest_error_handler.py
@@ -22,7 +22,6 @@ from gravitino.exceptions.base import RESTException
class RestErrorHandler(ErrorHandler):
-
def handle(self, error_response: ErrorResponse):
error_message = error_response.format_error_message()
code = error_response.code()
diff --git
a/clients/client-python/gravitino/exceptions/handlers/schema_error_handler.py
b/clients/client-python/gravitino/exceptions/handlers/schema_error_handler.py
index fa805de81d..42a0a8894b 100644
---
a/clients/client-python/gravitino/exceptions/handlers/schema_error_handler.py
+++
b/clients/client-python/gravitino/exceptions/handlers/schema_error_handler.py
@@ -28,9 +28,7 @@ from gravitino.exceptions.base import (
class SchemaErrorHandler(RestErrorHandler):
-
def handle(self, error_response: ErrorResponse):
-
error_message = error_response.format_error_message()
code = error_response.code()
exception_type = error_response.type()
diff --git a/clients/client-python/gravitino/rest/rest_utils.py
b/clients/client-python/gravitino/rest/rest_utils.py
index 4243922c6c..58c4d55b7a 100644
--- a/clients/client-python/gravitino/rest/rest_utils.py
+++ b/clients/client-python/gravitino/rest/rest_utils.py
@@ -20,7 +20,6 @@ from gravitino.exceptions.base import IllegalArgumentException
def encode_string(to_encode: str):
-
if to_encode is None:
raise IllegalArgumentException("Invalid string to encode: None")
diff --git a/clients/client-python/gravitino/utils/__init__.py
b/clients/client-python/gravitino/utils/__init__.py
index ee8b21b902..091797d89a 100644
--- a/clients/client-python/gravitino/utils/__init__.py
+++ b/clients/client-python/gravitino/utils/__init__.py
@@ -16,3 +16,5 @@
# under the License.
from gravitino.utils.http_client import Response, HTTPClient, unpack
+
+__all__ = ["Response", "HTTPClient", "unpack"]
diff --git a/clients/client-python/gravitino/utils/http_client.py
b/clients/client-python/gravitino/utils/http_client.py
index b3e38a5d35..1e51b4ef62 100644
--- a/clients/client-python/gravitino/utils/http_client.py
+++ b/clients/client-python/gravitino/utils/http_client.py
@@ -79,7 +79,6 @@ class Response:
class HTTPClient:
-
FORMDATA_HEADER = {
"Content-Type": "application/x-www-form-urlencoded",
"Accept": "application/vnd.gravitino.v1+json",
diff --git a/clients/client-python/gravitino/utils/__init__.py
b/clients/client-python/pyproject.toml
similarity index 86%
copy from clients/client-python/gravitino/utils/__init__.py
copy to clients/client-python/pyproject.toml
index ee8b21b902..e402f51007 100644
--- a/clients/client-python/gravitino/utils/__init__.py
+++ b/clients/client-python/pyproject.toml
@@ -15,4 +15,11 @@
# specific language governing permissions and limitations
# under the License.
-from gravitino.utils.http_client import Response, HTTPClient, unpack
+[tool.ruff]
+line-length = 88
+
+[tool.ruff.lint]
+ignore-init-module-imports = true
+
+[tool.ruff.format]
+quote-style = "double"
\ No newline at end of file
diff --git a/clients/client-python/requirements.txt
b/clients/client-python/requirements.txt
index 70699bc672..d2e67ee346 100644
--- a/clients/client-python/requirements.txt
+++ b/clients/client-python/requirements.txt
@@ -26,4 +26,7 @@ cachetools==5.5.2
gcsfs==2024.3.1
s3fs==2024.3.1
ossfs==2023.12.0
-adlfs==2023.12.0
\ No newline at end of file
+adlfs==2023.12.0
+black==24.4.2
+flake8==7.0.0
+pre-commit==3.5.0
\ No newline at end of file
diff --git a/clients/client-python/tests/integration/auth/test_auth_common.py
b/clients/client-python/tests/integration/auth/test_auth_common.py
index ede3a4e2ad..8c09664566 100644
--- a/clients/client-python/tests/integration/auth/test_auth_common.py
+++ b/clients/client-python/tests/integration/auth/test_auth_common.py
@@ -111,7 +111,6 @@ class TestCommonAuth:
os.environ["GRAVITINO_USER"] = ""
def init_test_env(self):
-
catalog = self.gravitino_client.create_catalog(
name=self.catalog_name,
catalog_type=Catalog.Type.FILESET,
diff --git a/clients/client-python/tests/integration/auth/test_oauth2_client.py
b/clients/client-python/tests/integration/auth/test_oauth2_client.py
index 0f4a1e4ca9..a3add01562 100644
--- a/clients/client-python/tests/integration/auth/test_oauth2_client.py
+++ b/clients/client-python/tests/integration/auth/test_oauth2_client.py
@@ -46,12 +46,10 @@ DOCKER_TEST = os.environ.get("DOCKER_TEST")
"Skipping tests when DOCKER_TEST=false",
)
class TestOAuth2(IntegrationTestEnv, TestCommonAuth):
-
oauth2_container: OAuth2Container = None
@classmethod
def setUpClass(cls):
-
cls._get_gravitino_home()
cls.oauth2_container = OAuth2Container()
@@ -90,7 +88,6 @@ class TestOAuth2(IntegrationTestEnv, TestCommonAuth):
@classmethod
def _get_default_sign_key(cls) -> str:
-
jwk_uri = f"{cls.oauth2_server_uri}/oauth2/jwks"
# Get JWK from OAuth2 Server
@@ -156,7 +153,6 @@ class TestOAuth2(IntegrationTestEnv, TestCommonAuth):
self.init_test_env()
def init_test_env(self):
-
self.gravitino_admin_client.create_metalake(
self.metalake_name, comment="", properties={}
)
diff --git
a/clients/client-python/tests/integration/auth/test_simple_auth_client.py
b/clients/client-python/tests/integration/auth/test_simple_auth_client.py
index 09309abb74..1d365e7673 100644
--- a/clients/client-python/tests/integration/auth/test_simple_auth_client.py
+++ b/clients/client-python/tests/integration/auth/test_simple_auth_client.py
@@ -31,7 +31,6 @@ logger = logging.getLogger(__name__)
class TestSimpleAuthClient(IntegrationTestEnv, TestCommonAuth):
-
def setUp(self):
os.environ["GRAVITINO_USER"] = self.creator
self.gravitino_admin_client = GravitinoAdminClient(
diff --git a/clients/client-python/tests/integration/base_hadoop_env.py
b/clients/client-python/tests/integration/base_hadoop_env.py
index c8e6cb9dae..ef820a0f57 100644
--- a/clients/client-python/tests/integration/base_hadoop_env.py
+++ b/clients/client-python/tests/integration/base_hadoop_env.py
@@ -30,7 +30,6 @@ PYTHON_BUILD_PATH = os.environ.get("PYTHON_BUILD_PATH")
class BaseHadoopEnvironment:
-
@classmethod
def init_hadoop_env(cls):
cls._unzip_hadoop_pack()
diff --git
a/clients/client-python/tests/integration/containers/hdfs_container.py
b/clients/client-python/tests/integration/containers/hdfs_container.py
index a3e0859434..3f34d65435 100644
--- a/clients/client-python/tests/integration/containers/hdfs_container.py
+++ b/clients/client-python/tests/integration/containers/hdfs_container.py
@@ -70,7 +70,6 @@ async def check_hdfs_container_status(hdfs_container):
class HDFSContainer(BaseContainer):
-
def __init__(self):
container_name = "python-hdfs"
image_name = os.environ.get("GRAVITINO_CI_HIVE_DOCKER_IMAGE")
diff --git
a/clients/client-python/tests/integration/containers/oauth2_container.py
b/clients/client-python/tests/integration/containers/oauth2_container.py
index 02ebc7d1d2..56fb504b7b 100644
--- a/clients/client-python/tests/integration/containers/oauth2_container.py
+++ b/clients/client-python/tests/integration/containers/oauth2_container.py
@@ -36,7 +36,6 @@ async def check_oauth2_container_status(oauth2_container:
"OAuth2Container"):
class OAuth2Container(BaseContainer):
-
def __init__(self):
container_name = "sample-auth-server"
image_name = os.environ.get("GRAVITINO_OAUTH2_SAMPLE_SERVER")
diff --git a/clients/client-python/tests/integration/test_fileset_catalog.py
b/clients/client-python/tests/integration/test_fileset_catalog.py
index 11e83c0b20..12d9d190e2 100644
--- a/clients/client-python/tests/integration/test_fileset_catalog.py
+++ b/clients/client-python/tests/integration/test_fileset_catalog.py
@@ -90,7 +90,6 @@ class TestFilesetCatalog(IntegrationTestEnv):
self.clean_test_data()
def clean_test_data(self):
-
self.gravitino_client = GravitinoClient(
uri="http://localhost:8090";, metalake_name=self.metalake_name
)
diff --git a/clients/client-python/tests/integration/test_gvfs_with_hdfs.py
b/clients/client-python/tests/integration/test_gvfs_with_hdfs.py
index f07a2e0328..7c8272a2e8 100644
--- a/clients/client-python/tests/integration/test_gvfs_with_hdfs.py
+++ b/clients/client-python/tests/integration/test_gvfs_with_hdfs.py
@@ -100,7 +100,6 @@ class TestGvfsWithHDFS(IntegrationTestEnv):
@classmethod
def setUpClass(cls):
-
cls._get_gravitino_home()
cls.hdfs_container = HDFSContainer()
@@ -161,9 +160,7 @@ class TestGvfsWithHDFS(IntegrationTestEnv):
schema_name=cls.schema_name, comment="", properties={}
)
- cls.fileset_storage_location: str = (
-
f"hdfs://{cls.hdfs_container.get_ip()}:9000/{cls.catalog_name}/{cls.schema_name}/{cls.fileset_name}"
- )
+ cls.fileset_storage_location: str =
f"hdfs://{cls.hdfs_container.get_ip()}:9000/{cls.catalog_name}/{cls.schema_name}/{cls.fileset_name}"
cls.fileset_gvfs_location = (
f"gvfs://fileset/{cls.catalog_name}/{cls.schema_name}/{cls.fileset_name}"
)
diff --git a/clients/client-python/tests/unittests/audit/test_caller_context.py
b/clients/client-python/tests/unittests/audit/test_caller_context.py
index 4243d2d5c2..f34abe1343 100644
--- a/clients/client-python/tests/unittests/audit/test_caller_context.py
+++ b/clients/client-python/tests/unittests/audit/test_caller_context.py
@@ -22,7 +22,6 @@ from gravitino.audit.caller_context import
CallerContextHolder, CallerContext
class TestCallerContext(unittest.TestCase):
-
def test_caller_context(self):
thread_names_and_values = [
("Thread1", {"k1": "v1", "k2": "v2"}),
diff --git
a/clients/client-python/tests/unittests/auth/test_oauth2_token_provider.py
b/clients/client-python/tests/unittests/auth/test_oauth2_token_provider.py
index 6388a292e4..dcc80bba56 100644
--- a/clients/client-python/tests/unittests/auth/test_oauth2_token_provider.py
+++ b/clients/client-python/tests/unittests/auth/test_oauth2_token_provider.py
@@ -31,9 +31,7 @@ OAUTH_PORT = 1082
class TestOAuth2TokenProvider(unittest.TestCase):
-
def test_provider_init_exception(self):
-
with self.assertRaises(IllegalArgumentException):
_ = DefaultOAuth2TokenProvider(uri="test")
@@ -48,7 +46,6 @@ class TestOAuth2TokenProvider(unittest.TestCase):
return_value=mock_base.mock_authentication_invalid_client_error(),
)
def test_authertication_invalid_client_error(self, *mock_methods):
-
with self.assertRaises(UnauthorizedException):
_ = DefaultOAuth2TokenProvider(
uri=f"http://127.0.0.1:{OAUTH_PORT}";,
@@ -62,7 +59,6 @@ class TestOAuth2TokenProvider(unittest.TestCase):
return_value=mock_base.mock_authentication_invalid_grant_error(),
)
def test_authertication_invalid_grant_error(self, *mock_methods):
-
with self.assertRaises(BadRequestException):
_ = DefaultOAuth2TokenProvider(
uri=f"http://127.0.0.1:{OAUTH_PORT}";,
@@ -76,7 +72,6 @@ class TestOAuth2TokenProvider(unittest.TestCase):
return_value=mock_base.mock_authentication_with_error_authentication_type(),
)
def test_authentication_with_error_authentication_type(self,
*mock_methods):
-
with self.assertRaises(IllegalArgumentException):
_ = DefaultOAuth2TokenProvider(
uri=f"http://127.0.0.1:{OAUTH_PORT}";,
diff --git
a/clients/client-python/tests/unittests/auth/test_simple_auth_provider.py
b/clients/client-python/tests/unittests/auth/test_simple_auth_provider.py
index d5e0f168a7..91d9a1a49c 100644
--- a/clients/client-python/tests/unittests/auth/test_simple_auth_provider.py
+++ b/clients/client-python/tests/unittests/auth/test_simple_auth_provider.py
@@ -25,7 +25,6 @@ from gravitino.auth.simple_auth_provider import
SimpleAuthProvider
class TestSimpleAuthProvider(unittest.TestCase):
-
def test_auth_provider(self):
os.environ["user.name"] = "test_auth1"
provider: AuthDataProvider = SimpleAuthProvider()
diff --git a/clients/client-python/tests/unittests/mock_base.py
b/clients/client-python/tests/unittests/mock_base.py
index dd3316b697..fb30715441 100644
--- a/clients/client-python/tests/unittests/mock_base.py
+++ b/clients/client-python/tests/unittests/mock_base.py
@@ -131,5 +131,4 @@ def mock_data(cls):
def mock_name_identifier_json(name, namespace):
-
return json.dumps({"name": name, "namespace": namespace}).encode("utf-8")
diff --git a/clients/client-python/tests/unittests/rel/test_distributions.py
b/clients/client-python/tests/unittests/rel/test_distributions.py
index a9e0637c57..85976aaf95 100644
--- a/clients/client-python/tests/unittests/rel/test_distributions.py
+++ b/clients/client-python/tests/unittests/rel/test_distributions.py
@@ -34,7 +34,6 @@ class MockExpression(Expression):
class TestDistributions(unittest.TestCase):
-
def setUp(self):
# Create mock expressions for testing
self.expr1 = MockExpression() # Use the MockExpression class
diff --git a/clients/client-python/tests/unittests/rel/test_types.py
b/clients/client-python/tests/unittests/rel/test_types.py
index bf5685c4ac..2acdbcf4ea 100644
--- a/clients/client-python/tests/unittests/rel/test_types.py
+++ b/clients/client-python/tests/unittests/rel/test_types.py
@@ -21,7 +21,6 @@ from gravitino.api.types.types import Types, Name
class TestTypes(unittest.TestCase):
-
def test_null_type(self):
instance: Types.NullType = Types.NullType.get()
self.assertIsInstance(instance, Types.NullType)
diff --git a/clients/client-python/tests/unittests/test_credential_api.py
b/clients/client-python/tests/unittests/test_credential_api.py
index 2811a226f9..bf96b1a25e 100644
--- a/clients/client-python/tests/unittests/test_credential_api.py
+++ b/clients/client-python/tests/unittests/test_credential_api.py
@@ -31,7 +31,6 @@ from tests.unittests import mock_base
@mock_base.mock_data
class TestCredentialApi(unittest.TestCase):
-
def test_get_credentials(self, *mock_method):
json_str = self._get_s3_token_str()
mock_resp = self._get_mock_http_resp(json_str)
diff --git a/clients/client-python/tests/unittests/test_credential_factory.py
b/clients/client-python/tests/unittests/test_credential_factory.py
index fddbbc098b..0e67e81fea 100644
--- a/clients/client-python/tests/unittests/test_credential_factory.py
+++ b/clients/client-python/tests/unittests/test_credential_factory.py
@@ -32,7 +32,6 @@ from gravitino.api.credential.azure_account_key_credential
import (
class TestCredentialFactory(unittest.TestCase):
-
def test_s3_token_credential(self):
s3_credential_info = {
S3TokenCredential._SESSION_ACCESS_KEY_ID: "access_key",
diff --git a/clients/client-python/tests/unittests/test_error_handler.py
b/clients/client-python/tests/unittests/test_error_handler.py
index a402ae111f..688e6c3480 100644
--- a/clients/client-python/tests/unittests/test_error_handler.py
+++ b/clients/client-python/tests/unittests/test_error_handler.py
@@ -49,9 +49,7 @@ from gravitino.exceptions.handlers.schema_error_handler
import SCHEMA_ERROR_HAND
class TestErrorHandler(unittest.TestCase):
-
def test_rest_error_handler(self):
-
with self.assertRaises(RESTException):
REST_ERROR_HANDLER.handle(
ErrorResponse.generate_error_response(RESTException, "mock
error")
@@ -106,7 +104,6 @@ class TestErrorHandler(unittest.TestCase):
)
def test_fileset_error_handler(self):
-
with self.assertRaises(NoSuchFilesetException):
FILESET_ERROR_HANDLER.handle(
ErrorResponse.generate_error_response(
@@ -132,7 +129,6 @@ class TestErrorHandler(unittest.TestCase):
)
def test_credential_error_handler(self):
-
with self.assertRaises(NoSuchCredentialException):
CREDENTIAL_ERROR_HANDLER.handle(
ErrorResponse.generate_error_response(
@@ -151,7 +147,6 @@ class TestErrorHandler(unittest.TestCase):
)
def test_metalake_error_handler(self):
-
with self.assertRaises(NoSuchMetalakeException):
METALAKE_ERROR_HANDLER.handle(
ErrorResponse.generate_error_response(
@@ -177,7 +172,6 @@ class TestErrorHandler(unittest.TestCase):
)
def test_catalog_error_handler(self):
-
with self.assertRaises(ConnectionFailedException):
CATALOG_ERROR_HANDLER.handle(
ErrorResponse.generate_error_response(
@@ -217,7 +211,6 @@ class TestErrorHandler(unittest.TestCase):
)
def test_schema_error_handler(self):
-
with self.assertRaises(NoSuchCatalogException):
SCHEMA_ERROR_HANDLER.handle(
ErrorResponse.generate_error_response(
diff --git a/clients/client-python/tests/unittests/test_fileset_catalog_api.py
b/clients/client-python/tests/unittests/test_fileset_catalog_api.py
index 06c27d3dfa..93fdf77205 100644
--- a/clients/client-python/tests/unittests/test_fileset_catalog_api.py
+++ b/clients/client-python/tests/unittests/test_fileset_catalog_api.py
@@ -31,7 +31,6 @@ from tests.unittests import mock_base
@mock_base.mock_data
class TestFilesetCatalogApi(unittest.TestCase):
-
def test_get_file_location(self, *mock_method):
json_data = {"code": 0, "fileLocation": "file:/test/1"}
json_str = json.dumps(json_data)
diff --git a/clients/client-python/tests/unittests/test_precondition.py
b/clients/client-python/tests/unittests/test_precondition.py
index 78a246597e..e7eeee00bc 100644
--- a/clients/client-python/tests/unittests/test_precondition.py
+++ b/clients/client-python/tests/unittests/test_precondition.py
@@ -24,7 +24,6 @@ from gravitino.utils.precondition import Precondition
class TestPrecondition(unittest.TestCase):
-
def test_check_argument(self):
with self.assertRaises(IllegalArgumentException):
Precondition.check_argument(False, "error")
diff --git a/dev/docker/iceberg-rest-server/rewrite_config.py
b/dev/docker/iceberg-rest-server/rewrite_config.py
index 9bb7d467d3..ac24ad9c08 100755
--- a/dev/docker/iceberg-rest-server/rewrite_config.py
+++ b/dev/docker/iceberg-rest-server/rewrite_config.py
@@ -19,66 +19,68 @@
import os
env_map = {
- "GRAVITINO_IO_IMPL" : "io-impl",
- "GRAVITINO_URI" : "uri",
- "GRAVITINO_CATALOG_BACKEND" : "catalog-backend",
- "GRAVITINO_JDBC_DRIVER": "jdbc-driver",
- "GRAVITINO_JDBC_USER": "jdbc-user",
- "GRAVITINO_JDBC_PASSWORD": "jdbc-password",
- "GRAVITINO_WAREHOUSE" : "warehouse",
- "GRAVITINO_CREDENTIAL_PROVIDER_TYPE" : "credential-providers",
- "GRAVITINO_CREDENTIAL_PROVIDERS" : "credential-providers",
- "GRAVITINO_GCS_CREDENTIAL_FILE_PATH" : "gcs-service-account-file",
- "GRAVITINO_GCS_SERVICE_ACCOUNT_FILE" : "gcs-service-account-file",
- "GRAVITINO_S3_ACCESS_KEY" : "s3-access-key-id",
- "GRAVITINO_S3_SECRET_KEY" : "s3-secret-access-key",
- "GRAVITINO_S3_ENDPOINT" : "s3-endpoint",
- "GRAVITINO_S3_REGION" : "s3-region",
- "GRAVITINO_S3_ROLE_ARN" : "s3-role-arn",
- "GRAVITINO_S3_EXTERNAL_ID" : "s3-external-id",
- "GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME" : "azure-storage-account-name",
- "GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY" : "azure-storage-account-key",
- "GRAVITINO_AZURE_TENANT_ID" : "azure-tenant-id",
- "GRAVITINO_AZURE_CLIENT_ID" : "azure-client-id",
- "GRAVITINO_AZURE_CLIENT_SECRET" : "azure-client-secret",
- "GRAVITINO_OSS_ACCESS_KEY": "oss-access-key-id",
- "GRAVITINO_OSS_SECRET_KEY": "oss-secret-access-key",
- "GRAVITINO_OSS_ENDPOINT": "oss-endpoint",
- "GRAVITINO_OSS_REGION": "oss-region",
- "GRAVITINO_OSS_ROLE_ARN": "oss-role-arn",
- "GRAVITINO_OSS_EXTERNAL_ID": "oss-external-id",
-
+ "GRAVITINO_IO_IMPL": "io-impl",
+ "GRAVITINO_URI": "uri",
+ "GRAVITINO_CATALOG_BACKEND" : "catalog-backend",
+ "GRAVITINO_JDBC_DRIVER": "jdbc-driver",
+ "GRAVITINO_JDBC_USER": "jdbc-user",
+ "GRAVITINO_JDBC_PASSWORD": "jdbc-password",
+ "GRAVITINO_WAREHOUSE": "warehouse",
+ "GRAVITINO_CREDENTIAL_PROVIDER_TYPE": "credential-providers",
+ "GRAVITINO_CREDENTIAL_PROVIDERS": "credential-providers",
+ "GRAVITINO_GCS_CREDENTIAL_FILE_PATH": "gcs-service-account-file",
+ "GRAVITINO_GCS_SERVICE_ACCOUNT_FILE": "gcs-service-account-file",
+ "GRAVITINO_S3_ACCESS_KEY": "s3-access-key-id",
+ "GRAVITINO_S3_SECRET_KEY": "s3-secret-access-key",
+ "GRAVITINO_S3_ENDPOINT": "s3-endpoint",
+ "GRAVITINO_S3_REGION": "s3-region",
+ "GRAVITINO_S3_ROLE_ARN": "s3-role-arn",
+ "GRAVITINO_S3_EXTERNAL_ID": "s3-external-id",
+ "GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME": "azure-storage-account-name",
+ "GRAVITINO_AZURE_STORAGE_ACCOUNT_KEY": "azure-storage-account-key",
+ "GRAVITINO_AZURE_TENANT_ID": "azure-tenant-id",
+ "GRAVITINO_AZURE_CLIENT_ID": "azure-client-id",
+ "GRAVITINO_AZURE_CLIENT_SECRET": "azure-client-secret",
+ "GRAVITINO_OSS_ACCESS_KEY": "oss-access-key-id",
+ "GRAVITINO_OSS_SECRET_KEY": "oss-secret-access-key",
+ "GRAVITINO_OSS_ENDPOINT": "oss-endpoint",
+ "GRAVITINO_OSS_REGION": "oss-region",
+ "GRAVITINO_OSS_ROLE_ARN": "oss-role-arn",
+ "GRAVITINO_OSS_EXTERNAL_ID": "oss-external-id",
}
init_config = {
- "catalog-backend" : "jdbc",
- "jdbc-driver" : "org.sqlite.JDBC",
- "uri" : "jdbc:sqlite::memory:",
- "jdbc-user" : "iceberg",
- "jdbc-password" : "iceberg",
- "jdbc-initialize" : "true",
- "jdbc.schema-version" : "V1"
+ "catalog-backend": "jdbc",
+ "jdbc-driver": "org.sqlite.JDBC",
+ "uri": "jdbc:sqlite::memory:",
+ "jdbc-user": "iceberg",
+ "jdbc-password": "iceberg",
+ "jdbc-initialize": "true",
+ "jdbc.schema-version": "V1",
}
-def parse_config_file(file_path):
- config_map = {}
- with open(file_path, 'r') as file:
- for line in file:
- stripped_line = line.strip()
- if stripped_line and not stripped_line.startswith('#'):
- key, value = stripped_line.split('=', 1)
- key = key.strip()
- value = value.strip()
- config_map[key] = value
- return config_map
+def parse_config_file(file_path):
+ config_map = {}
+ with open(file_path, "r") as file:
+ for line in file:
+ stripped_line = line.strip()
+ if stripped_line and not stripped_line.startswith("#"):
+ key, value = stripped_line.split("=", 1)
+ key = key.strip()
+ value = value.strip()
+ config_map[key] = value
+ return config_map
+
config_prefix = "gravitino.iceberg-rest."
+
def update_config(config, key, value):
config[config_prefix + key] = value
-
-config_file_path = 'conf/gravitino-iceberg-rest-server.conf'
+
+
+config_file_path = "conf/gravitino-iceberg-rest-server.conf"
config_map = parse_config_file(config_file_path)
for k, v in init_config.items():
@@ -87,11 +89,11 @@ for k, v in init_config.items():
for k, v in env_map.items():
if k in os.environ:
update_config(config_map, v, os.environ[k])
-
-if os.path.exists(config_file_path):
- os.remove(config_file_path)
-with open(config_file_path, 'w') as file:
- for key, value in config_map.items():
- line = "{} = {}\n".format(key, value)
- file.write(line)
+if os.path.exists(config_file_path):
+ os.remove(config_file_path)
+
+with open(config_file_path, "w") as file:
+ for key, value in config_map.items():
+ line = "{} = {}\n".format(key, value)
+ file.write(line)
