This is an automated email from the ASF dual-hosted git repository. beto pushed a commit to branch sl-1-extension-loading in repository https://gitbox.apache.org/repos/asf/superset.git
commit 7cb7988e80b9f9b68ee7a45c83151ab221b0d52a Author: Beto Dealmeida <[email protected]> AuthorDate: Mon Feb 2 17:23:53 2026 -0500 feat(extensions): improve extension loading with Pydantic compatibility - Add Pydantic v1/v2 compatibility for manifest validation - Simplify extension initialization in SupersetAppInitializer - Fix docker-compose-light.yml for development: - Add superset-core volume mount - Fix EXAMPLES_HOST to use db-light service - Change WEBPACK_DEVSERVER_HOST default to 0.0.0.0 - Add EXTENSIONS_PATH config support Co-Authored-By: Claude Opus 4.5 <[email protected]> --- docker-compose-light.yml | 8 ++- docker/pythonpath_dev/superset_config.py | 10 +++- superset/extensions/utils.py | 91 +++++++++++++++++++++++++++++++- superset/initialization/__init__.py | 27 ++-------- 4 files changed, 110 insertions(+), 26 deletions(-) diff --git a/docker-compose-light.yml b/docker-compose-light.yml index 09b0c65b0f8..4b805798a4e 100644 --- a/docker-compose-light.yml +++ b/docker-compose-light.yml @@ -64,9 +64,11 @@ x-superset-volumes: &superset-volumes # /app/pythonpath_docker will be appended to the PYTHONPATH in the final container - ./docker:/app/docker - ./superset:/app/superset + - ./superset-core:/app/superset-core - ./superset-frontend:/app/superset-frontend - superset_home_light:/app/superset_home - ./tests:/app/tests + - ./extensions:/app/extensions x-common-build: &common-build context: . target: ${SUPERSET_BUILD_TARGET:-dev} # can use `dev` (default) or `lean` @@ -138,6 +140,10 @@ services: DATABASE_DB: superset_light POSTGRES_DB: superset_light SUPERSET_CONFIG_PATH: /app/docker/pythonpath_dev/superset_config_docker_light.py + # Override examples host to use light DB service + EXAMPLES_HOST: db-light + # Skip example loading for faster startup + SUPERSET_LOAD_EXAMPLES: "no" healthcheck: disable: true @@ -160,7 +166,7 @@ services: # configuring the dev-server to use the host.docker.internal to connect to the backend superset: "http://superset-light:8088"; # Webpack dev server configuration - WEBPACK_DEVSERVER_HOST: "${WEBPACK_DEVSERVER_HOST:-127.0.0.1}" + WEBPACK_DEVSERVER_HOST: "${WEBPACK_DEVSERVER_HOST:-0.0.0.0}" WEBPACK_DEVSERVER_PORT: "${WEBPACK_DEVSERVER_PORT:-9000}" ports: - "${NODE_PORT:-9001}:9000" # Parameterized port, accessible on all interfaces diff --git a/docker/pythonpath_dev/superset_config.py b/docker/pythonpath_dev/superset_config.py index d88d9899c27..e647b7827ad 100644 --- a/docker/pythonpath_dev/superset_config.py +++ b/docker/pythonpath_dev/superset_config.py @@ -105,7 +105,15 @@ class CeleryConfig: CELERY_CONFIG = CeleryConfig -FEATURE_FLAGS = {"ALERT_REPORTS": True} +# Extensions configuration +# For local development, point to the extensions directory +# Note: If running in Docker, this path needs to be accessible from inside the container +EXTENSIONS_PATH = os.getenv("EXTENSIONS_PATH", "/app/extensions") + +FEATURE_FLAGS = { + "ALERT_REPORTS": True, + "ENABLE_EXTENSIONS": True, +} ALERT_REPORTS_NOTIFICATION_DRY_RUN = True WEBDRIVER_BASEURL = f"http://superset_app{os.environ.get('SUPERSET_APP_ROOT', '/')}/" # When using docker compose baseurl should be http://superset_nginx{ENV{BASEPATH}}/ # noqa: E501 # The base URL for the email report hyperlinks. diff --git a/superset/extensions/utils.py b/superset/extensions/utils.py index 883c9114728..e1a9f154292 100644 --- a/superset/extensions/utils.py +++ b/superset/extensions/utils.py @@ -25,6 +25,7 @@ from pathlib import Path from typing import Any, Generator, Iterable, Tuple from zipfile import ZipFile +import pydantic from flask import current_app from pydantic import ValidationError from superset_core.extensions.types import Manifest @@ -35,6 +36,22 @@ from superset.utils.core import check_is_safe_zip logger = logging.getLogger(__name__) +# Detect Pydantic version for compatibility +PYDANTIC_V2 = int(pydantic.VERSION.split(".")[0]) >= 2 + + +def _validate_manifest(data: dict[str, Any]) -> Manifest: + """ + Validate manifest data using the appropriate Pydantic method. + + Handles both Pydantic v1 and v2 compatibility. + """ + if PYDANTIC_V2: + return Manifest.model_validate(data) + else: + return Manifest.parse_obj(data) + + FRONTEND_REGEX = re.compile(r"^frontend/dist/([^/]+)$") BACKEND_REGEX = re.compile(r"^backend/src/(.+)$") @@ -171,7 +188,7 @@ def get_loaded_extension( if filename == "manifest.json": try: manifest_data = json.loads(content) - manifest = Manifest.model_validate(manifest_data) + manifest = _validate_manifest(manifest_data) except ValidationError as e: raise Exception(f"Invalid manifest.json: {e}") from e except Exception as e: @@ -223,6 +240,76 @@ def build_extension_data(extension: LoadedExtension) -> dict[str, Any]: return extension_data +def load_extension_backend(extension: LoadedExtension) -> None: + """ + Load an extension's backend code and register its entry points. + + This installs the extension's Python modules in-memory and registers + any entry points declared in the manifest (e.g., semantic layers). + """ + # Install backend modules in-memory if present + if extension.backend: + install_in_memory_importer( + extension.backend, + source_base_path=extension.source_base_path, + ) + + # Register entry points from manifest + manifest = extension.manifest + if manifest.backend: + for ep_str in manifest.backend.entryPoints: + _register_entry_point(ep_str, extension.name) + + +def _register_entry_point(ep_str: str, extension_name: str) -> None: + """ + Parse and register a single entry point string. + + Entry point format: "name = module:ClassName" + """ + if "=" not in ep_str: + logger.warning( + "Invalid entry point format in extension %s: %s", + extension_name, + ep_str, + ) + return + + name, _, target = ep_str.partition("=") + name = name.strip() + target = target.strip() + + if ":" not in target: + logger.warning( + "Invalid entry point target in extension %s: %s (expected module:Class)", + extension_name, + target, + ) + return + + module_path, _, class_name = target.partition(":") + + try: + module = eager_import(module_path) + cls = getattr(module, class_name) + + # Register with semantic layer registry + from superset.semantic_layers.registry import register_semantic_layer + + register_semantic_layer(name, cls) + logger.info( + "Registered entry point '%s' from extension %s", + name, + extension_name, + ) + except Exception: + logger.exception( + "Failed to load entry point '%s' from extension %s", + name, + extension_name, + ) + + def get_extensions() -> dict[str, LoadedExtension]: extensions: dict[str, LoadedExtension] = {} @@ -234,6 +321,7 @@ def get_extensions() -> dict[str, LoadedExtension]: extension = get_loaded_extension(files, source_base_path=abs_dist_path) extension_id = extension.manifest.id extensions[extension_id] = extension + load_extension_backend(extension) logger.info( "Loading extension %s (ID: %s) from local filesystem", extension.name, @@ -248,6 +336,7 @@ def get_extensions() -> dict[str, LoadedExtension]: extension_id = extension.manifest.id if extension_id not in extensions: # Don't override LOCAL_EXTENSIONS extensions[extension_id] = extension + load_extension_backend(extension) logger.info( "Loading extension %s (ID: %s) from discovery path", extension.name, diff --git a/superset/initialization/__init__.py b/superset/initialization/__init__.py index adc783c1e12..4f2eb9d7648 100644 --- a/superset/initialization/__init__.py +++ b/superset/initialization/__init__.py @@ -546,37 +546,18 @@ class SupersetAppInitializer: # pylint: disable=too-many-public-methods self.init_extensions() def init_extensions(self) -> None: - from superset.extensions.utils import ( - eager_import, - get_extensions, - install_in_memory_importer, - ) + from superset.extensions.utils import get_extensions try: - extensions = get_extensions() + # get_extensions() discovers and loads all extensions, + # including installing in-memory importers and registering entry points + get_extensions() except Exception: # pylint: disable=broad-except # noqa: S110 # If the db hasn't been initialized yet, an exception will be raised. # It's fine to ignore this, as in this case there are no extensions # present yet. return - for extension in extensions.values(): - if backend_files := extension.backend: - install_in_memory_importer( - backend_files, - source_base_path=extension.source_base_path, - ) - - backend = extension.manifest.backend - - if backend and (entrypoints := backend.entryPoints): - for entrypoint in entrypoints: - try: - eager_import(entrypoint) - except Exception as ex: # pylint: disable=broad-except # noqa: S110 - # Surface exceptions during initialization of extensions - print(ex) - def init_app_in_ctx(self) -> None: """ Runs init logic in the context of the app
