This is an automated email from the ASF dual-hosted git repository. beto pushed a commit to branch semantic-layer-stack in repository https://gitbox.apache.org/repos/asf/superset.git
commit acb8b63023fe002fafa629ac0b51274918b137f4 Author: Beto Dealmeida <[email protected]> AuthorDate: Tue Jan 27 12:10:53 2026 -0500 Make extension --- setup.py | 7 +- superset-snowflake-semantic-layer/README.md | 51 ++++++++ .../backend/pyproject.toml | 24 ++++ .../src/snowflake_semantic_layer}/__init__.py | 6 +- .../src/snowflake_semantic_layer}/schemas.py | 0 .../snowflake_semantic_layer}/semantic_layer.py | 16 +-- .../src/snowflake_semantic_layer}/semantic_view.py | 9 +- .../backend/src/snowflake_semantic_layer}/utils.py | 4 +- superset-snowflake-semantic-layer/extension.json | 15 +++ superset/extensions/utils.py | 69 +++++++++++ superset/semantic_layers/models.py | 14 +-- superset/semantic_layers/registry.py | 130 +++++++++++++++++++++ 12 files changed, 309 insertions(+), 36 deletions(-) diff --git a/setup.py b/setup.py index 6a62c552fac..9b834cc2417 100644 --- a/setup.py +++ b/setup.py @@ -30,9 +30,7 @@ with open(PACKAGE_JSON) as package_file: def get_git_sha() -> str: try: - output = subprocess.check_output( - ["git", "rev-parse", "HEAD"] - ) # noqa: S603, S607 + output = subprocess.check_output(["git", "rev-parse", "HEAD"]) # noqa: S603, S607 return output.decode().strip() except Exception: # pylint: disable=broad-except return "" @@ -60,9 +58,6 @@ setup( include_package_data=True, zip_safe=False, entry_points={ - "superset.semantic_layers": [ - "snowflake = superset.semantic_layers.snowflake:SnowflakeSemanticLayer" - ], "console_scripts": ["superset=superset.cli.main:superset"], # the `postgres` and `postgres+psycopg2://` schemes were removed in SQLAlchemy 1.4 # noqa: E501 # add an alias here to prevent breaking existing databases diff --git a/superset-snowflake-semantic-layer/README.md b/superset-snowflake-semantic-layer/README.md new file mode 100644 index 00000000000..7c1a5c2c45d --- /dev/null +++ b/superset-snowflake-semantic-layer/README.md @@ -0,0 +1,51 @@ +# Snowflake Semantic Layer Extension for Apache Superset + +This extension adds support for Snowflake Semantic Views to Apache Superset. + +## Installation + +### As a pip package + +```bash +pip install superset-snowflake-semantic-layer +``` + +### As a Superset extension (.supx bundle) + +1. Build the extension bundle: + ```bash + cd superset-snowflake-semantic-layer + superset-extensions bundle + ``` + +2. Copy the generated `.supx` file to your Superset extensions directory. + +3. Configure Superset to load extensions: + ```python + # superset_config.py + EXTENSIONS_PATH = "/path/to/extensions" + ``` + +## Configuration + +When adding a Snowflake Semantic Layer in Superset, you'll need to provide: + +- **Account Identifier**: Your Snowflake account identifier (e.g., `abc12345`) +- **Authentication**: Either username/password or private key authentication +- **Role** (optional): The default Snowflake role to use +- **Warehouse** (optional): The default warehouse to use +- **Database** (optional): The default database containing semantic views +- **Schema** (optional): The default schema containing semantic views + +## Features + +- Browse and query Snowflake Semantic Views +- Support for dimensions and metrics defined in semantic views +- Filtering and aggregation through the Superset UI +- Group limiting (top N) with optional "Other" grouping + +## Requirements + +- Apache Superset 4.0+ +- Snowflake account with semantic views +- `snowflake-connector-python` >= 3.0.0 diff --git a/superset-snowflake-semantic-layer/backend/pyproject.toml b/superset-snowflake-semantic-layer/backend/pyproject.toml new file mode 100644 index 00000000000..511868b407a --- /dev/null +++ b/superset-snowflake-semantic-layer/backend/pyproject.toml @@ -0,0 +1,24 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "superset-snowflake-semantic-layer" +version = "1.0.0" +description = "Snowflake Semantic Layer extension for Apache Superset" +readme = "README.md" +license = "Apache-2.0" +requires-python = ">=3.10" +dependencies = [ + "apache-superset", + "snowflake-connector-python>=3.0.0", + "snowflake-sqlalchemy>=1.5.0", + "pydantic>=2.0.0", + "cryptography>=41.0.0", +] + +[project.entry-points."superset.semantic_layers"] +snowflake = "snowflake_semantic_layer:SnowflakeSemanticLayer" + +[tool.hatch.build.targets.wheel] +packages = ["src/snowflake_semantic_layer"] diff --git a/superset/semantic_layers/snowflake/__init__.py b/superset-snowflake-semantic-layer/backend/src/snowflake_semantic_layer/__init__.py similarity index 78% rename from superset/semantic_layers/snowflake/__init__.py rename to superset-snowflake-semantic-layer/backend/src/snowflake_semantic_layer/__init__.py index 40b1a53fed8..7c4721a4f05 100644 --- a/superset/semantic_layers/snowflake/__init__.py +++ b/superset-snowflake-semantic-layer/backend/src/snowflake_semantic_layer/__init__.py @@ -15,9 +15,9 @@ # specific language governing permissions and limitations # under the License. -from superset.semantic_layers.snowflake.schemas import SnowflakeConfiguration -from superset.semantic_layers.snowflake.semantic_layer import SnowflakeSemanticLayer -from superset.semantic_layers.snowflake.semantic_view import SnowflakeSemanticView +from snowflake_semantic_layer.schemas import SnowflakeConfiguration +from snowflake_semantic_layer.semantic_layer import SnowflakeSemanticLayer +from snowflake_semantic_layer.semantic_view import SnowflakeSemanticView __all__ = [ "SnowflakeConfiguration", diff --git a/superset/semantic_layers/snowflake/schemas.py b/superset-snowflake-semantic-layer/backend/src/snowflake_semantic_layer/schemas.py similarity index 100% rename from superset/semantic_layers/snowflake/schemas.py rename to superset-snowflake-semantic-layer/backend/src/snowflake_semantic_layer/schemas.py diff --git a/superset/semantic_layers/snowflake/semantic_layer.py b/superset-snowflake-semantic-layer/backend/src/snowflake_semantic_layer/semantic_layer.py similarity index 94% rename from superset/semantic_layers/snowflake/semantic_layer.py rename to superset-snowflake-semantic-layer/backend/src/snowflake_semantic_layer/semantic_layer.py index f9d4e1a1481..c6f12c4a524 100644 --- a/superset/semantic_layers/snowflake/semantic_layer.py +++ b/superset-snowflake-semantic-layer/backend/src/snowflake_semantic_layer/semantic_layer.py @@ -24,9 +24,9 @@ from pydantic import create_model, Field from snowflake.connector import connect from snowflake.connector.connection import SnowflakeConnection -from superset.semantic_layers.snowflake.schemas import SnowflakeConfiguration -from superset.semantic_layers.snowflake.semantic_view import SnowflakeSemanticView -from superset.semantic_layers.snowflake.utils import get_connection_parameters +from snowflake_semantic_layer.schemas import SnowflakeConfiguration +from snowflake_semantic_layer.semantic_view import SnowflakeSemanticView +from snowflake_semantic_layer.utils import get_connection_parameters from superset.semantic_layers.types import ( SemanticLayerImplementation, ) @@ -210,10 +210,7 @@ class SnowflakeSemanticLayer( """ Get the semantic views available in the semantic layer. """ - # Avoid circular import - from superset.semantic_layers.snowflake.semantic_view import ( - SnowflakeSemanticView, - ) + from snowflake_semantic_layer.semantic_view import SnowflakeSemanticView # create a new configuration with the runtime parameters configuration = self.configuration.model_copy(update=runtime_configuration) @@ -242,10 +239,7 @@ class SnowflakeSemanticLayer( """ Get a specific semantic view by name. """ - # Avoid circular import - from superset.semantic_layers.snowflake.semantic_view import ( - SnowflakeSemanticView, - ) + from snowflake_semantic_layer.semantic_view import SnowflakeSemanticView # create a new configuration with the additional parameters configuration = self.configuration.model_copy(update=additional_configuration) diff --git a/superset/semantic_layers/snowflake/semantic_view.py b/superset-snowflake-semantic-layer/backend/src/snowflake_semantic_layer/semantic_view.py similarity index 99% rename from superset/semantic_layers/snowflake/semantic_view.py rename to superset-snowflake-semantic-layer/backend/src/snowflake_semantic_layer/semantic_view.py index 9f34e6f2025..91d0f835be5 100644 --- a/superset/semantic_layers/snowflake/semantic_view.py +++ b/superset-snowflake-semantic-layer/backend/src/snowflake_semantic_layer/semantic_view.py @@ -28,8 +28,8 @@ from pandas import DataFrame from snowflake.connector import connect, DictCursor from snowflake.sqlalchemy.snowdialect import SnowflakeDialect -from superset.semantic_layers.snowflake.schemas import SnowflakeConfiguration -from superset.semantic_layers.snowflake.utils import ( +from snowflake_semantic_layer.schemas import SnowflakeConfiguration +from snowflake_semantic_layer.utils import ( get_connection_parameters, substitute_parameters, validate_order_by, @@ -515,7 +515,10 @@ class SnowflakeSemanticView(SemanticViewImplementation): # Check if temporal dimension is already in the order if order: for element, _ in order: - if isinstance(element, Dimension) and element.id == temporal_dimension.id: + if ( + isinstance(element, Dimension) + and element.id == temporal_dimension.id + ): return order # Prepend temporal dimension to existing order return [(temporal_dimension, OrderDirection.ASC)] + list(order) diff --git a/superset/semantic_layers/snowflake/utils.py b/superset-snowflake-semantic-layer/backend/src/snowflake_semantic_layer/utils.py similarity index 98% rename from superset/semantic_layers/snowflake/utils.py rename to superset-snowflake-semantic-layer/backend/src/snowflake_semantic_layer/utils.py index 76251c0288a..d73f17ddbee 100644 --- a/superset/semantic_layers/snowflake/utils.py +++ b/superset-snowflake-semantic-layer/backend/src/snowflake_semantic_layer/utils.py @@ -24,12 +24,12 @@ from typing import Any, Sequence from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import serialization -from superset.exceptions import SupersetParseError -from superset.semantic_layers.snowflake.schemas import ( +from snowflake_semantic_layer.schemas import ( PrivateKeyAuth, SnowflakeConfiguration, UserPasswordAuth, ) +from superset.exceptions import SupersetParseError from superset.sql.parse import SQLStatement diff --git a/superset-snowflake-semantic-layer/extension.json b/superset-snowflake-semantic-layer/extension.json new file mode 100644 index 00000000000..0a6610654c6 --- /dev/null +++ b/superset-snowflake-semantic-layer/extension.json @@ -0,0 +1,15 @@ +{ + "id": "superset-snowflake-semantic-layer", + "name": "Snowflake Semantic Layer", + "version": "1.0.0", + "description": "Connect to semantic views stored in Snowflake.", + "permissions": [], + "backend": { + "entryPoints": [ + "snowflake = snowflake_semantic_layer:SnowflakeSemanticLayer" + ], + "files": [ + "backend/src/**/*.py" + ] + } +} diff --git a/superset/extensions/utils.py b/superset/extensions/utils.py index e288287d229..592af833236 100644 --- a/superset/extensions/utils.py +++ b/superset/extensions/utils.py @@ -185,6 +185,73 @@ def build_extension_data(extension: LoadedExtension) -> dict[str, Any]: return extension_data +def load_extension_backend(extension: LoadedExtension) -> None: + """ + Load an extension's backend code and register its entry points. + + This installs the extension's Python modules in-memory and registers + any entry points declared in the manifest (e.g., semantic layers). + """ + # Install backend modules in-memory if present + if extension.backend: + install_in_memory_importer(extension.backend) + + # Register entry points from manifest + manifest = extension.manifest + if backend := manifest.get("backend"): + for ep_str in backend.get("entryPoints", []): + _register_entry_point(ep_str, extension.name) + + +def _register_entry_point(ep_str: str, extension_name: str) -> None: + """ + Parse and register a single entry point string. + + Entry point format: "name = module:ClassName" + """ + if "=" not in ep_str: + logger.warning( + "Invalid entry point format in extension %s: %s", + extension_name, + ep_str, + ) + return + + name, _, target = ep_str.partition("=") + name = name.strip() + target = target.strip() + + if ":" not in target: + logger.warning( + "Invalid entry point target in extension %s: %s (expected module:Class)", + extension_name, + target, + ) + return + + module_path, _, class_name = target.partition(":") + + try: + module = eager_import(module_path) + cls = getattr(module, class_name) + + # Register with semantic layer registry + from superset.semantic_layers.registry import register_semantic_layer + + register_semantic_layer(name, cls) + logger.info( + "Registered entry point '%s' from extension %s", + name, + extension_name, + ) + except Exception: + logger.exception( + "Failed to load entry point '%s' from extension %s", + name, + extension_name, + ) + + def get_extensions() -> dict[str, LoadedExtension]: extensions: dict[str, LoadedExtension] = {} @@ -194,6 +261,7 @@ def get_extensions() -> dict[str, LoadedExtension]: extension = get_loaded_extension(files) extension_id = extension.manifest["id"] extensions[extension_id] = extension + load_extension_backend(extension) logger.info( "Loading extension %s (ID: %s) from local filesystem", extension.name, @@ -208,6 +276,7 @@ def get_extensions() -> dict[str, LoadedExtension]: extension_id = extension.manifest["id"] if extension_id not in extensions: # Don't override LOCAL_EXTENSIONS extensions[extension_id] = extension + load_extension_backend(extension) logger.info( "Loading extension %s (ID: %s) from discovery path", extension.name, diff --git a/superset/semantic_layers/models.py b/superset/semantic_layers/models.py index dc09e5fe6e7..8f325e7beef 100644 --- a/superset/semantic_layers/models.py +++ b/superset/semantic_layers/models.py @@ -23,7 +23,6 @@ import uuid from collections.abc import Hashable from dataclasses import dataclass from functools import cached_property -from importlib.metadata import entry_points from typing import Any, TYPE_CHECKING from flask_appbuilder import Model @@ -37,6 +36,7 @@ from superset.explorables.base import TimeGrainDict from superset.extensions import encrypted_field_factory from superset.models.helpers import AuditMixinNullable, QueryResult from superset.semantic_layers.mapper import get_results +from superset.semantic_layers.registry import get_semantic_layer from superset.semantic_layers.types import ( BINARY, BOOLEAN, @@ -141,19 +141,11 @@ class SemanticLayer(AuditMixinNullable, Model): """ Return semantic layer implementation. """ - entry_point = next( - iter( - entry_points( - group="superset.semantic_layers", - name=self.type, - ) - ) - ) - implementation_class = entry_point.load() + implementation_class = get_semantic_layer(self.type) if not issubclass(implementation_class, SemanticLayerImplementation): raise TypeError( - f"Entry point for semantic layer type '{self.type}' " + f"Semantic layer type '{self.type}' " "must be a subclass of SemanticLayerImplementation" ) diff --git a/superset/semantic_layers/registry.py b/superset/semantic_layers/registry.py new file mode 100644 index 00000000000..d4cf2c51c82 --- /dev/null +++ b/superset/semantic_layers/registry.py @@ -0,0 +1,130 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Semantic layer registry. + +This module provides a registry for semantic layer implementations that can be +populated from: +1. Standard Python entry points (for pip-installed packages) +2. Superset extensions (for .supx bundles) +""" + +from __future__ import annotations + +import logging +from importlib.metadata import entry_points +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from superset.semantic_layers.types import SemanticLayerImplementation + +logger = logging.getLogger(__name__) + +ENTRY_POINT_GROUP = "superset.semantic_layers" + +# Registry mapping semantic layer type names to implementation classes +_semantic_layer_registry: dict[str, type[SemanticLayerImplementation]] = {} +_initialized_from_entry_points = False + + +def _init_from_entry_points() -> None: + """ + Pre-populate the registry from installed packages' entry points. + + This is called lazily on first access to ensure all packages are loaded. + """ + global _initialized_from_entry_points + if _initialized_from_entry_points: + return + + for ep in entry_points(group=ENTRY_POINT_GROUP): + if ep.name not in _semantic_layer_registry: + try: + _semantic_layer_registry[ep.name] = ep.load() + logger.info( + "Registered semantic layer '%s' from entry point %s", + ep.name, + ep.value, + ) + except Exception: + logger.exception( + "Failed to load semantic layer '%s' from entry point %s", + ep.name, + ep.value, + ) + + _initialized_from_entry_points = True + + +def register_semantic_layer( + name: str, + cls: type[SemanticLayerImplementation], +) -> None: + """ + Register a semantic layer implementation. + + This is called by extensions to register their semantic layer implementations. + + Args: + name: The type name for the semantic layer (e.g., "snowflake") + cls: The implementation class + """ + if name in _semantic_layer_registry: + logger.warning( + "Semantic layer '%s' already registered, overwriting with %s", + name, + cls, + ) + _semantic_layer_registry[name] = cls + logger.info("Registered semantic layer '%s' from extension: %s", name, cls) + + +def get_semantic_layer(name: str) -> type[SemanticLayerImplementation]: + """ + Get a semantic layer implementation by name. + + Args: + name: The type name for the semantic layer (e.g., "snowflake") + + Returns: + The implementation class + + Raises: + KeyError: If no implementation is registered for the given name + """ + _init_from_entry_points() + + if name not in _semantic_layer_registry: + available = ", ".join(_semantic_layer_registry.keys()) or "(none)" + raise KeyError( + f"No semantic layer implementation registered for type '{name}'. " + f"Available types: {available}" + ) + + return _semantic_layer_registry[name] + + +def get_registered_semantic_layers() -> dict[str, type[SemanticLayerImplementation]]: + """ + Get all registered semantic layer implementations. + + Returns: + A dictionary mapping type names to implementation classes + """ + _init_from_entry_points() + return dict(_semantic_layer_registry)
