This is an automated email from the ASF dual-hosted git repository. elizabeth pushed a commit to branch elizabeth/fix-resize-bug in repository https://gitbox.apache.org/repos/asf/superset.git
commit 0a3d9cbc18b8371253a79c3acbcbc9a8f3caed26 Author: Maxime Beauchemin <[email protected]> AuthorDate: Mon Jul 28 09:27:07 2025 -0700 feat: introducing a docker-compose-light.yml for lighter development (#34324) --- docker-compose-image-tag.yml | 3 + docker-compose-light.yml | 157 +++++++++++++++++++++ docker-compose-non-dev.yml | 3 + docker-compose.yml | 3 + docker/.env | 6 +- docker/pythonpath_dev/.gitignore | 1 + docker/pythonpath_dev/superset_config.py | 2 +- .../pythonpath_dev/superset_config_docker_light.py | 37 +++++ docs/docs/installation/docker-compose.mdx | 34 ++++- superset-frontend/webpack.config.js | 6 + superset/config.py | 2 +- 11 files changed, 247 insertions(+), 7 deletions(-) diff --git a/docker-compose-image-tag.yml b/docker-compose-image-tag.yml index 4246d80cef..5bc2418979 100644 --- a/docker-compose-image-tag.yml +++ b/docker-compose-image-tag.yml @@ -20,6 +20,9 @@ # If you choose to use this type of deployment make sure to # create you own docker environment file (docker/.env) with your own # unique random secure passwords and SECRET_KEY. +# +# For verbose logging during development: +# - Set SUPERSET_LOG_LEVEL=debug in docker/.env-local for detailed Superset logs # ----------------------------------------------------------------------- x-superset-image: &superset-image apachesuperset.docker.scarf.sh/apache/superset:${TAG:-latest-dev} x-superset-volumes: diff --git a/docker-compose-light.yml b/docker-compose-light.yml new file mode 100644 index 0000000000..a15b789812 --- /dev/null +++ b/docker-compose-light.yml @@ -0,0 +1,157 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# ----------------------------------------------------------------------- +# Lightweight docker-compose for running multiple Superset instances +# This includes only essential services: database, Redis, and Superset app +# +# IMPORTANT: To run multiple instances in parallel: +# - Use different project names: docker-compose -p project1 -f docker-compose-light.yml up +# - Use different NODE_PORT values: NODE_PORT=9002 docker-compose -p project2 -f docker-compose-light.yml up +# - Volumes are isolated by project name (e.g., project1_db_home_light, project2_db_home_light) +# - Database name is intentionally different (superset_light) to prevent accidental cross-connections +# +# For verbose logging during development: +# - Set SUPERSET_LOG_LEVEL=debug in docker/.env-local for detailed Superset logs +# ----------------------------------------------------------------------- +x-superset-user: &superset-user root +x-superset-volumes: &superset-volumes + # /app/pythonpath_docker will be appended to the PYTHONPATH in the final container + - ./docker:/app/docker + - ./superset:/app/superset + - ./superset-frontend:/app/superset-frontend + - superset_home_light:/app/superset_home + - ./tests:/app/tests +x-common-build: &common-build + context: . + target: ${SUPERSET_BUILD_TARGET:-dev} # can use `dev` (default) or `lean` + cache_from: + - apache/superset-cache:3.10-slim-bookworm + args: + DEV_MODE: "true" + INCLUDE_CHROMIUM: ${INCLUDE_CHROMIUM:-false} + INCLUDE_FIREFOX: ${INCLUDE_FIREFOX:-false} + BUILD_TRANSLATIONS: ${BUILD_TRANSLATIONS:-false} + +services: + db-light: + env_file: + - path: docker/.env # default + required: true + - path: docker/.env-local # optional override + required: false + image: postgres:16 + restart: unless-stopped + # No host port mapping - only accessible within Docker network + volumes: + - db_home_light:/var/lib/postgresql/data + - ./docker/docker-entrypoint-initdb.d:/docker-entrypoint-initdb.d + environment: + # Override database name to avoid conflicts + POSTGRES_DB: superset_light + + superset-light: + env_file: + - path: docker/.env # default + required: true + - path: docker/.env-local # optional override + required: false + build: + <<: *common-build + command: ["/app/docker/docker-bootstrap.sh", "app"] + restart: unless-stopped + # No host port mapping - accessed via webpack dev server proxy + extra_hosts: + - "host.docker.internal:host-gateway" + user: *superset-user + depends_on: + superset-init-light: + condition: service_completed_successfully + volumes: *superset-volumes + environment: + # Override DB connection for light service + DATABASE_HOST: db-light + DATABASE_DB: superset_light + POSTGRES_DB: superset_light + EXAMPLES_HOST: db-light + EXAMPLES_DB: superset_light + EXAMPLES_USER: superset + EXAMPLES_PASSWORD: superset + # Use light-specific config that disables Redis + SUPERSET_CONFIG_PATH: /app/docker/pythonpath_dev/superset_config_docker_light.py + + superset-init-light: + build: + <<: *common-build + command: ["/app/docker/docker-init.sh"] + env_file: + - path: docker/.env # default + required: true + - path: docker/.env-local # optional override + required: false + depends_on: + db-light: + condition: service_started + user: *superset-user + volumes: *superset-volumes + environment: + # Override DB connection for light service + DATABASE_HOST: db-light + DATABASE_DB: superset_light + POSTGRES_DB: superset_light + EXAMPLES_HOST: db-light + EXAMPLES_DB: superset_light + EXAMPLES_USER: superset + EXAMPLES_PASSWORD: superset + # Use light-specific config that disables Redis + SUPERSET_CONFIG_PATH: /app/docker/pythonpath_dev/superset_config_docker_light.py + healthcheck: + disable: true + + superset-node-light: + build: + context: . + target: superset-node + args: + # This prevents building the frontend bundle since we'll mount local folder + # and build it on startup while firing docker-frontend.sh in dev mode, where + # it'll mount and watch local files and rebuild as you update them + DEV_MODE: "true" + BUILD_TRANSLATIONS: ${BUILD_TRANSLATIONS:-false} + environment: + # set this to false if you have perf issues running the npm i; npm run dev in-docker + # if you do so, you have to run this manually on the host, which should perform better! + BUILD_SUPERSET_FRONTEND_IN_DOCKER: true + NPM_RUN_PRUNE: false + SCARF_ANALYTICS: "${SCARF_ANALYTICS:-}" + # configuring the dev-server to use the host.docker.internal to connect to the backend + superset: "http://superset-light:8088"; + ports: + - "127.0.0.1:${NODE_PORT:-9001}:9000" # Parameterized port + command: ["/app/docker/docker-frontend.sh"] + env_file: + - path: docker/.env # default + required: true + - path: docker/.env-local # optional override + required: false + volumes: *superset-volumes + +volumes: + superset_home_light: + external: false + db_home_light: + external: false diff --git a/docker-compose-non-dev.yml b/docker-compose-non-dev.yml index cde5359825..78eb3c010d 100644 --- a/docker-compose-non-dev.yml +++ b/docker-compose-non-dev.yml @@ -20,6 +20,9 @@ # If you choose to use this type of deployment make sure to # create you own docker environment file (docker/.env) with your own # unique random secure passwords and SECRET_KEY. +# +# For verbose logging during development: +# - Set SUPERSET_LOG_LEVEL=debug in docker/.env-local for detailed Superset logs # ----------------------------------------------------------------------- x-superset-volumes: &superset-volumes # /app/pythonpath_docker will be appended to the PYTHONPATH in the final container diff --git a/docker-compose.yml b/docker-compose.yml index 12bcc8dd82..08b0c40701 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,6 +20,9 @@ # If you choose to use this type of deployment make sure to # create you own docker environment file (docker/.env) with your own # unique random secure passwords and SECRET_KEY. +# +# For verbose logging during development: +# - Set SUPERSET_LOG_LEVEL=debug in docker/.env-local for detailed Superset logs # ----------------------------------------------------------------------- x-superset-user: &superset-user root x-superset-volumes: &superset-volumes diff --git a/docker/.env b/docker/.env index 2ae0578bfc..a0cbb47deb 100644 --- a/docker/.env +++ b/docker/.env @@ -53,7 +53,12 @@ PYTHONPATH=/app/pythonpath:/app/docker/pythonpath_dev REDIS_HOST=redis REDIS_PORT=6379 +# Development and logging configuration +# FLASK_DEBUG: Enables Flask dev features (auto-reload, better error pages) - keep 'true' for development FLASK_DEBUG=true +# SUPERSET_LOG_LEVEL: Controls Superset application logging verbosity (debug, info, warning, error, critical) +SUPERSET_LOG_LEVEL=info + SUPERSET_APP_ROOT="/" SUPERSET_ENV=development SUPERSET_LOAD_EXAMPLES=yes @@ -66,4 +71,3 @@ SUPERSET_SECRET_KEY=TEST_NON_DEV_SECRET ENABLE_PLAYWRIGHT=false PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true BUILD_SUPERSET_FRONTEND_IN_DOCKER=true -SUPERSET_LOG_LEVEL=info diff --git a/docker/pythonpath_dev/.gitignore b/docker/pythonpath_dev/.gitignore index 376bb61ea9..97cac2072f 100644 --- a/docker/pythonpath_dev/.gitignore +++ b/docker/pythonpath_dev/.gitignore @@ -20,4 +20,5 @@ # DON'T ignore the .gitignore !.gitignore !superset_config.py +!superset_config_docker_light.py !superset_config_local.example diff --git a/docker/pythonpath_dev/superset_config.py b/docker/pythonpath_dev/superset_config.py index 705846ac48..2de8f03794 100644 --- a/docker/pythonpath_dev/superset_config.py +++ b/docker/pythonpath_dev/superset_config.py @@ -129,7 +129,7 @@ if os.getenv("CYPRESS_CONFIG") == "true": # try: import superset_config_docker - from superset_config_docker import * # noqa + from superset_config_docker import * # noqa: F403 logger.info( f"Loaded your Docker configuration at [{superset_config_docker.__file__}]" diff --git a/docker/pythonpath_dev/superset_config_docker_light.py b/docker/pythonpath_dev/superset_config_docker_light.py new file mode 100644 index 0000000000..9a5ae0ae67 --- /dev/null +++ b/docker/pythonpath_dev/superset_config_docker_light.py @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Configuration for docker-compose-light.yml - disables Redis and uses minimal services + +# Import all settings from the main config first +from flask_caching.backends.filesystemcache import FileSystemCache +from superset_config import * # noqa: F403 + +# Override caching to use simple in-memory cache instead of Redis +RESULTS_BACKEND = FileSystemCache("/app/superset_home/sqllab") + +CACHE_CONFIG = { + "CACHE_TYPE": "SimpleCache", + "CACHE_DEFAULT_TIMEOUT": 300, + "CACHE_KEY_PREFIX": "superset_light_", +} +DATA_CACHE_CONFIG = CACHE_CONFIG +THUMBNAIL_CACHE_CONFIG = CACHE_CONFIG + + +# Disable Celery entirely for lightweight mode +CELERY_CONFIG = None # type: ignore[assignment,misc] diff --git a/docs/docs/installation/docker-compose.mdx b/docs/docs/installation/docker-compose.mdx index 16be28975d..74145ff0b5 100644 --- a/docs/docs/installation/docker-compose.mdx +++ b/docs/docs/installation/docker-compose.mdx @@ -26,11 +26,14 @@ Superset locally is using Docker Compose on a Linux or Mac OSX computer. Superset does not have official support for Windows. It's also the easiest way to launch a fully functioning **development environment** quickly. -Note that there are 3 major ways we support to run `docker compose`: +Note that there are 4 major ways we support to run `docker compose`: 1. **docker-compose.yml:** for interactive development, where we mount your local folder with the frontend/backend files that you can edit and experience the changes you make in the app in real time +1. **docker-compose-light.yml:** a lightweight configuration with minimal services (database, + Superset app, and frontend dev server) for development. Uses in-memory caching instead of Redis + and is designed for running multiple instances simultaneously 1. **docker-compose-non-dev.yml** where we just build a more immutable image based on the local branch and get all the required images running. Changes in the local branch at the time you fire this up will be reflected, but changes to the code @@ -44,7 +47,7 @@ Note that there are 3 major ways we support to run `docker compose`: The `dev` builds include the `psycopg2-binary` required to connect to the Postgres database launched as part of the `docker compose` builds. -More on these two approaches after setting up the requirements for either. +More on these approaches after setting up the requirements for either. ## Requirements @@ -103,13 +106,36 @@ and help you start fresh. In the context of `docker compose` setting from within docker. This will slow down the startup, but will fix various npm-related issues. ::: -### Option #2 - build a set of immutable images from the local branch +### Option #2 - lightweight development with multiple instances + +For a lighter development setup that uses fewer resources and supports running multiple instances: + +```bash +# Single lightweight instance (default port 9001) +docker compose -f docker-compose-light.yml up + +# Multiple instances with different ports +NODE_PORT=9001 docker compose -p superset-1 -f docker-compose-light.yml up +NODE_PORT=9002 docker compose -p superset-2 -f docker-compose-light.yml up +NODE_PORT=9003 docker compose -p superset-3 -f docker-compose-light.yml up +``` + +This configuration includes: +- PostgreSQL database (internal network only) +- Superset application server +- Frontend development server with webpack hot reloading +- In-memory caching (no Redis) +- Isolated volumes and networks per instance + +Access each instance at `http://localhost:{NODE_PORT}` (e.g., `http://localhost:9001`). + +### Option #3 - build a set of immutable images from the local branch ```bash docker compose -f docker-compose-non-dev.yml up ``` -### Option #3 - boot up an official release +### Option #4 - boot up an official release ```bash # Set the version you want to run diff --git a/superset-frontend/webpack.config.js b/superset-frontend/webpack.config.js index fed420f2fa..f315c11451 100644 --- a/superset-frontend/webpack.config.js +++ b/superset-frontend/webpack.config.js @@ -53,6 +53,7 @@ const { measure = false, nameChunks = false, } = parsedArgs; + const isDevMode = mode !== 'production'; const isDevServer = process.argv[1].includes('webpack-dev-server'); @@ -535,6 +536,11 @@ if (isDevMode) { runtimeErrors: error => !/ResizeObserver/.test(error.message), }, logging: 'error', + webSocketURL: { + hostname: '0.0.0.0', + pathname: '/ws', + port: 0, + }, }, static: { directory: path.join(process.cwd(), '../static/assets'), diff --git a/superset/config.py b/superset/config.py index f03d3c216f..273f2232da 100644 --- a/superset/config.py +++ b/superset/config.py @@ -1155,7 +1155,7 @@ class CeleryConfig: # pylint: disable=too-few-public-methods } -CELERY_CONFIG: type[CeleryConfig] = CeleryConfig +CELERY_CONFIG: type[CeleryConfig] | None = CeleryConfig # Set celery config to None to disable all the above configuration # CELERY_CONFIG = None
