This is an automated email from the ASF dual-hosted git repository.
pierrejeambrun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new a915216434c perf(api): optimize /ui/dags endpoint serialization
(#61483)
a915216434c is described below
commit a915216434cf583b0b54eee129cc9a93b96492b2
Author: john-rodriguez-mgni
<[email protected]>
AuthorDate: Fri Mar 6 07:44:03 2026 -0800
perf(api): optimize /ui/dags endpoint serialization (#61483)
This PR addresses a significant performance issue in the /ui/dags endpoint
where page load times scaled poorly with the number of DAGs (12-16 seconds
for just 25 DAGs in our testing).
Two optimizations are implemented:
1. Cache URLSafeSerializer for file_token generation
- Previously, a new URLSafeSerializer was instantiated and
conf.get_mandatory_value() was called for every DAG
- Now uses @lru_cache to create the serializer once and reuse it
2. Eliminate redundant Pydantic validation in response construction
- The original pattern used model_validate -> model_dump ->
model_validate
which caused triple serialization overhead per DAG
- Now validates once with DAGResponse.model_validate(), then uses
model_construct() to build DAGWithLatestDagRunsResponse
Together, these changes reduced page load time from 12-16 seconds to
~130ms in our dev environment.
Co-authored-by: Cursor <[email protected]>
---
.../airflow/api_fastapi/core_api/datamodels/dags.py | 17 +++++++++++++++--
.../airflow/api_fastapi/core_api/routes/ui/dags.py | 19 +++++++++++++------
2 files changed, 28 insertions(+), 8 deletions(-)
diff --git a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dags.py
b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dags.py
index 6c07f1ed8a8..5bda20af0bf 100644
--- a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dags.py
+++ b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/dags.py
@@ -20,6 +20,7 @@ from __future__ import annotations
import inspect
from collections.abc import Iterable, Mapping
from datetime import datetime, timedelta
+from functools import cache
from typing import TYPE_CHECKING, Any
from itsdangerous import URLSafeSerializer
@@ -42,6 +43,19 @@ from airflow.utils.types import DagRunType
if TYPE_CHECKING:
from airflow.serialization.definitions.param import SerializedParamsDict
+
+@cache
+def _get_file_token_serializer() -> URLSafeSerializer:
+ """
+ Return a cached URLSafeSerializer instance.
+
+ Uses @cache for lazy initialization - the serializer is created on first
+ call rather than at module import time. This avoids issues if the module
+ is imported before configuration is fully loaded.
+ """
+ return URLSafeSerializer(conf.get_mandatory_value("api", "secret_key"))
+
+
DAG_ALIAS_MAPPING: dict[str, str] = {
# The keys are the names in the response, the values are the original
names in the model
# This is used to map the names in the response to the names in the model
@@ -119,12 +133,11 @@ class DAGResponse(BaseModel):
@property
def file_token(self) -> str:
"""Return file token."""
- serializer = URLSafeSerializer(conf.get_mandatory_value("api",
"secret_key"))
payload = {
"bundle_name": self.bundle_name,
"relative_fileloc": self.relative_fileloc,
}
- return serializer.dumps(payload)
+ return _get_file_token_serializer().dumps(payload)
class DAGPatchBody(StrictBaseModel):
diff --git a/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/dags.py
b/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/dags.py
index 755d926defd..41da300e5d8 100644
--- a/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/dags.py
+++ b/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/dags.py
@@ -52,7 +52,7 @@ from airflow.api_fastapi.common.parameters import (
filter_param_factory,
)
from airflow.api_fastapi.common.router import AirflowRouter
-from airflow.api_fastapi.core_api.datamodels.dags import DAGResponse
+from airflow.api_fastapi.core_api.datamodels.dags import DAG_ALIAS_MAPPING,
DAGResponse
from airflow.api_fastapi.core_api.datamodels.ui.dag_runs import
DAGRunLightResponse
from airflow.api_fastapi.core_api.datamodels.ui.dags import (
DAGWithLatestDagRunsCollectionResponse,
@@ -234,18 +234,25 @@ def get_dags(
pending_actions_by_dag_id[dag_id].append(hitl_detail)
# aggregate rows by dag_id
- dag_runs_by_dag_id: dict[str, DAGWithLatestDagRunsResponse] = {
- dag.dag_id: DAGWithLatestDagRunsResponse.model_validate(
+ # Build the dict dynamically from DAGResponse.model_fields so that new
fields
+ # added to DAGResponse are picked up automatically without code changes
here.
+ dag_runs_by_dag_id: dict[str, DAGWithLatestDagRunsResponse] = {}
+ for dag in dags:
+ dag_data = {
+ DAG_ALIAS_MAPPING.get(field_name, field_name): getattr(
+ dag, DAG_ALIAS_MAPPING.get(field_name, field_name)
+ )
+ for field_name in DAGResponse.model_fields
+ }
+ dag_data.update(
{
- **DAGResponse.model_validate(dag).model_dump(),
"asset_expression": dag.asset_expression,
"latest_dag_runs": [],
"pending_actions": pending_actions_by_dag_id[dag.dag_id],
"is_favorite": dag.dag_id in favorite_dag_ids,
}
)
- for dag in dags
- }
+ dag_runs_by_dag_id[dag.dag_id] =
DAGWithLatestDagRunsResponse.model_validate(dag_data)
for row in recent_dag_runs:
dag_run_response = DAGRunLightResponse.model_validate(row)
