This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 293bc55dd74 Deselect DB tests at collection time instead of skipping
at runtime (#63791)
293bc55dd74 is described below
commit 293bc55dd747aff2280c9a22fc6b68046a3ffe3b
Author: Dev-iL <[email protected]>
AuthorDate: Thu Mar 19 02:13:56 2026 +0200
Deselect DB tests at collection time instead of skipping at runtime (#63791)
Move DB/non-DB test filtering from pytest_runtest_setup (where each xdist
worker had to import test modules only to skip them) to
`pytest_collection_modifyitems` (where items are deselected before
distribution). This avoids distributing ~6350 unwanted items to workers
entirely, reducing memory usage and unnecessary imports.
---
devel-common/src/tests_common/pytest_plugin.py | 72 +++++++++++++-------------
1 file changed, 37 insertions(+), 35 deletions(-)
diff --git a/devel-common/src/tests_common/pytest_plugin.py
b/devel-common/src/tests_common/pytest_plugin.py
index 53abf77844d..a4b590c5452 100644
--- a/devel-common/src/tests_common/pytest_plugin.py
+++ b/devel-common/src/tests_common/pytest_plugin.py
@@ -635,37 +635,6 @@ def skip_quarantined_test(item):
)
-def skip_db_test(item):
- if next(item.iter_markers(name="db_test"), None):
- if next(item.iter_markers(name="non_db_test_override"), None):
- # non_db_test can override the db_test set for example on module
or class level
- return
- pytest.skip(
- f"The test is skipped as it is DB test and --skip-db-tests is flag
is passed to pytest. {item}"
- )
- if next(item.iter_markers(name="backend"), None):
- # also automatically skip tests marked with `backend` marker as they
are implicitly
- # db tests
- pytest.skip(
- f"The test is skipped as it is DB test and --skip-db-tests is flag
is passed to pytest. {item}"
- )
-
-
-def only_run_db_test(item):
- if next(item.iter_markers(name="db_test"), None) and not next(
- item.iter_markers(name="non_db_test_override"), None
- ):
- # non_db_test at individual level can override the db_test set for
example on module or class level
- return
- if next(item.iter_markers(name="backend"), None):
- # Also do not skip the tests marked with `backend` marker - as it is
implicitly a db test
- return
- pytest.skip(
- f"The test is skipped as it is not a DB tests "
- f"and --run-db-tests-only flag is passed to pytest. {item}"
- )
-
-
def skip_if_integration_disabled(marker, item):
integration_name = marker.args[0]
environment_variable_name = "INTEGRATION_" + integration_name.upper()
@@ -712,6 +681,43 @@ def skip_if_credential_file_missing(item):
pytest.skip(f"The test requires credential file {credential_path}:
{item}")
+def _is_db_test(item) -> bool:
+ """Check if a test item should be treated as a DB test."""
+ if next(item.iter_markers(name="db_test"), None):
+ if next(item.iter_markers(name="non_db_test_override"), None):
+ return False
+ return True
+ if next(item.iter_markers(name="backend"), None):
+ return True
+ return False
+
+
+def pytest_collection_modifyitems(config, items):
+ """Deselect DB/non-DB tests early so pytest-xdist workers never receive
them.
+
+ Previously these tests were skipped at runtime via pytest_runtest_setup,
which
+ still required every xdist worker to import the test modules. With
thousands of
+ DB tests being skipped in non-DB runs, this caused excessive memory usage
— enough
+ to OOM on Python 3.14 CI runners. Deselecting during collection avoids
distributing
+ these items to workers entirely.
+ """
+ if not skip_db_tests and not run_db_tests_only:
+ return
+
+ selected = []
+ deselected = []
+ for item in items:
+ is_db = _is_db_test(item)
+ if (skip_db_tests and is_db) or (run_db_tests_only and not is_db):
+ deselected.append(item)
+ else:
+ selected.append(item)
+
+ if deselected:
+ config.hook.pytest_deselected(items=deselected)
+ items[:] = selected
+
+
def pytest_runtest_setup(item):
selected_integrations_list = item.config.option.integration
@@ -737,10 +743,6 @@ def pytest_runtest_setup(item):
skip_long_running_test(item)
if not include_quarantined:
skip_quarantined_test(item)
- if skip_db_tests:
- skip_db_test(item)
- if run_db_tests_only:
- only_run_db_test(item)
skip_if_credential_file_missing(item)
