This is an automated email from the ASF dual-hosted git repository.
gopidesu pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 9b665c52733 CI: enable e2e remote logging elasticsearch test (#63693)
9b665c52733 is described below
commit 9b665c527339e27e1b77f165854eae040075136a
Author: Owen Leung <[email protected]>
AuthorDate: Fri Mar 20 17:42:19 2026 +0800
CI: enable e2e remote logging elasticsearch test (#63693)
* CI: enable e2e remote logging elasticsearch test
* CI: fix failing CI test
* CI: exclude changes to existing aws provider ci behaviour
---------
Co-authored-by: Jarek Potiuk <[email protected]>
---
.github/workflows/additional-prod-image-tests.yml | 21 +++++++++
.github/workflows/ci-amd-arm.yml | 4 ++
.../src/airflow_breeze/utils/selective_checks.py | 35 ++++++++++++++
dev/breeze/tests/test_selective_checks.py | 55 ++++++++++++++++++++++
4 files changed, 115 insertions(+)
diff --git a/.github/workflows/additional-prod-image-tests.yml
b/.github/workflows/additional-prod-image-tests.yml
index ad3f60042c9..03a94c84179 100644
--- a/.github/workflows/additional-prod-image-tests.yml
+++ b/.github/workflows/additional-prod-image-tests.yml
@@ -36,6 +36,14 @@ on: # yamllint disable-line rule:truthy
description: "Whether to run Task SDK integration tests (true/false)"
required: true
type: string
+ run-remote-logging-s3-e2e-tests:
+ description: "Whether to run S3 remote logging e2e tests (true/false)"
+ required: true
+ type: string
+ run-remote-logging-elasticsearch-e2e-tests:
+ description: "Whether to run Elasticsearch remote logging e2e tests
(true/false)"
+ required: true
+ type: string
constraints-branch:
description: "Branch used to construct constraints URL from."
required: true
@@ -225,6 +233,19 @@ jobs:
default-python-version: "${{ inputs.default-python-version }}"
use-uv: ${{ inputs.use-uv }}
e2e_test_mode: "remote_log"
+ if: inputs.canary-run == 'true' || inputs.run-remote-logging-s3-e2e-tests
== 'true'
+
+ test-e2e-integration-tests-remote-log-elasticsearch:
+ name: "Elasticsearch remote logging tests with PROD image"
+ uses: ./.github/workflows/airflow-e2e-tests.yml
+ with:
+ workflow-name: "Elasticsearch remote logging e2e test"
+ runners: ${{ inputs.runners }}
+ platform: ${{ inputs.platform }}
+ default-python-version: "${{ inputs.default-python-version }}"
+ use-uv: ${{ inputs.use-uv }}
+ e2e_test_mode: "remote_log_elasticsearch"
+ if: inputs.canary-run == 'true' ||
inputs.run-remote-logging-elasticsearch-e2e-tests == 'true'
test-e2e-integration-tests-xcom-object-storage:
name: "XCom object storage backend tests with PROD image"
diff --git a/.github/workflows/ci-amd-arm.yml b/.github/workflows/ci-amd-arm.yml
index 90b060d560f..db5f6ec43d0 100644
--- a/.github/workflows/ci-amd-arm.yml
+++ b/.github/workflows/ci-amd-arm.yml
@@ -116,6 +116,8 @@ jobs:
run-helm-tests: ${{ steps.selective-checks.outputs.run-helm-tests }}
run-kubernetes-tests: ${{
steps.selective-checks.outputs.run-kubernetes-tests }}
run-mypy: ${{ steps.selective-checks.outputs.run-mypy }}
+ run-remote-logging-elasticsearch-e2e-tests: ${{
steps.selective-checks.outputs.run-remote-logging-elasticsearch-e2e-tests }}
+ run-remote-logging-s3-e2e-tests: ${{
steps.selective-checks.outputs.run-remote-logging-s3-e2e-tests }}
run-system-tests: ${{ steps.selective-checks.outputs.run-system-tests }}
run-task-sdk-tests: ${{
steps.selective-checks.outputs.run-task-sdk-tests }}
run-task-sdk-integration-tests: ${{
steps.selective-checks.outputs.run-task-sdk-integration-tests }}
@@ -772,6 +774,8 @@ jobs:
default-python-version: "${{
needs.build-info.outputs.default-python-version }}"
run-task-sdk-integration-tests: ${{
needs.build-info.outputs.run-task-sdk-integration-tests }}
canary-run: ${{ needs.build-info.outputs.canary-run }}
+ run-remote-logging-elasticsearch-e2e-tests: ${{
needs.build-info.outputs.run-remote-logging-elasticsearch-e2e-tests }}
+ run-remote-logging-s3-e2e-tests: ${{
needs.build-info.outputs.run-remote-logging-s3-e2e-tests }}
use-uv: ${{ needs.build-info.outputs.use-uv }}
run-ui-e2e-tests: ${{ needs.build-info.outputs.run-ui-e2e-tests }}
run-airflow-ctl-integration-tests: ${{
needs.build-info.outputs.run-airflow-ctl-integration-tests }}
diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
index 259a381f9ec..1985da6ece1 100644
--- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py
+++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py
@@ -122,6 +122,9 @@ class FileGroupForCi(Enum):
AIRFLOW_CTL_FILES = auto()
AIRFLOW_CTL_INTEGRATION_TEST_FILES = auto()
BREEZE_INTEGRATION_TEST_FILES = auto()
+ REMOTE_LOGGING_E2E_SHARED_FILES = auto()
+ REMOTE_LOGGING_E2E_S3_FILES = auto()
+ REMOTE_LOGGING_E2E_ELASTICSEARCH_FILES = auto()
ALL_PYPROJECT_TOML_FILES = auto()
ALL_PYTHON_FILES = auto()
ALL_SOURCE_FILES = auto()
@@ -176,6 +179,24 @@ CI_FILE_GROUP_MATCHES: HashableDict[FileGroupForCi] =
HashableDict(
r"^dev/breeze/pyproject\.toml",
r"^dev/breeze/uv\.lock",
],
+ FileGroupForCi.REMOTE_LOGGING_E2E_SHARED_FILES: [
+
r"^airflow-core/src/airflow/config_templates/airflow_local_settings\.py$",
+ r"^airflow-core/src/airflow/logging/.*",
+ r"^airflow-core/src/airflow/logging_config\.py$",
+
r"^airflow-core/src/airflow/api_fastapi/core_api/routes/public/log\.py$",
+
r"^airflow-core/src/airflow/api_fastapi/core_api/datamodels/log\.py$",
+ r"^airflow-core/src/airflow/utils/log/.*",
+ r"^airflow-e2e-tests/.*",
+ r"^shared/logging/.*",
+ ],
+ FileGroupForCi.REMOTE_LOGGING_E2E_S3_FILES: [
+ r"^airflow-e2e-tests/tests/airflow_e2e_tests/remote_log_tests/.*",
+
r"^providers/amazon/src/airflow/providers/amazon/aws/log/s3_task_handler\.py$",
+ ],
+ FileGroupForCi.REMOTE_LOGGING_E2E_ELASTICSEARCH_FILES: [
+
r"^airflow-e2e-tests/tests/airflow_e2e_tests/remote_log_elasticsearch_tests/.*",
+ r"^providers/elasticsearch/.*",
+ ],
FileGroupForCi.PYTHON_PRODUCTION_FILES: [
r"^airflow-core/src/airflow/.*\.py",
r"^providers/.*\.py",
@@ -925,6 +946,18 @@ class SelectiveChecks:
def run_ui_e2e_tests(self) -> bool:
return self._should_be_run(FileGroupForCi.UI_FILES)
+ @cached_property
+ def run_remote_logging_s3_e2e_tests(self) -> bool:
+ return
self._should_be_run(FileGroupForCi.REMOTE_LOGGING_E2E_SHARED_FILES) or
self._should_be_run(
+ FileGroupForCi.REMOTE_LOGGING_E2E_S3_FILES
+ )
+
+ @cached_property
+ def run_remote_logging_elasticsearch_e2e_tests(self) -> bool:
+ return
self._should_be_run(FileGroupForCi.REMOTE_LOGGING_E2E_SHARED_FILES) or
self._should_be_run(
+ FileGroupForCi.REMOTE_LOGGING_E2E_ELASTICSEARCH_FILES
+ )
+
@cached_property
def run_amazon_tests(self) -> bool:
if self.providers_test_types_list_as_strings_in_json == "[]":
@@ -1028,6 +1061,8 @@ class SelectiveChecks:
or self.run_helm_tests
or self.run_task_sdk_integration_tests
or self.run_airflow_ctl_integration_tests
+ or self.run_remote_logging_s3_e2e_tests
+ or self.run_remote_logging_elasticsearch_e2e_tests
or self.run_ui_e2e_tests
)
diff --git a/dev/breeze/tests/test_selective_checks.py
b/dev/breeze/tests/test_selective_checks.py
index 135841d47b7..a870616b06f 100644
--- a/dev/breeze/tests/test_selective_checks.py
+++ b/dev/breeze/tests/test_selective_checks.py
@@ -1250,6 +1250,61 @@ def assert_outputs_are_printed(expected_outputs:
dict[str, str], stderr: str):
id="All mypy checks should run when
devel-common/pyproject.toml changes",
)
),
+ (
+ pytest.param(
+
("providers/amazon/src/airflow/providers/amazon/aws/log/s3_task_handler.py",),
+ {
+ "run-remote-logging-s3-e2e-tests": "true",
+ "run-remote-logging-elasticsearch-e2e-tests": "false",
+ "prod-image-build": "true",
+ },
+ id="S3 remote logging changes enable only S3 e2e",
+ )
+ ),
+ (
+ pytest.param(
+
("providers/elasticsearch/src/airflow/providers/elasticsearch/log/es_task_handler.py",),
+ {
+ "run-remote-logging-s3-e2e-tests": "false",
+ "run-remote-logging-elasticsearch-e2e-tests": "true",
+ "prod-image-build": "true",
+ },
+ id="Elasticsearch remote logging changes enable only
Elasticsearch e2e",
+ )
+ ),
+ (
+ pytest.param(
+
("providers/elasticsearch/src/airflow/providers/elasticsearch/log/es_json_formatter.py",),
+ {
+ "run-remote-logging-s3-e2e-tests": "false",
+ "run-remote-logging-elasticsearch-e2e-tests": "true",
+ "prod-image-build": "true",
+ },
+ id="Elasticsearch helper changes enable Elasticsearch e2e",
+ )
+ ),
+ (
+ pytest.param(
+
("airflow-core/src/airflow/config_templates/airflow_local_settings.py",),
+ {
+ "run-remote-logging-s3-e2e-tests": "true",
+ "run-remote-logging-elasticsearch-e2e-tests": "true",
+ "prod-image-build": "true",
+ },
+ id="Shared remote logging changes enable both remote logging
e2e jobs",
+ )
+ ),
+ (
+ pytest.param(
+ ("shared/logging/src/airflow_shared/logging/remote.py",),
+ {
+ "run-remote-logging-s3-e2e-tests": "true",
+ "run-remote-logging-elasticsearch-e2e-tests": "true",
+ "prod-image-build": "true",
+ },
+ id="Shared logging library changes enable both remote logging
e2e jobs",
+ )
+ ),
],
)
def test_expected_output_pull_request_main(
