This is an automated email from the ASF dual-hosted git repository.
bbovenzi pushed a change to branch mapped-task-action-ux
in repository https://gitbox.apache.org/repos/asf/airflow.git
from 53dc1dc51b Remove table selection for now
add 0ae0f7e244 Support unknown backends in entrypoint_prod.sh (#22883)
add 6933022e94 Fix new MyPy errors in main (#22884)
add aa8c08db38 Databricks: Correctly handle HTTP exception (#22885)
add 7be57eb256 Databricks SQL operators are now Python 3.10 compatible
(#22886)
add 10078287e2 Replace old Breeze with Python based implementation (#22880)
add 81bbb5531f Remove unnecessary python 3.6 conditionals (#20549)
add 7c64b921f2 Push CI image using new Python Breeze (#22888)
add 3f63e9d685 Fix "force_answers" parameter to be "yes" rather than
"true" (#22891)
add 03e1c9b152 MSSQLToGCSOperator fails: datetime is not JSON Serializable
(#22882)
add 4201e6e0da update INTHEWILD (#22896)
add a111a79c0d make operator's execution_timeout configurable (#22389)
add 1a8b8f521c Add test case for clearTaskInstance call with invalid Task
IDs. (#22894)
add 8af77127f1 Allow using mapped upstream's aggregated XCom (#22849)
add a3dd8473e4 Handle invalid JSON metadata in get_logs_with_metadata
endpoint. (#22898)
add 73a87f8885 Add concept doc for Dynamic Task Mapping (#22867)
add f662b7de8c Catch error in Breeze when docker is not running (#22901)
add 2a2841bf6f Merge branch 'main' into mapped-task-action-ux
No new revisions were added by this update.
Summary of changes:
.github/workflows/build-images.yml | 30 +-
.github/workflows/ci.yml | 22 +-
.gitignore | 4 +-
.gitpod.yml | 2 +-
.pre-commit-config.yaml | 26 +-
BREEZE.rst | 2111 ++---------
Breeze2 | 42 -
CI.rst | 10 +-
CONTRIBUTING.rst | 22 +-
CONTRIBUTORS_QUICK_START.rst | 38 +-
Dockerfile | 8 +-
IMAGES.rst | 28 +-
INTHEWILD.md | 1 -
LOCAL_VIRTUALENV.rst | 4 +-
STATIC_CODE_CHECKS.rst | 24 +-
TESTING.rst | 90 +-
airflow/__init__.py | 6 -
airflow/cli/cli_parser.py | 5 +-
airflow/cli/commands/celery_command.py | 2 +-
airflow/config_templates/config.yml | 9 +
airflow/config_templates/default_airflow.cfg | 5 +
airflow/configuration.py | 42 +-
airflow/logging_config.py | 20 -
airflow/migrations/db_types.py | 8 -
airflow/models/abstractoperator.py | 6 +-
airflow/models/baseoperator.py | 9 +-
airflow/models/mappedoperator.py | 39 +-
airflow/models/taskinstance.py | 92 +-
airflow/operators/bash.py | 2 +-
airflow/operators/dummy.py | 2 +-
airflow/operators/python.py | 2 +-
.../amazon/aws/transfers/redshift_to_s3.py | 2 -
.../amazon/aws/transfers/s3_to_redshift.py | 1 -
airflow/providers/asana/hooks/asana.py | 4 +-
airflow/providers/cloudant/hooks/cloudant.py | 2 +-
.../cncf/kubernetes/operators/kubernetes_pod.py | 2 +-
.../providers/cncf/kubernetes/utils/__init__.py | 1 +
airflow/providers/databricks/hooks/databricks.py | 20 +-
.../providers/databricks/hooks/databricks_base.py | 14 +-
.../providers/databricks/hooks/databricks_sql.py | 4 +-
airflow/providers/databricks/provider.yaml | 3 -
airflow/providers/datadog/hooks/datadog.py | 2 +-
.../dbt/cloud/example_dags/example_dbt_cloud.py | 6 +-
airflow/providers/docker/hooks/docker.py | 6 +-
airflow/providers/docker/operators/docker.py | 8 +-
airflow/providers/docker/operators/docker_swarm.py | 7 +-
.../providers/elasticsearch/log/es_task_handler.py | 2 +-
.../github/example_dags/example_github.py | 2 +-
airflow/providers/github/sensors/github.py | 2 +-
.../google/cloud/example_dags/example_vision.py | 2 +
airflow/providers/google/cloud/hooks/automl.py | 29 +-
.../providers/google/cloud/hooks/bigquery_dts.py | 9 +-
.../providers/google/cloud/hooks/cloud_build.py | 23 +-
.../providers/google/cloud/hooks/cloud_composer.py | 16 +-
.../google/cloud/hooks/cloud_memorystore.py | 31 +-
.../providers/google/cloud/hooks/datacatalog.py | 45 +-
airflow/providers/google/cloud/hooks/dataplex.py | 12 +-
airflow/providers/google/cloud/hooks/dataproc.py | 44 +-
.../google/cloud/hooks/dataproc_metastore.py | 26 +-
airflow/providers/google/cloud/hooks/dlp.py | 61 +-
airflow/providers/google/cloud/hooks/kms.py | 5 +-
.../google/cloud/hooks/kubernetes_engine.py | 14 +-
.../google/cloud/hooks/natural_language.py | 13 +-
airflow/providers/google/cloud/hooks/os_login.py | 3 +-
airflow/providers/google/cloud/hooks/pubsub.py | 13 +-
.../providers/google/cloud/hooks/speech_to_text.py | 3 +-
.../providers/google/cloud/hooks/stackdriver.py | 27 +-
airflow/providers/google/cloud/hooks/tasks.py | 27 +-
.../providers/google/cloud/hooks/text_to_speech.py | 3 +-
.../google/cloud/hooks/vertex_ai/auto_ml.py | 18 +-
.../cloud/hooks/vertex_ai/batch_prediction_job.py | 13 +-
.../google/cloud/hooks/vertex_ai/custom_job.py | 43 +-
.../google/cloud/hooks/vertex_ai/dataset.py | 27 +-
.../cloud/hooks/vertex_ai/endpoint_service.py | 21 +-
.../hooks/vertex_ai/hyperparameter_tuning_job.py | 13 +-
.../google/cloud/hooks/vertex_ai/model_service.py | 15 +-
.../google/cloud/hooks/video_intelligence.py | 3 +-
airflow/providers/google/cloud/hooks/vision.py | 37 +-
airflow/providers/google/cloud/hooks/workflows.py | 19 +-
airflow/providers/google/cloud/operators/automl.py | 27 +-
.../google/cloud/operators/bigquery_dts.py | 7 +-
.../google/cloud/operators/cloud_build.py | 23 +-
.../google/cloud/operators/cloud_composer.py | 13 +-
.../google/cloud/operators/cloud_memorystore.py | 37 +-
.../google/cloud/operators/datacatalog.py | 43 +-
.../providers/google/cloud/operators/dataplex.py | 9 +-
.../providers/google/cloud/operators/dataproc.py | 29 +-
.../google/cloud/operators/dataproc_metastore.py | 21 +-
airflow/providers/google/cloud/operators/dlp.py | 61 +-
.../google/cloud/operators/natural_language.py | 9 +-
airflow/providers/google/cloud/operators/pubsub.py | 9 +-
.../google/cloud/operators/speech_to_text.py | 3 +-
.../google/cloud/operators/stackdriver.py | 23 +-
airflow/providers/google/cloud/operators/tasks.py | 27 +-
.../google/cloud/operators/text_to_speech.py | 3 +-
.../google/cloud/operators/vertex_ai/auto_ml.py | 5 +-
.../operators/vertex_ai/batch_prediction_job.py | 7 +-
.../google/cloud/operators/vertex_ai/custom_job.py | 5 +-
.../google/cloud/operators/vertex_ai/dataset.py | 15 +-
.../cloud/operators/vertex_ai/endpoint_service.py | 15 +-
.../vertex_ai/hyperparameter_tuning_job.py | 7 +-
.../cloud/operators/vertex_ai/model_service.py | 9 +-
.../google/cloud/operators/video_intelligence.py | 7 +-
airflow/providers/google/cloud/operators/vision.py | 35 +-
.../providers/google/cloud/operators/workflows.py | 19 +-
.../providers/google/cloud/sensors/bigquery_dts.py | 3 +-
airflow/providers/google/cloud/sensors/dataplex.py | 3 +-
.../providers/google/cloud/sensors/workflows.py | 3 +-
.../google/cloud/transfers/mssql_to_gcs.py | 5 +-
.../azure/example_dags/example_adf_run_pipeline.py | 8 +-
airflow/providers/trino/hooks/trino.py | 4 +-
airflow/utils/cli.py | 2 +-
airflow/utils/helpers.py | 2 +-
airflow/utils/log/log_reader.py | 2 +-
airflow/utils/yaml.py | 7 -
airflow/www/views.py | 11 +-
breeze | 3698 +-------------------
breeze-complete | 7 +-
breeze => breeze-legacy | 0
dev/PROVIDER_PACKAGE_DETAILS.md | 18 +-
dev/README_RELEASE_AIRFLOW.md | 8 +-
dev/README_RELEASE_HELM_CHART.md | 2 +-
dev/README_RELEASE_PROVIDER_PACKAGES.md | 23 +-
dev/REFRESHING_CI_CACHE.md | 2 +-
dev/TRACKING_BACKTRACKING_ISSUES.md | 7 +-
dev/breeze/README.md | 2 +-
...e2-complete-bash.sh => breeze-complete-bash.sh} | 10 +-
...e2-complete-fish.sh => breeze-complete-fish.sh} | 6 +-
...eze2-complete-zsh.sh => breeze-complete-zsh.sh} | 10 +-
dev/breeze/doc/BREEZE2.md | 141 -
.../0002-implement-standalone-python-command.md | 6 +-
.../adr/0003-bootstrapping-virtual-environment.md | 28 +-
dev/breeze/setup.cfg | 2 +-
dev/breeze/src/airflow_breeze/__init__.py | 4 +-
dev/breeze/src/airflow_breeze/breeze.py | 395 ++-
.../build_image/ci/build_ci_image.py | 25 +-
.../build_image/ci/build_ci_params.py | 7 +-
.../build_image/prod/build_prod_image.py | 31 +-
.../build_image/prod/build_prod_params.py | 12 +-
dev/breeze/src/airflow_breeze/global_constants.py | 4 +-
dev/breeze/src/airflow_breeze/pre_commit_ids.py | 1 +
dev/breeze/src/airflow_breeze/shell/enter_shell.py | 14 +-
.../src/airflow_breeze/shell/shell_params.py | 4 +-
dev/breeze/src/airflow_breeze/utils/cache.py | 1 -
dev/breeze/src/airflow_breeze/utils/confirm.py | 29 +-
dev/breeze/src/airflow_breeze/utils/console.py | 13 +-
.../airflow_breeze/utils/docker_command_utils.py | 91 +-
dev/breeze/src/airflow_breeze/utils/path_utils.py | 1 -
dev/breeze/src/airflow_breeze/utils/registry.py | 11 +-
dev/breeze/src/airflow_breeze/utils/run_utils.py | 6 +-
dev/breeze/src/airflow_breeze/utils/visuals.py | 14 +-
dev/provider_packages/README.md | 2 +-
dev/refresh_images.sh | 6 +-
docs/apache-airflow-providers-databricks/index.rst | 2 +-
.../concepts/dynamic-task-mapping.rst | 261 ++
docs/apache-airflow/concepts/index.rst | 1 +
.../howto/dynamic-dag-generation.rst | 2 +
docs/apache-airflow/img/mapping-simple-graph.png | Bin 0 -> 7676 bytes
docs/apache-airflow/img/mapping-simple-grid.png | Bin 0 -> 179670 bytes
docs/build_docs.py | 6 +-
images/breeze/output-breeze-commands.svg | 136 +
images/breeze/output-build-docs.svg | 168 +
images/breeze/output-build-image.svg | 167 +
images/breeze/output-build-prod-image.svg | 180 +
images/breeze/output-cleanup.svg | 136 +
images/breeze/output-commands.svg | 165 +
images/breeze/output-config.svg | 138 +
images/breeze/output-self-upgrade.svg | 133 +
images/breeze/output-setup-autocomplete.svg | 138 +
images/breeze/output-shell.svg | 151 +
images/breeze/output-start-airflow.svg | 153 +
images/breeze/output-static-checks.svg | 158 +
images/breeze/output-stop.svg | 136 +
images/breeze/output-version.svg | 132 +
scripts/ci/images/ci_push_ci_images.sh | 33 -
scripts/ci/images/ci_push_empty_ci_images.sh | 37 -
scripts/ci/images/ci_push_empty_prod_images.sh | 38 -
scripts/ci/libraries/_build_images.sh | 4 +-
.../ci/pre_commit/pre_commit_breeze_cmd_line.py | 82 +
.../ci/pre_commit/pre_commit_breeze_cmd_line.sh | 82 -
scripts/docker/entrypoint_prod.sh | 8 +-
scripts/tools/check_if_limited_dependencies.py | 52 +
scripts/tools/confirm | 14 +-
setup.cfg | 1 -
setup.py | 4 +-
.../endpoints/test_task_instance_endpoint.py | 20 +
tests/cli/commands/test_triggerer_command.py | 4 -
tests/core/test_configuration.py | 65 +
tests/jobs/test_triggerer_job.py | 12 -
tests/models/test_taskinstance.py | 41 +-
.../databricks/operators/test_databricks_sql.py | 8 +-
tests/providers/google/cloud/hooks/test_automl.py | 27 +-
.../google/cloud/hooks/test_bigquery_dts.py | 13 +-
.../google/cloud/hooks/test_cloud_build.py | 42 +-
.../google/cloud/hooks/test_cloud_composer.py | 4 +-
.../google/cloud/hooks/test_cloud_memorystore.py | 35 +-
.../google/cloud/hooks/test_datacatalog.py | 5 +-
.../providers/google/cloud/hooks/test_dataplex.py | 10 +-
.../providers/google/cloud/hooks/test_dataproc.py | 68 +-
.../google/cloud/hooks/test_dataproc_metastore.py | 44 +-
tests/providers/google/cloud/hooks/test_dlp.py | 93 +-
tests/providers/google/cloud/hooks/test_gcs.py | 19 +-
tests/providers/google/cloud/hooks/test_kms.py | 10 +-
.../google/cloud/hooks/test_natural_language.py | 13 +-
.../providers/google/cloud/hooks/test_os_login.py | 5 +-
tests/providers/google/cloud/hooks/test_pubsub.py | 29 +-
.../google/cloud/hooks/test_speech_to_text.py | 4 +-
tests/providers/google/cloud/hooks/test_tasks.py | 27 +-
.../google/cloud/hooks/test_text_to_speech.py | 4 +-
.../google/cloud/hooks/test_video_intelligence.py | 5 +-
tests/providers/google/cloud/hooks/test_vision.py | 97 +-
.../google/cloud/hooks/vertex_ai/test_auto_ml.py | 14 +-
.../hooks/vertex_ai/test_batch_prediction_job.py | 14 +-
.../cloud/hooks/vertex_ai/test_custom_job.py | 42 +-
.../google/cloud/hooks/vertex_ai/test_dataset.py | 42 +-
.../cloud/hooks/vertex_ai/test_endpoint_service.py | 30 +-
.../vertex_ai/test_hyperparameter_tuning_job.py | 14 +-
.../cloud/hooks/vertex_ai/test_model_service.py | 18 +-
.../google/cloud/operators/test_automl.py | 27 +-
.../google/cloud/operators/test_bigquery_dts.py | 7 +-
.../google/cloud/operators/test_cloud_build.py | 25 +-
.../google/cloud/operators/test_cloud_composer.py | 3 +-
.../cloud/operators/test_cloud_memorystore.py | 17 +-
.../google/cloud/operators/test_datacatalog.py | 15 +-
.../google/cloud/operators/test_dataplex.py | 10 +-
.../cloud/operators/test_dataproc_metastore.py | 5 +-
tests/providers/google/cloud/operators/test_dlp.py | 62 +-
.../google/cloud/operators/test_pubsub.py | 15 +-
.../google/cloud/operators/test_speech_to_text.py | 3 +-
.../providers/google/cloud/operators/test_tasks.py | 27 +-
.../google/cloud/operators/test_text_to_speech.py | 3 +-
.../google/cloud/operators/test_vertex_ai.py | 5 +-
.../cloud/operators/test_video_intelligence.py | 7 +-
.../google/cloud/operators/test_vision.py | 47 +-
.../google/cloud/sensors/test_bigquery_dts.py | 5 +-
.../google/cloud/sensors/test_dataplex.py | 5 +-
.../google/cloud/transfers/test_mssql_to_gcs.py | 25 +
tests/triggers/test_temporal.py | 2 -
tests/www/views/test_views_log.py | 20 +
239 files changed, 5124 insertions(+), 7380 deletions(-)
delete mode 100755 Breeze2
copy breeze => breeze-legacy (100%)
rename dev/breeze/autocomplete/{Breeze2-complete-bash.sh =>
breeze-complete-bash.sh} (73%)
rename dev/breeze/autocomplete/{Breeze2-complete-fish.sh =>
breeze-complete-fish.sh} (66%)
rename dev/breeze/autocomplete/{Breeze2-complete-zsh.sh =>
breeze-complete-zsh.sh} (81%)
delete mode 100644 dev/breeze/doc/BREEZE2.md
create mode 100644 docs/apache-airflow/concepts/dynamic-task-mapping.rst
create mode 100644 docs/apache-airflow/img/mapping-simple-graph.png
create mode 100644 docs/apache-airflow/img/mapping-simple-grid.png
create mode 100644 images/breeze/output-breeze-commands.svg
create mode 100644 images/breeze/output-build-docs.svg
create mode 100644 images/breeze/output-build-image.svg
create mode 100644 images/breeze/output-build-prod-image.svg
create mode 100644 images/breeze/output-cleanup.svg
create mode 100644 images/breeze/output-commands.svg
create mode 100644 images/breeze/output-config.svg
create mode 100644 images/breeze/output-self-upgrade.svg
create mode 100644 images/breeze/output-setup-autocomplete.svg
create mode 100644 images/breeze/output-shell.svg
create mode 100644 images/breeze/output-start-airflow.svg
create mode 100644 images/breeze/output-static-checks.svg
create mode 100644 images/breeze/output-stop.svg
create mode 100644 images/breeze/output-version.svg
delete mode 100755 scripts/ci/images/ci_push_ci_images.sh
delete mode 100755 scripts/ci/images/ci_push_empty_ci_images.sh
delete mode 100755 scripts/ci/images/ci_push_empty_prod_images.sh
create mode 100755 scripts/ci/pre_commit/pre_commit_breeze_cmd_line.py
delete mode 100755 scripts/ci/pre_commit/pre_commit_breeze_cmd_line.sh
create mode 100755 scripts/tools/check_if_limited_dependencies.py
