jason810496 commented on code in PR #62696:
URL: https://github.com/apache/airflow/pull/62696#discussion_r2894577658
##########
shared/configuration/src/airflow_shared/configuration/parser.py:
##########
@@ -213,8 +248,69 @@ def _lookup_sequence(self) -> list[Callable]:
self._get_option_from_commands,
self._get_option_from_secrets,
self._get_option_from_defaults,
+ self._get_option_from_provider_cfg_config_fallbacks,
+ self._get_option_from_provider_metadata_config_fallbacks,
]
+ def _get_config_sources_for_as_dict(self) -> list[tuple[str,
ConfigParser]]:
+ """Override the base method to add provider fallbacks."""
+ return [
+ ("provider-cfg-fallback-defaults",
self._provider_cfg_config_fallback_default_values),
+ ("provider-metadata-fallback-defaults",
self._provider_metadata_config_fallback_default_values),
+ ("default", self._default_values),
+ ("airflow.cfg", self),
+ ]
+
+ def _get_option_from_provider_cfg_config_fallbacks(
+ self,
+ deprecated_key: str | None,
+ deprecated_section: str | None,
+ key: str,
+ section: str,
+ issue_warning: bool = True,
+ extra_stacklevel: int = 0,
+ **kwargs,
+ ) -> str | ValueNotFound:
+ """Get config option from provider fallback defaults."""
+ if self.get_from_provider_cfg_config_fallback_defaults(section, key)
is not None:
+ # no expansion needed
+ return
self.get_from_provider_cfg_config_fallback_defaults(section, key, **kwargs)
+ return VALUE_NOT_FOUND_SENTINEL
+
+ def _get_option_from_provider_metadata_config_fallbacks(
+ self,
+ deprecated_key: str | None,
+ deprecated_section: str | None,
+ key: str,
+ section: str,
+ issue_warning: bool = True,
+ extra_stacklevel: int = 0,
+ **kwargs,
+ ) -> str | ValueNotFound:
+ """Get config option from provider metadata fallback defaults."""
+ if self.get_from_provider_metadata_config_fallback_defaults(section,
key) is not None:
+ # no expansion needed
+ return
self.get_from_provider_metadata_config_fallback_defaults(section, key, **kwargs)
+ return VALUE_NOT_FOUND_SENTINEL
+
+ def get_from_provider_cfg_config_fallback_defaults(self, section: str,
key: str, **kwargs) -> Any:
+ """Get provider config fallback default values."""
+ return self._provider_cfg_config_fallback_default_values.get(section,
key, fallback=None, **kwargs)
+
+ @cached_property
+ def _provider_metadata_config_fallback_default_values(self) ->
ConfigParser:
+ """Return Provider metadata config fallback default values."""
+ base_configuration_description: dict[str, dict[str, Any]] = {}
+ for _, config in self.provider_manager_type().provider_configs:
+ base_configuration_description.update(config)
Review Comment:
This behavior is intentional for the SDK, since it is an essential part of
resolving the blocker in #60000. Without initializing
`ProvidersManagerTaskRuntime`, we couldn’t honor the default configuration
values from the provider metadata.
The `ProvidersManagerTaskRuntime` is initialized only when necessary (if a
provider operator really needs `conf.get` to obtain default configuration
values from providers), so I intentionally chose to use `cached_property` to
avoid initializing `ProvidersManagerTaskRuntime` when constructing
`AirflowConfigParser`, using lazy initialization.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
