feng-tao closed pull request #4340: [AIRFLOW-3540] Respect environment config
when looking up config file.
URL: https://github.com/apache/incubator-airflow/pull/4340
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/UPDATING.md b/UPDATING.md
index 986d3a23c1..575fc0a3c5 100644
--- a/UPDATING.md
+++ b/UPDATING.md
@@ -24,6 +24,14 @@ assists users migrating to a new version.
## Airflow Master
+### Modification to config file discovery
+
+If the `AIRFLOW_CONFIG` environment variable was not set and the
+`~/airflow/airflow.cfg` file existed, airflow previously used
+`~/airflow/airflow.cfg` instead of `$AIRFLOW_HOME/airflow.cfg`. Now airflow
+will discover its config file using the `$AIRFLOW_CONFIG` and `$AIRFLOW_HOME`
+environment variables rather than checking for the presence of a file.
+
### Modification to `ts_nodash` macro
`ts_nodash` previously contained TimeZone information alongwith execution
date. For Example: `20150101T000000+0000`. This is not user-friendly for file
or folder names which was a popular use case for `ts_nodash`. Hence this
behavior has been changed and using `ts_nodash` will no longer contain TimeZone
information, restoring the pre-1.10 behavior of this macro. And a new macro
`ts_nodash_with_tz` has been added which can be used to get a string with
execution date and timezone info without dashes.
diff --git a/airflow/configuration.py b/airflow/configuration.py
index 3662df8d06..332c069a3a 100644
--- a/airflow/configuration.py
+++ b/airflow/configuration.py
@@ -441,23 +441,23 @@ def mkdir_p(path):
'Error creating {}: {}'.format(path, exc.strerror))
-# Setting AIRFLOW_HOME and AIRFLOW_CONFIG from environment variables, using
-# "~/airflow" and "~/airflow/airflow.cfg" respectively as defaults.
+def get_airflow_home():
+ return expand_env_var(os.environ.get('AIRFLOW_HOME', '~/airflow'))
+
+
+def get_airflow_config(airflow_home):
+ if 'AIRFLOW_CONFIG' not in os.environ:
+ return os.path.join(airflow_home, 'airflow.cfg')
+ return expand_env_var(os.environ['AIRFLOW_CONFIG'])
-if 'AIRFLOW_HOME' not in os.environ:
- AIRFLOW_HOME = expand_env_var('~/airflow')
-else:
- AIRFLOW_HOME = expand_env_var(os.environ['AIRFLOW_HOME'])
+# Setting AIRFLOW_HOME and AIRFLOW_CONFIG from environment variables, using
+# "~/airflow" and "$AIRFLOW_HOME/airflow.cfg" respectively as defaults.
+
+AIRFLOW_HOME = get_airflow_home()
+AIRFLOW_CONFIG = get_airflow_config(AIRFLOW_HOME)
mkdir_p(AIRFLOW_HOME)
-if 'AIRFLOW_CONFIG' not in os.environ:
- if os.path.isfile(expand_env_var('~/airflow.cfg')):
- AIRFLOW_CONFIG = expand_env_var('~/airflow.cfg')
- else:
- AIRFLOW_CONFIG = AIRFLOW_HOME + '/airflow.cfg'
-else:
- AIRFLOW_CONFIG = expand_env_var(os.environ['AIRFLOW_CONFIG'])
# Set up dags folder for unit tests
# this directory won't exist if users install via pip
diff --git a/tests/test_configuration.py b/tests/test_configuration.py
index acebd5732c..9f903f58b3 100644
--- a/tests/test_configuration.py
+++ b/tests/test_configuration.py
@@ -21,6 +21,7 @@
from __future__ import unicode_literals
import os
+import contextlib
from collections import OrderedDict
import six
@@ -35,6 +36,23 @@
import unittest
[email protected]
+def env_vars(**vars):
+ original = {}
+ for key, value in vars.items():
+ original[key] = os.environ.get(key)
+ if value is not None:
+ os.environ[key] = value
+ else:
+ os.environ.pop(key, None)
+ yield
+ for key, value in original.items():
+ if value is not None:
+ os.environ[key] = value
+ else:
+ os.environ.pop(key, None)
+
+
class ConfTest(unittest.TestCase):
@classmethod
@@ -49,6 +67,30 @@ def tearDownClass(cls):
del os.environ['AIRFLOW__TESTSECTION__TESTKEY']
del os.environ['AIRFLOW__TESTSECTION__TESTPERCENT']
+ def test_airflow_home_default(self):
+ with env_vars(AIRFLOW_HOME=None):
+ self.assertEqual(
+ configuration.get_airflow_home(),
+ configuration.expand_env_var('~/airflow'))
+
+ def test_airflow_home_override(self):
+ with env_vars(AIRFLOW_HOME='/path/to/airflow'):
+ self.assertEqual(
+ configuration.get_airflow_home(),
+ '/path/to/airflow')
+
+ def test_airflow_config_default(self):
+ with env_vars(AIRFLOW_CONFIG=None):
+ self.assertEqual(
+ configuration.get_airflow_config('/home/airflow'),
+ configuration.expand_env_var('/home/airflow/airflow.cfg'))
+
+ def test_airflow_config_override(self):
+ with env_vars(AIRFLOW_CONFIG='/path/to/airflow/airflow.cfg'):
+ self.assertEqual(
+ configuration.get_airflow_config('/home//airflow'),
+ '/path/to/airflow/airflow.cfg')
+
def test_env_var_config(self):
opt = conf.get('testsection', 'testkey')
self.assertEqual(opt, 'testvalue')
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
