Public bug reported: It happened at least twice recently that functional tests failed due to missing qrouter- namespace. Examples:
https://zuul.opendev.org/t/openstack/build/f8510ec1f1ca4329a5bb28fb1a38614c https://zuul.openstack.org/build/31bd6bca81f34c6c9af385daf6f64ee8 Stacktrace: 2022-03-02 11:34:08.097415 | controller | Captured traceback: 2022-03-02 11:34:08.097427 | controller | ~~~~~~~~~~~~~~~~~~~ 2022-03-02 11:34:08.097439 | controller | Traceback (most recent call last): 2022-03-02 11:34:08.097451 | controller | 2022-03-02 11:34:08.097462 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/common/utils.py", line 718, in wait_until_true 2022-03-02 11:34:08.097475 | controller | eventlet.sleep(sleep) 2022-03-02 11:34:08.097486 | controller | 2022-03-02 11:34:08.097507 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/eventlet/greenthread.py", line 36, in sleep 2022-03-02 11:34:08.097520 | controller | hub.switch() 2022-03-02 11:34:08.097531 | controller | 2022-03-02 11:34:08.097542 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/eventlet/hubs/hub.py", line 313, in switch 2022-03-02 11:34:08.097554 | controller | return self.greenlet.switch() 2022-03-02 11:34:08.097565 | controller | 2022-03-02 11:34:08.097575 | controller | eventlet.timeout.Timeout: 60 seconds 2022-03-02 11:34:08.097586 | controller | 2022-03-02 11:34:08.097597 | controller | 2022-03-02 11:34:08.097607 | controller | During handling of the above exception, another exception occurred: 2022-03-02 11:34:08.097619 | controller | 2022-03-02 11:34:08.097629 | controller | 2022-03-02 11:34:08.097640 | controller | Traceback (most recent call last): 2022-03-02 11:34:08.097650 | controller | 2022-03-02 11:34:08.097661 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/base.py", line 183, in func 2022-03-02 11:34:08.097672 | controller | return f(self, *args, **kwargs) 2022-03-02 11:34:08.097683 | controller | 2022-03-02 11:34:08.097693 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/base.py", line 183, in func 2022-03-02 11:34:08.097704 | controller | return f(self, *args, **kwargs) 2022-03-02 11:34:08.097714 | controller | 2022-03-02 11:34:08.097725 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/functional/agent/l3/test_ha_router.py", line 88, in test_ha_router_lifecycle 2022-03-02 11:34:08.097736 | controller | router_info = self._router_lifecycle(enable_ha=True) 2022-03-02 11:34:08.097747 | controller | 2022-03-02 11:34:08.097758 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/functional/agent/l3/framework.py", line 336, in _router_lifecycle 2022-03-02 11:34:08.097769 | controller | common_utils.wait_until_true(lambda: router.ha_state == 'primary') 2022-03-02 11:34:08.097780 | controller | 2022-03-02 11:34:08.097791 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/common/utils.py", line 723, in wait_until_true 2022-03-02 11:34:08.097807 | controller | raise WaitTimeout(_("Timed out after %d seconds") % timeout) 2022-03-02 11:34:08.097819 | controller | 2022-03-02 11:34:08.097830 | controller | neutron.common.utils.WaitTimeout: Timed out after 60 seconds 2022-03-02 11:34:08.097840 | controller | 2022-03-02 11:34:08.097851 | controller | 2022-03-02 11:34:08.097861 | controller | Captured stderr: 2022-03-02 11:34:08.097872 | controller | ~~~~~~~~~~~~~~~~ 2022-03-02 11:34:08.097882 | controller | Traceback (most recent call last): 2022-03-02 11:34:08.097899 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/eventlet/hubs/hub.py", line 476, in fire_timers 2022-03-02 11:34:08.097911 | controller | timer() 2022-03-02 11:34:08.097923 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/eventlet/hubs/timer.py", line 59, in __call__ 2022-03-02 11:34:08.097934 | controller | cb(*args, **kw) 2022-03-02 11:34:08.097945 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/eventlet/greenthread.py", line 221, in main 2022-03-02 11:34:08.097956 | controller | result = function(*args, **kwargs) 2022-03-02 11:34:08.097967 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/oslo_concurrency/lockutils.py", line 391, in inner 2022-03-02 11:34:08.097979 | controller | return f(*args, **kwargs) 2022-03-02 11:34:08.097992 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/agent/linux/external_process.py", line 254, in _check_child_processes 2022-03-02 11:34:08.098004 | controller | self._execute_action(service_id) 2022-03-02 11:34:08.098015 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/agent/linux/external_process.py", line 265, in _execute_action 2022-03-02 11:34:08.098026 | controller | action_function(service_id) 2022-03-02 11:34:08.098037 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/agent/linux/external_process.py", line 271, in _respawn_action 2022-03-02 11:34:08.098048 | controller | self._monitored_processes[service_id].enable() 2022-03-02 11:34:08.098068 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/agent/linux/external_process.py", line 88, in enable 2022-03-02 11:34:08.098081 | controller | ip_wrapper.netns.execute(cmd, addl_env=self.cmd_addl_env, 2022-03-02 11:34:08.098092 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/agent/linux/ip_lib.py", line 741, in execute 2022-03-02 11:34:08.098104 | controller | return utils.execute(cmd, check_exit_code=check_exit_code, 2022-03-02 11:34:08.098115 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/agent/linux/utils.py", line 156, in execute 2022-03-02 11:34:08.098126 | controller | raise exceptions.ProcessExecutionError(msg, 2022-03-02 11:34:08.098138 | controller | neutron_lib.exceptions.ProcessExecutionError: Exit code: 255; Cmd: ['ip', 'netns', 'exec', 'qrouter-c150c299-8c24-4f21-97a1-8b8021c87e10', 'haproxy', '-f', '/tmp/tmpsv705_is/tmpacwjfd97/ns-metadata-proxy/c150c299-8c24-4f21-97a1-8b8021c87e10.conf']; Stdin: ; Stdout: ; Stderr: Cannot open network namespace "qrouter-c150c299-8c24-4f21-97a1-8b8021c87e10": No such file or directory ** Affects: neutron Importance: Critical Status: Confirmed ** Tags: functional-tests gate-failure -- You received this bug notification because you are a member of Yahoo! Engineering Team, which is subscribed to neutron. https://bugs.launchpad.net/bugs/1964313 Title: Functional tests fails due to missing namespace Status in neutron: Confirmed Bug description: It happened at least twice recently that functional tests failed due to missing qrouter- namespace. Examples: https://zuul.opendev.org/t/openstack/build/f8510ec1f1ca4329a5bb28fb1a38614c https://zuul.openstack.org/build/31bd6bca81f34c6c9af385daf6f64ee8 Stacktrace: 2022-03-02 11:34:08.097415 | controller | Captured traceback: 2022-03-02 11:34:08.097427 | controller | ~~~~~~~~~~~~~~~~~~~ 2022-03-02 11:34:08.097439 | controller | Traceback (most recent call last): 2022-03-02 11:34:08.097451 | controller | 2022-03-02 11:34:08.097462 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/common/utils.py", line 718, in wait_until_true 2022-03-02 11:34:08.097475 | controller | eventlet.sleep(sleep) 2022-03-02 11:34:08.097486 | controller | 2022-03-02 11:34:08.097507 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/eventlet/greenthread.py", line 36, in sleep 2022-03-02 11:34:08.097520 | controller | hub.switch() 2022-03-02 11:34:08.097531 | controller | 2022-03-02 11:34:08.097542 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/eventlet/hubs/hub.py", line 313, in switch 2022-03-02 11:34:08.097554 | controller | return self.greenlet.switch() 2022-03-02 11:34:08.097565 | controller | 2022-03-02 11:34:08.097575 | controller | eventlet.timeout.Timeout: 60 seconds 2022-03-02 11:34:08.097586 | controller | 2022-03-02 11:34:08.097597 | controller | 2022-03-02 11:34:08.097607 | controller | During handling of the above exception, another exception occurred: 2022-03-02 11:34:08.097619 | controller | 2022-03-02 11:34:08.097629 | controller | 2022-03-02 11:34:08.097640 | controller | Traceback (most recent call last): 2022-03-02 11:34:08.097650 | controller | 2022-03-02 11:34:08.097661 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/base.py", line 183, in func 2022-03-02 11:34:08.097672 | controller | return f(self, *args, **kwargs) 2022-03-02 11:34:08.097683 | controller | 2022-03-02 11:34:08.097693 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/base.py", line 183, in func 2022-03-02 11:34:08.097704 | controller | return f(self, *args, **kwargs) 2022-03-02 11:34:08.097714 | controller | 2022-03-02 11:34:08.097725 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/functional/agent/l3/test_ha_router.py", line 88, in test_ha_router_lifecycle 2022-03-02 11:34:08.097736 | controller | router_info = self._router_lifecycle(enable_ha=True) 2022-03-02 11:34:08.097747 | controller | 2022-03-02 11:34:08.097758 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/functional/agent/l3/framework.py", line 336, in _router_lifecycle 2022-03-02 11:34:08.097769 | controller | common_utils.wait_until_true(lambda: router.ha_state == 'primary') 2022-03-02 11:34:08.097780 | controller | 2022-03-02 11:34:08.097791 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/common/utils.py", line 723, in wait_until_true 2022-03-02 11:34:08.097807 | controller | raise WaitTimeout(_("Timed out after %d seconds") % timeout) 2022-03-02 11:34:08.097819 | controller | 2022-03-02 11:34:08.097830 | controller | neutron.common.utils.WaitTimeout: Timed out after 60 seconds 2022-03-02 11:34:08.097840 | controller | 2022-03-02 11:34:08.097851 | controller | 2022-03-02 11:34:08.097861 | controller | Captured stderr: 2022-03-02 11:34:08.097872 | controller | ~~~~~~~~~~~~~~~~ 2022-03-02 11:34:08.097882 | controller | Traceback (most recent call last): 2022-03-02 11:34:08.097899 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/eventlet/hubs/hub.py", line 476, in fire_timers 2022-03-02 11:34:08.097911 | controller | timer() 2022-03-02 11:34:08.097923 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/eventlet/hubs/timer.py", line 59, in __call__ 2022-03-02 11:34:08.097934 | controller | cb(*args, **kw) 2022-03-02 11:34:08.097945 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/eventlet/greenthread.py", line 221, in main 2022-03-02 11:34:08.097956 | controller | result = function(*args, **kwargs) 2022-03-02 11:34:08.097967 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/.tox/dsvm-functional/lib/python3.8/site-packages/oslo_concurrency/lockutils.py", line 391, in inner 2022-03-02 11:34:08.097979 | controller | return f(*args, **kwargs) 2022-03-02 11:34:08.097992 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/agent/linux/external_process.py", line 254, in _check_child_processes 2022-03-02 11:34:08.098004 | controller | self._execute_action(service_id) 2022-03-02 11:34:08.098015 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/agent/linux/external_process.py", line 265, in _execute_action 2022-03-02 11:34:08.098026 | controller | action_function(service_id) 2022-03-02 11:34:08.098037 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/agent/linux/external_process.py", line 271, in _respawn_action 2022-03-02 11:34:08.098048 | controller | self._monitored_processes[service_id].enable() 2022-03-02 11:34:08.098068 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/agent/linux/external_process.py", line 88, in enable 2022-03-02 11:34:08.098081 | controller | ip_wrapper.netns.execute(cmd, addl_env=self.cmd_addl_env, 2022-03-02 11:34:08.098092 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/agent/linux/ip_lib.py", line 741, in execute 2022-03-02 11:34:08.098104 | controller | return utils.execute(cmd, check_exit_code=check_exit_code, 2022-03-02 11:34:08.098115 | controller | File "/home/zuul/src/opendev.org/openstack/neutron/neutron/agent/linux/utils.py", line 156, in execute 2022-03-02 11:34:08.098126 | controller | raise exceptions.ProcessExecutionError(msg, 2022-03-02 11:34:08.098138 | controller | neutron_lib.exceptions.ProcessExecutionError: Exit code: 255; Cmd: ['ip', 'netns', 'exec', 'qrouter-c150c299-8c24-4f21-97a1-8b8021c87e10', 'haproxy', '-f', '/tmp/tmpsv705_is/tmpacwjfd97/ns-metadata-proxy/c150c299-8c24-4f21-97a1-8b8021c87e10.conf']; Stdin: ; Stdout: ; Stderr: Cannot open network namespace "qrouter-c150c299-8c24-4f21-97a1-8b8021c87e10": No such file or directory To manage notifications about this bug go to: https://bugs.launchpad.net/neutron/+bug/1964313/+subscriptions -- Mailing list: https://launchpad.net/~yahoo-eng-team Post to : yahoo-eng-team@lists.launchpad.net Unsubscribe : https://launchpad.net/~yahoo-eng-team More help : https://help.launchpad.net/ListHelp
