Public bug reported:
These tests were added as part of
https://opendev.org/openstack/neutron/commit/f9a54350e6b77dcb03a6e1eeedb547f276813c99
We seeing random failures in these tests, this bug to track these
failures.
Example failures:-
- test_ha_router_failover_graceful
-
https://7f1261c1f7c867223017-371138c00916065f498de90723b300c8.ssl.cf5.rackcdn.com/periodic/opendev.org/openstack/neutron/master/neutron-fullstack-fips/af92329/testr_results.html
ft1.7:
neutron.tests.fullstack.test_l3_agent.TestHAL3Agent.test_ha_router_failover_gracefultesttools.testresult.real._StringException:
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/base.py",
line 178, in func
return f(self, *args, **kwargs)
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/fullstack/test_l3_agent.py",
line 579, in test_ha_router_failover_graceful
self._test_ha_router_failover('shutdown')
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/fullstack/test_l3_agent.py",
line 541, in _test_ha_router_failover
netcat_tcp.test_connectivity()
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/common/net_helpers.py",
line 575, in test_connectivity
message = self.server_process.read_stdout(READ_TIMEOUT).strip()
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/common/net_helpers.py",
line 336, in read_stdout
return self._read_stream(self.stdout, timeout)
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/common/net_helpers.py",
line 343, in _read_stream
raise RuntimeError('No output in %.2f seconds' % timeout)
RuntimeError: No output in 5.00 seconds
-
https://storage.gra.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_0e2/944979/2/check/neutron-fullstack/0e2e9a0/testr_results.html
ft1.8:
neutron.tests.fullstack.test_l3_agent.TestHAL3Agent.test_ha_router_failover_gracefultesttools.testresult.real._StringException:
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/base.py",
line 178, in func
return f(self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/fullstack/test_l3_agent.py",
line 579, in test_ha_router_failover_graceful
self._test_ha_router_failover('shutdown')
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/fullstack/test_l3_agent.py",
line 542, in _test_ha_router_failover
netcat_udp.test_connectivity()
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/common/net_helpers.py",
line 575, in test_connectivity
message = self.server_process.read_stdout(READ_TIMEOUT).strip()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/common/net_helpers.py",
line 336, in read_stdout
return self._read_stream(self.stdout, timeout)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/common/net_helpers.py",
line 343, in _read_stream
raise RuntimeError('No output in %.2f seconds' % timeout)
RuntimeError: No output in 5.00 seconds
-
https://962baa9ccd0a43825d0b-0a894e3582385412e867858a97ee7e54.ssl.cf2.rackcdn.com/943404/10/check/neutron-fullstack/f89edf5/testr_results.html
ft1.8:
neutron.tests.fullstack.test_l3_agent.TestHAL3Agent.test_ha_router_failover_gracefultesttools.testresult.real._StringException:
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/base.py",
line 178, in func
return f(self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/fullstack/test_l3_agent.py",
line 579, in test_ha_router_failover_graceful
self._test_ha_router_failover('shutdown')
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/fullstack/test_l3_agent.py",
line 542, in _test_ha_router_failover
netcat_udp.test_connectivity()
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/common/net_helpers.py",
line 575, in test_connectivity
message = self.server_process.read_stdout(READ_TIMEOUT).strip()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/common/net_helpers.py",
line 336, in read_stdout
return self._read_stream(self.stdout, timeout)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/common/net_helpers.py",
line 343, in _read_stream
raise RuntimeError('No output in %.2f seconds' % timeout)
RuntimeError: No output in 5.00 seconds
-
https://4656218b6b9f64be6a53-b8f5baac1536449624f55cad0c5791e8.ssl.cf1.rackcdn.com/944803/1/gate/neutron-fullstack/dfdcc58/testr_results.html
ft1.6:
neutron.tests.fullstack.test_l3_agent.TestHAL3Agent.test_ha_router_failover_gracefultesttools.testresult.real._StringException:
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/base.py",
line 178, in func
return f(self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/fullstack/test_l3_agent.py",
line 579, in test_ha_router_failover_graceful
self._test_ha_router_failover('shutdown')
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/fullstack/test_l3_agent.py",
line 537, in _test_ha_router_failover
vm.block_until_ping(external.ip)
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/common/machine_fixtures.py",
line 67, in block_until_ping
utils.wait_until_true(
File "/home/zuul/src/opendev.org/openstack/neutron/neutron/common/utils.py",
line 734, in wait_until_true
raise exception if exception else WaitTimeout(
neutron.tests.common.machine_fixtures.FakeMachineException: No ICMP reply
obtained from IP address 42.0.0.165
- test_ha_router_failover_host_failure
-
https://59cf76542d9f261862f7-56852c2a39b5b5caf6602061c30b0cc0.ssl.cf2.rackcdn.com/944979/2/check/neutron-fullstack/a9d8a2d/testr_results.html
ft1.10:
neutron.tests.fullstack.test_l3_agent.TestHAL3Agent.test_ha_router_failover_host_failuretesttools.testresult.real._StringException:
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/base.py",
line 178, in func
return f(self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/fullstack/test_l3_agent.py",
line 582, in test_ha_router_failover_host_failure
self._test_ha_router_failover('kill')
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/fullstack/test_l3_agent.py",
line 496, in _test_ha_router_failover
vm.block_until_ping(external.ip)
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/common/machine_fixtures.py",
line 67, in block_until_ping
utils.wait_until_true(
File "/home/zuul/src/opendev.org/openstack/neutron/neutron/common/utils.py",
line 734, in wait_until_true
raise exception if exception else WaitTimeout(
neutron.tests.common.machine_fixtures.FakeMachineException: No ICMP reply
obtained from IP address 42.0.0.226
-
https://d99bd7feea5bac8bf2c7-371f69b7cb442b1ed4a26cfca1c7e4db.ssl.cf2.rackcdn.com/944879/1/gate/neutron-fullstack-with-uwsgi/9f7dbcb/testr_results.html
ft1.8:
neutron.tests.fullstack.test_l3_agent.TestHAL3Agent.test_ha_router_failover_host_failuretesttools.testresult.real._StringException:
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/base.py",
line 178, in func
return f(self, *args, **kwargs)
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/fullstack/test_l3_agent.py",
line 556, in test_ha_router_failover_host_failure
self._test_ha_router_failover('kill')
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/fullstack/test_l3_agent.py",
line 550, in _test_ha_router_failover
self.assertLess(lost, threshold, message)
File "/usr/lib/python3.10/unittest/case.py", line 1232, in assertLess
self.fail(self._formatMessage(msg, standardMsg))
File "/usr/lib/python3.10/unittest/case.py", line 675, in fail
raise self.failureException(msg)
AssertionError: 98 not less than 90 : Sent 336 packets, received 238 packets,
lost 98 packets
-
https://0c63ab9652170854bf26-a09d1a3317eb4b9b558e42ad19c25861.ssl.cf2.rackcdn.com/944013/2/check/neutron-fullstack-7/518678e/testr_results.html
ft1.8:
neutron.tests.fullstack.test_l3_agent.TestHAL3Agent.test_ha_router_failover_host_failuretesttools.testresult.real._StringException:
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/base.py",
line 178, in func
return f(self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/fullstack/test_l3_agent.py",
line 582, in test_ha_router_failover_host_failure
self._test_ha_router_failover('kill')
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/fullstack/test_l3_agent.py",
line 496, in _test_ha_router_failover
vm.block_until_ping(external.ip)
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/common/machine_fixtures.py",
line 67, in block_until_ping
utils.wait_until_true(
File "/home/zuul/src/opendev.org/openstack/neutron/neutron/common/utils.py",
line 734, in wait_until_true
raise exception if exception else WaitTimeout(
neutron.tests.common.machine_fixtures.FakeMachineException: No ICMP reply
obtained from IP address 42.0.0.122
** Affects: neutron
Importance: Undecided
Status: New
--
You received this bug notification because you are a member of Yahoo!
Engineering Team, which is subscribed to neutron.
https://bugs.launchpad.net/bugs/2104019
Title:
random failures in L3 HA fullstack failover tests
Status in neutron:
New
Bug description:
These tests were added as part of
https://opendev.org/openstack/neutron/commit/f9a54350e6b77dcb03a6e1eeedb547f276813c99
We seeing random failures in these tests, this bug to track these
failures.
Example failures:-
- test_ha_router_failover_graceful
-
https://7f1261c1f7c867223017-371138c00916065f498de90723b300c8.ssl.cf5.rackcdn.com/periodic/opendev.org/openstack/neutron/master/neutron-fullstack-fips/af92329/testr_results.html
ft1.7:
neutron.tests.fullstack.test_l3_agent.TestHAL3Agent.test_ha_router_failover_gracefultesttools.testresult.real._StringException:
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/base.py",
line 178, in func
return f(self, *args, **kwargs)
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/fullstack/test_l3_agent.py",
line 579, in test_ha_router_failover_graceful
self._test_ha_router_failover('shutdown')
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/fullstack/test_l3_agent.py",
line 541, in _test_ha_router_failover
netcat_tcp.test_connectivity()
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/common/net_helpers.py",
line 575, in test_connectivity
message = self.server_process.read_stdout(READ_TIMEOUT).strip()
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/common/net_helpers.py",
line 336, in read_stdout
return self._read_stream(self.stdout, timeout)
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/common/net_helpers.py",
line 343, in _read_stream
raise RuntimeError('No output in %.2f seconds' % timeout)
RuntimeError: No output in 5.00 seconds
-
https://storage.gra.cloud.ovh.net/v1/AUTH_dcaab5e32b234d56b626f72581e3644c/zuul_opendev_logs_0e2/944979/2/check/neutron-fullstack/0e2e9a0/testr_results.html
ft1.8:
neutron.tests.fullstack.test_l3_agent.TestHAL3Agent.test_ha_router_failover_gracefultesttools.testresult.real._StringException:
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/base.py",
line 178, in func
return f(self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/fullstack/test_l3_agent.py",
line 579, in test_ha_router_failover_graceful
self._test_ha_router_failover('shutdown')
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/fullstack/test_l3_agent.py",
line 542, in _test_ha_router_failover
netcat_udp.test_connectivity()
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/common/net_helpers.py",
line 575, in test_connectivity
message = self.server_process.read_stdout(READ_TIMEOUT).strip()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/common/net_helpers.py",
line 336, in read_stdout
return self._read_stream(self.stdout, timeout)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/common/net_helpers.py",
line 343, in _read_stream
raise RuntimeError('No output in %.2f seconds' % timeout)
RuntimeError: No output in 5.00 seconds
-
https://962baa9ccd0a43825d0b-0a894e3582385412e867858a97ee7e54.ssl.cf2.rackcdn.com/943404/10/check/neutron-fullstack/f89edf5/testr_results.html
ft1.8:
neutron.tests.fullstack.test_l3_agent.TestHAL3Agent.test_ha_router_failover_gracefultesttools.testresult.real._StringException:
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/base.py",
line 178, in func
return f(self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/fullstack/test_l3_agent.py",
line 579, in test_ha_router_failover_graceful
self._test_ha_router_failover('shutdown')
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/fullstack/test_l3_agent.py",
line 542, in _test_ha_router_failover
netcat_udp.test_connectivity()
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/common/net_helpers.py",
line 575, in test_connectivity
message = self.server_process.read_stdout(READ_TIMEOUT).strip()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/common/net_helpers.py",
line 336, in read_stdout
return self._read_stream(self.stdout, timeout)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/common/net_helpers.py",
line 343, in _read_stream
raise RuntimeError('No output in %.2f seconds' % timeout)
RuntimeError: No output in 5.00 seconds
-
https://4656218b6b9f64be6a53-b8f5baac1536449624f55cad0c5791e8.ssl.cf1.rackcdn.com/944803/1/gate/neutron-fullstack/dfdcc58/testr_results.html
ft1.6:
neutron.tests.fullstack.test_l3_agent.TestHAL3Agent.test_ha_router_failover_gracefultesttools.testresult.real._StringException:
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/base.py",
line 178, in func
return f(self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/fullstack/test_l3_agent.py",
line 579, in test_ha_router_failover_graceful
self._test_ha_router_failover('shutdown')
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/fullstack/test_l3_agent.py",
line 537, in _test_ha_router_failover
vm.block_until_ping(external.ip)
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/common/machine_fixtures.py",
line 67, in block_until_ping
utils.wait_until_true(
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/common/utils.py", line
734, in wait_until_true
raise exception if exception else WaitTimeout(
neutron.tests.common.machine_fixtures.FakeMachineException: No ICMP reply
obtained from IP address 42.0.0.165
- test_ha_router_failover_host_failure
-
https://59cf76542d9f261862f7-56852c2a39b5b5caf6602061c30b0cc0.ssl.cf2.rackcdn.com/944979/2/check/neutron-fullstack/a9d8a2d/testr_results.html
ft1.10:
neutron.tests.fullstack.test_l3_agent.TestHAL3Agent.test_ha_router_failover_host_failuretesttools.testresult.real._StringException:
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/base.py",
line 178, in func
return f(self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/fullstack/test_l3_agent.py",
line 582, in test_ha_router_failover_host_failure
self._test_ha_router_failover('kill')
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/fullstack/test_l3_agent.py",
line 496, in _test_ha_router_failover
vm.block_until_ping(external.ip)
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/common/machine_fixtures.py",
line 67, in block_until_ping
utils.wait_until_true(
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/common/utils.py", line
734, in wait_until_true
raise exception if exception else WaitTimeout(
neutron.tests.common.machine_fixtures.FakeMachineException: No ICMP reply
obtained from IP address 42.0.0.226
-
https://d99bd7feea5bac8bf2c7-371f69b7cb442b1ed4a26cfca1c7e4db.ssl.cf2.rackcdn.com/944879/1/gate/neutron-fullstack-with-uwsgi/9f7dbcb/testr_results.html
ft1.8:
neutron.tests.fullstack.test_l3_agent.TestHAL3Agent.test_ha_router_failover_host_failuretesttools.testresult.real._StringException:
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/base.py",
line 178, in func
return f(self, *args, **kwargs)
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/fullstack/test_l3_agent.py",
line 556, in test_ha_router_failover_host_failure
self._test_ha_router_failover('kill')
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/fullstack/test_l3_agent.py",
line 550, in _test_ha_router_failover
self.assertLess(lost, threshold, message)
File "/usr/lib/python3.10/unittest/case.py", line 1232, in assertLess
self.fail(self._formatMessage(msg, standardMsg))
File "/usr/lib/python3.10/unittest/case.py", line 675, in fail
raise self.failureException(msg)
AssertionError: 98 not less than 90 : Sent 336 packets, received 238 packets,
lost 98 packets
-
https://0c63ab9652170854bf26-a09d1a3317eb4b9b558e42ad19c25861.ssl.cf2.rackcdn.com/944013/2/check/neutron-fullstack-7/518678e/testr_results.html
ft1.8:
neutron.tests.fullstack.test_l3_agent.TestHAL3Agent.test_ha_router_failover_host_failuretesttools.testresult.real._StringException:
Traceback (most recent call last):
File "/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/base.py",
line 178, in func
return f(self, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/fullstack/test_l3_agent.py",
line 582, in test_ha_router_failover_host_failure
self._test_ha_router_failover('kill')
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/fullstack/test_l3_agent.py",
line 496, in _test_ha_router_failover
vm.block_until_ping(external.ip)
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/tests/common/machine_fixtures.py",
line 67, in block_until_ping
utils.wait_until_true(
File
"/home/zuul/src/opendev.org/openstack/neutron/neutron/common/utils.py", line
734, in wait_until_true
raise exception if exception else WaitTimeout(
neutron.tests.common.machine_fixtures.FakeMachineException: No ICMP reply
obtained from IP address 42.0.0.122
To manage notifications about this bug go to:
https://bugs.launchpad.net/neutron/+bug/2104019/+subscriptions
--
Mailing list: https://launchpad.net/~yahoo-eng-team
Post to : yahoo-eng-team@lists.launchpad.net
Unsubscribe : https://launchpad.net/~yahoo-eng-team
More help : https://help.launchpad.net/ListHelp
