Public bug reported:
Failure statement : LB health monitor deletion fails with exception
"Server-side error: "'NoneType' object has no attribute
'load_balancer_id'"
Test executed : 1) Create LB, Virtual servers (HTTP & HTTPS), Pool, Members &
Health monitors
2) Delete LB objects randomly
3) If there is a dependency to a object while its deleting,
then handle exception
BadRequestException, ConflictException
Below is the HM tried to delete:
DELETE /v2.0/lbaas/healthmonitors/50ffba87-3e85-4d59-8c46-d782a8cbe8eb
octavia log trace:
Oct 04 01:29:01 controller-ch9q6cgc8v octavia-api[805]: 2020-10-04 01:29:01.801
1 DEBUG vmware_nsx.services.lbaas.octavia.octavia_driver [req-
9c7ab75f-3922-4cb7-81c0-a82b9f645a0c - a0a882c114b741c6a50e2fddc68a67db -
default default] vmware_nsx.services.lbaas.octavia.octavia_driver.NS
XOctaviaDriver method __init__ called with arguments () {} wrapper
/usr/lib/python3.7/site-packages/oslo_log/helpers.py:66
Oct 04 01:29:01 controller-ch9q6cgc8v octavia-api[805]: 2020-10-04 01:29:01.802
1 DEBUG vmware_nsx.services.lbaas.octavia.octavia_driver [req-
9c7ab75f-3922-4cb7-81c0-a82b9f645a0c - a0a882c114b741c6a50e2fddc68a67db -
default default] vmware_nsx.services.lbaas.octavia.octavia_driver.NS
XOctaviaDriver method _init_rpc_messaging called with arguments () {} wrapper
/usr/lib/python3.7/site-packages/oslo_log/helpers.py:66
Oct 04 01:29:01 controller-ch9q6cgc8v octavia-api[805]: 2020-10-04 01:29:01.808
1 ERROR wsme.api [req-9c7ab75f-3922-4cb7-81c0-a82b9f645a0c -
a0a882c114b741c6a50e2fddc68a67db - default default] Server-side error:
"'NoneType' object has no attribute 'load_balancer_id'". Detail:
Oct 04 01:29:01 controller-ch9q6cgc8v octavia-api[805]: Traceback (most recent
call last):
Oct 04 01:29:01 controller-ch9q6cgc8v octavia-api[805]:
Oct 04 01:29:01 controller-ch9q6cgc8v octavia-api[805]: File
"/usr/lib/python3.7/site-packages/wsmeext/pecan.py", line 85, in callfunction
Oct 04 01:29:01 controller-ch9q6cgc8v octavia-api[805]: result = f(self,
*args, **kwargs)
Oct 04 01:29:01 controller-ch9q6cgc8v octavia-api[805]:
Oct 04 01:29:01 controller-ch9q6cgc8v octavia-api[805]: File
"/usr/lib/python3.7/sitepackages/octavia/api/v2/controllers/health_monitor.py",
line 413, in delete
Oct 04 01:29:01 controller-ch9q6cgc8v octavia-api[805]:
self._test_lb_and_listener_and_pool_statuses(lock_session, db_hm)
Oct 04 01:29:01 controller-ch9q6cgc8v octavia-api[805]:
Oct 04 01:29:01 controller-ch9q6cgc8v octavia-api[805]: File
"/usr/lib/python3.7/site-packages/octavia/api/v2/controllers/health_monitor.py"
, line 94, in _test_lb_and_listener_and_pool_statuses
Oct 04 01:29:01 controller-ch9q6cgc8v octavia-api[805]: load_balancer_id =
pool.load_balancer_id
Oct 04 01:29:01 controller-ch9q6cgc8v octavia-api[805]:
Oct 04 01:29:01 controller-ch9q6cgc8v octavia-api[805]: AttributeError:
'NoneType' object has no attribute 'load_balancer_id'
Oct 04 01:29:01 controller-ch9q6cgc8v octavia-api[805]:
Oct 04 01:29:23 controller-ch9q6cgc8v octavia-api[805]: 2020-10-04 01:29:23.215
1 DEBUG vmware_nsx.services.lbaas.octavia.octavia_driver [req-
b2a8b23c-485c-4257-9f00-5a8f581a5387 - a0a882c114b741c6a50e2fddc68a67db -
default default] vmware_nsx.services.lbaas.octavia.octavia_driver.NS
XOctaviaDriver method __init__ called with arguments () {} wrapper
/usr/lib/python3.7/site packages/oslo_log/helpers.py:66
Test executed:
@pytest.mark.esxitest
@pytest.mark.usefixtures("do_cleanup")
def test_lb_and_delete_in_random_order_TC27(lb, nsxvc, cloud):
''' As part of this test case a LB topology will be brought up.
Different components of the LB will be tried to be deleted
in random order '''
_test_case = inspect.stack()[0][3].split("_")[-1]
router, network, subnet =\
ru.create_net_subnet_rtr_lb(cloud, _test_case)
lb, lsnr, pool, hm, lbfip, mem1, mem2 =\
create_general_roundrobin_lb_topo(_test_case,
network, subnet, router, cloud, nsxvc)
# Creating a dictionary of the LB resources
# Format of dictionary is same as clenaup.py since we are going to
# use delete functions of cleanup.py
clean_lb_res = Clean(cloud, _test_case)
clean_lb_res.resources = {"lbs": [lb],
"listeners": [lsnr],
"pools": [pool],
"health_monitors": [hm],
"pool_members": [{pool.id: [mem1, mem2]}]}
# Below loop will try for 5 mins to randomly delete lb resources.
# After 5 mins the test case will fail.
lb_keys = clean_lb_res.resources.keys()
start_time = time.time()
while lb_keys and time.time() - start_time < 300:
# Randomly select one lb resource
indx = randrange(0, len(lb_keys))
lb_res = lb_keys[indx]
# Start deleting the resource
try:
func_name = "delete_%s" % (lb_res)
getattr(clean_lb_res, func_name)()
# removing element from list since successfully deleted
lb_keys.remove(lb_res)
except AttributeError as e:
log.exception(e)
assert False, "Function with name %s doesn't exist" % (func_name)
except (exc.BadRequestException, exc.ConflictException) as e:
log.info("Message=%s, Response=%s, Details=%s, request_id=%s" % \
(e.message, e.response, e.details, e.request_id))
log.info("%s didn't work, selecting some other lb resource to"
" delete" % (func_name))
assert not lb_keys, "Still some resources left to be cleaned"
Test failure:
getattr(clean_lb_res, func_name)()
tests/test_load_balancer.py:1126:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
cleanup.py:205: in delete_health_monitors
self.cloud.delete_health_monitor(hm)
resources.py:619: in delete_health_monitor
return self.conn.load_balancer.delete_health_monitor(healthmonitor)
systest_env/local/lib/python2.7/site-packages/openstack/load_balancer/v2/_proxy.py:471:
in delete_health_monitor
ignore_missing=ignore_missing)
systest_env/local/lib/python2.7/site-packages/openstack/proxy.py:41: in check
return method(self, expected, actual, *args, **kwargs)
systest_env/local/lib/python2.7/site-packages/openstack/proxy.py:144: in _delete
rv = res.delete(self)
systest_env/local/lib/python2.7/site-packages/openstack/resource.py:876: in
delete
self._translate_response(response, has_body=False, **kwargs)
systest_env/local/lib/python2.7/site-packages/openstack/resource.py:695: in
_translate_response
exceptions.raise_from_response(response, error_message=error_message)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
response = <Response [500]>, error_message = None
def raise_from_response(response, error_message=None):
"""Raise an instance of an HTTPException based on keystoneauth
response."""
if response.status_code < 400:
return
if response.status_code == 409:
cls = ConflictException
elif response.status_code == 404:
cls = NotFoundException
elif response.status_code == 400:
cls = BadRequestException
else:
cls = HttpException
details = None
content_type = response.headers.get('content-type', '')
if response.content and 'application/json' in content_type:
# Iterate over the nested objects to retrieve "message" attribute.
# TODO(shade) Add exception handling for times when the content type
# is lying.
try:
content = response.json()
messages = [obj.get('message') for obj in content.values()
if isinstance(obj, dict)]
# Join all of the messages together nicely and filter out any
# objects that don't have a "message" attr.
details = '\n'.join(msg for msg in messages if msg)
except Exception:
details = response.text
elif response.content and 'text/html' in content_type:
# Split the lines, strip whitespace and inline HTML from the
response.
details = [re.sub(r'<.+?>', '', i.strip())
for i in response.text.splitlines()]
details = list(set([msg for msg in details if msg]))
# Return joined string separated by colons.
details = ': '.join(details)
if not details and response.reason:
details = response.reason
else:
details = response.text
http_status = response.status_code
request_id = response.headers.get('x-openstack-request-id')
raise cls(
message=error_message, response=response, details=details,
http_status=http_status, request_id=request_id
)
E HttpException: HttpException: 500: Server Error for url:
https://172.18.150.100:9876/v2.0/lbaas/healthmonitors/50ffba87-3e85-4d59-8c46-d782a8cbe8eb,
Internal Server Error
** Affects: neutron
Importance: Undecided
Status: New
--
You received this bug notification because you are a member of Yahoo!
Engineering Team, which is subscribed to neutron.
https://bugs.launchpad.net/bugs/1898657
Title:
LB health monitor deletion fails with exception "Server-side error:
"'NoneType' object has no attribute 'load_balancer_id'"
Status in neutron:
New
Bug description:
Failure statement : LB health monitor deletion fails with exception
"Server-side error: "'NoneType' object has no attribute
'load_balancer_id'"
Test executed : 1) Create LB, Virtual servers (HTTP & HTTPS), Pool, Members &
Health monitors
2) Delete LB objects randomly
3) If there is a dependency to a object while its deleting,
then handle exception
BadRequestException, ConflictException
Below is the HM tried to delete:
DELETE /v2.0/lbaas/healthmonitors/50ffba87-3e85-4d59-8c46-d782a8cbe8eb
octavia log trace:
Oct 04 01:29:01 controller-ch9q6cgc8v octavia-api[805]: 2020-10-04
01:29:01.801 1 DEBUG vmware_nsx.services.lbaas.octavia.octavia_driver [req-
9c7ab75f-3922-4cb7-81c0-a82b9f645a0c - a0a882c114b741c6a50e2fddc68a67db -
default default] vmware_nsx.services.lbaas.octavia.octavia_driver.NS
XOctaviaDriver method __init__ called with arguments () {} wrapper
/usr/lib/python3.7/site-packages/oslo_log/helpers.py:66
Oct 04 01:29:01 controller-ch9q6cgc8v octavia-api[805]: 2020-10-04
01:29:01.802 1 DEBUG vmware_nsx.services.lbaas.octavia.octavia_driver [req-
9c7ab75f-3922-4cb7-81c0-a82b9f645a0c - a0a882c114b741c6a50e2fddc68a67db -
default default] vmware_nsx.services.lbaas.octavia.octavia_driver.NS
XOctaviaDriver method _init_rpc_messaging called with arguments () {} wrapper
/usr/lib/python3.7/site-packages/oslo_log/helpers.py:66
Oct 04 01:29:01 controller-ch9q6cgc8v octavia-api[805]: 2020-10-04
01:29:01.808 1 ERROR wsme.api [req-9c7ab75f-3922-4cb7-81c0-a82b9f645a0c -
a0a882c114b741c6a50e2fddc68a67db - default default] Server-side error:
"'NoneType' object has no attribute 'load_balancer_id'". Detail:
Oct 04 01:29:01 controller-ch9q6cgc8v octavia-api[805]: Traceback (most
recent call last):
Oct 04 01:29:01 controller-ch9q6cgc8v octavia-api[805]:
Oct 04 01:29:01 controller-ch9q6cgc8v octavia-api[805]: File
"/usr/lib/python3.7/site-packages/wsmeext/pecan.py", line 85, in callfunction
Oct 04 01:29:01 controller-ch9q6cgc8v octavia-api[805]: result = f(self,
*args, **kwargs)
Oct 04 01:29:01 controller-ch9q6cgc8v octavia-api[805]:
Oct 04 01:29:01 controller-ch9q6cgc8v octavia-api[805]: File
"/usr/lib/python3.7/sitepackages/octavia/api/v2/controllers/health_monitor.py",
line 413, in delete
Oct 04 01:29:01 controller-ch9q6cgc8v octavia-api[805]:
self._test_lb_and_listener_and_pool_statuses(lock_session, db_hm)
Oct 04 01:29:01 controller-ch9q6cgc8v octavia-api[805]:
Oct 04 01:29:01 controller-ch9q6cgc8v octavia-api[805]: File
"/usr/lib/python3.7/site-packages/octavia/api/v2/controllers/health_monitor.py"
, line 94, in _test_lb_and_listener_and_pool_statuses
Oct 04 01:29:01 controller-ch9q6cgc8v octavia-api[805]: load_balancer_id
= pool.load_balancer_id
Oct 04 01:29:01 controller-ch9q6cgc8v octavia-api[805]:
Oct 04 01:29:01 controller-ch9q6cgc8v octavia-api[805]: AttributeError:
'NoneType' object has no attribute 'load_balancer_id'
Oct 04 01:29:01 controller-ch9q6cgc8v octavia-api[805]:
Oct 04 01:29:23 controller-ch9q6cgc8v octavia-api[805]: 2020-10-04
01:29:23.215 1 DEBUG vmware_nsx.services.lbaas.octavia.octavia_driver [req-
b2a8b23c-485c-4257-9f00-5a8f581a5387 - a0a882c114b741c6a50e2fddc68a67db -
default default] vmware_nsx.services.lbaas.octavia.octavia_driver.NS
XOctaviaDriver method __init__ called with arguments () {} wrapper
/usr/lib/python3.7/site packages/oslo_log/helpers.py:66
Test executed:
@pytest.mark.esxitest
@pytest.mark.usefixtures("do_cleanup")
def test_lb_and_delete_in_random_order_TC27(lb, nsxvc, cloud):
''' As part of this test case a LB topology will be brought up.
Different components of the LB will be tried to be deleted
in random order '''
_test_case = inspect.stack()[0][3].split("_")[-1]
router, network, subnet =\
ru.create_net_subnet_rtr_lb(cloud, _test_case)
lb, lsnr, pool, hm, lbfip, mem1, mem2 =\
create_general_roundrobin_lb_topo(_test_case,
network, subnet, router, cloud, nsxvc)
# Creating a dictionary of the LB resources
# Format of dictionary is same as clenaup.py since we are going to
# use delete functions of cleanup.py
clean_lb_res = Clean(cloud, _test_case)
clean_lb_res.resources = {"lbs": [lb],
"listeners": [lsnr],
"pools": [pool],
"health_monitors": [hm],
"pool_members": [{pool.id: [mem1, mem2]}]}
# Below loop will try for 5 mins to randomly delete lb resources.
# After 5 mins the test case will fail.
lb_keys = clean_lb_res.resources.keys()
start_time = time.time()
while lb_keys and time.time() - start_time < 300:
# Randomly select one lb resource
indx = randrange(0, len(lb_keys))
lb_res = lb_keys[indx]
# Start deleting the resource
try:
func_name = "delete_%s" % (lb_res)
getattr(clean_lb_res, func_name)()
# removing element from list since successfully deleted
lb_keys.remove(lb_res)
except AttributeError as e:
log.exception(e)
assert False, "Function with name %s doesn't exist" % (func_name)
except (exc.BadRequestException, exc.ConflictException) as e:
log.info("Message=%s, Response=%s, Details=%s, request_id=%s" % \
(e.message, e.response, e.details, e.request_id))
log.info("%s didn't work, selecting some other lb resource to"
" delete" % (func_name))
assert not lb_keys, "Still some resources left to be cleaned"
Test failure:
getattr(clean_lb_res, func_name)()
tests/test_load_balancer.py:1126:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
_
cleanup.py:205: in delete_health_monitors
self.cloud.delete_health_monitor(hm)
resources.py:619: in delete_health_monitor
return self.conn.load_balancer.delete_health_monitor(healthmonitor)
systest_env/local/lib/python2.7/site-packages/openstack/load_balancer/v2/_proxy.py:471:
in delete_health_monitor
ignore_missing=ignore_missing)
systest_env/local/lib/python2.7/site-packages/openstack/proxy.py:41: in check
return method(self, expected, actual, *args, **kwargs)
systest_env/local/lib/python2.7/site-packages/openstack/proxy.py:144: in
_delete
rv = res.delete(self)
systest_env/local/lib/python2.7/site-packages/openstack/resource.py:876: in
delete
self._translate_response(response, has_body=False, **kwargs)
systest_env/local/lib/python2.7/site-packages/openstack/resource.py:695: in
_translate_response
exceptions.raise_from_response(response, error_message=error_message)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
_
response = <Response [500]>, error_message = None
def raise_from_response(response, error_message=None):
"""Raise an instance of an HTTPException based on keystoneauth
response."""
if response.status_code < 400:
return
if response.status_code == 409:
cls = ConflictException
elif response.status_code == 404:
cls = NotFoundException
elif response.status_code == 400:
cls = BadRequestException
else:
cls = HttpException
details = None
content_type = response.headers.get('content-type', '')
if response.content and 'application/json' in content_type:
# Iterate over the nested objects to retrieve "message" attribute.
# TODO(shade) Add exception handling for times when the content
type
# is lying.
try:
content = response.json()
messages = [obj.get('message') for obj in content.values()
if isinstance(obj, dict)]
# Join all of the messages together nicely and filter out any
# objects that don't have a "message" attr.
details = '\n'.join(msg for msg in messages if msg)
except Exception:
details = response.text
elif response.content and 'text/html' in content_type:
# Split the lines, strip whitespace and inline HTML from the
response.
details = [re.sub(r'<.+?>', '', i.strip())
for i in response.text.splitlines()]
details = list(set([msg for msg in details if msg]))
# Return joined string separated by colons.
details = ': '.join(details)
if not details and response.reason:
details = response.reason
else:
details = response.text
http_status = response.status_code
request_id = response.headers.get('x-openstack-request-id')
raise cls(
message=error_message, response=response, details=details,
http_status=http_status, request_id=request_id
)
E HttpException: HttpException: 500: Server Error for url:
https://172.18.150.100:9876/v2.0/lbaas/healthmonitors/50ffba87-3e85-4d59-8c46-d782a8cbe8eb,
Internal Server Error
To manage notifications about this bug go to:
https://bugs.launchpad.net/neutron/+bug/1898657/+subscriptions
--
Mailing list: https://launchpad.net/~yahoo-eng-team
Post to : yahoo-eng-team@lists.launchpad.net
Unsubscribe : https://launchpad.net/~yahoo-eng-team
More help : https://help.launchpad.net/ListHelp
