Public bug reported: Seen here:
http://logs.openstack.org/65/416765/1/check/gate-tempest-dsvm-neutron- identity-v3-only-full-ubuntu-xenial- nv/faf1363/logs/screen-n-cpu.txt.gz#_2017-01-04_23_15_06_674 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager [req-832f822c-02b3-453b-b80b-61ad7f06f401 - -] Error updating resources for node ubuntu-xenial-osic-cloud1-disk-6483179. 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager Traceback (most recent call last): 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/opt/stack/new/nova/nova/compute/manager.py", line 6537, in update_available_resource_for_node 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager rt.update_available_resource(context, nodename) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/opt/stack/new/nova/nova/compute/resource_tracker.py", line 540, in update_available_resource 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager self._update_available_resource(context, resources) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/usr/local/lib/python2.7/dist-packages/oslo_concurrency/lockutils.py", line 271, in inner 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager return f(*args, **kwargs) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/opt/stack/new/nova/nova/compute/resource_tracker.py", line 564, in _update_available_resource 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager self._init_compute_node(context, resources) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/opt/stack/new/nova/nova/compute/resource_tracker.py", line 451, in _init_compute_node 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager self.scheduler_client.update_resource_stats(self.compute_node) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/opt/stack/new/nova/nova/scheduler/client/__init__.py", line 60, in update_resource_stats 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager self.reportclient.update_resource_stats(compute_node) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/opt/stack/new/nova/nova/scheduler/client/__init__.py", line 37, in __run_method 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager return getattr(self.instance, __name)(*args, **kwargs) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/opt/stack/new/nova/nova/scheduler/client/report.py", line 462, in update_resource_stats 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager compute_node.hypervisor_hostname) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/opt/stack/new/nova/nova/scheduler/client/report.py", line 282, in _ensure_resource_provider 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager rp = self._get_resource_provider(uuid) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/opt/stack/new/nova/nova/scheduler/client/report.py", line 47, in wrapper 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager return f(self, *a, **k) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/opt/stack/new/nova/nova/scheduler/client/report.py", line 195, in _get_resource_provider 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager resp = self.get("/resource_providers/%s" % uuid) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/opt/stack/new/nova/nova/scheduler/client/report.py", line 160, in get 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager endpoint_filter=self.ks_filter, raise_exc=False) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/usr/local/lib/python2.7/dist-packages/keystoneauth1/session.py", line 710, in get 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager return self.request(url, 'GET', **kwargs) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/usr/local/lib/python2.7/dist-packages/positional/__init__.py", line 101, in inner 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager return wrapped(*args, **kwargs) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/usr/local/lib/python2.7/dist-packages/keystoneauth1/session.py", line 467, in request 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager auth_headers = self.get_auth_headers(auth) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/usr/local/lib/python2.7/dist-packages/keystoneauth1/session.py", line 770, in get_auth_headers 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager return auth.get_headers(self, **kwargs) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/usr/local/lib/python2.7/dist-packages/keystoneauth1/plugin.py", line 90, in get_headers 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager token = self.get_token(session) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/usr/local/lib/python2.7/dist-packages/keystoneauth1/identity/base.py", line 90, in get_token 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager return self.get_access(session).auth_token 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/usr/local/lib/python2.7/dist-packages/keystoneauth1/identity/base.py", line 136, in get_access 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager self.auth_ref = self.get_auth_ref(session) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/usr/local/lib/python2.7/dist-packages/keystoneauth1/identity/generic/base.py", line 198, in get_auth_ref 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager return self._plugin.get_auth_ref(session, **kwargs) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/usr/local/lib/python2.7/dist-packages/keystoneauth1/identity/v3/base.py", line 167, in get_auth_ref 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager authenticated=False, log=False, **rkwargs) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/usr/local/lib/python2.7/dist-packages/keystoneauth1/session.py", line 718, in post 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager return self.request(url, 'POST', **kwargs) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/usr/local/lib/python2.7/dist-packages/positional/__init__.py", line 101, in inner 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager return wrapped(*args, **kwargs) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/usr/local/lib/python2.7/dist-packages/keystoneauth1/session.py", line 607, in request 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager raise exceptions.from_response(resp, method, url) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager Unauthorized: The request you have made requires authentication. (HTTP 401) (Request-ID: req-38ce6df3-1dca-4b66-a680-d2691bb4955f) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager The scheduler report client's safe_connect method isn't handle Unauthorized exceptions, which are happening in the keystone v3 jobs now with the placement API, at least until this devstack patch is merged: https://review.openstack.org/#/c/416599/ We handle Unauthorized in the new nova-status upgrade check command but not the safe_connect() decorator, which we need to do. ** Affects: nova Importance: High Assignee: Matt Riedemann (mriedem) Status: Triaged ** Affects: nova/newton Importance: Undecided Status: Confirmed ** Tags: compute placement scheduler ** Changed in: nova Assignee: (unassigned) => Matt Riedemann (mriedem) ** Changed in: nova Importance: Undecided => High ** Changed in: nova Status: New => Triaged ** Also affects: nova/newton Importance: Undecided Status: New -- You received this bug notification because you are a member of Yahoo! Engineering Team, which is subscribed to OpenStack Compute (nova). https://bugs.launchpad.net/bugs/1654107 Title: resource tracking fails with Unauthorized from placement API (keystone v3) Status in OpenStack Compute (nova): Triaged Status in OpenStack Compute (nova) newton series: Confirmed Bug description: Seen here: http://logs.openstack.org/65/416765/1/check/gate-tempest-dsvm-neutron- identity-v3-only-full-ubuntu-xenial- nv/faf1363/logs/screen-n-cpu.txt.gz#_2017-01-04_23_15_06_674 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager [req-832f822c-02b3-453b-b80b-61ad7f06f401 - -] Error updating resources for node ubuntu-xenial-osic-cloud1-disk-6483179. 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager Traceback (most recent call last): 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/opt/stack/new/nova/nova/compute/manager.py", line 6537, in update_available_resource_for_node 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager rt.update_available_resource(context, nodename) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/opt/stack/new/nova/nova/compute/resource_tracker.py", line 540, in update_available_resource 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager self._update_available_resource(context, resources) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/usr/local/lib/python2.7/dist-packages/oslo_concurrency/lockutils.py", line 271, in inner 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager return f(*args, **kwargs) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/opt/stack/new/nova/nova/compute/resource_tracker.py", line 564, in _update_available_resource 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager self._init_compute_node(context, resources) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/opt/stack/new/nova/nova/compute/resource_tracker.py", line 451, in _init_compute_node 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager self.scheduler_client.update_resource_stats(self.compute_node) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/opt/stack/new/nova/nova/scheduler/client/__init__.py", line 60, in update_resource_stats 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager self.reportclient.update_resource_stats(compute_node) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/opt/stack/new/nova/nova/scheduler/client/__init__.py", line 37, in __run_method 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager return getattr(self.instance, __name)(*args, **kwargs) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/opt/stack/new/nova/nova/scheduler/client/report.py", line 462, in update_resource_stats 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager compute_node.hypervisor_hostname) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/opt/stack/new/nova/nova/scheduler/client/report.py", line 282, in _ensure_resource_provider 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager rp = self._get_resource_provider(uuid) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/opt/stack/new/nova/nova/scheduler/client/report.py", line 47, in wrapper 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager return f(self, *a, **k) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/opt/stack/new/nova/nova/scheduler/client/report.py", line 195, in _get_resource_provider 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager resp = self.get("/resource_providers/%s" % uuid) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/opt/stack/new/nova/nova/scheduler/client/report.py", line 160, in get 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager endpoint_filter=self.ks_filter, raise_exc=False) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/usr/local/lib/python2.7/dist-packages/keystoneauth1/session.py", line 710, in get 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager return self.request(url, 'GET', **kwargs) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/usr/local/lib/python2.7/dist-packages/positional/__init__.py", line 101, in inner 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager return wrapped(*args, **kwargs) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/usr/local/lib/python2.7/dist-packages/keystoneauth1/session.py", line 467, in request 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager auth_headers = self.get_auth_headers(auth) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/usr/local/lib/python2.7/dist-packages/keystoneauth1/session.py", line 770, in get_auth_headers 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager return auth.get_headers(self, **kwargs) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/usr/local/lib/python2.7/dist-packages/keystoneauth1/plugin.py", line 90, in get_headers 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager token = self.get_token(session) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/usr/local/lib/python2.7/dist-packages/keystoneauth1/identity/base.py", line 90, in get_token 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager return self.get_access(session).auth_token 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/usr/local/lib/python2.7/dist-packages/keystoneauth1/identity/base.py", line 136, in get_access 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager self.auth_ref = self.get_auth_ref(session) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/usr/local/lib/python2.7/dist-packages/keystoneauth1/identity/generic/base.py", line 198, in get_auth_ref 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager return self._plugin.get_auth_ref(session, **kwargs) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/usr/local/lib/python2.7/dist-packages/keystoneauth1/identity/v3/base.py", line 167, in get_auth_ref 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager authenticated=False, log=False, **rkwargs) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/usr/local/lib/python2.7/dist-packages/keystoneauth1/session.py", line 718, in post 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager return self.request(url, 'POST', **kwargs) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/usr/local/lib/python2.7/dist-packages/positional/__init__.py", line 101, in inner 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager return wrapped(*args, **kwargs) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager File "/usr/local/lib/python2.7/dist-packages/keystoneauth1/session.py", line 607, in request 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager raise exceptions.from_response(resp, method, url) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager Unauthorized: The request you have made requires authentication. (HTTP 401) (Request-ID: req-38ce6df3-1dca-4b66-a680-d2691bb4955f) 2017-01-04 23:15:06.674 7096 ERROR nova.compute.manager The scheduler report client's safe_connect method isn't handle Unauthorized exceptions, which are happening in the keystone v3 jobs now with the placement API, at least until this devstack patch is merged: https://review.openstack.org/#/c/416599/ We handle Unauthorized in the new nova-status upgrade check command but not the safe_connect() decorator, which we need to do. To manage notifications about this bug go to: https://bugs.launchpad.net/nova/+bug/1654107/+subscriptions -- Mailing list: https://launchpad.net/~yahoo-eng-team Post to : yahoo-eng-team@lists.launchpad.net Unsubscribe : https://launchpad.net/~yahoo-eng-team More help : https://help.launchpad.net/ListHelp
