Public bug reported:
openstack queens:
Turn on NUMA scheduling:
vi /etc/nova/nova.conf
enabled_filters =....,NUMATopologyFilter
(openstack) flavor show p1
+----------------------------+--------------------------------------------------------------------------------------+
| Field | Value
|
+----------------------------+--------------------------------------------------------------------------------------+
| OS-FLV-DISABLED:disabled | False
|
| OS-FLV-EXT-DATA:ephemeral | 0
|
| access_project_ids | None
|
| disk | 10
|
| id | ab9f4851-c4a0-48e4-affe-e780ad8a87a1
|
| name | p1
|
| os-flavor-access:is_public | True
|
| properties | hw:mem_page_size='1024', hw:numa_cpus.1='20',
hw:numa_mem.1='512', hw:numa_nodes='1' |
| ram | 512
|
| rxtx_factor | 1.0
|
| swap |
|
| vcpus | 1
|
+----------------------------+--------------------------------------------------------------------------------------+
[root@controller ~]# numactl --hardware
available: 2 nodes (0-1)
node 0 cpus: 0 1 2 3 4 5 6 7 16 17 18 19 20 21 22 23
node 0 size: 130669 MB
node 0 free: 116115 MB
node 1 cpus: 8 9 10 11 12 13 14 15 24 25 26 27 28 29 30 31
node 1 size: 131072 MB
node 1 free: 114675 MB
node distances:
node 0 1
0: 10 21
1: 21 10
Error log
tail -f /var/log/nova/nova-conductor.log ::
default default] Failed to compute_task_build_instances: No valid host was
found. There are not enough hosts available.
Traceback (most recent call last):
File "/usr/lib/python2.7/site-packages/oslo_messaging/rpc/server.py", line
226, in inner
return func(*args, **kwargs)
File "/usr/lib/python2.7/site-packages/nova/scheduler/manager.py", line 154,
in select_destinations
allocation_request_version, return_alternates)
File "/usr/lib/python2.7/site-packages/nova/scheduler/filter_scheduler.py",
line 91, in select_destinations
allocation_request_version, return_alternates)
File "/usr/lib/python2.7/site-packages/nova/scheduler/filter_scheduler.py",
line 243, in _schedule
claimed_instance_uuids)
File "/usr/lib/python2.7/site-packages/nova/scheduler/filter_scheduler.py",
line 280, in _ensure_sufficient_hosts
raise exception.NoValidHost(reason=reason)
NoValidHost: No valid host was found. There are not enough hosts available.
: NoValidHost_Remote: No valid host was found. There are not enough hosts
available.
Error log
tail -f /var/log/nova/nova-scheduler.log::
2018-06-15 16:52:33.457 5829 DEBUG nova.virt.hardware
[req-be251765-6c3b-46aa-ae05-6c2e12ae8661 7e909565a4b847fe81cd6d1cf778c893
b2760ba26e5645bf9856669d560d91c7 - default default] Attempting to fit instance
cell
InstanceNUMACell(cpu_pinning_raw=None,cpu_policy=None,cpu_thread_policy=None,cpu_topology=<?>,cpuset=set([0]),cpuset_reserved=None,id=0,memory=512,pagesize=1024)
on host_cell
NUMACell(cpu_usage=0,cpuset=set([8,9,10,11,12,13,14,15,24,25,26,27,28,29,30,31]),id=1,memory=131072,memory_usage=0,mempages=[NUMAPagesTopology,NUMAPagesTopology],pinned_cpus=set([]),siblings=[set([8,24]),set([14,30]),set([15,31]),set([11,27]),set([10,26]),set([12,28]),set([9,25]),set([13,29])])
_numa_fit_instance_cell
/usr/lib/python2.7/site-packages/nova/virt/hardware.py:974
2018-06-15 16:52:33.458 5829 DEBUG nova.virt.hardware
[req-be251765-6c3b-46aa-ae05-6c2e12ae8661 7e909565a4b847fe81cd6d1cf778c893
b2760ba26e5645bf9856669d560d91c7 - default default] No pinning requested,
considering limitations on usable cpu and memory _numa_fit_instance_cell
/usr/lib/python2.7/site-packages/nova/virt/hardware.py:1003
2018-06-15 16:52:33.459 5829 DEBUG nova.scheduler.filters.numa_topology_filter
[req-be251765-6c3b-46aa-ae05-6c2e12ae8661 7e909565a4b847fe81cd6d1cf778c893
b2760ba26e5645bf9856669d560d91c7 - default default] [instance:
b1dff78a-ff24-4337-9cdb-edcecc9f9d00] controller, controller fails NUMA
topology requirements. The instance does not fit on this host. host_passes
/usr/lib/python2.7/site-packages/nova/scheduler/filters/numa_topology_filter.py:104
2018-06-15 16:52:33.459 5829 INFO nova.filters
[req-be251765-6c3b-46aa-ae05-6c2e12ae8661 7e909565a4b847fe81cd6d1cf778c893
b2760ba26e5645bf9856669d560d91c7 - default default] Filter NUMATopologyFilter
returned 0 hosts
2018-06-15 16:52:33.459 5829 DEBUG nova.filters
[req-be251765-6c3b-46aa-ae05-6c2e12ae8661 7e909565a4b847fe81cd6d1cf778c893
b2760ba26e5645bf9856669d560d91c7 - default default] Filtering removed all hosts
for the request with instance ID 'b1dff78a-ff24-4337-9cdb-edcecc9f9d00'. Filter
results: [('RetryFilter', [(u'controller', u'controller')]),
('AvailabilityZoneFilter', [(u'controller', u'controller')]), ('ComputeFilter',
[(u'controller', u'controller')]), ('ComputeCapabilitiesFilter',
[(u'controller', u'controller')]), ('ImagePropertiesFilter', [(u'controller',
u'controller')]), ('ServerGroupAntiAffinityFilter', [(u'controller',
u'controller')]), ('ServerGroupAffinityFilter', [(u'controller',
u'controller')]), ('NUMATopologyFilter', None)] get_filtered_objects
/usr/lib/python2.7/site-packages/nova/filters.py:129
2018-06-15 16:52:33.460 5829 INFO nova.filters
[req-be251765-6c3b-46aa-ae05-6c2e12ae8661 7e909565a4b847fe81cd6d1cf778c893
b2760ba26e5645bf9856669d560d91c7 - default default] Filtering removed all hosts
for the request with instance ID 'b1dff78a-ff24-4337-9cdb-edcecc9f9d00'. Filter
results: ['RetryFilter: (start: 1, end: 1)', 'AvailabilityZoneFilter: (start:
1, end: 1)', 'ComputeFilter: (start: 1, end: 1)', 'ComputeCapabilitiesFilter:
(start: 1, end: 1)', 'ImagePropertiesFilter: (start: 1, end: 1)',
'ServerGroupAntiAffinityFilter: (start: 1, end: 1)',
'ServerGroupAffinityFilter: (start: 1, end: 1)', 'NUMATopologyFilter: (start:
1, end: 0)']
2018-06-15 16:52:33.460 5829 DEBUG nova.scheduler.filter_scheduler
[req-be251765-6c3b-46aa-ae05-6c2e12ae8661 7e909565a4b847fe81cd6d1cf778c893
b2760ba26e5645bf9856669d560d91c7 - default default] Filtered []
_get_sorted_hosts
/usr/lib/python2.7/site-packages/nova/scheduler/filter_scheduler.py:404
2018-06-15 16:52:33.460 5829 DEBUG nova.scheduler.filter_scheduler
[req-be251765-6c3b-46aa-ae05-6c2e12ae8661 7e909565a4b847fe81cd6d1cf778c893
b2760ba26e5645bf9856669d560d91c7 - default default] There are 0 hosts available
but 1 instances requested to build. _ensure_sufficient_hosts
/usr/lib/python2.7/site-packages/nova/scheduler/filter_scheduler.py:278
2018-06-15 16:52:42.945 5829 DEBUG oslo_service.periodic_task
[req-be251765-6c3b-46aa-ae05-6c2e12ae8661 7e909565a4b847fe81cd6d1cf778c893
b2760ba26e5645bf9856669d560d91c7 - default default] Running periodic task
SchedulerManager._run_periodic_tasks run_periodic_tasks
/usr/lib/python2.7/site-packages/oslo_service/periodic_task.py:215
This error is where I did not configure it?
** Affects: nova
Importance: Undecided
Status: New
--
You received this bug notification because you are a member of Yahoo!
Engineering Team, which is subscribed to OpenStack Compute (nova).
https://bugs.launchpad.net/bugs/1777089
Title:
controller fails NUMA topology requirements. The instance does not fit
on this host. host_passes
Status in OpenStack Compute (nova):
New
Bug description:
openstack queens:
Turn on NUMA scheduling:
vi /etc/nova/nova.conf
enabled_filters =....,NUMATopologyFilter
(openstack) flavor show p1
+----------------------------+--------------------------------------------------------------------------------------+
| Field | Value
|
+----------------------------+--------------------------------------------------------------------------------------+
| OS-FLV-DISABLED:disabled | False
|
| OS-FLV-EXT-DATA:ephemeral | 0
|
| access_project_ids | None
|
| disk | 10
|
| id | ab9f4851-c4a0-48e4-affe-e780ad8a87a1
|
| name | p1
|
| os-flavor-access:is_public | True
|
| properties | hw:mem_page_size='1024', hw:numa_cpus.1='20',
hw:numa_mem.1='512', hw:numa_nodes='1' |
| ram | 512
|
| rxtx_factor | 1.0
|
| swap |
|
| vcpus | 1
|
+----------------------------+--------------------------------------------------------------------------------------+
[root@controller ~]# numactl --hardware
available: 2 nodes (0-1)
node 0 cpus: 0 1 2 3 4 5 6 7 16 17 18 19 20 21 22 23
node 0 size: 130669 MB
node 0 free: 116115 MB
node 1 cpus: 8 9 10 11 12 13 14 15 24 25 26 27 28 29 30 31
node 1 size: 131072 MB
node 1 free: 114675 MB
node distances:
node 0 1
0: 10 21
1: 21 10
Error log
tail -f /var/log/nova/nova-conductor.log ::
default default] Failed to compute_task_build_instances: No valid host was
found. There are not enough hosts available.
Traceback (most recent call last):
File "/usr/lib/python2.7/site-packages/oslo_messaging/rpc/server.py", line
226, in inner
return func(*args, **kwargs)
File "/usr/lib/python2.7/site-packages/nova/scheduler/manager.py", line
154, in select_destinations
allocation_request_version, return_alternates)
File "/usr/lib/python2.7/site-packages/nova/scheduler/filter_scheduler.py",
line 91, in select_destinations
allocation_request_version, return_alternates)
File "/usr/lib/python2.7/site-packages/nova/scheduler/filter_scheduler.py",
line 243, in _schedule
claimed_instance_uuids)
File "/usr/lib/python2.7/site-packages/nova/scheduler/filter_scheduler.py",
line 280, in _ensure_sufficient_hosts
raise exception.NoValidHost(reason=reason)
NoValidHost: No valid host was found. There are not enough hosts available.
: NoValidHost_Remote: No valid host was found. There are not enough hosts
available.
Error log
tail -f /var/log/nova/nova-scheduler.log::
2018-06-15 16:52:33.457 5829 DEBUG nova.virt.hardware
[req-be251765-6c3b-46aa-ae05-6c2e12ae8661 7e909565a4b847fe81cd6d1cf778c893
b2760ba26e5645bf9856669d560d91c7 - default default] Attempting to fit instance
cell
InstanceNUMACell(cpu_pinning_raw=None,cpu_policy=None,cpu_thread_policy=None,cpu_topology=<?>,cpuset=set([0]),cpuset_reserved=None,id=0,memory=512,pagesize=1024)
on host_cell
NUMACell(cpu_usage=0,cpuset=set([8,9,10,11,12,13,14,15,24,25,26,27,28,29,30,31]),id=1,memory=131072,memory_usage=0,mempages=[NUMAPagesTopology,NUMAPagesTopology],pinned_cpus=set([]),siblings=[set([8,24]),set([14,30]),set([15,31]),set([11,27]),set([10,26]),set([12,28]),set([9,25]),set([13,29])])
_numa_fit_instance_cell
/usr/lib/python2.7/site-packages/nova/virt/hardware.py:974
2018-06-15 16:52:33.458 5829 DEBUG nova.virt.hardware
[req-be251765-6c3b-46aa-ae05-6c2e12ae8661 7e909565a4b847fe81cd6d1cf778c893
b2760ba26e5645bf9856669d560d91c7 - default default] No pinning requested,
considering limitations on usable cpu and memory _numa_fit_instance_cell
/usr/lib/python2.7/site-packages/nova/virt/hardware.py:1003
2018-06-15 16:52:33.459 5829 DEBUG
nova.scheduler.filters.numa_topology_filter
[req-be251765-6c3b-46aa-ae05-6c2e12ae8661 7e909565a4b847fe81cd6d1cf778c893
b2760ba26e5645bf9856669d560d91c7 - default default] [instance:
b1dff78a-ff24-4337-9cdb-edcecc9f9d00] controller, controller fails NUMA
topology requirements. The instance does not fit on this host. host_passes
/usr/lib/python2.7/site-packages/nova/scheduler/filters/numa_topology_filter.py:104
2018-06-15 16:52:33.459 5829 INFO nova.filters
[req-be251765-6c3b-46aa-ae05-6c2e12ae8661 7e909565a4b847fe81cd6d1cf778c893
b2760ba26e5645bf9856669d560d91c7 - default default] Filter NUMATopologyFilter
returned 0 hosts
2018-06-15 16:52:33.459 5829 DEBUG nova.filters
[req-be251765-6c3b-46aa-ae05-6c2e12ae8661 7e909565a4b847fe81cd6d1cf778c893
b2760ba26e5645bf9856669d560d91c7 - default default] Filtering removed all hosts
for the request with instance ID 'b1dff78a-ff24-4337-9cdb-edcecc9f9d00'. Filter
results: [('RetryFilter', [(u'controller', u'controller')]),
('AvailabilityZoneFilter', [(u'controller', u'controller')]), ('ComputeFilter',
[(u'controller', u'controller')]), ('ComputeCapabilitiesFilter',
[(u'controller', u'controller')]), ('ImagePropertiesFilter', [(u'controller',
u'controller')]), ('ServerGroupAntiAffinityFilter', [(u'controller',
u'controller')]), ('ServerGroupAffinityFilter', [(u'controller',
u'controller')]), ('NUMATopologyFilter', None)] get_filtered_objects
/usr/lib/python2.7/site-packages/nova/filters.py:129
2018-06-15 16:52:33.460 5829 INFO nova.filters
[req-be251765-6c3b-46aa-ae05-6c2e12ae8661 7e909565a4b847fe81cd6d1cf778c893
b2760ba26e5645bf9856669d560d91c7 - default default] Filtering removed all hosts
for the request with instance ID 'b1dff78a-ff24-4337-9cdb-edcecc9f9d00'. Filter
results: ['RetryFilter: (start: 1, end: 1)', 'AvailabilityZoneFilter: (start:
1, end: 1)', 'ComputeFilter: (start: 1, end: 1)', 'ComputeCapabilitiesFilter:
(start: 1, end: 1)', 'ImagePropertiesFilter: (start: 1, end: 1)',
'ServerGroupAntiAffinityFilter: (start: 1, end: 1)',
'ServerGroupAffinityFilter: (start: 1, end: 1)', 'NUMATopologyFilter: (start:
1, end: 0)']
2018-06-15 16:52:33.460 5829 DEBUG nova.scheduler.filter_scheduler
[req-be251765-6c3b-46aa-ae05-6c2e12ae8661 7e909565a4b847fe81cd6d1cf778c893
b2760ba26e5645bf9856669d560d91c7 - default default] Filtered []
_get_sorted_hosts
/usr/lib/python2.7/site-packages/nova/scheduler/filter_scheduler.py:404
2018-06-15 16:52:33.460 5829 DEBUG nova.scheduler.filter_scheduler
[req-be251765-6c3b-46aa-ae05-6c2e12ae8661 7e909565a4b847fe81cd6d1cf778c893
b2760ba26e5645bf9856669d560d91c7 - default default] There are 0 hosts available
but 1 instances requested to build. _ensure_sufficient_hosts
/usr/lib/python2.7/site-packages/nova/scheduler/filter_scheduler.py:278
2018-06-15 16:52:42.945 5829 DEBUG oslo_service.periodic_task
[req-be251765-6c3b-46aa-ae05-6c2e12ae8661 7e909565a4b847fe81cd6d1cf778c893
b2760ba26e5645bf9856669d560d91c7 - default default] Running periodic task
SchedulerManager._run_periodic_tasks run_periodic_tasks
/usr/lib/python2.7/site-packages/oslo_service/periodic_task.py:215
This error is where I did not configure it?
To manage notifications about this bug go to:
https://bugs.launchpad.net/nova/+bug/1777089/+subscriptions
--
Mailing list: https://launchpad.net/~yahoo-eng-team
Post to : yahoo-eng-team@lists.launchpad.net
Unsubscribe : https://launchpad.net/~yahoo-eng-team
More help : https://help.launchpad.net/ListHelp
