Hi guys, i get a strange problem on all my compute hosts with volume attachments by Cinder. All volumes are attached with multipath iSCSI.
* Ubuntu 12.04 * Icehouse * KVM how to reproduce: * Attach volume * mount volume and fill with files * umount * detach volume root@comp8:~# multipath -ll 36006016013503c00a91d1364f910e511 dm-1 DGC,VRAID size=3.1T features='1 queue_if_no_path' hwhandler='1 emc' wp=rw |-+- policy='round-robin 0' prio=1 status=active | `- 13:0:0:7 sdg 8:96 active ready running `-+- policy='round-robin 0' prio=0 status=enabled `- 12:0:0:7 sdf 8:80 active ready running Nova Compute log: 2015-06-26 08:17:01.994 13059 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.7/dist-packages/nova/openstack/common/excutils.py", line 68, in __exit__ 2015-06-26 08:17:01.994 13059 TRACE oslo.messaging.rpc.dispatcher six.reraise(self.type_, self.value, self.tb) 2015-06-26 08:17:01.994 13059 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.7/dist-packages/nova/compute/manager.py", line 307, in decorated_function 2015-06-26 08:17:01.994 13059 TRACE oslo.messaging.rpc.dispatcher return function(self, context, *args, **kwargs) 2015-06-26 08:17:01.994 13059 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.7/dist-packages/nova/compute/manager.py", line 4296, in detach_volume 2015-06-26 08:17:01.994 13059 TRACE oslo.messaging.rpc.dispatcher self._detach_volume(context, instance, bdm) 2015-06-26 08:17:01.994 13059 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.7/dist-packages/nova/compute/manager.py", line 4243, in _detach_volume 2015-06-26 08:17:01.994 13059 TRACE oslo.messaging.rpc.dispatcher connection_info = jsonutils.loads(bdm.connection_info) 2015-06-26 08:17:01.994 13059 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.7/dist-packages/nova/openstack/common/jsonutils.py", line 164, in loads 2015-06-26 08:17:01.994 13059 TRACE oslo.messaging.rpc.dispatcher return json.loads(s) 2015-06-26 08:17:01.994 13059 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.7/json/__init__.py", line 326, in loads 2015-06-26 08:17:01.994 13059 TRACE oslo.messaging.rpc.dispatcher return _default_decoder.decode(s) 2015-06-26 08:17:01.994 13059 TRACE oslo.messaging.rpc.dispatcher File "/usr/lib/python2.7/json/decoder.py", line 366, in decode 2015-06-26 08:17:01.994 13059 TRACE oslo.messaging.rpc.dispatcher obj, end = self.raw_decode(s, idx=_w(s, 0).end()) 2015-06-26 08:17:01.994 13059 TRACE oslo.messaging.rpc.dispatcher TypeError: expected string or buffer 2015-06-26 08:17:01.994 13059 TRACE oslo.messaging.rpc.dispatcher 2015-06-26 08:17:01.999 13059 ERROR oslo.messaging._drivers.common [-] Returning exception expected string or buffer to caller 2015-06-26 08:17:02.000 13059 ERROR oslo.messaging._drivers.common [-] ['Traceback (most recent call last):\n', ' File "/usr/lib/python2.7/dist-packages/oslo/messaging/rpc/dispatcher.py", line 133, in _dispatch_and_reply\n incoming.message))\n', ' File "/usr/lib/python2.7/dist-packages/oslo/messaging/rpc/dispatcher.py", line 176, in _dispatch\n return self._do_dispatch(endpoint, method, ctxt, args)\n', ' File "/usr/lib/python2.7/dist-packages/oslo/messaging/rpc/dispatcher.py", line 122, in _do_dispatch\n result = getattr(endpoint, method)(ctxt, **new_args)\n', ' File "/usr/lib/python2.7/dist-packages/nova/exception.py", line 88, in wrapped\n payload)\n', ' File "/usr/lib/python2.7/dist-packages/nova/openstack/common/excutils.py", line 68, in __exit__\n six.reraise(self.type_, self.value, self.tb)\n', ' File "/usr/lib/python2.7/dist-packages/nova/exception.py", line 71, in wrapped\n return f(self, context, *args, **kw)\n', ' File "/usr/lib/python2.7/dist-packages/nova/compute/manager.py", line 291, in decorated_function\n LOG.warning(msg, e, instance_uuid=instance_uuid)\n', ' File "/usr/lib/python2.7/dist-packages/nova/openstack/common/excutils.py", line 68, in __exit__\n six.reraise(self.type_, self.value, self.tb)\n', ' File "/usr/lib/python2.7/dist-packages/nova/compute/manager.py", line 268, in decorated_function\n return function(self, context, *args, **kwargs)\n', ' File "/usr/lib/python2.7/dist-packages/nova/compute/manager.py", line 320, in decorated_function\n e, sys.exc_info())\n', ' File "/usr/lib/python2.7/dist-packages/nova/openstack/common/excutils.py", line 68, in __exit__\n six.reraise(self.type_, self.value, self.tb)\n', ' File "/usr/lib/python2.7/dist-packages/nova/compute/manager.py", line 307, in decorated_function\n return function(self, context, *args, **kwargs)\n', ' File "/usr/lib/python2.7/dist-packages/nova/compute/manager.py", line 4296, in detach_volume\n self._detach_volume(context, instance, bdm)\n', ' File "/usr/lib/python2.7/dist-packages/nova/compute/manager.py", line 4243, in _detach_volume\n connection_info = jsonutils.loads(bdm.connection_info)\n', ' File "/usr/lib/python2.7/dist-packages/nova/openstack/common/jsonutils.py", line 164, in loads\n return json.loads(s)\n', ' File "/usr/lib/python2.7/json/__init__.py", line 326, in loads\n return _default_decoder.decode(s)\n', ' File "/usr/lib/python2.7/json/decoder.py", line 366, in decode\n obj, end = self.raw_decode(s, idx=_w(s, 0).end())\n', 'TypeError: expected string or buffer\n'] Nova-Compute.conf: [DEFAULT] libvirt_type=kvm libvirt_ovs_bridge=br-int libvirt_vif_type=ethernet libvirt_vif_driver=nova.virt.libvirt.vif.LibvirtGenericVIFDriver vif_plugging_is_fatal=True libvirt_nonblocking = True libvirt_use_virtio_for_bridges=True libvirt_cpu_mode=host-passthrough disk_cachemodes="file=writeback,block=none" running_deleted_instance_action=reap libvirt_inject_partition = -2 vif_plugging_timeout = 0 compute_driver=nova.virt.libvirt.LibvirtDriver libvirt_iscsi_use_multipath=True Anynone has seen this error before ? Is there any solution ? Cheers and thanks Heiko -- anynines.com
signature.asc
Description: OpenPGP digital signature
_______________________________________________ Mailing list: http://lists.openstack.org/cgi-bin/mailman/listinfo/openstack Post to : openstack@lists.openstack.org Unsubscribe : http://lists.openstack.org/cgi-bin/mailman/listinfo/openstack
