Hi:
We found a deadloop in qemu-system-aarch64, the backtrace is:
(gdb)
Id Target Id Frame
* 1 Thread 0xffffb4bee010 (LWP 1616556) "qemu-system-aar" 0x0000ffffb5d58d64
in __lll_lock_wait () from target:/lib64/libpthread.so.0
2 Thread 0xffffb4bea950 (LWP 1616625) "qemu-system-aar" 0x0000ffffb5ca1310
in syscall () from target:/lib64/libc.so.6
3 Thread 0xffffafffe950 (LWP 1616631) "qemu-system-aar" 0x0000ffffb5c069a0
in sigtimedwait () from target:/lib64/libc.so.6
4 Thread 0xffffae91c950 (LWP 1616637) "IO mon_iothread" 0x0000ffffb5c9b668
in poll () from target:/lib64/libc.so.6
5 Thread 0xffffade18950 (LWP 1616690) "CPU 0/KVM" 0x0000ffffb5c9b774
in ppoll () from target:/lib64/libc.so.6
6 Thread 0xffff579fe950 (LWP 1616731) "vnc_worker" 0x0000ffffb5d557f0
in pthread_cond_wait@@GLIBC_2.17 () from target:/lib64/libpthread.so.0
(gdb) thread app all bt
(gdb) bt
#0 0x0000ffffb5d58d64 in __lll_lock_wait () from target:/lib64/libpthread.so.0
#1 0x0000ffffb5d51db8 in pthread_mutex_lock () from
target:/lib64/libpthread.so.0
#2 0x0000aaaab4d0d2b4 in qemu_mutex_lock_impl (mutex=0xaaaab528b5c0
<qemu_global_mutex>, file=0xaaaab4e907d8 "../util/main-loop.c", line=242) at
../util/qemu-thread-posix.c:79
#3 0x0000aaaab4badf68 in qemu_mutex_lock_iothread_impl
(file=file@entry=0xaaaab4e907d8 "../util/main-loop.c", line=line@entry=242) at
../softmmu/cpus.c:485
#4 0x0000aaaab4d225b4 in os_host_main_loop_wait (timeout=0) at
../util/main-loop.c:242
#5 main_loop_wait (nonblocking=nonblocking@entry=0) at ../util/main-loop.c:520
#6 0x0000aaaab4b08b78 in qemu_main_loop () at ../softmmu/vl.c:1678
#7 0x0000aaaab4745db0 in main (argc=<optimized out>, argv=<optimized out>,
envp=<optimized out>) at ../softmmu/main.c:50
(gdb) thread app all bt
Thread 6 (Thread 0xffff579fe950 (LWP 1616731)):
#0 0x0000ffffb5d557f0 in pthread_cond_wait@@GLIBC_2.17 () from
target:/lib64/libpthread.so.0
#1 0x0000aaaab4d0d60c in qemu_cond_wait_impl (cond=<optimized out>,
mutex=0xaaaae73a7a78, file=0xaaaab4da2ea0 "../ui/vnc-jobs.c", line=215) at
../util/qemu-thread-posix.c:174
#2 0x0000aaaab4848b94 in vnc_worker_thread_loop
(queue=queue@entry=0xaaaae73a7a40) at ../ui/vnc-jobs.c:215
#3 0x0000aaaab4849110 in vnc_worker_thread (arg=0xaaaae73a7a40) at
../ui/vnc-jobs.c:325
#4 0x0000aaaab4d0d17c in qemu_thread_start (args=0xaaaae73a7ae0) at
../util/qemu-thread-posix.c:521
#5 0x0000ffffb5d4f718 in start_thread () from target:/lib64/libpthread.so.0
#6 0x0000ffffb5ca507c in thread_start () from target:/lib64/libc.so.6
Thread 5 (Thread 0xffffade18950 (LWP 1616690)):
#0 0x0000ffffb5c9b774 in ppoll () from target:/lib64/libc.so.6
#1 0x0000aaaab4d0f86c in ppoll (__ss=0x0, __timeout=0xffffade179c8,
__nfds=<optimized out>, __fds=<optimized out>) at /usr/include/bits/poll2.h:77
#2 qemu_poll_ns (fds=<optimized out>, nfds=<optimized out>,
timeout=timeout@entry=599999132090) at ../util/qemu-timer.c:349
#3 0x0000aaaab4d26630 in fdmon_poll_wait (ctx=0xaaaae2c7b0a0,
ready_list=0xffffade17ac0, timeout=599999132090) at ../util/fdmon-poll.c:80
#4 0x0000aaaab4d16050 in aio_poll (ctx=0xaaaae2c7b0a0,
blocking=blocking@entry=true) at ../util/aio-posix.c:607
#5 0x0000aaaab4c42308 in bdrv_drain_all_begin () at ../block/io.c:629
#6 bdrv_drain_all_begin () at ../block/io.c:594
#7 0x0000aaaab4c423ec in bdrv_drain_all () at ../block/io.c:680
#8 0x0000aaaab4c25668 in qmp_transaction (dev_list=0xffff504f2490,
has_props=false, props=0xffff50891f50, errp=errp@entry=0xffffade17bd8) at
../blockdev.c:2361
#9 0x0000aaaab4ca7348 in qmp_marshal_transaction (args=<optimized out>,
ret=<optimized out>, errp=0xffffb43eaec8) at qapi/qapi-commands-transaction.c:43
#10 0x0000aaaab4d17b24 in do_qmp_dispatch_bh (opaque=0xffffb43eaed8) at
../qapi/qmp-dispatch.c:110
#11 0x0000aaaab4d13050 in aio_bh_poll (ctx=ctx@entry=0xaaaae2c7b0a0) at
../util/async.c:164
#12 0x0000aaaab4d15bb8 in aio_poll (ctx=ctx@entry=0xaaaae2c7b0a0,
blocking=blocking@entry=true) at ../util/aio-posix.c:659
#13 0x0000aaaab4c6fc14 in blk_prw (blk=0xaaaae2ce95f0,
offset=offset@entry=10240, buf=<optimized out>, bytes=bytes@entry=512,
co_entry=co_entry@entry=0xaaaab4c70530 <blk_write_entry>, flags=flags@entry=0)
at ../block/block-backend.c:1335
#14 0x0000aaaab4c6fda4 in blk_pwrite (blk=<optimized out>,
offset=offset@entry=10240, buf=<optimized out>, count=count@entry=512,
flags=flags@entry=0) at ../block/block-backend.c:1501
#15 0x0000aaaab4936044 in pflash_update (offset=10240, offset@entry=10744,
size=size@entry=4, pfl=<optimized out>, pfl=<optimized out>) at
../hw/block/pflash_cfi01.c:398
#16 0x0000aaaab4936b64 in pflash_write (be=<optimized out>, width=4,
value=<optimized out>, offset=10744, pfl=0xaaaae2c7def0) at
../hw/block/pflash_cfi01.c:527
#17 pflash_mem_write_with_attrs (opaque=0xaaaae2c7def0, addr=10744,
value=<optimized out>, len=4, attrs=...) at ../hw/block/pflash_cfi01.c:685
#18 0x0000aaaab4ba65fc in access_with_adjusted_size (addr=addr@entry=10744,
value=value@entry=0xffffade17ef8, size=4, access_size_min=<optimized out>,
access_size_max=<optimized out>,
access_fn=access_fn@entry=0xaaaab4ba6990
<memory_region_write_with_attrs_accessor>, mr=0xaaaae2c7e2a0, attrs=...) at
../softmmu/memory.c:552
#19 0x0000aaaab4ba99a4 in memory_region_dispatch_write
(mr=mr@entry=0xaaaae2c7e2a0, addr=10744, data=<optimized out>, op=MO_32,
attrs=attrs@entry=...) at ../softmmu/memory.c:1507
#20 0x0000aaaab4b478a0 in flatview_write_continue (fv=fv@entry=0xffff5058cc80,
addr=addr@entry=67119608, attrs=..., ptr=ptr@entry=0xffffb6e6e028,
len=len@entry=4, addr1=<optimized out>, l=<optimized out>, mr=0xaaaae2c7e2a0)
at /usr/src/debug/qemu-5.2.0-1.aarch64/include/qemu/host-utils.h:164
#21 0x0000aaaab4b47a94 in flatview_write (fv=0xffff5058cc80, addr=67119608,
attrs=..., buf=0xffffb6e6e028, len=4) at ../softmmu/physmem.c:2799
#22 0x0000aaaab4b4a8f8 in address_space_write (as=<optimized out>,
addr=<optimized out>, attrs=..., buf=buf@entry=0xffffb6e6e028, len=<optimized
out>) at ../softmmu/physmem.c:2891
#23 0x0000aaaab4b4a9ac in address_space_rw (as=<optimized out>, addr=<optimized
out>, attrs=..., attrs@entry=..., buf=buf@entry=0xffffb6e6e028, len=<optimized
out>, is_write=<optimized out>) at ../softmmu/physmem.c:2901
#24 0x0000aaaab4bed810 in kvm_cpu_exec (cpu=cpu@entry=0xaaaae2d41bf0) at
../accel/kvm/kvm-all.c:2541
#25 0x0000aaaab4bb38cc in kvm_vcpu_thread_fn (arg=0xaaaae2d41bf0) at
../accel/kvm/kvm-cpus.c:49
#26 0x0000aaaab4d0d17c in qemu_thread_start (args=0xaaaae2da0300) at
../util/qemu-thread-posix.c:521
#27 0x0000ffffb5d4f718 in start_thread () from target:/lib64/libpthread.so.0
#28 0x0000ffffb5ca507c in thread_start () from target:/lib64/libc.so.6
Thread 4 (Thread 0xffffae91c950 (LWP 1616637)):
#0 0x0000ffffb5c9b668 in poll () from target:/lib64/libc.so.6
#1 0x0000ffffb63cbc94 in g_main_context_iterate.isra () from
target:/lib64/libglib-2.0.so.0
#2 0x0000ffffb63cc0b8 in g_main_loop_run () from target:/lib64/libglib-2.0.so.0
#3 0x0000aaaab4c27de4 in iothread_run (opaque=0xaaaae2a47d20) at
../iothread.c:80
#4 0x0000aaaab4d0d17c in qemu_thread_start (args=0xaaaae2d064c0) at
../util/qemu-thread-posix.c:521
#5 0x0000ffffb5d4f718 in start_thread () from target:/lib64/libpthread.so.0
#6 0x0000ffffb5ca507c in thread_start () from target:/lib64/libc.so.6
Thread 3 (Thread 0xffffafffe950 (LWP 1616631)):
#0 0x0000ffffb5c069a0 in sigtimedwait () from target:/lib64/libc.so.6
#1 0x0000ffffb5d5a174 in sigwait () from target:/lib64/libpthread.so.0
--Type <RET> for more, q to quit, c to continue without paging--
#2 0x0000aaaab4d2744c in sigwait_compat (opaque=0xaaaae2c6b670) at
../util/compatfd.c:37
#3 0x0000aaaab4d0d17c in qemu_thread_start (args=0xaaaae2c7add0) at
../util/qemu-thread-posix.c:521
#4 0x0000ffffb5d4f718 in start_thread () from target:/lib64/libpthread.so.0
#5 0x0000ffffb5ca507c in thread_start () from target:/lib64/libc.so.6
Thread 2 (Thread 0xffffb4bea950 (LWP 1616625)):
#0 0x0000ffffb5ca1310 in syscall () from target:/lib64/libc.so.6
#1 0x0000aaaab4d0dbd0 in qemu_futex_wait (val=<optimized out>, f=<optimized
out>) at /usr/src/debug/qemu-5.2.0-1.aarch64/include/qemu/futex.h:29
#2 qemu_event_wait (ev=0xaaaab52a69f0 <rcu_gp_event>) at
../util/qemu-thread-posix.c:460
#3 0x0000aaaab4cfefac in wait_for_readers () at ../util/rcu.c:135
#4 synchronize_rcu () at ../util/rcu.c:171
#5 0x0000aaaab4cff3b0 in call_rcu_thread (opaque=<optimized out>) at
../util/rcu.c:265
#6 0x0000aaaab4d0d17c in qemu_thread_start (args=0xaaaae2907b30) at
../util/qemu-thread-posix.c:521
#7 0x0000ffffb5d4f718 in start_thread () from target:/lib64/libpthread.so.0
#8 0x0000ffffb5ca507c in thread_start () from target:/lib64/libc.so.6
Thread 1 (Thread 0xffffb4bee010 (LWP 1616556)):
#0 0x0000ffffb5d58d64 in __lll_lock_wait () from target:/lib64/libpthread.so.0
#1 0x0000ffffb5d51db8 in pthread_mutex_lock () from
target:/lib64/libpthread.so.0
#2 0x0000aaaab4d0d2b4 in qemu_mutex_lock_impl (mutex=0xaaaab528b5c0
<qemu_global_mutex>, file=0xaaaab4e907d8 "../util/main-loop.c", line=242) at
../util/qemu-thread-posix.c:79
#3 0x0000aaaab4badf68 in qemu_mutex_lock_iothread_impl
(file=file@entry=0xaaaab4e907d8 "../util/main-loop.c", line=line@entry=242) at
../softmmu/cpus.c:485
#4 0x0000aaaab4d225b4 in os_host_main_loop_wait (timeout=0) at
../util/main-loop.c:242
#5 main_loop_wait (nonblocking=nonblocking@entry=0) at ../util/main-loop.c:520
#6 0x0000aaaab4b08b78 in qemu_main_loop () at ../softmmu/vl.c:1678
#7 0x0000aaaab4745db0 in main (argc=<optimized out>, argv=<optimized out>,
envp=<optimized out>) at ../softmmu/main.c:50
This deadloop can be reproduced by following steps:
1 Start up the qemu-system-aarch64;
2 virsh blockcopy --pivot;
Analyzing:
In libvirt, qemuDomainBlockPivot calls qmp command "transaction" to the qemu,
then qemu monitor in thread 4 while create a BH for "qmp_marshal_transaction",
and insert it to the global qemu_aio_context->bh_list;
The the qemu-system-aarch64 start with uefi and call pflash_write in thread 5,
which will call the aio_bh_poll. In aio_bh_poll, get the BH
"qmp_marshal_transaction".
Becouse in the qmp_marshal_transaction, it will call aio_poll again, and the
condition is "wait bdrv_drain_all_poll all return false", but in blk_prw , the
blk->in_flight is increased, so the aio_pool called by bdrv_drain_all_begin
can not exit.
Addtional information:
gdb:
(gdb) f 2
#2 0x0000aaaab4d0d2b4 in qemu_mutex_lock_impl (mutex=0xaaaab528b5c0
<qemu_global_mutex>, file=0xaaaab4e907d8 "../util/main-loop.c", line=242)
at ../util/qemu-thread-posix.c:79
79 err = pthread_mutex_lock(&mutex->lock);
(gdb) p mutex
$1 = (QemuMutex *) 0xaaaab528b5c0 <qemu_global_mutex>
(gdb) p mutex->lock
$2 = {__data = {__lock = 2, __count = 0, __owner = 1616690, __nusers = 1,
__kind = 0, __spins = 0, __list = {__prev = 0x0, __next = 0x0}},
__size = "\002\000\000\000\000\000\000\000\062\253\030\000\001", '\000'
<repeats 34 times>, __align = 2}
(gdb)
(gdb) f 13
#13 0x0000aaaab4c6fc14 in blk_prw (blk=0xaaaae2ce95f0,
offset=offset@entry=10240, buf=<optimized out>, bytes=bytes@entry=512,
co_entry=co_entry@entry=0xaaaab4c70530 <blk_write_entry>,
flags=flags@entry=0) at ../block/block-backend.c:1335
1335 BDRV_POLL_WHILE(blk_bs(blk), rwco.ret == NOT_DONE);
(gdb) p blk
$4 = (BlockBackend *) 0xaaaae2ce95f0
(gdb) p blk->in_flight
$5 = 1
(gdb) p blk->root->bs->in_flight
$6 = 0
(gdb)f 13
#13 0x0000aaaab4c6fc14 in blk_prw (blk=0xaaaae2ce95f0,
offset=offset@entry=10240, buf=<optimized out>, bytes=bytes@entry=512,
co_entry=co_entry@entry=0xaaaab4c70530 <blk_write_entry>,
flags=flags@entry=0) at ../block/block-backend.c:1335
1335 BDRV_POLL_WHILE(blk_bs(blk), rwco.ret == NOT_DONE);
(gdb) p blk
$4 = (BlockBackend *) 0xaaaae2ce95f0
(gdb) p blk->root->bs->in_flight
$6 = 0
(gdb) p blk->root->bs->filename
$7 = "/var/lib/libvirt/qemu/nvram/vm_2_1040_VARS.fd", '\000' <repeats 4050
times>
(gdb) p *blk->root->klass
$21 = {stay_at_node = false, parent_is_bds = false, inherit_options =
0xaaaab4745d88 <blk_root_inherit_options>,
change_media = 0xaaaab4c6f7a8 <blk_root_change_media>, resize =
0xaaaab4c6e9d0 <blk_root_resize>, get_name = 0xaaaab4c6ee10 <blk_root_get_name>,
get_parent_desc = 0xaaaab4c6f510 <blk_root_get_parent_desc>, drained_begin =
0xaaaab4c6eb40 <blk_root_drained_begin>,
drained_end = 0xaaaab4c6ea80 <blk_root_drained_end>, drained_poll =
0xaaaab4c6ec30 <blk_root_drained_poll>,
activate = 0xaaaab4c6f338 <blk_root_activate>, inactivate = 0xaaaab4c6ed88
<blk_root_inactivate>, attach = 0xaaaab4c6ea38 <blk_root_attach>,
detach = 0xaaaab4c6e9f0 <blk_root_detach>, update_filename = 0x0,
can_set_aio_ctx = 0xaaaab4c6ebb8 <blk_root_can_set_aio_ctx>,
set_aio_ctx = 0xaaaab4c6f048 <blk_root_set_aio_ctx>}
(gdb) p blk->in_flight
$5 = 1
static bool blk_root_drained_poll(BdrvChild *child)
{
BlockBackend *blk = child->opaque;
assert(blk->quiesce_counter);
return !!blk->in_flight;
}
xml:
[root@HS-22 ~]# virsh dumpxml vm_2_1040
<domain type='kvm'>
<name>vm_2_1040</name>
<uuid>ec0c6db1-3641-4435-af90-4ccf854bc158</uuid>
<metadata>
<hypercenter:hc
xmlns:hypercenter="http://libvirt.org/schemas/domain/qemu/1.0";>
<vmName>YXBpX3Rlc3RfbXVsdF9kaXNrXzE=</vmName>
<vmDesc>YXBpLXRlc3Q=</vmDesc>
<createVmType>0</createVmType>
</hypercenter:hc>
</metadata>
<maxMemory slots='16' unit='KiB'>4294967296</maxMemory>
<memory unit='KiB'>1048576</memory>
<currentMemory unit='KiB'>1048576</currentMemory>
<vcpu placement='static' current='1'>240</vcpu>
<os>
<type arch='aarch64' machine='virt-5.2'>hvm</type>
<loader readonly='yes'
type='pflash'>/usr/share/edk2/aarch64/QEMU_EFI-silent-pflash.raw</loader>
<nvram>/var/lib/libvirt/qemu/nvram/vm_2_1040_VARS.fd</nvram>
<boot dev='hd'/>
<boot dev='cdrom'/>
</os>
<features>
<acpi/>
<apic/>
<pae/>
<gic version='3'/>
</features>
<cpu mode='host-passthrough' check='none'>
<topology sockets='240' dies='1' cores='1' threads='1'/>
<numa>
<cell id='0' cpus='0-239' memory='1048576' unit='KiB'/>
</numa>
</cpu>
<clock offset='utc'/>
<on_poweroff>destroy</on_poweroff>
<on_reboot>restart</on_reboot>
<on_crash>destroy</on_crash>
<devices>
<emulator>/usr/bin/qemu-system-aarch64</emulator>
<disk type='file' device='disk'>
<driver name='qemu' type='qcow2' cache='none'/>
<source file='/data/vms/api_test_mult_disk_1'/>
<backingStore/>
<target dev='vda' bus='virtio'/>
<address type='pci' domain='0x0000' bus='0x05' slot='0x00'
function='0x0'/>
</disk>
<disk type='file' device='disk'>
<driver name='qemu' type='qcow2' cache='none'/>
<source file='/data/vms/move_snap.1661953145.6960313'/>
<backingStore type='file'>
<format type='raw'/>
<source file='/imgs/api_autotest_ocfs2/api_test_mult_disk_1_virtio_1'/>
</backingStore>
<target dev='vdb' bus='virtio'/>
<address type='pci' domain='0x0000' bus='0x08' slot='0x00'
function='0x0'/>
</disk>
<disk type='file' device='disk'>
<driver name='qemu' type='qcow2' cache='none'/>
<source file='/data/vms/move_snap.1661953145.6960402'/>
<backingStore type='file'>
<format type='raw'/>
<source file='/imgs/api_autotest_ocfs2/api_test_mult_disk_1_virtio_2'/>
</backingStore>
<target dev='vdc' bus='virtio'/>
<address type='pci' domain='0x0000' bus='0x09' slot='0x00'
function='0x0'/>
</disk>
<disk type='file' device='cdrom'>
<driver name='qemu' type='raw'/>
<target dev='sdz' bus='scsi'/>
<readonly/>
<address type='drive' controller='0' bus='0' target='25' unit='0'/>
</disk>
<controller type='scsi' index='0' model='virtio-scsi'>
<address type='pci' domain='0x0000' bus='0x02' slot='0x00'
function='0x0'/>
</controller>
<controller type='pci' index='0' model='pcie-root'/>
<controller type='pci' index='1' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='1' port='0x8'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x0'
multifunction='on'/>
</controller>
<controller type='pci' index='2' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='2' port='0x9'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01'
function='0x1'/>
</controller>
<controller type='pci' index='3' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='3' port='0xa'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01'
function='0x2'/>
</controller>
<controller type='pci' index='4' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='4' port='0xb'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01'
function='0x3'/>
</controller>
<controller type='pci' index='5' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='5' port='0xc'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01'
function='0x4'/>
</controller>
<controller type='pci' index='6' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='6' port='0xd'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01'
function='0x5'/>
</controller>
<controller type='pci' index='7' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='7' port='0xe'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01'
function='0x6'/>
</controller>
<controller type='pci' index='8' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='8' port='0xf'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01'
function='0x7'/>
</controller>
<controller type='pci' index='9' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='9' port='0x10'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'
multifunction='on'/>
</controller>
<controller type='pci' index='10' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='10' port='0x11'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02'
function='0x1'/>
</controller>
<controller type='pci' index='11' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='11' port='0x12'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02'
function='0x2'/>
</controller>
<controller type='pci' index='12' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='12' port='0x13'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02'
function='0x3'/>
</controller>
<controller type='pci' index='13' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='13' port='0x14'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02'
function='0x4'/>
</controller>
<controller type='pci' index='14' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='14' port='0x15'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02'
function='0x5'/>
</controller>
<controller type='pci' index='15' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='15' port='0x16'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02'
function='0x6'/>
</controller>
<controller type='pci' index='16' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='16' port='0x17'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02'
function='0x7'/>
</controller>
<controller type='pci' index='17' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='17' port='0x18'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'
multifunction='on'/>
</controller>
<controller type='pci' index='18' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='18' port='0x19'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03'
function='0x1'/>
</controller>
<controller type='pci' index='19' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='19' port='0x1a'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03'
function='0x2'/>
</controller>
<controller type='pci' index='20' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='20' port='0x1b'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03'
function='0x3'/>
</controller>
<controller type='pci' index='21' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='21' port='0x1c'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03'
function='0x4'/>
</controller>
<controller type='pci' index='22' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='22' port='0x1d'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03'
function='0x5'/>
</controller>
<controller type='pci' index='23' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='23' port='0x1e'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03'
function='0x6'/>
</controller>
<controller type='pci' index='24' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='24' port='0x1f'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03'
function='0x7'/>
</controller>
<controller type='pci' index='25' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='25' port='0x20'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'
multifunction='on'/>
</controller>
<controller type='pci' index='26' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='26' port='0x21'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x04'
function='0x1'/>
</controller>
<controller type='pci' index='27' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='27' port='0x22'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x04'
function='0x2'/>
</controller>
<controller type='pci' index='28' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='28' port='0x23'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x04'
function='0x3'/>
</controller>
<controller type='pci' index='29' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='29' port='0x24'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x04'
function='0x4'/>
</controller>
<controller type='pci' index='30' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='30' port='0x25'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x04'
function='0x5'/>
</controller>
<controller type='pci' index='31' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='31' port='0x26'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x04'
function='0x6'/>
</controller>
<controller type='pci' index='32' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='32' port='0x27'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x04'
function='0x7'/>
</controller>
<controller type='pci' index='33' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='33' port='0x28'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'
multifunction='on'/>
</controller>
<controller type='pci' index='34' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='34' port='0x29'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x05'
function='0x1'/>
</controller>
<controller type='pci' index='35' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='35' port='0x2a'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x05'
function='0x2'/>
</controller>
<controller type='pci' index='36' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='36' port='0x2b'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x05'
function='0x3'/>
</controller>
<controller type='pci' index='37' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='37' port='0x2c'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x05'
function='0x4'/>
</controller>
<controller type='pci' index='38' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='38' port='0x2d'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x05'
function='0x5'/>
</controller>
<controller type='pci' index='39' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='39' port='0x2e'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x05'
function='0x6'/>
</controller>
<controller type='pci' index='40' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='40' port='0x2f'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x05'
function='0x7'/>
</controller>
<controller type='pci' index='41' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='41' port='0x30'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x06' function='0x0'
multifunction='on'/>
</controller>
<controller type='pci' index='42' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='42' port='0x31'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x06'
function='0x1'/>
</controller>
<controller type='pci' index='43' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='43' port='0x32'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x06'
function='0x2'/>
</controller>
<controller type='pci' index='44' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='44' port='0x33'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x06'
function='0x3'/>
</controller>
<controller type='pci' index='45' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='45' port='0x34'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x06'
function='0x4'/>
</controller>
<controller type='pci' index='46' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='46' port='0x35'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x06'
function='0x5'/>
</controller>
<controller type='pci' index='47' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='47' port='0x36'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x06'
function='0x6'/>
</controller>
<controller type='pci' index='48' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='48' port='0x37'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x06'
function='0x7'/>
</controller>
<controller type='pci' index='49' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='49' port='0x38'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x07' function='0x0'
multifunction='on'/>
</controller>
<controller type='pci' index='50' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='50' port='0x39'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x07'
function='0x1'/>
</controller>
<controller type='pci' index='51' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='51' port='0x3a'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x07'
function='0x2'/>
</controller>
<controller type='pci' index='52' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='52' port='0x3b'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x07'
function='0x3'/>
</controller>
<controller type='usb' index='0' model='qemu-xhci'>
<address type='pci' domain='0x0000' bus='0x03' slot='0x00'
function='0x0'/>
</controller>
<controller type='virtio-serial' index='0'>
<address type='pci' domain='0x0000' bus='0x04' slot='0x00'
function='0x0'/>
</controller>
<interface type='bridge'>
<mac address='20:ac:9c:0c:4e:5b'/>
<source bridge='ovsbr0'/>
<virtualport type='openvswitch'>
<parameters interfaceid='f9712d0e-d689-40c7-a6cc-ccb70f0acb7c'/>
</virtualport>
<target dev='vm-20ac9c0c4e5b'/>
<model type='virtio'/>
<driver name='vhost'/>
<mtu size='1500'/>
<address type='pci' domain='0x0000' bus='0x01' slot='0x00'
function='0x0'/>
</interface>
<serial type='pty'>
<log file='/var/log/vm_2_1040.log'/>
<target type='system-serial' port='0'>
<model name='pl011'/>
</target>
</serial>
<console type='pty'>
<log file='/var/log/vm_2_1040.log'/>
<target type='serial' port='0'/>
</console>
<channel type='unix'>
<source mode='bind' path='/var/lib/libvirt/qemu/vm_2_1040.agent'/>
<target type='virtio' name='org.qemu.guest_agent.0'/>
<address type='virtio-serial' controller='0' bus='0' port='1'/>
</channel>
<input type='tablet' bus='usb'>
<address type='usb' bus='0' port='2'/>
</input>
<input type='mouse' bus='usb'>
<address type='usb' bus='0' port='3'/>
</input>
<input type='keyboard' bus='usb'>
<address type='usb' bus='0' port='4'/>
</input>
<graphics type='vnc' port='-1' autoport='yes' listen='0.0.0.0'
keymap='en-us'>
<listen type='address' address='0.0.0.0'/>
</graphics>
<video>
<model type='virtio' heads='1' primary='yes'/>
<address type='pci' domain='0x0000' bus='0x07' slot='0x00'
function='0x0'/>
</video>
<hub type='usb'>
<address type='usb' bus='0' port='1'/>
</hub>
<memballoon model='virtio'>
<stats period='10'/>
<address type='pci' domain='0x0000' bus='0x06' slot='0x00'
function='0x0'/>
</memballoon>
</devices>
</domain>
libvirtd.log:
2022-09-03 19:19:46.733+0000: 11179: warning :
qemuDomainObjBeginJobInternal:953 : Cannot start job (query, none, none) for
domain vm_2_1040; current job is (modify, none, none) owned by (11176
remoteDispatchDomainBlockJobAbort, 0 <null>, 0 <null> (flags=0x0)) for
(279639s, 0s, 0s)
2022-09-03 19:19:46.733+0000: 11179: error : qemuDomainObjBeginJobInternal:975
: Timed out during operation: cannot acquire state change lock (held by
monitor=remoteDispatchDomainBlockJobAbort)
2022-09-03 19:19:55.158+0000: 3515465: warning :
qemuDomainObjBeginJobInternal:953 : Cannot start job (query, none, none) for
domain vm_2_1040; current job is (modify, none, none) owned by (11176
remoteDispatchDomainBlockJobAbort, 0 <null>, 0 <null> (flags=0x0)) for
(279648s, 0s, 0s)
2022-09-03 19:19:55.158+0000: 3515465: error :
qemuDomainObjBeginJobInternal:975 : Timed out during operation: cannot acquire
state change lock (held by monitor=remoteDispatchDomainBlockJobAbort)
2022-09-03 19:19:57.065+0000: 2012406: warning :
qemuDomainObjBeginJobInternal:953 : Cannot start job (query, none, none) for
domain vm_2_1040; current job is (modify, none, none) owned by (11176
remoteDispatchDomainBlockJobAbort, 0 <null>, 0 <null> (flags=0x0)) for
(279650s, 0s, 0s)
2022-09-03 19:19:57.065+0000: 2012406: error :
qemuDomainObjBeginJobInternal:975 : Timed out during operation: cannot acquire
state change lock (held by monitor=remoteDispatchDomainBlockJobAbort)
wang...@chinatelecom.cn
