Hi everyone,
I have a VM right now that is setup as my gaming machine with video
pass-through, it works perfectly except when there's I/O on the host.
When there is I/O, the VM will have micro-freeze events where everything
locks up for milliseconds up to a second at a time at frequent intervals,
the host is on NVMe, CPUs are pinned with a dedicated core for emulator.
Whatever I put, virtio, virtio-scsi, cache=none, cache=writeback, threads
or native, as soon as there is I/O, I get the same problem, to the point
where it causes disconnections on the virtual USB bus.
VM is setup with hugepages, CPU pinning, I tried making qemu threads
realtime with no success (even blue screens). I tried with dedicated cores
(isolcpus), i440/q35, now I'm out of ideas. I do run a custom kernel to be
able to use pcie_acs_override=downstream,multifunction in GRUB because I
can't use passthrough otherwise.
This problem has been following me from my old setup to my new setup
Old setup:
ASUS B150 PLUS USB3
Core i7 6700K
16 GB DDR3
SATA raw LVM volume for Windows
New setup
ASRock Z390 Taichi
Core i7 9700K
48 GB DDR4
NVMe raw LVM volume for Windows
Any ideas would be appreciated.
Here is the libvirt XML definition (I excluded the buses):
<domain type='kvm'>
<name>win10-jerome</name>
<uuid>fc19f3c1-b8c6-4184-9b43-159157eff5b5</uuid>
<memory unit='KiB'>15360000</memory>
<currentMemory unit='KiB'>15360000</currentMemory>
<memoryBacking>
<hugepages/>
<nosharepages/>
<locked/>
</memoryBacking>
<vcpu placement='static'>4</vcpu>
<cputune>
<vcpupin vcpu='0' cpuset='0'/>
<vcpupin vcpu='1' cpuset='1'/>
<vcpupin vcpu='2' cpuset='2'/>
<vcpupin vcpu='3' cpuset='3'/>
<emulatorpin cpuset='7'/>
</cputune>
<resource>
<partition>/machine</partition>
</resource>
<os>
<type arch='x86_64' machine='pc-q35-4.0'>hvm</type>
<boot dev='hd'/>
<bootmenu enable='yes'/>
</os>
<features>
<acpi/>
<apic/>
<hyperv>
<relaxed state='on'/>
<vapic state='on'/>
<spinlocks state='on' retries='8191'/>
<vpindex state='on'/>
<runtime state='on'/>
<synic state='on'/>
<stimer state='on'/>
<reset state='on'/>
<frequencies state='on'/>
</hyperv>
<ioapic driver='kvm'/>
</features>
<cpu mode='host-passthrough' check='none'>
<topology sockets='1' cores='4' threads='1'/>
<feature policy='require' name='invtsc'/>
</cpu>
<clock offset='utc'>
<timer name='hypervclock' present='yes'/>
<timer name='hpet' present='yes'/>
</clock>
<devices>
<emulator>/usr/bin/kvm-spice</emulator>
<disk type='block' device='disk'>
<driver name='qemu' type='raw' cache='none' io='native'/>
<source dev='/dev/vgP4Root/win10jerome'/>
<backingStore/>
<target dev='sda' bus='sata'/>
<address type='drive' controller='0' bus='0' target='0' unit='0'/>
</disk>
<disk type='block' device='disk'>
<driver name='qemu' type='raw' cache='none' io='native'
discard='ignore' detect_zeroes='off'/>
<source dev='/dev/vgP4Root/win10data'/>
<target dev='sdb' bus='scsi'/>
<address type='drive' controller='0' bus='0' target='0' unit='1'/>
</disk>
<disk type='block' device='disk'>
<driver name='qemu' type='raw' cache='none' io='native'
discard='ignore' detect_zeroes='off'/>
<source dev='/dev/vgP4Root/win10swap'/>
<target dev='sdd' bus='scsi'/>
<address type='drive' controller='0' bus='0' target='0' unit='3'/>
</disk>
<interface type='direct'>
<mac address='52:54:00:31:1e:f9'/>
<source dev='eth0' mode='bridge'/>
<model type='virtio'/>
<address type='pci' domain='0x0000' bus='0x04' slot='0x00'
function='0x0'/>
</interface>
<input type='mouse' bus='ps2'/>
<input type='keyboard' bus='ps2'/>
<graphics type='spice' autoport='yes' listen='127.0.0.1'>
<listen type='address' address='127.0.0.1'/>
</graphics>
<video>
<model type='qxl' ram='65536' vram='16384' vgamem='16384' heads='1'
primary='yes'/>
<address type='pci' domain='0x00 <hostdev mode='subsystem'
type='pci' managed='yes'>
</video>
<hostdev mode='subsystem' type='pci' managed='yes'>
<source>
<address domain='0x0000' bus='0x02' slot='0x00' function='0x0'/>
</source>
<rom bar='on'/>
<address type='pci' domain='0x0000' bus='0x02' slot='0x00'
function='0x0' multifunction='on'/>
</hostdev>
<hostdev mode='subsystem' type='pci' managed='yes'>
<source>
<address domain='0x0000' bus='0x02' slot='0x00' function='0x1'/>
</source>
<rom bar='on'/>
<address type='pci' domain='0x0000' bus='0x02' slot='0x00'
function='0x1'/>
</hostdev>
00' bus='0x00' slot='0x01' function='0x0'/>
<rng model='virtio'>
<backend model='random'>/dev/urandom</backend>
<address type='pci' domain='0x0000' bus='0x07' slot='0x00'
function='0x0'/>
</rng>
</devices>
</domain>
Generated command line:
qemu-system-x86_64 -enable-kvm -name guest=win10-jerome,debug-threads=on -S
-object
secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-22-win10-jerome/master-key.aes
-machine
pc-q35-4.0,accel=kvm,usb=off,dump-guest-core=off,mem-merge=off,kernel_irqchip=on
-cpu
host,invtsc=on,hv_time,hv_relaxed,hv_vapic,hv_spinlocks=0x1fff,hv_vpindex,hv_runtime,hv_synic,hv_stimer,hv_reset,hv_frequencies
-m 15000 -mem-prealloc -mem-path
/dev/hugepages/libvirt/qemu/22-win10-jerome -overcommit mem-lock=on -smp
4,sockets=1,cores=4,threads=1 -uuid fc19f3c1-b8c6-4184-9b43-159157eff5b5
-no-user-config -nodefaults -chardev
socket,id=charmonitor,fd=25,server,nowait -mon
chardev=charmonitor,id=monitor,mode=control -rtc base=utc -no-shutdown
-global ICH9-LPC.disable_s3=0 -global ICH9-LPC.disable_s4=0 -boot
menu=on,strict=on -device
pcie-root-port,port=0x10,chassis=1,id=pci.1,bus=pcie.0,multifunction=on,addr=0x2
-device pcie-root-port,port=0x11,chassis=2,id=pci.2,bus=pcie.0,addr=0x2.0x1
-device pcie-root-port,port=0x12,chassis=3,id=pci.3,bus=pcie.0,addr=0x2.0x2
-device pcie-root-port,port=0x13,chassis=4,id=pci.4,bus=pcie.0,addr=0x2.0x3
-device pcie-root-port,port=0x14,chassis=5,id=pci.5,bus=pcie.0,addr=0x2.0x4
-device pcie-root-port,port=0x15,chassis=6,id=pci.6,bus=pcie.0,addr=0x2.0x5
-device pcie-root-port,port=0x16,chassis=7,id=pci.7,bus=pcie.0,addr=0x2.0x6
-device pcie-root-port,port=0x17,chassis=8,id=pci.8,bus=pcie.0,addr=0x2.0x7
-device i82801b11-bridge,id=pci.9,bus=pcie.0,addr=0x1e -device
pci-bridge,chassis_nr=10,id=pci.10,bus=pci.9,addr=0x0 -device
ich9-usb-ehci1,id=usb,bus=pcie.0,addr=0x1d.0x7 -device
ich9-usb-uhci1,masterbus=usb.0,firstport=0,bus=pcie.0,multifunction=on,addr=0x1d
-device ich9-usb-uhci2,masterbus=usb.0,firstport=2,bus=pcie.0,addr=0x1d.0x1
-device ich9-usb-uhci3,masterbus=usb.0,firstport=4,bus=pcie.0,addr=0x1d.0x2
-device virtio-scsi-pci,id=scsi0,bus=pci.5,addr=0x0 -device
virtio-serial-pci,id=virtio-serial0,bus=pci.1,addr=0x0 -drive
file=/dev/vgP4Root/win10jerome,format=raw,if=none,id=drive-sata0-0-0,cache=none,aio=native
-device
ide-hd,bus=ide.0,drive=drive-sata0-0-0,id=sata0-0-0,bootindex=1,write-cache=on
-drive
file=/dev/vgP4Root/win10data,format=raw,if=none,id=drive-scsi0-0-0-1,cache=none,discard=ignore,detect-zeroes=off,aio=native
-device
scsi-hd,bus=scsi0.0,channel=0,scsi-id=0,lun=1,device_id=drive-scsi0-0-0-1,drive=drive-scsi0-0-0-1,id=scsi0-0-0-1,write-cache=on
-drive
file=/dev/vgP4Root/win10swap,format=raw,if=none,id=drive-scsi0-0-0-3,cache=none,discard=ignore,detect-zeroes=off,aio=native
-device
scsi-hd,bus=scsi0.0,channel=0,scsi-id=0,lun=3,device_id=drive-scsi0-0-0-3,drive=drive-scsi0-0-0-3,id=scsi0-0-0-3,write-cache=on
-netdev tap,fd=27,id=hostnet0,vhost=on,vhostfd=28 -device
virtio-net-pci,netdev=hostnet0,id=net0,mac=52:54:00:31:1e:f9,bus=pci.4,addr=0x0
-spice port=5901,addr=127.0.0.1,disable-ticketing,seamless-migration=on
-device
qxl-vga,id=video0,ram_size=67108864,vram_size=16777216,vram64_size_mb=0,vgamem_mb=16,max_outputs=1,bus=pcie.0,addr=0x1
-chardev spicevmc,id=charredir0,name=usbredir -device
usb-redir,chardev=charredir0,id=redir0,bus=usb.0,port=2 -chardev
spicevmc,id=charredir1,name=usbredir -device
usb-redir,chardev=charredir1,id=redir1,bus=usb.0,port=3 -device
vfio-pci,host=02:00.0,id=hostdev0,bus=pci.2,multifunction=on,addr=0x0,rombar=1
-device vfio-pci,host=02:00.1,id=hostdev1,bus=pci.2,addr=0x0.0x1,rombar=1
-device usb-host,hostbus=1,hostaddr=7,id=hostdev2,bus=usb.0,port=5 -device
usb-host,hostbus=1,hostaddr=56,id=hostdev3,bus=usb.0,port=1 -device
usb-host,hostbus=1,hostaddr=25,id=hostdev4,bus=usb.0,port=4 -device
virtio-balloon-pci,id=balloon0,bus=pci.3,addr=0x0 -object
rng-random,id=objrng0,filename=/dev/urandom -device
virtio-rng-pci,rng=objrng0,id=rng0,bus=pci.7,addr=0x0 -sandbox
on,obsolete=deny,elevateprivileges=deny,spawn=deny,resourcecontrol=deny
-msg timestamp=on
