Hi,
I am struggling with an issue for some time now, and still not able to
figure out what the issue is
My setup:
Host kernel - Linux 4.18.0-408.el8.x86_64 x86_64
Host OS - Centos 8 stream
Running KVM, and openstack on a 3 node cluster, I have a vm running on KVM
outside openstack as well. Few windows vms running on kvm through openstack
as well.
VM running outside openstack on KVM is Fedora 28
The setup works all well in case of no IO or no load.
Once I start certain workload on my windows vm or fedora vms. The vms start
to freeze over a span of a week. Mostly the Fedora vms which are running
outside openstack, but sometimes even windows vm running on openstack
freeze.
Attaching the virt xml of the fedora vm.
One thing to note is, this occurs only on systems with lesser memory I have
seen this issue on hosts running 96G and even 128G. If hosts has large
memory it works fine, host 192G worked fine for me. Because of this fact my
focus has always been on memory. But everytime I check memory on the Host,
there is plenty of free memory on the host. Another thing is the fedora vm
has direct disk passthrough, I am not sure if that has something to do with
this. I have tried reading the logs but nothing conclusive there.
Is this some known issue? Am I missing a driver here. Let me know if any
other information is needed
If any one has observed this, please let me know. Thanks in advance for
your help.
Thanks,
Shrey
<domain type='kvm' id='1'>
<name>Acuity-VM</name>
<uuid>8c161131-97bf-49d7-8d54-73882f8290ad</uuid>
<title>J100E4RV</title>
<description>srv01:J100E4RV</description>
<metadata>
<libosinfo:libosinfo xmlns:libosinfo="http://libosinfo.org/xmlns/libvirt/domain/1.0";>
<libosinfo:os id="http://fedoraproject.org/fedora/28"/>
</libosinfo:libosinfo>
</metadata>
<memory unit='KiB'>25165824</memory>
<currentMemory unit='KiB'>25165824</currentMemory>
<vcpu placement='static'>10</vcpu>
<resource>
<partition>/machine</partition>
</resource>
<os>
<type arch='x86_64' machine='pc-q35-rhel8.6.0'>hvm</type>
<boot dev='hd'/>
</os>
<features>
<acpi/>
<apic/>
<vmport state='off'/>
</features>
<cpu mode='custom' match='exact' check='full'>
<model fallback='forbid'>Cascadelake-Server</model>
<vendor>Intel</vendor>
<feature policy='require' name='ss'/>
<feature policy='require' name='vmx'/>
<feature policy='require' name='pdcm'/>
<feature policy='require' name='hypervisor'/>
<feature policy='require' name='tsc_adjust'/>
<feature policy='require' name='umip'/>
<feature policy='require' name='pku'/>
<feature policy='require' name='md-clear'/>
<feature policy='require' name='stibp'/>
<feature policy='require' name='arch-capabilities'/>
<feature policy='require' name='xsaves'/>
<feature policy='require' name='ibpb'/>
<feature policy='require' name='ibrs'/>
<feature policy='require' name='amd-stibp'/>
<feature policy='require' name='amd-ssbd'/>
<feature policy='require' name='rdctl-no'/>
<feature policy='require' name='ibrs-all'/>
<feature policy='require' name='skip-l1dfl-vmentry'/>
<feature policy='require' name='mds-no'/>
<feature policy='require' name='pschange-mc-no'/>
<feature policy='require' name='tsx-ctrl'/>
<feature policy='disable' name='hle'/>
<feature policy='disable' name='rtm'/>
<feature policy='disable' name='mpx'/>
</cpu>
<clock offset='utc'>
<timer name='rtc' tickpolicy='catchup'/>
<timer name='pit' tickpolicy='delay'/>
<timer name='hpet' present='no'/>
</clock>
<on_poweroff>destroy</on_poweroff>
<on_reboot>restart</on_reboot>
<on_crash>destroy</on_crash>
<pm>
<suspend-to-mem enabled='no'/>
<suspend-to-disk enabled='no'/>
</pm>
<devices>
<emulator>/usr/libexec/qemu-kvm</emulator>
<disk type='file' device='disk'>
<driver name='qemu' type='qcow2'/>
<source file='/acuity_utils/Pivot3_VM.qcow2' index='11'/>
<backingStore/>
<target dev='sda' bus='scsi'/>
<alias name='scsi0-0-0-0'/>
<address type='drive' controller='0' bus='0' target='0' unit='0'/>
</disk>
<disk type='file' device='disk'>
<driver name='qemu' type='qcow2'/>
<source file='/acuity_utils/Pivot3_VM_1.qcow2' index='10'/>
<backingStore/>
<target dev='sdb' bus='scsi'/>
<alias name='scsi0-0-0-1'/>
<address type='drive' controller='0' bus='0' target='0' unit='1'/>
</disk>
<disk type='block' device='disk'>
<driver name='qemu' type='raw' cache='none' io='native'/>
<source dev='/dev/LogT0Diskj' index='9'/>
<backingStore/>
<target dev='sdc' bus='scsi'/>
<alias name='scsi0-0-1-0'/>
<address type='drive' controller='0' bus='0' target='1' unit='0'/>
</disk>
<disk type='block' device='disk'>
<driver name='qemu' type='raw' cache='none' io='native'/>
<source dev='/dev/LogT1Diskb' index='8'/>
<backingStore/>
<target dev='sdd' bus='scsi'/>
<alias name='scsi0-0-2-0'/>
<address type='drive' controller='0' bus='0' target='2' unit='0'/>
</disk>
<disk type='block' device='disk'>
<driver name='qemu' type='raw' cache='none' io='native'/>
<source dev='/dev/LogT1Diskc' index='7'/>
<backingStore/>
<target dev='sde' bus='scsi'/>
<alias name='scsi0-0-2-1'/>
<address type='drive' controller='0' bus='0' target='2' unit='1'/>
</disk>
<disk type='block' device='disk'>
<driver name='qemu' type='raw' cache='none' io='native'/>
<source dev='/dev/LogT1Diskd' index='6'/>
<backingStore/>
<target dev='sdf' bus='scsi'/>
<alias name='scsi0-0-2-2'/>
<address type='drive' controller='0' bus='0' target='2' unit='2'/>
</disk>
<disk type='block' device='disk'>
<driver name='qemu' type='raw' cache='none' io='native'/>
<source dev='/dev/LogT1Diske' index='5'/>
<backingStore/>
<target dev='sdg' bus='scsi'/>
<alias name='scsi0-0-2-3'/>
<address type='drive' controller='0' bus='0' target='2' unit='3'/>
</disk>
<disk type='block' device='disk'>
<driver name='qemu' type='raw' cache='none' io='native'/>
<source dev='/dev/LogT1Diskf' index='4'/>
<backingStore/>
<target dev='sdh' bus='scsi'/>
<alias name='scsi0-0-2-4'/>
<address type='drive' controller='0' bus='0' target='2' unit='4'/>
</disk>
<disk type='block' device='disk'>
<driver name='qemu' type='raw' cache='none' io='native'/>
<source dev='/dev/LogT1Diskg' index='3'/>
<backingStore/>
<target dev='sdi' bus='scsi'/>
<alias name='scsi0-0-2-5'/>
<address type='drive' controller='0' bus='0' target='2' unit='5'/>
</disk>
<disk type='block' device='disk'>
<driver name='qemu' type='raw' cache='none' io='native'/>
<source dev='/dev/LogT1Diskh' index='2'/>
<backingStore/>
<target dev='sdj' bus='scsi'/>
<alias name='scsi0-0-2-6'/>
<address type='drive' controller='0' bus='0' target='2' unit='6'/>
</disk>
<disk type='block' device='disk'>
<driver name='qemu' type='raw' cache='none' io='native'/>
<source dev='/dev/LogT1Diski' index='1'/>
<backingStore/>
<target dev='sdk' bus='scsi'/>
<alias name='scsi0-0-2-7'/>
<address type='drive' controller='0' bus='0' target='2' unit='7'/>
</disk>
<controller type='scsi' index='0' model='virtio-scsi'>
<driver queues='10'/>
<alias name='scsi0'/>
<address type='pci' domain='0x0000' bus='0x01' slot='0x00' function='0x0'/>
</controller>
<controller type='usb' index='0' model='qemu-xhci' ports='15'>
<alias name='usb'/>
<address type='pci' domain='0x0000' bus='0x02' slot='0x00' function='0x0'/>
</controller>
<controller type='sata' index='0'>
<alias name='ide'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x1f' function='0x2'/>
</controller>
<controller type='pci' index='0' model='pcie-root'>
<alias name='pcie.0'/>
</controller>
<controller type='pci' index='1' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='1' port='0x10'/>
<alias name='pci.1'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0' multifunction='on'/>
</controller>
<controller type='pci' index='2' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='2' port='0x11'/>
<alias name='pci.2'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x1'/>
</controller>
<controller type='pci' index='3' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='3' port='0x12'/>
<alias name='pci.3'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x2'/>
</controller>
<controller type='pci' index='4' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='4' port='0x13'/>
<alias name='pci.4'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x3'/>
</controller>
<controller type='pci' index='5' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='5' port='0x14'/>
<alias name='pci.5'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x4'/>
</controller>
<controller type='pci' index='6' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='6' port='0x15'/>
<alias name='pci.6'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x5'/>
</controller>
<controller type='pci' index='7' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='7' port='0x16'/>
<alias name='pci.7'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x6'/>
</controller>
<controller type='pci' index='8' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='8' port='0x17'/>
<alias name='pci.8'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x7'/>
</controller>
<controller type='pci' index='9' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='9' port='0x18'/>
<alias name='pci.9'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0' multifunction='on'/>
</controller>
<controller type='pci' index='10' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='10' port='0x19'/>
<alias name='pci.10'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x1'/>
</controller>
<controller type='pci' index='11' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='11' port='0x1a'/>
<alias name='pci.11'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x2'/>
</controller>
<controller type='pci' index='12' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='12' port='0x1b'/>
<alias name='pci.12'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x3'/>
</controller>
<controller type='pci' index='13' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='13' port='0x1c'/>
<alias name='pci.13'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x4'/>
</controller>
<controller type='virtio-serial' index='0'>
<alias name='virtio-serial0'/>
<address type='pci' domain='0x0000' bus='0x03' slot='0x00' function='0x0'/>
</controller>
<interface type='bridge' trustGuestRxFilters='yes'>
<mac address='52:54:00:af:23:29'/>
<source bridge='brsan0'/>
<target dev='vnet0'/>
<model type='virtio'/>
<driver name='vhost' queues='10'/>
<alias name='net0'/>
<address type='pci' domain='0x0000' bus='0x0a' slot='0x00' function='0x0'/>
</interface>
<interface type='bridge' trustGuestRxFilters='yes'>
<mac address='52:54:00:d7:40:07'/>
<source bridge='brsan1'/>
<target dev='vnet1'/>
<model type='virtio'/>
<driver name='vhost' queues='10'/>
<alias name='net1'/>
<address type='pci' domain='0x0000' bus='0x0b' slot='0x00' function='0x0'/>
</interface>
<interface type='network'>
<mac address='52:54:00:9d:e9:4b'/>
<source network='Pivot3_Network' portid='41baad3a-b293-469a-9fb0-f73a74f56957' bridge='p3_dednet4'/>
<target dev='vnet2'/>
<model type='virtio'/>
<alias name='net2'/>
<address type='pci' domain='0x0000' bus='0x0c' slot='0x00' function='0x0'/>
</interface>
<interface type='bridge' trustGuestRxFilters='yes'>
<mac address='52:54:00:f0:40:7b'/>
<source bridge='brmanage'/>
<target dev='vnet3'/>
<model type='virtio'/>
<driver name='vhost' queues='10'/>
<alias name='net3'/>
<address type='pci' domain='0x0000' bus='0x0d' slot='0x00' function='0x0'/>
</interface>
<serial type='pty'>
<source path='/dev/pts/0'/>
<target type='isa-serial' port='0'>
<model name='isa-serial'/>
</target>
<alias name='serial0'/>
</serial>
<console type='pty' tty='/dev/pts/0'>
<source path='/dev/pts/0'/>
<target type='serial' port='0'/>
<alias name='serial0'/>
</console>
<channel type='unix'>
<source mode='bind' path='/var/lib/libvirt/qemu/channel/target/domain-1-Acuity-VM/org.qemu.guest_agent.0'/>
<target type='virtio' name='org.qemu.guest_agent.0' state='connected'/>
<alias name='channel0'/>
<address type='virtio-serial' controller='0' bus='0' port='1'/>
</channel>
<channel type='spicevmc'>
<target type='virtio' name='com.redhat.spice.0' state='disconnected'/>
<alias name='channel1'/>
<address type='virtio-serial' controller='0' bus='0' port='2'/>
</channel>
<input type='tablet' bus='usb'>
<alias name='input0'/>
<address type='usb' bus='0' port='1'/>
</input>
<input type='mouse' bus='ps2'>
<alias name='input1'/>
</input>
<input type='keyboard' bus='ps2'>
<alias name='input2'/>
</input>
<graphics type='spice' port='5900' tlsPort='5901' autoport='yes' listen='127.0.0.1'>
<listen type='address' address='127.0.0.1'/>
<image compression='off'/>
</graphics>
<sound model='ich9'>
<alias name='sound0'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x1b' function='0x0'/>
</sound>
<audio id='1' type='spice'/>
<video>
<model type='qxl' ram='65536' vram='65536' vgamem='16384' heads='1' primary='yes'/>
<alias name='video0'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x0'/>
</video>
<redirdev bus='usb' type='spicevmc'>
<alias name='redir0'/>
<address type='usb' bus='0' port='2'/>
</redirdev>
<redirdev bus='usb' type='spicevmc'>
<alias name='redir1'/>
<address type='usb' bus='0' port='3'/>
</redirdev>
<memballoon model='none'/>
<rng model='virtio'>
<backend model='random'>/dev/urandom</backend>
<alias name='rng0'/>
<address type='pci' domain='0x0000' bus='0x04' slot='0x00' function='0x0'/>
</rng>
</devices>
<seclabel type='dynamic' model='dac' relabel='yes'>
<label>+107:+107</label>
<imagelabel>+107:+107</imagelabel>
</seclabel>
</domain>
