Hi,
I am struggling with an issue for some time now, and still not able to figure
out what the issue is
My setup:
Host kernel - Linux 4.18.0-408.el8.x86_64 x86_64
Host OS - Centos 8 stream
Running KVM, and openstack on a 3 node cluster, I have a vm running on KVM
outside openstack as well. Few windows vms running on kvm through openstack as
well.
VM running outside openstack on KVM is Fedora 28
The setup works all well in case of no IO or no load.
Once I start certain workload on my windows vm or fedora vms. The vms start to
freeze over a span of a week. Mostly the Fedora vms which are running outside
openstack, but sometimes even windows vm running on openstack freeze.
Attaching the virt xml of the fedora vm.
One thing to note is, this occurs only on systems with lesser memory I have
seen this issue on hosts running 96G and even 128G. If hosts has large memory
it works fine, host 192G worked fine for me. Because of this fact my focus has
always been on memory. But everytime I check memory on the Host, there is
plenty of free memory on the host. Another thing is the fedora vm has direct
disk passthrough, I am not sure if that has something to do with this. I have
tried reading the logs but nothing conclusive there.
Is this some known issue? Am I missing a driver here. Let me know if any other
information is needed
If any one has observed this, please let me know. Thanks in advance for your
help.
Thanks,
Shrey
The information contained in this transmission may be confidential. Any
disclosure, copying, or further distribution of confidential information is not
permitted unless such privilege is explicitly granted in writing by Quantum.
Quantum reserves the right to have electronic communications, including email
and attachments, sent across its networks filtered through security software
programs and retain such messages in order to comply with applicable data
security and retention requirements. Quantum is not responsible for the proper
and complete transmission of the substance of this communication or for any
delay in its receipt.
<domain type='kvm' id='1'>
<name>Acuity-VM</name>
<uuid>8c161131-97bf-49d7-8d54-73882f8290ad</uuid>
<title>J100E4RV</title>
<description>srv01:J100E4RV</description>
<metadata>
<libosinfo:libosinfo xmlns:libosinfo="http://libosinfo.org/xmlns/libvirt/domain/1.0";>
<libosinfo:os id="http://fedoraproject.org/fedora/28"/>
</libosinfo:libosinfo>
</metadata>
<memory unit='KiB'>25165824</memory>
<currentMemory unit='KiB'>25165824</currentMemory>
<vcpu placement='static'>10</vcpu>
<resource>
<partition>/machine</partition>
</resource>
<os>
<type arch='x86_64' machine='pc-q35-rhel8.6.0'>hvm</type>
<boot dev='hd'/>
</os>
<features>
<acpi/>
<apic/>
<vmport state='off'/>
</features>
<cpu mode='custom' match='exact' check='full'>
<model fallback='forbid'>Cascadelake-Server</model>
<vendor>Intel</vendor>
<feature policy='require' name='ss'/>
<feature policy='require' name='vmx'/>
<feature policy='require' name='pdcm'/>
<feature policy='require' name='hypervisor'/>
<feature policy='require' name='tsc_adjust'/>
<feature policy='require' name='umip'/>
<feature policy='require' name='pku'/>
<feature policy='require' name='md-clear'/>
<feature policy='require' name='stibp'/>
<feature policy='require' name='arch-capabilities'/>
<feature policy='require' name='xsaves'/>
<feature policy='require' name='ibpb'/>
<feature policy='require' name='ibrs'/>
<feature policy='require' name='amd-stibp'/>
<feature policy='require' name='amd-ssbd'/>
<feature policy='require' name='rdctl-no'/>
<feature policy='require' name='ibrs-all'/>
<feature policy='require' name='skip-l1dfl-vmentry'/>
<feature policy='require' name='mds-no'/>
<feature policy='require' name='pschange-mc-no'/>
<feature policy='require' name='tsx-ctrl'/>
<feature policy='disable' name='hle'/>
<feature policy='disable' name='rtm'/>
<feature policy='disable' name='mpx'/>
</cpu>
<clock offset='utc'>
<timer name='rtc' tickpolicy='catchup'/>
<timer name='pit' tickpolicy='delay'/>
<timer name='hpet' present='no'/>
</clock>
<on_poweroff>destroy</on_poweroff>
<on_reboot>restart</on_reboot>
<on_crash>destroy</on_crash>
<pm>
<suspend-to-mem enabled='no'/>
<suspend-to-disk enabled='no'/>
</pm>
<devices>
<emulator>/usr/libexec/qemu-kvm</emulator>
<disk type='file' device='disk'>
<driver name='qemu' type='qcow2'/>
<source file='/acuity_utils/Pivot3_VM.qcow2' index='11'/>
<backingStore/>
<target dev='sda' bus='scsi'/>
<alias name='scsi0-0-0-0'/>
<address type='drive' controller='0' bus='0' target='0' unit='0'/>
</disk>
<disk type='file' device='disk'>
<driver name='qemu' type='qcow2'/>
<source file='/acuity_utils/Pivot3_VM_1.qcow2' index='10'/>
<backingStore/>
<target dev='sdb' bus='scsi'/>
<alias name='scsi0-0-0-1'/>
<address type='drive' controller='0' bus='0' target='0' unit='1'/>
</disk>
<disk type='block' device='disk'>
<driver name='qemu' type='raw' cache='none' io='native'/>
<source dev='/dev/LogT0Diskj' index='9'/>
<backingStore/>
<target dev='sdc' bus='scsi'/>
<alias name='scsi0-0-1-0'/>
<address type='drive' controller='0' bus='0' target='1' unit='0'/>
</disk>
<disk type='block' device='disk'>
<driver name='qemu' type='raw' cache='none' io='native'/>
<source dev='/dev/LogT1Diskb' index='8'/>
<backingStore/>
<target dev='sdd' bus='scsi'/>
<alias name='scsi0-0-2-0'/>
<address type='drive' controller='0' bus='0' target='2' unit='0'/>
</disk>
<disk type='block' device='disk'>
<driver name='qemu' type='raw' cache='none' io='native'/>
<source dev='/dev/LogT1Diskc' index='7'/>
<backingStore/>
<target dev='sde' bus='scsi'/>
<alias name='scsi0-0-2-1'/>
<address type='drive' controller='0' bus='0' target='2' unit='1'/>
</disk>
<disk type='block' device='disk'>
<driver name='qemu' type='raw' cache='none' io='native'/>
<source dev='/dev/LogT1Diskd' index='6'/>
<backingStore/>
<target dev='sdf' bus='scsi'/>
<alias name='scsi0-0-2-2'/>
<address type='drive' controller='0' bus='0' target='2' unit='2'/>
</disk>
<disk type='block' device='disk'>
<driver name='qemu' type='raw' cache='none' io='native'/>
<source dev='/dev/LogT1Diske' index='5'/>
<backingStore/>
<target dev='sdg' bus='scsi'/>
<alias name='scsi0-0-2-3'/>
<address type='drive' controller='0' bus='0' target='2' unit='3'/>
</disk>
<disk type='block' device='disk'>
<driver name='qemu' type='raw' cache='none' io='native'/>
<source dev='/dev/LogT1Diskf' index='4'/>
<backingStore/>
<target dev='sdh' bus='scsi'/>
<alias name='scsi0-0-2-4'/>
<address type='drive' controller='0' bus='0' target='2' unit='4'/>
</disk>
<disk type='block' device='disk'>
<driver name='qemu' type='raw' cache='none' io='native'/>
<source dev='/dev/LogT1Diskg' index='3'/>
<backingStore/>
<target dev='sdi' bus='scsi'/>
<alias name='scsi0-0-2-5'/>
<address type='drive' controller='0' bus='0' target='2' unit='5'/>
</disk>
<disk type='block' device='disk'>
<driver name='qemu' type='raw' cache='none' io='native'/>
<source dev='/dev/LogT1Diskh' index='2'/>
<backingStore/>
<target dev='sdj' bus='scsi'/>
<alias name='scsi0-0-2-6'/>
<address type='drive' controller='0' bus='0' target='2' unit='6'/>
</disk>
<disk type='block' device='disk'>
<driver name='qemu' type='raw' cache='none' io='native'/>
<source dev='/dev/LogT1Diski' index='1'/>
<backingStore/>
<target dev='sdk' bus='scsi'/>
<alias name='scsi0-0-2-7'/>
<address type='drive' controller='0' bus='0' target='2' unit='7'/>
</disk>
<controller type='scsi' index='0' model='virtio-scsi'>
<driver queues='10'/>
<alias name='scsi0'/>
<address type='pci' domain='0x0000' bus='0x01' slot='0x00' function='0x0'/>
</controller>
<controller type='usb' index='0' model='qemu-xhci' ports='15'>
<alias name='usb'/>
<address type='pci' domain='0x0000' bus='0x02' slot='0x00' function='0x0'/>
</controller>
<controller type='sata' index='0'>
<alias name='ide'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x1f' function='0x2'/>
</controller>
<controller type='pci' index='0' model='pcie-root'>
<alias name='pcie.0'/>
</controller>
<controller type='pci' index='1' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='1' port='0x10'/>
<alias name='pci.1'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0' multifunction='on'/>
</controller>
<controller type='pci' index='2' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='2' port='0x11'/>
<alias name='pci.2'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x1'/>
</controller>
<controller type='pci' index='3' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='3' port='0x12'/>
<alias name='pci.3'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x2'/>
</controller>
<controller type='pci' index='4' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='4' port='0x13'/>
<alias name='pci.4'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x3'/>
</controller>
<controller type='pci' index='5' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='5' port='0x14'/>
<alias name='pci.5'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x4'/>
</controller>
<controller type='pci' index='6' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='6' port='0x15'/>
<alias name='pci.6'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x5'/>
</controller>
<controller type='pci' index='7' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='7' port='0x16'/>
<alias name='pci.7'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x6'/>
</controller>
<controller type='pci' index='8' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='8' port='0x17'/>
<alias name='pci.8'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x7'/>
</controller>
<controller type='pci' index='9' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='9' port='0x18'/>
<alias name='pci.9'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0' multifunction='on'/>
</controller>
<controller type='pci' index='10' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='10' port='0x19'/>
<alias name='pci.10'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x1'/>
</controller>
<controller type='pci' index='11' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='11' port='0x1a'/>
<alias name='pci.11'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x2'/>
</controller>
<controller type='pci' index='12' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='12' port='0x1b'/>
<alias name='pci.12'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x3'/>
</controller>
<controller type='pci' index='13' model='pcie-root-port'>
<model name='pcie-root-port'/>
<target chassis='13' port='0x1c'/>
<alias name='pci.13'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x4'/>
</controller>
<controller type='virtio-serial' index='0'>
<alias name='virtio-serial0'/>
<address type='pci' domain='0x0000' bus='0x03' slot='0x00' function='0x0'/>
</controller>
<interface type='bridge' trustGuestRxFilters='yes'>
<mac address='52:54:00:af:23:29'/>
<source bridge='brsan0'/>
<target dev='vnet0'/>
<model type='virtio'/>
<driver name='vhost' queues='10'/>
<alias name='net0'/>
<address type='pci' domain='0x0000' bus='0x0a' slot='0x00' function='0x0'/>
</interface>
<interface type='bridge' trustGuestRxFilters='yes'>
<mac address='52:54:00:d7:40:07'/>
<source bridge='brsan1'/>
<target dev='vnet1'/>
<model type='virtio'/>
<driver name='vhost' queues='10'/>
<alias name='net1'/>
<address type='pci' domain='0x0000' bus='0x0b' slot='0x00' function='0x0'/>
</interface>
<interface type='network'>
<mac address='52:54:00:9d:e9:4b'/>
<source network='Pivot3_Network' portid='41baad3a-b293-469a-9fb0-f73a74f56957' bridge='p3_dednet4'/>
<target dev='vnet2'/>
<model type='virtio'/>
<alias name='net2'/>
<address type='pci' domain='0x0000' bus='0x0c' slot='0x00' function='0x0'/>
</interface>
<interface type='bridge' trustGuestRxFilters='yes'>
<mac address='52:54:00:f0:40:7b'/>
<source bridge='brmanage'/>
<target dev='vnet3'/>
<model type='virtio'/>
<driver name='vhost' queues='10'/>
<alias name='net3'/>
<address type='pci' domain='0x0000' bus='0x0d' slot='0x00' function='0x0'/>
</interface>
<serial type='pty'>
<source path='/dev/pts/0'/>
<target type='isa-serial' port='0'>
<model name='isa-serial'/>
</target>
<alias name='serial0'/>
</serial>
<console type='pty' tty='/dev/pts/0'>
<source path='/dev/pts/0'/>
<target type='serial' port='0'/>
<alias name='serial0'/>
</console>
<channel type='unix'>
<source mode='bind' path='/var/lib/libvirt/qemu/channel/target/domain-1-Acuity-VM/org.qemu.guest_agent.0'/>
<target type='virtio' name='org.qemu.guest_agent.0' state='connected'/>
<alias name='channel0'/>
<address type='virtio-serial' controller='0' bus='0' port='1'/>
</channel>
<channel type='spicevmc'>
<target type='virtio' name='com.redhat.spice.0' state='disconnected'/>
<alias name='channel1'/>
<address type='virtio-serial' controller='0' bus='0' port='2'/>
</channel>
<input type='tablet' bus='usb'>
<alias name='input0'/>
<address type='usb' bus='0' port='1'/>
</input>
<input type='mouse' bus='ps2'>
<alias name='input1'/>
</input>
<input type='keyboard' bus='ps2'>
<alias name='input2'/>
</input>
<graphics type='spice' port='5900' tlsPort='5901' autoport='yes' listen='127.0.0.1'>
<listen type='address' address='127.0.0.1'/>
<image compression='off'/>
</graphics>
<sound model='ich9'>
<alias name='sound0'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x1b' function='0x0'/>
</sound>
<audio id='1' type='spice'/>
<video>
<model type='qxl' ram='65536' vram='65536' vgamem='16384' heads='1' primary='yes'/>
<alias name='video0'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x0'/>
</video>
<redirdev bus='usb' type='spicevmc'>
<alias name='redir0'/>
<address type='usb' bus='0' port='2'/>
</redirdev>
<redirdev bus='usb' type='spicevmc'>
<alias name='redir1'/>
<address type='usb' bus='0' port='3'/>
</redirdev>
<memballoon model='none'/>
<rng model='virtio'>
<backend model='random'>/dev/urandom</backend>
<alias name='rng0'/>
<address type='pci' domain='0x0000' bus='0x04' slot='0x00' function='0x0'/>
</rng>
</devices>
<seclabel type='dynamic' model='dac' relabel='yes'>
<label>+107:+107</label>
<imagelabel>+107:+107</imagelabel>
</seclabel>
</domain>
