Hi Christian,
I cleared off the host leaving just the hung VM on it before running
those commands. Because the VM is hung up, I'm not able to grab
/proc/kallsyms and /proc/modules from it. I tried grabbing it from
another VM using the same image/kernel, but it has kernel page tables
isolation enabled, so I presume that's why it hasn't worked properly. If
this output it critical, I would need to stage some VMs and grab those
files and hope they fail live migration.
Below is the output you requested. This is new to me, so hopefully it's
correct.
Thank you,
Corey Melanson
$ sudo perf kvm stat --live
Analyze events for all VMs, all VCPUs:
VM-EXIT Samples Samples% Time% Min Time Max
Time Avg time
MSR_WRITE 269190 38.63% 42.74% 0.65us 855.99us
1.60us ( +- 2.06% )
PREEMPTION_TIMER 222622 31.95% 28.06% 0.61us 946.08us
1.27us ( +- 2.46% )
PENDING_INTERRUPT 203862 29.25% 29.11% 0.62us 799.59us
1.44us ( +- 2.57% )
EXTERNAL_INTERRUPT 1209 0.17% 0.09% 0.43us 123.06us
0.78us ( +- 13.36% )
Total Samples:696883, Total events handled time:1009576.31us.
$ sudo perf kvm stat live -d 30 --vcpu 0 --pid=53454
Analyze events for pid(s) 53454, VCPU 0:
VM-EXIT Samples Samples% Time% Min Time Max
Time Avg time
MSR_WRITE 2227999 50.05% 54.64% 0.64us 1228.11us
1.78us ( +- 0.74% )
PENDING_INTERRUPT 1112941 25.00% 25.20% 0.61us 1080.49us
1.64us ( +- 1.10% )
PREEMPTION_TIMER 1109447 24.92% 20.13% 0.63us 1017.27us
1.32us ( +- 1.07% )
EXTERNAL_INTERRUPT 1350 0.03% 0.03% 0.44us 366.12us
1.56us ( +- 28.55% )
Total Samples:4451737, Total events handled time:7264173.95us.
$ sudo perf stat -e 'kvm:*' sleep 30s
Performance counter stats for 'sleep 30s':
0 kvm:kvm_entry
0 kvm:kvm_hypercall
0 kvm:kvm_hv_hypercall
0 kvm:kvm_pio
0 kvm:kvm_fast_mmio
0 kvm:kvm_cpuid
0 kvm:kvm_apic
0 kvm:kvm_exit
0 kvm:kvm_inj_virq
0 kvm:kvm_inj_exception
0 kvm:kvm_page_fault
0 kvm:kvm_msr
0 kvm:kvm_cr
0 kvm:kvm_pic_set_irq
0 kvm:kvm_apic_ipi
0 kvm:kvm_apic_accept_irq
0 kvm:kvm_eoi
0 kvm:kvm_pv_eoi
0 kvm:kvm_nested_vmrun
0 kvm:kvm_nested_intercepts
0 kvm:kvm_nested_vmexit
0 kvm:kvm_nested_vmexit_inject
0 kvm:kvm_nested_intr_vmexit
0 kvm:kvm_invlpga
0 kvm:kvm_skinit
0 kvm:kvm_emulate_insn
0 kvm:vcpu_match_mmio
0 kvm:kvm_write_tsc_offset
0 kvm:kvm_update_master_clock
0 kvm:kvm_track_tsc
0 kvm:kvm_pml_full
0 kvm:kvm_ple_window
0 kvm:kvm_pvclock_update
0 kvm:kvm_wait_lapic_expire
0 kvm:kvm_enter_smm
0 kvm:kvm_pi_irte_update
0 kvm:kvm_hv_notify_acked_sint
0 kvm:kvm_hv_synic_set_irq
0 kvm:kvm_hv_synic_send_eoi
0 kvm:kvm_hv_synic_set_msr
0 kvm:kvm_hv_stimer_set_config
0 kvm:kvm_hv_stimer_set_count
0 kvm:kvm_hv_stimer_start_periodic
0 kvm:kvm_hv_stimer_start_one_shot
0 kvm:kvm_hv_stimer_callback
0 kvm:kvm_hv_stimer_expiration
0 kvm:kvm_hv_stimer_cleanup
0 kvm:kvm_avic_incomplete_ipi
0 kvm:kvm_avic_unaccelerated_access
0 kvm:kvm_hv_timer_state
0 kvm:kvm_hv_flush_tlb
0 kvm:kvm_hv_flush_tlb_ex
0 kvm:kvm_userspace_exit
0 kvm:kvm_vcpu_wakeup
0 kvm:kvm_set_irq
0 kvm:kvm_ioapic_set_irq
0 kvm:kvm_ioapic_delayed_eoi_inj
0 kvm:kvm_msi_set_irq
0 kvm:kvm_ack_irq
0 kvm:kvm_mmio
0 kvm:kvm_fpu
0 kvm:kvm_age_page
0 kvm:kvm_try_async_get_page
0 kvm:kvm_async_pf_doublefault
0 kvm:kvm_async_pf_not_present
0 kvm:kvm_async_pf_ready
0 kvm:kvm_async_pf_completed
0 kvm:kvm_halt_poll_ns
30.000754261 seconds time elapsed
0.000695000 seconds user
0.000000000 seconds sys
$ sudo perf kvm --host report -i perf.data.kvm
Samples: 3M of event 'cycles:ppp', Event count (approx.): 2508025040032
Overhead Command Shared Object Symbol
17.06% CPU 2/KVM [kernel.kallsyms] [k] vmx_vcpu_run
16.76% CPU 0/KVM [kernel.kallsyms] [k] vmx_vcpu_run
16.32% CPU 4/KVM [kernel.kallsyms] [k] vmx_vcpu_run
16.25% CPU 1/KVM [kernel.kallsyms] [k] vmx_vcpu_run
0.77% CPU 1/KVM [kernel.kallsyms] [k] vcpu_enter_guest
0.74% CPU 2/KVM [kernel.kallsyms] [k] native_write_msr
0.73% CPU 1/KVM [kernel.kallsyms] [k] native_write_msr
0.73% CPU 2/KVM [kernel.kallsyms] [k] vcpu_enter_guest
0.71% CPU 4/KVM [kernel.kallsyms] [k] vcpu_enter_guest
0.71% CPU 4/KVM [kernel.kallsyms] [k] native_write_msr
0.63% CPU 0/KVM [kernel.kallsyms] [k] native_write_msr
0.61% CPU 0/KVM [kernel.kallsyms] [k] vcpu_enter_guest
0.41% CPU 2/KVM [kernel.kallsyms] [k] vmexit_fill_RSB
0.40% CPU 1/KVM [kernel.kallsyms] [k] vmexit_fill_RSB
0.40% CPU 4/KVM [kernel.kallsyms] [k] vmexit_fill_RSB
0.36% CPU 0/KVM [kernel.kallsyms] [k] vmexit_fill_RSB
0.26% swapper [kernel.kallsyms] [k] intel_idle
0.24% CPU 4/KVM [kernel.kallsyms] [k] __indirect_thunk_start
0.24% CPU 4/KVM [kernel.kallsyms] [k] vmx_read_l1_tsc_offset
0.24% CPU 2/KVM [kernel.kallsyms] [k] __srcu_read_lock
0.23% CPU 4/KVM [kernel.kallsyms] [k] __srcu_read_lock
0.23% CPU 4/KVM [kernel.kallsyms] [k] vmx_set_hv_timer
0.23% CPU 1/KVM [kernel.kallsyms] [k] __srcu_read_lock
0.23% CPU 1/KVM [kernel.kallsyms] [k]
kvm_arch_vcpu_ioctl_run
0.22% CPU 4/KVM [kernel.kallsyms] [k]
kvm_arch_vcpu_ioctl_run
0.22% CPU 2/KVM [kernel.kallsyms] [k] intel_guest_get_msrs
0.22% CPU 2/KVM [kernel.kallsyms] [k]
kvm_arch_vcpu_ioctl_run
0.22% CPU 1/KVM [kernel.kallsyms] [k] intel_guest_get_msrs
0.22% CPU 1/KVM [kernel.kallsyms] [k] __indirect_thunk_start
0.22% CPU 3/KVM [kernel.kallsyms] [k] vmx_vcpu_run
0.21% CPU 0/KVM [kernel.kallsyms] [k] vmx_set_hv_timer
0.21% CPU 2/KVM [kernel.kallsyms] [k] __indirect_thunk_start
0.21% CPU 4/KVM [kernel.kallsyms] [k] vmx_set_msr
0.21% CPU 4/KVM [kernel.kallsyms] [k] intel_guest_get_msrs
0.21% CPU 1/KVM [kernel.kallsyms] [k] update_cr8_intercept
0.20% CPU 0/KVM [kernel.kallsyms] [k] __srcu_read_lock
0.20% CPU 2/KVM [kernel.kallsyms] [k] update_cr8_intercept
0.20% CPU 0/KVM [kernel.kallsyms] [k] vmx_set_msr
0.19% CPU 0/KVM [kernel.kallsyms] [k] __apic_accept_irq
0.19% CPU 4/KVM [kernel.kallsyms] [k] update_cr8_intercept
0.19% CPU 0/KVM [kernel.kallsyms] [k] vmx_read_l1_tsc_offset
0.18% CPU 2/KVM [kernel.kallsyms] [k] vmx_read_l1_tsc_offset
0.18% CPU 1/KVM [kernel.kallsyms] [k] __apic_accept_irq
0.18% CPU 1/KVM [kernel.kallsyms] [k] vmx_read_l1_tsc_offset
0.18% CPU 0/KVM [kernel.kallsyms] [k] intel_guest_get_msrs
0.18% CPU 0/KVM [kernel.kallsyms] [k] __indirect_thunk_start
0.17% CPU 4/KVM [kernel.kallsyms] [k]
skip_emulated_instruction
0.17% CPU 0/KVM [kernel.kallsyms] [k]
kvm_arch_vcpu_ioctl_run
0.17% CPU 4/KVM [kernel.kallsyms] [k] vmx_cache_reg
0.17% CPU 4/KVM [kernel.kallsyms] [k] apic_set_eoi
0.17% CPU 4/KVM [kernel.kallsyms] [k] __apic_accept_irq
0.17% CPU 0/KVM [kernel.kallsyms] [k] update_cr8_intercept
0.16% CPU 2/KVM [kernel.kallsyms] [k] __apic_accept_irq
0.16% CPU 2/KVM [kernel.kallsyms] [k] vmx_interrupt_allowed
0.15% CPU 1/KVM [kernel.kallsyms] [k] vmx_interrupt_allowed
0.15% CPU 4/KVM [kernel.kallsyms] [k] kvm_get_apic_interrupt
0.15% CPU 2/KVM [kernel.kallsyms] [k] vmx_set_msr
0.15% CPU 1/KVM [kernel.kallsyms] [k] vmx_save_host_state
0.15% CPU 4/KVM [kernel.kallsyms] [k] read_tsc
0.15% CPU 2/KVM [kernel.kallsyms] [k] kvm_get_apic_interrupt
0.15% CPU 1/KVM [kernel.kallsyms] [k] vmx_set_msr
0.15% CPU 0/KVM [kernel.kallsyms] [k] apic_set_eoi
0.14% CPU 0/KVM [kernel.kallsyms] [k] kvm_get_apic_interrupt
0.14% CPU 4/KVM [kernel.kallsyms] [k]
kvm_write_guest_offset_cached
(truncated)
$ sudo perf kvm --guest report -i perf.data.kvm
Samples: 3M of event 'cycles:ppp', Event count (approx.): 2508025040032
Overhead Command Shared Object Symbol
0.13% :53459 [kernel.vmlinux] [g] 0xffffffff81053538
0.11% :53454 [kernel.vmlinux] [g] 0xffffffff81053538
0.11% :53459 [kernel.vmlinux] [g] 0xffffffff810e476a
0.08% :53457 [kernel.vmlinux] [g] 0xffffffff81053538
0.08% :53456 [kernel.vmlinux] [g] 0xffffffff81053538
0.07% :53456 [kernel.vmlinux] [g] 0xffffffff8101e18f
0.05% :53454 [kernel.vmlinux] [g] 0xffffffff81053676
0.05% :53456 [kernel.vmlinux] [g] 0xffffffff8137a8eb
0.04% :53454 [kernel.vmlinux] [g] 0xffffffff81053677
0.04% :53457 [kernel.vmlinux] [g] 0xffffffff8101e1a0
0.01% :53457 [kernel.vmlinux] [g] 0xffffffff8174e251
0.01% :53457 [kernel.vmlinux] [g] 0xffffffff8101eefd
0.01% :53456 [kernel.vmlinux] [g] 0xffffffff8137a8ef
(truncated)
--
You received this bug notification because you are a member of Ubuntu
Bugs, which is subscribed to Ubuntu.
https://bugs.launchpad.net/bugs/1826051
Title:
VMs go to 100% CPU after live migration from Trusty to Bionic
To manage notifications about this bug go to:
https://bugs.launchpad.net/ubuntu/+source/qemu/+bug/1826051/+subscriptions
--
ubuntu-bugs mailing list
[email protected]
https://lists.ubuntu.com/mailman/listinfo/ubuntu-bugs
