** Changed in: ubuntu-realtime
Status: In Progress => Fix Released
** No longer affects: kdump-tools (Ubuntu)
--
You received this bug notification because you are a member of Kernel
Packages, which is subscribed to kdump-tools in Ubuntu.
Matching subscriptions: Maintainer
https://bugs.launchpad.net/bugs/1995270
Title:
Kernel crash dump not getting generated
Status in ubuntu-realtime:
Fix Released
Bug description:
I have to enable the kernel crash dump feature to investigate certain
system hang issues. My intention is to enable the kernel crash dump
feature (https://ubuntu.com/server/docs/kernel-crash-dump) along with
kernel.hung_task_panic and kernel.hung_task_timeout_secs parameters
(in /etc/sysctl.conf), so that a kernel vmcore is generated when
kernel detects a hung task for more than 5 minutes. I will be then
able to send the vmcore files to Kernel engineers in Canonical for
investigation.
However, While trying to enable kernel crash dump, I'm unable to get a
vmcore generate using the test procedure described in "Testing the
Crash Dump Mechanism" section in:
https://ubuntu.com/server/docs/kernel-crash-dump
When I run the command: "echo c > /proc/sysrq-trigger", all I see is a
backtrace (image attached). I do not see the print .. "Begin: Saving
vmcore from kernel crash ..."
Upon system reboot there is no vmcore file under the /var/crash
directory. could you please help pointing out what steps i'm missing?
Here are some system information:
=======================================
Last login: Mon Oct 31 11:35:21 2022
root@vran-server-1:~# cat /proc/cmdline
BOOT_IMAGE=/vmlinuz-5.15.0-1025-realtime
root=/dev/mapper/ubuntu--vg-ubuntu--lv ro rhgb quiet skew_tick=1 nohz=on
nohz_full=2-23,26-47,50-71,74-95 rcu_nocbs=2-23,26-47,50-71,74-95
intel_pstate=disable nosoftlockup intel_iommu=on iommu=pt
usbcore.autosuspend=-1 selinux=0 enforcing=0 nmi_watchdog=0 softlockup_panic=0
audit=0 cgroup_memory=1 cgroup_enable=memory mce=off idle=poll
default_hugepagesz=1G skew_tick=1 idle=poll processor.max_cstate=1
intel_idle.max_cstate=0 rcu_nocb_poll kthread_cpus=0,1,72,73,48,49,24,25
irqaffinity=0,1,72,73,48,49,24,25 nosoftlockup tsc=nowatchdog
isolcpus=managed_irq,domain,2-23,26-47,50-71,74-95
systemd.cpu_affinity=0,1,72,73,48,49,24,25 cgroup.memory=nokmem
crashkernel=512M-:256M
root@vran-server-1:~# uname -a
Linux vran-server-1 5.15.0-1025-realtime #26 SMP PREEMPT_RT Thu Oct 20
18:14:07 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux
root@vran-server-1:~# kdump-config show
DUMP_MODE: kdump
USE_KDUMP: 1
KDUMP_COREDIR: /var/crash
crashkernel addr: 0x49000000
/var/lib/kdump/vmlinuz: symbolic link to /boot/vmlinuz-5.15.0-1025-realtime
kdump initrd:
/var/lib/kdump/initrd.img: symbolic link to
/var/lib/kdump/initrd.img-5.15.0-1025-realtime
current state: ready to kdump
kexec command:
/sbin/kexec -p --command-line="BOOT_IMAGE=/vmlinuz-5.15.0-1025-realtime
root=/dev/mapper/ubuntu--vg-ubuntu--lv ro rhgb quiet skew_tick=1 nohz=on
nohz_full=2-23,26-47,50-71,74-95 rcu_nocbs=2-23,26-47,50-71,74-95
intel_pstate=disable nosoftlockup intel_iommu=on iommu=pt
usbcore.autosuspend=-1 selinux=0 enforcing=0 nmi_watchdog=0 softlockup_panic=0
audit=0 cgroup_memory=1 cgroup_enable=memory mce=off idle=poll
default_hugepagesz=1G skew_tick=1 idle=poll processor.max_cstate=1
intel_idle.max_cstate=0 rcu_nocb_poll kthread_cpus=0,1,72,73,48,49,24,25
irqaffinity=0,1,72,73,48,49,24,25 nosoftlockup tsc=nowatchdog
isolcpus=managed_irq,domain,2-23,26-47,50-71,74-95
systemd.cpu_affinity=0,1,72,73,48,49,24,25 cgroup.memory=nokmem reset_devices
systemd.unit=kdump-tools-dump.service nr_cpus=1 irqpoll nousb"
--initrd=/var/lib/kdump/initrd.img /var/lib/kdump/vmlinuz
root@vran-server-1:~# cat /etc/default/kexec
# Defaults for kexec initscript
# sourced by /etc/init.d/kexec and /etc/init.d/kexec-load
# Load a kexec kernel (true/false)
LOAD_KEXEC=true
# Kernel and initrd image
KERNEL_IMAGE="/vmlinuz"
INITRD="/initrd.img"
# If empty, use current /proc/cmdline
APPEND=""
# Load the default kernel from grub config (true/false)
USE_GRUB_CONFIG=false
root@vran-server-1:~# cat /etc/default/kdump-tools
# kdump-tools configuration
# ---------------------------------------------------------------------------
# USE_KDUMP - controls kdump will be configured
# 0 - kdump kernel will not be loaded
# 1 - kdump kernel will be loaded and kdump is configured
#
USE_KDUMP=1
# ---------------------------------------------------------------------------
# Kdump Kernel:
# KDUMP_KERNEL - A full pathname to a kdump kernel.
# KDUMP_INITRD - A full pathname to the kdump initrd (if used).
# If these are not set, kdump-config will try to use the current kernel
# and initrd if it is relocatable. Otherwise, you will need to specify
# these manually.
KDUMP_KERNEL=/var/lib/kdump/vmlinuz
KDUMP_INITRD=/var/lib/kdump/initrd.img
# ---------------------------------------------------------------------------
# vmcore Handling:
# KDUMP_COREDIR - local path to save the vmcore to.
# KDUMP_FAIL_CMD - This variable can be used to cause a reboot or
# start a shell if saving the vmcore fails. If not set, "reboot -f"
# is the default.
# Example - start a shell if the vmcore copy fails:
# KDUMP_FAIL_CMD="echo 'makedumpfile FAILED.'; /bin/bash; reboot -f"
# KDUMP_DUMP_DMESG - This variable controls if the dmesg buffer is dumped.
# If unset or set to 1, the dmesg buffer is dumped. If set to 0, the dmesg
# buffer is not dumped.
# KDUMP_NUM_DUMPS - This variable controls how many dump files are kept on
# the machine to prevent running out of disk space. If set to 0 or unset,
# the variable is ignored and no dump files are automatically purged.
# KDUMP_COMPRESSION - Compress the dumpfile. No compression is used by
default.
# Supported compressions: bzip2, gzip, lz4, xz
KDUMP_COREDIR="/var/crash"
#KDUMP_FAIL_CMD="reboot -f"
#KDUMP_DUMP_DMESG=
#KDUMP_NUM_DUMPS=
#KDUMP_COMPRESSION=
# ---------------------------------------------------------------------------
# Makedumpfile options:
# MAKEDUMP_ARGS - extra arguments passed to makedumpfile (8). The default,
# if unset, is to pass '-c -d 31' telling makedumpfile to use compression
# and reduce the corefile to in-use kernel pages only.
#MAKEDUMP_ARGS="-c -d 31"
# ---------------------------------------------------------------------------
# Kexec/Kdump args
# KDUMP_KEXEC_ARGS - Additional arguments to the kexec command used to load
# the kdump kernel
# Example - Use this option on x86 systems with PAE and more than
# 4 gig of memory:
# KDUMP_KEXEC_ARGS="--elf64-core-headers"
# KDUMP_CMDLINE - The default is to use the contents of /proc/cmdline.
# Set this variable to override /proc/cmdline.
# KDUMP_CMDLINE_APPEND - Additional arguments to append to the command line
# for the kdump kernel. If unset, it defaults to
# "reset_devices systemd.unit=kdump-tools-dump.service nr_cpus=1 irqpoll
nousb"
#KDUMP_KEXEC_ARGS=""
#KDUMP_CMDLINE=""
#KDUMP_CMDLINE_APPEND="reset_devices systemd.unit=kdump-tools-dump.service
nr_cpus=1 irqpoll nousb"
# ---------------------------------------------------------------------------
# Architecture specific Overrides:
# ---------------------------------------------------------------------------
# Remote dump facilities:
# HOSTTAG - Select if hostname of IP address will be used as a prefix to the
# timestamped directory when sending files to the remote server.
# 'ip' is the default.
#HOSTTAG="hostname|[ip]"
# NFS - Hostname and mount point of the NFS server configured to receive
# the crash dump. The syntax must be {HOSTNAME}:{MOUNTPOINT}
# (e.g. remote:/var/crash)
# NFS_TIMEO - Timeout before NFS retries a request. See man nfs(5) for
details.
# NFS_RETRANS - Number of times NFS client retries a request. See man nfs(5)
for details.
#NFS="<nfs mount>"
#NFS_TIMEO="600"
#NFS_RETRANS="3"
# FTP - Hostname and path of the FTP server configured to receive the crash
dump.
# The syntax is {HOSTNAME}[:{PATH}] with PATH defaulting to /.
# FTP_USER - FTP username. A anonomous upload will be used if not set.
# FTP_PASSWORD - password for the FTP user
# FTP_PORT=21 - FTP port. Port 21 will be used by default.
#FTP="<server>:<path>"
#FTP_USER=""
#FTP_PASSWORD=""
#FTP_PORT=21
# SSH - username and hostname of the remote server that will receive the dump
# and dmesg files.
# SSH_KEY - Full path of the ssh private key to be used to login to the remote
# server. use kdump-config propagate to send the public key to the
# remote server
#SSH="<user at server>"
#SSH_KEY="<path>"
root@vran-server-1:~#
root@vran-server-1:~# cat /proc/sys/kernel/sysrq
176
root@vran-server-1:~# dmesg | grep -i crash
[ 0.000000] Command line: BOOT_IMAGE=/vmlinuz-5.15.0-1025-realtime
root=/dev/mapper/ubuntu--vg-ubuntu--lv ro rhgb quiet skew_tick=1 nohz=on
nohz_full=2-23,26-47,50-71,74-95 rcu_nocbs=2-23,26-47,50-71,74-95
intel_pstate=disable nosoftlockup intel_iommu=on iommu=pt
usbcore.autosuspend=-1 selinux=0 enforcing=0 nmi_watchdog=0 softlockup_panic=0
audit=0 cgroup_memory=1 cgroup_enable=memory mce=off idle=poll
default_hugepagesz=1G skew_tick=1 idle=poll processor.max_cstate=1
intel_idle.max_cstate=0 rcu_nocb_poll kthread_cpus=0,1,72,73,48,49,24,25
irqaffinity=0,1,72,73,48,49,24,25 nosoftlockup tsc=nowatchdog
isolcpus=managed_irq,domain,2-23,26-47,50-71,74-95
systemd.cpu_affinity=0,1,72,73,48,49,24,25 cgroup.memory=nokmem
crashkernel=512M-:256M
[ 0.012353] Reserving 256MB of memory at 1168MB for crashkernel (System
RAM: 195278MB)
[ 0.551413] Kernel command line: BOOT_IMAGE=/vmlinuz-5.15.0-1025-realtime
root=/dev/mapper/ubuntu--vg-ubuntu--lv ro rhgb quiet skew_tick=1 nohz=on
nohz_full=2-23,26-47,50-71,74-95 rcu_nocbs=2-23,26-47,50-71,74-95
intel_pstate=disable nosoftlockup intel_iommu=on iommu=pt
usbcore.autosuspend=-1 selinux=0 enforcing=0 nmi_watchdog=0 softlockup_panic=0
audit=0 cgroup_memory=1 cgroup_enable=memory mce=off idle=poll
default_hugepagesz=1G skew_tick=1 idle=poll processor.max_cstate=1
intel_idle.max_cstate=0 rcu_nocb_poll kthread_cpus=0,1,72,73,48,49,24,25
irqaffinity=0,1,72,73,48,49,24,25 nosoftlockup tsc=nowatchdog
isolcpus=managed_irq,domain,2-23,26-47,50-71,74-95
systemd.cpu_affinity=0,1,72,73,48,49,24,25 cgroup.memory=nokmem
crashkernel=512M-:256M
[ 3.536582] pstore: Using crash dump compression: deflate
[ 3.992799] megaraid_sas 0000:5e:00.0: firmware crash dump : yes
To manage notifications about this bug go to:
https://bugs.launchpad.net/ubuntu-realtime/+bug/1995270/+subscriptions
--
Mailing list: https://launchpad.net/~kernel-packages
Post to : kernel-packages@lists.launchpad.net
Unsubscribe : https://launchpad.net/~kernel-packages
More help : https://help.launchpad.net/ListHelp
