Linux Kernel Crashes and kdump: Installing a Flight Recorder for Your Servers

┌──────────────────────────────────────────────┐
│            Physical Memory Layout             │
│                                              │
│  ┌────────────────┐  ┌───────────────────┐  │
│  │  Primary Kernel │  │  Reserved Memory  │  │
│  │  (running)      │  │  (crashkernel)    │  │
│  │                 │  │                   │  │
│  │  User processes │  │  Capture kernel + │  │
│  │  Kernel modules │  │  initramfs        │  │
│  │  Page cache...  │  │  (standing by)    │  │
│  └────────────────┘  └───────────────────┘  │
│                                              │
│  Primary crashes → kexec switch → capture    │
│  kernel boots → reads /proc/vmcore → writes  │
│  to /var/crash/                              │
└──────────────────────────────────────────────┘

# Install kexec-tools (includes kdump service and kexec tool)
sudo dnf install kexec-tools

# Install crash tool (for post-mortem vmcore analysis)
sudo dnf install crash

# Install kernel debuginfo package (required by crash, package name matches kernel version)
sudo dnf install kernel-debuginfo-$(uname -r)

sudo apt install kexec-tools makedumpfile crash
sudo apt install linux-image-$(uname -r)-dbg

# Edit GRUB configuration
sudo vi /etc/default/grub

# Add crashkernel parameter to GRUB_CMDLINE_LINUX line
# Common patterns:
#   crashkernel=256M              Fixed reservation of 256M
#   crashkernel=auto              Auto-calculate (RHEL supports this, but can be inaccurate)
#   crashkernel=128M@64M          Reserve 128M starting at offset 64M
#   crashkernel=2G-16G:256M,16G-64G:512M  Tiered reservation based on system memory
GRUB_CMDLINE_LINUX="crashkernel=256M rd.lvm.lv=centos/root rhgb quiet"

# RHEL/CentOS 7
sudo grub2-mkconfig -o /boot/grub2/grub.cfg

# RHEL/CentOS 8+ / Fedora
sudo grub2-mkconfig -o /boot/grub2/grub.cfg
# Or (EFI systems)
sudo grub2-mkconfig -o /boot/efi/EFI/centos/grub.cfg

# Ubuntu / Debian
sudo update-grub

# Reboot to activate crashkernel reservation
sudo reboot

# Check if kernel boot parameters include crashkernel
cat /proc/cmdline | grep crashkernel

# Check reserved memory region
dmesg | grep -i "crashkernel"
# Expected output:
# Reserving 256MB of memory at 800MB for crashkernel

# Check kdump service status
systemctl status kdump.service

# /etc/kdump.conf

# vmcore storage path (default: /var/crash)
path /var/crash

# Core collector: makedumpfile
# -d 8: Filter out these page types (bitmask):
#   1=zero pages, 2=cache pages, 4=cache private, 8=user data,
#   16=free pages
# -c: Compress output
# Use -d 31 to filter all filterable pages, significantly reducing vmcore size
core_collector makedumpfile -l --message-level 1 -d 31

# Action after dump completes (default: reboot)
# Options: reboot / halt / poweroff / shell
# shell mode drops to a shell after dumping, useful for manual inspection
default reboot

# Dump target override (optional)
# If /var/crash is on a small partition, specify a different target
# ext4 /dev/sdb1
# nfs my.nfs.server:/export/crashdumps

# Behavior on dump failure
# failure_action options: continue / halt / reboot / shell
# Recommended for production: shell — at least gives you a chance to troubleshoot
failure_action shell

# /etc/kdump.conf

# SSH remote dumping
ssh root@crash-collector.internal
sshkey /root/.ssh/id_rsa
path /data/crashdumps
core_collector makedumpfile -l --message-level 1 -d 31
default reboot

# RHEL/CentOS
sudo kdumpctl restart

# Ubuntu/Debian
sudo systemctl restart kdump-tools

# Check service status
systemctl status kdump

# Check if capture kernel is loaded into reserved memory
# Method 1: Check kexec load status
kexec -p -l /boot/vmlinuz-$(uname -r) 2>&1
# If already loaded, it will say "kexec_load failed: File exists" or similar

# Method 2: Check kdump kernel load log
dmesg | grep -i kdump
# Expected output:
# kdump: Loaded kdump kernel at 0x...
# kdump: kexec: kdump kernel loaded

# Method 3: Check /sys/kernel/kexec_loaded
cat /sys/kernel/kexec_loaded
# 0 = not loaded, 1 = loaded (should be 1 when kdump is working)

# Check kexec_crash_loaded (whether crash capture kernel is loaded)
cat /sys/kernel/kexec_crash_loaded
# 1 = crash capture kernel is in place

# Step 1: Confirm kdump service is healthy
systemctl status kdump

# Step 2: Confirm panic_on_oops is enabled
sysctl kernel.panic_on_oops
# Should be 1; if not:
sudo sysctl -w kernel.panic_on_oops=1

# Step 3: Ensure auto-reboot after panic
sysctl kernel.panic
# Value > 0 means reboot after N seconds; 0 means no auto-reboot
# Recommended:
sudo sysctl -w kernel.panic=10

# Step 4: Sync disk data to avoid filesystem corruption
sync && sync && sync

# Step 5: Trigger kernel crash!
# Method A: Via SysRq (recommended)
echo c > /proc/sysrq-trigger

# Method B: If SysRq is disabled, enable it first
echo 1 > /proc/sys/kernel/sysrq
echo c > /proc/sysrq-trigger

# Check if vmcore was saved
ls -lh /var/crash/
# Expected to see something like:
# drwxr-xr-x 2 root root  4096 Jul 17 09:20 127.0.0.1-2026-07-17-09:20:01
# Inside should be vmlinux (or vmlinuz) and vmcore files

# Check vmcore file size
ls -lh /var/crash/*/vmcore*
# Compressed vmcore is typically between a few hundred MB to a few GB

# View specific errors
journalctl -u kdump.service -e

# Most common cause: crashkernel parameter not set or insufficient reservation
# Check:
cat /proc/cmdline | grep crashkernel
# If no crashkernel= parameter, GRUB config wasn't updated

# Check if memory reservation took effect
dmesg | grep -i crashkernel
# If you see "crashkernel reservation failed", the reservation failed
# Possible causes: too little memory, or reserved address conflicts with another region

# Check if there are kdump logs
journalctl -b -1 | grep -i kdump
# -b -1 means previous boot's logs

# Check if /var/crash has space
df -h /var/crash

# Check console output (if you have serial console or IPMI logs)
# Common causes:
#   1. Capture kernel's initramfs missing disk drivers
#   2. /var/crash partition out of space
#   3. makedumpfile parameter errors

# Basic usage: crash <vmcore> <vmlinux-debuginfo>
# vmlinux-debuginfo is typically at /usr/lib/debug/lib/modules/$(uname -r)/vmlinux

crash /var/crash/127.0.0.1-2026-07-17-09:20:01/vmcore \
      /usr/lib/debug/lib/modules/$(uname -r)/vmlinux

# RHEL/CentOS
sudo dnf install kernel-debuginfo-$(uname -r)

# Ubuntu
sudo apt install linux-image-$(uname -r)-dbg

# Find vmlinux file location
find /usr/lib/debug -name vmlinux

crash> bt            # Print kernel call trace at crash time (most used)
crash> bt -a         # Print call traces for all CPUs
crash> ps            # List all processes at crash time
crash> ps | grep -i "D"  # Filter D-state (uninterruptible sleep) processes
crash> log           # Print kernel log buffer (dmesg content)
crash> sys           # Show system info (kernel version, CPU, memory, etc.)
crash> files         # List open files at crash time
crash> vm            # Show virtual memory info
crash> kmem -i       # Kernel memory usage overview
crash> dev -l        # List loaded device drivers
crash> mod           # List loaded kernel modules
crash> mod -s <name>  # Load debug info for a specific module
crash> struct <type> <addr>  # Inspect struct contents at given address
crash> rd <addr> <count>  # Read memory
crash> dis <addr>     # Disassemble code at given address

# 1. First look at the call trace to determine where the crash occurred
crash> bt
PID: 0      TASK: ffffffff81c10480  CPU: 0   COMMAND: "swapper/0"
 #0 [ffff88003fc03c90] machine_kexec at ffffffff8105f7a0
 #1 [ffff88003fc03ce0] crash_kexec at ffffffff810b0a72
 #2 [ffff88003fc03db0] oops_end at ffffffff81009524
 #3 [ffff88003fc03dd0] no_context at ffffffff8104b6a5
 #4 [ffff88003fc03e20] __bad_area_nosemaphore at ffffffff8104b6e5
 #5 [ffff88003fc03e70] bad_area at ffffffff8104b810
 #6 [ffff88003fc03ea0] do_page_fault at ffffffff8104bd76
 #7 [ffff88003fc03f30] page_fault at ffffffff816012b8
    [exception RIP: my_driver_write+42]
    RIP: ffffffffa0001234  RSP: ffff88003fc03fe8  RFLAGS: 00010246
    RAX: 0000000000000000 RBX: ffff88003e8a0000 RCX: 0000000000000000
    RDX: 0000000000000100 RSI: ffff88003e8a1000 RDI: 0000000000000000
    RBP: ffff88003fc03ff0
 #8 [ffff88003fc03ff8] sys_write at ffffffff811df3a2

# Call trace interpretation:
# 1. sys_write was called (userspace write syscall)
# 2. Entered my_driver_write (custom driver module's write function)
# 3. page_fault occurred (page fault exception)
# 4. bad_area → no_context → oops_end → crash_kexec → machine_kexec
# Conclusion: my_driver_write accessed an invalid memory address, triggering a page fault

# 2. Disassemble the code at the crash location
crash> dis ffffffffa0001234
0xffffffffa0001234 <my_driver_write+42>:  mov    %rax,(%rdi)
# RDI = 0 (from register info above), mov %rax,(0) is a null pointer dereference

# 3. Check which module this belongs to
crash> mod -s my_driver
     MODULE       NAME                 SIZE  OBJECT FILE
ffff88003e8a0000  my_driver           16384  /lib/modules/.../my_driver.ko

# 4. View the source code in the module (requires debuginfo)
crash> sym ffffffffa0001234
ffffffffa0001230 (t) my_driver_write+38 /usr/src/my_driver/write.c: 42

# Located write.c line 42 — found an uninitialized pointer being written to

# Find D-state processes
crash> ps | grep "UN"
   PID    PPID  CPU       TASK        ST  %MEM     VSZ    RSS  COMM
  12345      1   0  ffff88003e5b8000   UN   0.2  262144   8192  mysql

# UN = Uninterruptible Sleep

# View this process's call trace
crash> bt ffff88003e5b8000
PID: 12345  TASK: ffff88003e5b8000  CPU: 0  COMMAND: "mysql"
 #0 [ffff88003e8a3d80] __schedule at ffffffff8109a2a3
 #1 [ffff88003e8a3dd0] schedule at ffffffff8109a3a5
 #2 [ffff88003e8a3e00] schedule_timeout at ffffffff8109a6b0
 #3 [ffff88003e8a3e50] wait_for_completion at ffffffff8109a8c0
 #4 [ffff88003e8a3ea0] flush_work at ffffffff81098123
 #5 [ffff88003e8a3ef0] __cancel_work_timer at ffffffff81098456
 #6 [ffff88003e8a3f50] cancel_work_sync at ffffffff81098501

# This process is stuck in cancel_work_sync → wait_for_completion
# Meaning a work_struct never completed execution

# Check what this process is waiting for
crash> struct task_struct ffff88003e5b8000
struct task_struct {
  ...
  state = 2,  // TASK_UNINTERRUPTIBLE
  ...
}

# /etc/sysctl.d/99-kdump.conf

# Trigger kdump on panic (instead of just rebooting)
kernel.panic_on_oops = 1

# Auto-reboot N seconds after panic (gives kdump time to complete dump)
# Too short may cut off the dump before it finishes
kernel.panic = 10

# SysRq functionality (allows manual crash trigger via /proc/sysrq-trigger)
kernel.sysrq = 1

# Hung task detection: processes in D state beyond this many seconds trigger a warning
# Default 120 seconds, adjust based on your workload
kernel.hung_task_timeout_secs = 120

# Hung task detection triggers panic (default: warning only, no panic)
# Set to 1 if you want hung tasks to trigger kdump capture
kernel.hung_task_panic = 1

# Hardware NMI (non-maskable interrupt) triggers panic
# For hangs caused by hardware failures, NMI is the last resort
kernel.unknown_nmi_panic = 1

# Soft lockup detection: CPU not yielding for too long triggers a warning
kernel.softlockup_panic = 1

# Physical servers typically have more memory, reserve a bit more
# GRUB parameter:
crashkernel=512M

# Configure IPMI to view console output during crash
# Even if kdump fails to save vmcore, IPMI System Event Log
# may record hardware information from before the crash

# Consider configuring serial console to output panic info to serial port
# Add to GRUB parameters:
console=tty0 console=ttyS0,115200

# VMs typically have less memory, reserve less
crashkernel=128M

# Ensure the VM is configured with enough reserved memory
# Check memory and currentMemory settings in libvirt XML

# VMs can leverage hypervisor memory snapshots as a supplement
# virsh dump <domain> <file> --memory-only
# This command is especially useful when the VM is completely unresponsive

# Host configuration is the same as a regular server
crashkernel=256M

# Critical: ensure kdump's initramfs includes storage driver modules
# If using overlay2, verify relevant modules are in initramfs
# Add to /etc/dracut.conf.d/kdump.conf:
add_drivers+=" overlay br_netfilter nf_conntrack "

# Notify Kubernetes after dump completes
# Configure kdump_post script in /etc/kdump.conf:
# kdump_post /usr/local/bin/kdump-notify.sh

#!/bin/bash
# /usr/local/bin/kdump-notify.sh
# Executed after kdump dump completes, $1 is status (0=success, 1=failure)

if [ "$1" -eq 0 ]; then
    # Dump successful, send notification
    logger -t kdump "vmcore captured successfully at $(date)"
    # Can call WeChat Work/Feishu/DingTalk webhook here
    curl -s -X POST "https://your-webhook.example.com/notify" \
         -H "Content-Type: application/json" \
         -d "{\"event\":\"kernel_panic\",\"host\":\"$(hostname)\",\"vmcore\":\"$(ls -t /var/crash/ | head -1)\"}"
else
    # Dump failed
    logger -t kdump "vmcore capture FAILED at $(date)"
fi

#!/bin/bash
# /usr/local/bin/cleanup-vmcore.sh
# Keep the most recent 5 vmcores, automatically delete older ones

CRASH_DIR="/var/crash"
KEEP_COUNT=5

# Sort by modification time, delete the oldest
cd "$CRASH_DIR" || exit 1
ls -dt */ 2>/dev/null | tail -n +$((KEEP_COUNT + 1)) | while read dir; do
    echo "Removing old vmcore: $dir"
    rm -rf "$dir"
done

# Check disk usage, delete oldest if over 80%
USAGE=$(df "$CRASH_DIR" | awk 'NR==2{print $5}' | tr -d '%')
while [ "$USAGE" -gt 80 ]; do
    OLDEST=$(ls -dt */ 2>/dev/null | tail -1)
    if [ -z "$OLDEST" ]; then
        break
    fi
    echo "Disk usage ${USAGE}%, removing: $OLDEST"
    rm -rf "$OLDEST"
    USAGE=$(df "$CRASH_DIR" | awk 'NR==2{print $5}' | tr -d '%')
done

# Run cleanup daily at 3 AM
0 3 * * * /usr/local/bin/cleanup-vmcore.sh >> /var/log/vmcore-cleanup.log 2>&1

#!/bin/bash
# kdump production configuration script
# For RHEL/CentOS 8+, adjust package names for other distros

set -euo pipefail

echo "=== 1. Install components ==="
dnf install -y kexec-tools crash
dnf install -y "kernel-debuginfo-$(uname -r)" 2>/dev/null || \
    echo "Warning: kernel-debuginfo not found, crash analysis may be limited"

echo "=== 2. Configure crashkernel parameter ==="
GRUB_FILE="/etc/default/grub"
if ! grep -q "crashkernel=" "$GRUB_FILE"; then
    sed -i 's/GRUB_CMDLINE_LINUX="/GRUB_CMDLINE_LINUX="crashkernel=256M /' "$GRUB_FILE"
    echo "Added crashkernel=256M to GRUB"
else
    echo "crashkernel already configured"
fi

echo "=== 3. Configure sysctl parameters ==="
cat > /etc/sysctl.d/99-kdump.conf << 'EOF'
kernel.panic_on_oops = 1
kernel.panic = 10
kernel.sysrq = 1
kernel.hung_task_timeout_secs = 120
kernel.hung_task_panic = 1
kernel.unknown_nmi_panic = 1
kernel.softlockup_panic = 1
EOF
sysctl --system

echo "=== 4. Configure kdump.conf ==="
cat > /etc/kdump.conf << 'EOF'
path /var/crash
core_collector makedumpfile -l --message-level 1 -d 31
default reboot
failure_action shell
EOF

echo "=== 5. Enable and start kdump ==="
systemctl enable kdump
systemctl restart kdump

echo "=== 6. Create vmcore cleanup cron ==="
cat > /usr/local/bin/cleanup-vmcore.sh << 'SCRIPT'
#!/bin/bash
CRASH_DIR="/var/crash"
KEEP_COUNT=5
cd "$CRASH_DIR" || exit 1
ls -dt */ 2>/dev/null | tail -n +$((KEEP_COUNT + 1)) | while read dir; do
    rm -rf "$dir"
done
USAGE=$(df "$CRASH_DIR" | awk 'NR==2{print $5}' | tr -d '%')
while [ "$USAGE" -gt 80 ]; do
    OLDEST=$(ls -dt */ 2>/dev/null | tail -1)
    [ -z "$OLDEST" ] && break
    rm -rf "$OLDEST"
    USAGE=$(df "$CRASH_DIR" | awk 'NR==2{print $5}' | tr -d '%')
done
SCRIPT
chmod +x /usr/local/bin/cleanup-vmcore.sh
echo "0 3 * * * root /usr/local/bin/cleanup-vmcore.sh >> /var/log/vmcore-cleanup.log 2>&1" > /etc/cron.d/vmcore-cleanup

echo "=== 7. Update GRUB ==="
grub2-mkconfig -o /boot/grub2/grub.cfg

echo ""
echo "=== Configuration complete ==="
echo "Reboot the system to activate crashkernel: sudo reboot"
echo "After reboot, verify:"
echo "  1. cat /proc/cmdline | grep crashkernel"
echo "  2. systemctl status kdump"
echo "  3. cat /sys/kernel/kexec_crash_loaded (should be 1)"
echo ""
echo "Test kdump (will trigger crash and reboot):"
echo "  echo c > /proc/sysrq-trigger"