Linux Boot Process Explained: From Firmware to Userspace

[Power On]
    │
    ▼
[1. Firmware Stage] BIOS / UEFI
    │  POST (Power-On Self Test)
    │  Hardware initialization
    │  Find boot device
    ▼
[2. Bootloader Stage] GRUB2
    │  Load GRUB into memory
    │  Read grub.cfg
    │  Load kernel and initramfs
    ▼
[3. Kernel Stage] Linux Kernel
    │  Kernel decompress and initialize
    │  Hardware detection and driver loading
    │  Mount initramfs
    │  Start init (systemd)
    ▼
[4. Userspace Stage] systemd
    │  Read default.target
    │  Start services in dependency order
    │  Start login service
    ▼
[5. Login Stage]
    │  getty / display-manager
    │  User authentication
    │  Shell startup
    ▼
[System Ready]

1. POST (Power-On Self Test)
   ├── CPU initialization
   ├── Memory detection
   ├── Hardware detection
   └── BIOS settings check

2. Boot device selection
   ├── Follow BIOS boot order
   ├── First bootable device
   └── Read MBR (first 512 bytes)

3. MBR structure
   ├── 446 bytes: Boot code
   ├── 64 bytes: Partition table (4 primary partitions)
   └── 2 bytes: Magic number (0x55AA)
   
4. Execute boot code in MBR
   └── Load GRUB's core.img

1. POST
   └── Same as BIOS

2. UEFI firmware initialization
   ├── Initialize UEFI drivers
   ├── Run UEFI applications
   └── Secure Boot verification

3. Boot manager
   ├── Read BootOrder variable
   ├── Find EFI System Partition (ESP)
   │   └── Usually /dev/sda1 (FAT32, ~512MB)
   └── Execute EFI application: /EFI/<distro>/grubx64.efi

4. Secure Boot
   ├── Verify EFI application signature
   ├── Use kernel built-in keys
   └── Reject unsigned or invalidly signed bootloaders

# View UEFI boot entries
$ efibootmgr
BootCurrent: 0000
BootOrder: 0000,0001,0002
Boot0000* ubuntu    HD(1,GPT,...)/File(\EFI\ubuntu\grubx64.efi)
Boot0001* UEFI:USB   ...
Boot0002* UEFI:CDROM ...

# Add boot entry
$ efibootmgr -c -d /dev/sda -p 1 -L "My Linux" -l '\EFI\mylinux\grubx64.efi'

# Modify boot order
$ efibootmgr -o 0000,0001,0002

# Delete boot entry
$ efibootmgr -b 0002 -B

# View Secure Boot status
$ mokutil --sb-state
SecureBoot enabled

# View enrolled keys
$ mokutil --list-enrolled

# Enroll a new key (requires reboot into MokManager)
$ mokutil --import /path/to/key.der

# Disable Secure Boot (must be done in UEFI settings)
# Reboot → Enter UEFI settings → Security → Secure Boot → Disabled

1. boot.img (446 bytes)
   ├── Located in MBR or ESP
   └── Loads core.img

2. core.img (32KB)
   ├── diskboot.img: Loads from disk
   ├── lzma_decompress.img: Decompresses
   └── kernel.img: GRUB core

3. GRUB normal operation
   ├── Reads /boot/grub/grub.cfg
   ├── Displays boot menu
   └── Loads selected kernel

/etc/default/grub          ← Main config file (user edits)
         │
         ▼
/etc/grub.d/               ← Script directory
  ├── 00_header             ← Header configuration
  ├── 10_linux              ← Linux menu entries
  ├── 20_memtest            ← Memtest menu entry
  ├── 30_os-prober          ← Other OS detection
  └── 40_custom             ← Custom menu entries
         │
         ▼
/boot/grub/grub.cfg        ← Generated config (do not edit manually)

# /etc/default/grub

# Boot menu timeout (seconds)
GRUB_TIMEOUT=5

# Default boot entry (0 = first entry)
GRUB_DEFAULT=0

# Menu display style
GRUB_TIMEOUT_STYLE=menu    # menu/countdown/hidden

# Kernel parameters
GRUB_CMDLINE_LINUX="quiet splash"
GRUB_CMDLINE_LINUX_DEFAULT="quiet splash"

# Disable submenu (show all kernels)
GRUB_DISABLE_SUBMENU=y

# Disable recovery mode
GRUB_DISABLE_RECOVERY=true

# Partition UUID
GRUB_DISABLE_LINUX_UUID=false

# Graphical terminal
GRUB_TERMINAL=gfxterm
GRUB_GFXMODE=1920x1080

# Background image
GRUB_BACKGROUND="/boot/grub/background.png"

# Font
GRUB_FONT="/boot/grub/fonts/unicode.pf2"

# Serial console (for servers)
#GRUB_TERMINAL="serial console"
#GRUB_SERIAL_COMMAND="serial --speed=115200 --unit=0 --word=8 --parity=no --stop=1"

# /etc/default/grub
GRUB_CMDLINE_LINUX=""

# Common kernel parameters:
# quiet         - Hide boot messages
# splash        - Show splash screen
# nomodeset     - Don't load graphics driver (troubleshooting display issues)
# text          - Boot to text mode
# single / 1    - Single user mode
# rescue        - Rescue mode
# init=/bin/bash - Replace init (emergency recovery)
# systemd.unit=rescue.target - Boot to rescue mode
# systemd.unit=emergency.target - Boot to emergency mode

# Debug parameters:
# debug         - Enable kernel debugging
# loglevel=8    - Kernel log level
# print-fatal-signals=1 - Print signal info

# Performance parameters:
# nohz_full=1-7 - Tickless CPU
# isolcpus=1-7  - Isolate CPUs
# transparent_hugepage=never - Disable THP

# Security parameters:
# apparmor=1    - Enable AppArmor
# audit=1       - Enable auditing

# Debian/Ubuntu
$ update-grub
# Or
$ grub-mkconfig -o /boot/grub/grub.cfg

# RHEL/CentOS
$ grub2-mkconfig -o /boot/grub2/grub.cfg

# UEFI systems (RHEL/CentOS)
$ grub2-mkconfig -o /boot/efi/EFI/centos/grub.cfg

# /etc/grub.d/40_custom
#!/bin/sh
exec tail -n +3 $0

menuentry "Custom Linux Rescue" {
    set root='(hd0,msdos1)'
    linux /boot/vmlinuz-rescue root=/dev/sda1 ro single
    initrd /boot/initramfs-rescue.img
}

menuentry "Windows 10" {
    insmod part_gpt
    insmod chain
    set root='(hd0,gpt1)'
    chainloader /EFI/Microsoft/Boot/bootmgfw.efi
}

menuentry "Memtest86+" {
    linux16 /boot/memtest86+.bin
}

# 1. Generate password hash
$ grub-mkpasswd-pbkdf2
Enter password: 
Reenter password: 
PBKDF2 hash of your password is grub.pbkdf2.sha512.10000.ABC123...

# 2. Add to /etc/grub.d/40_custom
set superusers="admin"
password_pbkdf2 admin grub.pbkdf2.sha512.10000.ABC123...

# 3. Add protection to specific menu entries
# In /etc/grub.d/10_linux, add --users "" to menuentry
# Allow booting default entry without password, but require password for editing

# 4. Regenerate configuration
$ update-grub

# Press 'c' at the GRUB menu to enter command line

# View available commands
grub> help

# List disks
grub> ls
(hd0) (hd0,msdos1) (hd0,msdos2)

# List partition contents
grub> ls (hd0,msdos1)/
boot/  etc/  home/  ...

# Manual boot
grub> set root=(hd0,msdos1)
grub> linux /boot/vmlinuz-6.6.0 root=/dev/sda1 ro
grub> initrd /boot/initramfs-6.6.0.img
grub> boot

# Search for files
grub> search.file /boot/grub/grub.cfg

Kernel boot → Mount initramfs → Load drivers → Mount real root → switch_root → systemd

# View initramfs contents
$ lsinitramfs /boot/initrd.img-$(uname -r) | head -30
.
bin
bin/busybox
conf
conf/conf.d
conf/init
cryptroot
etc
etc/ld.so.conf
etc/ld.so.conf.d
...
init
lib
lib/modules/6.6.0/kernel/drivers/
...
scripts
scripts/init-premount
scripts/local-top

# initramfs /init script (simplified)
#!/bin/sh

# 1. Mount basic filesystems
mount -t proc proc /proc
mount -t sysfs sysfs /sys
mount -t devtmpfs devtmpfs /dev

# 2. Load kernel modules
modprobe ext4
modprobe sd_mod
modprobe nvme

# 3. Parse kernel command line parameters
# root=/dev/sda1
# Or root=UUID=xxx
# Or root=LABEL=root

# 4. Find and mount root filesystem
mount /dev/sda1 /mnt/root

# 5. Switch to real root filesystem
exec switch_root /mnt/root /sbin/init

# Debian/Ubuntu
$ update-initramfs -u            # Update current kernel
$ update-initramfs -u -k all     # Update all kernels
$ update-initramfs -c -k 6.6.0   # Create for specific kernel

# RHEL/CentOS
$ dracut -f                      # Force regenerate
$ dracut -f /boot/initramfs-6.6.0.img 6.6.0

# View included modules
$ lsinitramfs /boot/initrd.img-$(uname -r) | grep "kernel/drivers/"
# Or
$ dracut --list-modules

# Debian/Ubuntu: /etc/initramfs-tools/modules
# Add required module names
nvme
nvme-core
ext4
xfs

# Regenerate
$ update-initramfs -u

# RHEL/CentOS: /etc/dracut.conf.d/custom.conf
add_drivers+=" nvme nvme-core ext4 xfs "
force_drivers+=" my_custom_driver "

$ dracut -f

# 1. Add debug parameter in GRUB
# GRUB_CMDLINE_LINUX="rd.debug"

# 2. Add shell to initramfs
# GRUB_CMDLINE_LINUX="rd.break"
# Pause at initramfs stage, enter shell

# 3. Common issues
# - Root filesystem not found: Missing storage driver
# - LVM/LUKS cannot unlock: Missing lvm2/cryptsetup
# - NFS root mount failure: Missing nfs module

1. Kernel decompression
   ├── Decompress from bzImage into memory
   └── Jump to kernel entry point

2. Early initialization
   ├── CPU initialization
   ├── Memory subsystem initialization
   ├── Page table setup
   └── Interrupt controller initialization

3. Hardware detection
   ├── PCI bus scan
   ├── Device tree parsing (ARM)
   ├── ACPI table parsing (x86)
   └── Load built-in drivers

4. Memory management initialization
   ├── Buddy allocator
   ├── Slab allocator
   └── vmalloc area

5. Scheduler initialization
   ├── Create init task (PID 0)
   └── Start idle process

6. Mount initramfs
   ├── Decompress cpio archive
   └── Execute /init

7. Mount real root filesystem
   └── switch_root to /sbin/init

# Pass kernel parameters in GRUB
# /etc/default/grub
GRUB_CMDLINE_LINUX="parameter list"

# Temporarily modify in GRUB menu
# Press 'e' to edit → append parameter to linux line → Ctrl+X to boot

# View current boot parameters
$ cat /proc/cmdline
BOOT_IMAGE=/boot/vmlinuz-6.6.0 root=UUID=xxx ro quiet splash

# Common troubleshooting parameters:
# single          - Single user mode
# rescue          - Rescue mode
# emergency       - Emergency mode
# init=/bin/bash  - Boot directly to bash (skip systemd)
# systemd.unit=rescue.target
# rd.break        - Pause at initramfs stage
# root=/dev/sda1  - Specify root partition
# rootflags=subvol=root - Root partition mount options
# nomodeset       - Don't load graphics driver
# acpi=off        - Disable ACPI
# noapic          - Disable APIC
# nodmraid        - Disable dmraid
# rd.shell        - Provide emergency shell

# View current boot kernel logs
$ dmesg
# Or
$ journalctl -k

# View previous boot kernel logs
$ journalctl -k -b -1

# View boot time
$ systemd-analyze
Startup finished in 5.123s (kernel) + 12.456s (userspace) = 17.579s

# View per-service startup time
$ systemd-analyze blame | head -20
12.345s systemd-journal-flush.service
 8.123s networkd-dispatcher.service
 5.456s docker.service
 3.789s sshd.service
 ...

1. Early initialization
   ├── Read /etc/systemd/system.conf
   ├── Initialize core functionality
   ├── Mount /proc /sys /dev
   └── Start basic services

2. Load default target
   ├── Read default.target
   ├── Typically graphical.target or multi-user.target
   └── Resolve dependencies

3. Start by dependency
   ├── Pull up all Before= units
   ├── Start dependency-free services in parallel
   └── Process Requires=/Wants= relationships

4. Boot complete
   ├── All target dependencies satisfied
   ├── Start login service
   └── System ready

# View current target
$ systemctl get-default
multi-user.target

# View all targets
$ systemctl list-units --type=target

# Change default target
$ systemctl set-default graphical.target

# Temporarily switch target
$ systemctl isolate rescue.target
$ systemctl isolate multi-user.target

graphical.target
    │
    ├── Wants: multi-user.target
    │
    └── multi-user.target
            │
            ├── Wants: basic.target
            │
            └── basic.target
                    │
                    ├── Wants: sysinit.target
                    │
                    └── sysinit.target
                            │
                            ├── systemd-update-utmp.service
                            ├── systemd-tmpfiles-setup.service
                            └── ...

1. sysinit.target stage (earliest)
   ├── systemd-journald.service    # Logging service
   ├── systemd-modules-load.service # Load kernel modules
   ├── systemd-tmpfiles-setup-dev.service # /dev
   └── kmod-static-nodes.service

2. basic.target stage (base ready)
   ├── paths.target
   ├── slices.target
   ├── sockets.target
   ├── timers.target
   └── systemd-udevd.service

3. multi-user.target stage (multi-user ready)
   ├── sshd.service
   ├── network.service / NetworkManager.service
   ├── cron.service
   ├── getty.target                # Login terminals
   └── docker.service

4. graphical.target stage (graphical ready)
   ├── display-manager.service     # GDM/LightDM
   └── multi-user.target

# View service dependencies
$ systemctl list-dependencies nginx.service

# View reverse dependencies (who depends on this service)
$ systemctl list-dependencies --reverse nginx.service

# View service startup order
$ systemctl list-dependencies --after nginx.service
$ systemctl list-dependencies --before nginx.service

# /etc/systemd/system/myservice.service
[Unit]
Description=My Service
Requires=network.target       # Hard dependency (must start)
Wants=remote-fs.target         # Soft dependency (try to start)
After=network.target           # Start after network
Before=multi-user.target       # Start before multi-user
Conflicts=sendmail.service     # Mutually exclusive service

[Service]
Type=notify
ExecStart=/usr/bin/myservice
Restart=on-failure

[Install]
WantedBy=multi-user.target

# Overall time
$ systemd-analyze

# Per-service time
$ systemd-analyze blame | head -20

# Critical path analysis
$ systemd-analyze critical-chain
# Shows services on the critical boot path

# Critical path for specific service
$ systemd-analyze critical-chain docker.service

# Generate boot chart
$ systemd-analyze plot > boot-analysis.svg
# Open the SVG file in a browser for detailed timeline

# Analyze per-stage time
$ systemd-analyze --profile

# View all enabled services
$ systemctl list-unit-files --state=enabled

# Disable unnecessary services
$ systemctl disable bluetooth.service
$ systemctl disable cups.service
$ systemctl disable modemmanager.service
$ systemctl disable avahi-daemon.service
$ systemctl disable speech-dispatcher.service

# DHCP timeout optimization
# /etc/dhcp/dhclient.conf
timeout 10;
retry 3;

# Or use NetworkManager
# /etc/NetworkManager/conf.d/dhcp.conf
[connection]
ipv4.dhcp-timeout=10

# View loaded modules
$ lsmod | wc -l

# Disable unneeded modules
# /etc/modprobe.d/blacklist.conf
blacklist bluetooth
blacklist firewire-core
blacklist thunderbolt
blacklist uas

# Use journaling filesystem (ext4/xfs) to reduce fsck time
# Adjust fsck frequency
# /etc/fstab
# Column 6: pass (0=no check, 1=root partition first, 2=other partitions)
/dev/sda1  /  ext4  defaults  0  1
/dev/sda2  /home  ext4  defaults  0  2

# Check if service has unnecessary After=
$ systemctl cat docker.service | grep -E "After=|Requires="

# Remove unnecessary serial dependencies
# Create override
$ systemctl edit docker.service
[Unit]
After=          # Clear default After
After=network.target  # Keep only necessary ones

# systemd-networkd starts faster
$ systemctl disable NetworkManager
$ systemctl enable systemd-networkd
$ systemctl enable systemd-resolved

# Configuration
# /etc/systemd/network/20-wired.network
[Match]
Name=eth0

[Network]
DHCP=yes
# Or static IP
# Address=192.168.1.100/24
# Gateway=192.168.1.1
# DNS=8.8.8.8

# Before optimization
$ systemd-analyze
Startup finished in 5.1s (kernel) + 25.3s (userspace) = 30.4s

# After optimization
$ systemd-analyze
Startup finished in 4.8s (kernel) + 8.2s (userspace) = 13.0s

# Continuous monitoring
$ systemd-analyze plot > before-optimization.svg
# After optimization
$ systemd-analyze plot > after-optimization.svg

# Behavior on kernel panic
$ sysctl kernel.panic
kernel.panic = 0
# 0 = No auto-reboot (hang)
# N = Auto-reboot after N seconds

# Recommended: 10-second auto-reboot for production
$ sysctl -w kernel.panic=10

# 1. Install
$ apt install kdump-tools        # Debian/Ubuntu
$ dnf install kexec-tools        # RHEL/CentOS

# 2. Configure
# /etc/default/kdump-tools
KDUMP_SYSCTL="kernel.panic=10"
KDUMP_COREDIR="/var/crash"

# RHEL/CentOS: /etc/default/kexec
KDUMP_KERNEL="/boot/vmlinuz-$(uname -r)kdump"
KDUMP_INITRD="/boot/initramfs-$(uname -r)kdump.img"

# 3. Enable
$ systemctl enable kdump
$ systemctl start kdump

# 4. Verify
$ kdump-config show
# Or
$ kexec -p --status

# 5. Test (trigger panic)
# WARNING: This will crash the system!
$ echo 1 > /proc/sys/kernel/sysrq
$ echo c > /proc/sysrq-trigger

# View crash dumps
$ ls /var/crash/
202607101530/   # Directory organized by timestamp

# Analyze dump file
$ crash /usr/lib/debug/boot/vmlinux-$(uname -r) /var/crash/202607101530/dump.202607101530

# Common crash commands
crash> bt         # View crash call stack
crash> ps         # View process list
crash> log        # View kernel logs
crash> sys        # System info
crash> vm <pid>   # View process memory
crash> dev        # View device info

# Add in GRUB
systemd.unit=rescue.target

# Or after boot
$ systemctl rescue

# rescue mode:
# - Mounts root filesystem (read-only)
# - Starts minimal service set
# - Requires root password
# - Can edit files

# Add in GRUB
systemd.unit=emergency.target

# Or
$ systemctl emergency

# emergency mode:
# - Only mounts root filesystem (read-only)
# - Starts no services
# - Requires root password
# - Minimal environment

# Add in GRUB
init=/bin/bash

# Or
linux /boot/vmlinuz-6.6.0 root=/dev/sda1 ro init=/bin/bash

# Effect:
# - Skip systemd
# - Drop directly to bash
# - Root filesystem is read-only
# - No network, no services

# Remount root filesystem as read-write
mount -o remount,rw /

# Change password
passwd root

# Edit config files
vi /etc/fstab

# Reboot
exec /sbin/init
# Or
mount -o remount,ro /
reboot -f

# Add in GRUB
rd.break

# Effect:
# - Pause at initramfs stage
# - Root filesystem not mounted
# - Can repair GRUB, initramfs, etc.

# Mount real root
switch_root /sysroot

# Change root password
mount -o remount,rw /sysroot
chroot /sysroot
passwd root
touch /.autorelabel    # SELinux relabel
exit
exit

# Enable SysRq
$ echo 1 > /proc/sys/kernel/sysrq

# Common SysRq combinations (Alt+SysRq+letter)
# Alt+SysRq+s  - Sync filesystems
# Alt+SysRq+u  - Remount read-only
# Alt+SysRq+b  - Immediate reboot
# Alt+SysRq+e  - Terminate all processes
# Alt+SysRq+i  - Force kill
# Alt+SysRq+k  - Kill all processes on current terminal
# Alt+SysRq+t  - Show task state
# Alt+SysRq+w  - Show blocked tasks
# Alt+SysRq+p  - Show registers
# Alt+SysRq+c  - Trigger panic (kdump test)

# Safe reboot sequence (mnemonic: Raising Elephants Is So Utterly Boring)
# Alt+SysRq+r  - Take back raw mode
# Alt+SysRq+e  - Terminate processes
# Alt+SysRq+i  - Force kill
# Alt+SysRq+s  - Sync
# Alt+SysRq+u  - Remount read-only
# Alt+SysRq+b  - Reboot

# Command-line method
$ echo s > /proc/sysrq-trigger   # Sync
$ echo u > /proc/sysrq-trigger   # Read-only
$ echo b > /proc/sysrq-trigger   # Reboot

# Scenario: GRUB corrupted, cannot boot

# 1. Boot from Live USB
# 2. Mount root partition
$ mount /dev/sda1 /mnt
# If there's an EFI partition
$ mount /dev/sda2 /mnt/boot/efi

# 3. Mount virtual filesystems
$ mount --bind /dev /mnt/dev
$ mount --bind /proc /mnt/proc
$ mount --bind /sys /mnt/sys

# 4. chroot into the system
$ chroot /mnt

# 5. Reinstall GRUB
# BIOS:
$ grub-install /dev/sda
# UEFI:
$ grub-install --target=x86_64-efi --efi-directory=/boot/efi

# 6. Regenerate configuration
$ update-grub        # Debian/Ubuntu
$ grub2-mkconfig -o /boot/grub2/grub.cfg  # RHEL

# 7. Exit and reboot
$ exit
$ reboot

# Method 1: init=/bin/bash
# 1. Press 'e' at GRUB menu
# 2. Append to linux line: init=/bin/bash
# 3. Ctrl+X to boot
# 4. Remount
mount -o remount,rw /
# 5. Change password
passwd root
# 6. SELinux relabel
touch /.autorelabel
# 7. Reboot
exec /sbin/init

# Method 2: rd.break
# 1. Press 'e' at GRUB menu
# 2. Append to linux line: rd.break
# 3. Ctrl+X to boot
# 4. Remount sysroot
mount -o remount,rw /sysroot
chroot /sysroot
passwd root
touch /.autorelabel
exit
exit

# Scenario: /etc/fstab misconfiguration prevents boot
# systemd drops to emergency mode

# 1. Enter root password for emergency shell
# 2. Remount as read-write
mount -o remount,rw /

# 3. Check fstab
cat /etc/fstab
# Find the erroneous line

# 4. Fix fstab
vi /etc/fstab
# Comment out the erroneous line

# 5. Verify mount
mount -a
# No errors means fix is successful

# 6. Reboot
reboot

# Scenario: Cannot boot after kernel upgrade, reports root partition not found
# Cause: initramfs missing NVMe driver

# 1. Boot from old kernel (select Advanced options in GRUB menu)

# 2. Add driver to initramfs
# Debian/Ubuntu:
echo "nvme" >> /etc/initramfs-tools/modules
echo "nvme-core" >> /etc/initramfs-tools/modules
update-initramfs -u -k $(uname -r)

# RHEL/CentOS:
echo 'add_drivers+=" nvme nvme-core "' > /etc/dracut.conf.d/nvme.conf
dracut -f

# 3. Reboot and verify
reboot

#!/bin/bash
# boot-check.sh - Boot health check

echo "=== Boot Health Check ==="

echo ""
echo "1. Boot Mode"
[ -d /sys/firmware/efi ] && echo "   UEFI" || echo "   Legacy BIOS"

echo ""
echo "2. Boot Time"
systemd-analyze

echo ""
echo "3. Slowest Services (Top 10)"
systemd-analyze blame | head -10

echo ""
echo "4. Default Target"
systemctl get-default

echo ""
echo "5. Failed Services"
systemctl --failed

echo ""
echo "6. GRUB Configuration"
echo "   Kernel parameters: $(cat /proc/cmdline)"

echo ""
echo "7. Kernel Version"
uname -r

echo ""
echo "8. Recent Boot Records"
journalctl --list-boots | tail -5

echo ""
echo "9. Errors from Last Boot"
journalctl -b -1 -p err --no-pager 2>/dev/null | tail -10

echo ""
echo "10. kdump Status"
systemctl is-active kdump 2>/dev/null || echo "   kdump not installed/not enabled"