eBPF Programming Guide: From Principles to Production-Grade Tooling

User-space Program
    │
    ▼
┌─────────────┐
│  Write BPF C │
│  source code │
└──────┬──────┘
       │ clang -target bpf
       ▼
┌─────────────┐
│  eBPF bytecode│
│  (ELF file)  │
└──────┬──────┘
       │ bpf() syscall
       ▼
┌─────────────────┐
│  Kernel Verifier │
│  - CFG analysis  │
│  - Type checking │
│  - Safety checks │
└──────┬──────────┘
       │ Verified
       ▼
┌─────────────┐
│  JIT Compiler │
│  bytecode→native│
└──────┬──────┘
       │
       ▼
┌─────────────────┐
│  Kernel Hook Point│
│  (kprobe/tracepoint│
│   /XDP/etc.)     │
└─────────────────┘

# kprobe: dynamically trace kernel function entry
# Pros: can trace any kernel function, flexible
# Cons: depends on kernel symbols, function signature changes may cause issues

# tracepoint: static tracepoints
# Pros: stable kernel ABI, won't break across versions
# Cons: limited coverage, only at predefined kernel locations

# Rule of thumb: prefer tracepoints; use kprobes only when tracepoints don't cover your needs

# List all available tracepoints
sudo ls /sys/kernel/debug/tracing/events/

# View tracepoints for a specific subsystem
sudo ls /sys/kernel/debug/tracing/events/sched/
# common-data  sched_process_exec  sched_process_fork  sched_process_exit
# sched_switch  sched_waking        sched_wakeup_new    ...

# View the argument format of a specific tracepoint
sudo cat /sys/kernel/debug/tracing/events/sched/sched_switch/format
# name: sched_switch
# ID: 325
# format:
#     field:char prev_comm[16];       offset:8;   size:16;  signed:0;
#     field:pid_t prev_pid;            offset:24;  size:4;   signed:1;
#     field:int   prev_prio;           offset:28;  size:4;   signed:1;
#     field:long  prev_state;          offset:32;  size:8;   signed:1;
#     ...

# Check kernel version (5.4+ recommended)
uname -r
# 5.15.0-91-generic

# Check BTF support (prerequisite for CO-RE)
ls -la /sys/kernel/btf/vmlinux
# -r--r--r-- 1 root root 5283547 Jul 11 10:00 /sys/kernel/btf/vmlinux

# Check BPF-related kernel config
zcat /proc/config.gz | grep CONFIG_DEBUG_INFO_BTF
# CONFIG_DEBUG_INFO_BTF=y

# If /proc/config.gz doesn't exist, try:
grep CONFIG_DEBUG_INFO_BTF /boot/config-$(uname -r)

# Ubuntu / Debian
sudo apt-get update
sudo apt-get install -y \
    clang llvm \
    libbpf-dev \
    libelf-dev \
    zlib1g-dev \
    linux-tools-common \
    linux-tools-$(uname -r) \
    bpftool

# CentOS / RHEL / Rocky Linux
sudo dnf install -y \
    clang llvm \
    libbpf-devel \
    elfutils-libelf-devel \
    zlib-devel \
    bpftool

# Verify installation
clang --version       # needs clang 10+
llc --version
bpftool version

BCC Tool Execution Flow:
┌──────────┐    ┌──────────┐    ┌──────────┐    ┌──────────┐
│ Python   │───▶│ Embedded  │───▶│ Clang/LLVM│───▶│ Load into│
│ script   │    │ BPF C    │    │ JIT compile│    │ kernel  │
└──────────┘    └──────────┘    └──────────┘    └──────────┘

libbpf+CO-RE Tool Flow:
┌──────────┐    ┌──────────┐    ┌──────────┐
│ Dev machine│───▶│ Precompiled│───▶│ Distribute to│
│ clang build│   │ BPF bytecode│   │ production   │
└──────────┘    │ + BTF    │    └────┬─────┘
                └──────────┘         │
                                     ▼
                              ┌──────────────┐
                              │ libbpf loads  │
                              │ BTF relocate  │
                              │ field offset  │
                              └──────────────┘

# Choose BCC when:
# - Quick prototyping, no production deployment needed
# - Dev and prod kernel versions are identical
# - Team is more comfortable with Python
# - Temporary troubleshooting, one-time use

# Choose libbpf+CO-RE when:
# - Long-term production deployment required
# - Cross-kernel version portability needed
# - Resource-sensitive environments (containers, edge devices)
# - Distributable standalone binary required
# - Building operations tooling

┌──────────────────────────────────────────────────────┐
│  Dev machine (kernel 5.15)                            │
│  struct task_struct {                                  │
│      int   pid;          // offset 0x4                │
│      char  comm[16];      // offset 0x2c8              │
│      ...                                               │
│  }                                                    │
│                                                       │
│  clang compiles → BPF bytecode references task_struct  │
│                → generated BPF instructions embed     │
│                  field offsets                         │
└───────────────────────┬──────────────────────────────┘
                        │ Distribute
                        ▼
┌──────────────────────────────────────────────────────┐
│  Production machine (kernel 6.1)                      │
│  struct task_struct {                                  │
│      int   pid;          // offset 0x4  (unchanged)    │
│      ...                                               │
│      char  comm[16];      // offset 0x2d0 (changed!)   │
│      ...                                               │
│  }                                                    │
│                                                       │
│  libbpf reads /sys/kernel/btf/vmlinux at load time    │
│  → obtains real layout of task_struct in kernel 6.1    │
│  → automatically relocates comm offset 0x2c8 → 0x2d0  │
│  → BPF program runs correctly on new kernel            │
└──────────────────────────────────────────────────────┘

# Check kernel BTF file size
ls -lh /sys/kernel/btf/vmlinux
# -r--r--r-- 1 root root 5.1M Jul 11 10:00 /sys/kernel/btf/vmlinux

# Use bpftool to view a struct's BTF definition
bpftool btf dump file /sys/kernel/btf/vmlinux format c \
    | grep -A 20 "struct task_struct"

# View BTF function signatures
bpftool btf dump file /sys/kernel/btf/vmlinux format c \
    | grep "FUNC.*do_unlinkat"

# Enable in kernel config
CONFIG_DEBUG_INFO_BTF=y

# Install pahole for BTF generation
sudo apt-get install dwarves
# or
sudo dnf install dwarves

# Recompile kernel
make olddefconfig
make -j$(nproc)
sudo make modules_install
sudo make install
sudo reboot

my-ebpf-tool/
├── Makefile
├── src/
│   ├── bpf/
│   │   └── exec_monitor.bpf.c    # Kernel-side BPF program
│   └── user/
│       └── exec_monitor.c        # User-space loader
├── include/
│   └── exec_monitor.h           # Shared header
└── vmlinux.h                    # Kernel type definitions (generated by bpftool)

# Generate vmlinux.h from current kernel's BTF
bpftool btf dump file /sys/kernel/btf/vmlinux format c > vmlinux.h

# vmlinux.h contains all kernel data structure definitions
# CO-RE programs no longer need #include <linux/sched.h>, etc.
# Instead, they directly #include "vmlinux.h"
wc -l vmlinux.h
# 140000+ vmlinux.h

// exec_monitor.bpf.c
// Monitor all execve syscalls, record executed commands

#include "vmlinux.h"
#include <bpf/bpf_helpers.h>
#include <bpf/bpf_tracing.h>
#include "exec_monitor.h"

// BPF Map: ring buffer for passing events to user space
struct {
    __uint(type, BPF_MAP_TYPE_RINGBUF);
    __uint(max_entries, 256 * 1024);  // 256 KB
} events SEC(".maps");

// BPF Map: config table to control monitoring on/off
struct {
    __uint(type, BPF_MAP_TYPE_ARRAY);
    __uint(max_entries, 1);
    __type(key, u32);
    __type(value, u32);
} config SEC(".maps");

// tracepoint: syscalls:sys_enter_execve
// Monitor process creation events
SEC("tracepoint/syscalls/sys_enter_execve")
int trace_execve(struct trace_event_raw_sys_enter *ctx)
{
    // Read config: check if monitoring is enabled
    u32 key = 0;
    u32 *enabled = bpf_map_lookup_elem(&config, &key);
    if (!enabled || !*enabled) {
        return 0;
    }

    // Get current process info
    struct task_struct *task = (struct task_struct *)bpf_get_current_task();

    // Reserve space in the ring buffer
    struct event *e;
    e = bpf_ringbuf_reserve(&events, sizeof(*e), 0);
    if (!e) {
        return 0;
    }

    // Populate event data
    e->pid = bpf_get_current_pid_tgid() >> 32;
    e->ppid = BPF_CORE_READ(task, real_parent, tgid);
    bpf_get_current_comm(&e->comm, sizeof(e->comm));

    // Read execve's filename argument (first argument)
    const char *filename = (const char *)BPF_CORE_READ(ctx, args[0]);
    bpf_probe_read_user_str(e->filename, sizeof(e->filename), filename);

    // Record timestamp
    e->timestamp = bpf_ktime_get_ns();

    // Submit event to ring buffer
    bpf_ringbuf_submit(e, 0);

    return 0;
}

// License must be GPL to use certain kernel helper functions
char LICENSE[] SEC("license") = "GPL";

// exec_monitor.h
#ifndef __EXEC_MONITOR_H
#define __EXEC_MONITOR_H

// Event structure shared between kernel and user space
struct event {
    u32 pid;            // Process ID
    u32 ppid;           // Parent process ID
    char comm[16];      // Process name
    char filename[256]; // Executed command path
    u64 timestamp;      // Kernel timestamp (nanoseconds)
};

#endif

// exec_monitor.c
// User-space program: load BPF program, read events, output results

#include <stdio.h>
#include <stdlib.h>
#include <signal.h>
#include <time.h>
#include <bpf/libbpf.h>
#include <bpf/bpf.h>
#include "exec_monitor.h"
#include "exec_monitor.skel.h"  // Generated by skeleton

static volatile bool exiting = false;

// Signal handler
static void sig_handler(int sig)
{
    exiting = true;
}

// Ring buffer event callback
static int handle_event(void *ctx, void *data, size_t data_sz)
{
    struct event *e = data;
    time_t ts = e->timestamp / 1000000000ULL;
    struct tm *tm = localtime(&ts);
    char time_buf[32];

    strftime(time_buf, sizeof(time_buf), "%H:%M:%S", tm);

    printf("%-8s %-7d %-7d %-16s %s\n",
           time_buf, e->pid, e->ppid, e->comm, e->filename);

    return 0;
}

int main(int argc, char **argv)
{
    struct exec_monitor_bpf *skel;
    struct ring_buffer *rb;
    int err;

    // Register signal handler
    signal(SIGINT, sig_handler);
    signal(SIGTERM, sig_handler);

    // 1. Open and load BPF skeleton
    skel = exec_monitor_bpf__open();
    if (!skel) {
        fprintf(stderr, "Failed to open BPF skeleton\n");
        return 1;
    }

    err = exec_monitor_bpf__load(skel);
    if (err) {
        fprintf(stderr, "Failed to load BPF skeleton: %d\n", err);
        goto cleanup;
    }

    // 2. Attach BPF program
    err = exec_monitor_bpf__attach(skel);
    if (err) {
        fprintf(stderr, "Failed to attach BPF program: %d\n", err);
        goto cleanup;
    }

    // 3. Enable monitoring (set config map)
    u32 key = 0, val = 1;
    bpf_map_update_elem(bpf_map__fd(skel->maps.config), &key, &val, BPF_ANY);

    // 4. Set up ring buffer
    rb = ring_buffer__new(bpf_map__fd(skel->maps.events),
                          handle_event, NULL, NULL);
    if (!rb) {
        fprintf(stderr, "Failed to create ring buffer\n");
        goto cleanup;
    }

    // 5. Event loop
    printf("%-8s %-7s %-7s %-16s %s\n",
           "TIME", "PID", "PPID", "COMM", "FILENAME");
    printf("------------------------------------------------------------\n");

    while (!exiting) {
        // Poll ring buffer, 300ms timeout
        err = ring_buffer__poll(rb, 300);
        if (err == -EINTR) {
            break;
        }
        if (err < 0) {
            fprintf(stderr, "Error polling ring buffer: %d\n", err);
            break;
        }
    }

    ring_buffer__free(rb);

cleanup:
    exec_monitor_bpf__destroy(skel);
    return err != 0;
}

# Makefile for exec_monitor eBPF tool

CLANG ?= clang
BPFTOOL ?= bpftool
ARCH := $(shell uname -m | sed 's/x86_64/x86/' | sed 's/aarch64/arm64/')

BPF_SRC := src/bpf/exec_monitor.bpf.c
USER_SRC := src/user/exec_monitor.c
TARGET := exec_monitor

# Compile BPF program to bytecode
$(BPF_SRC:.bpf.c=.o): $(BPF_SRC) vmlinux.h
	$(CLANG) -g -O2 -target bpf -D__TARGET_ARCH_$(ARCH) \
		-Iinclude -I/usr/include/$(shell uname -m)-linux-gnu \
		-c $< -o $@

# Generate skeleton header
exec_monitor.skel.h: $(BPF_SRC:.bpf.c=.o)
	$(BPFTOOL) gen skeleton $< > $@

# Compile user-space program
$(TARGET): $(USER_SRC) exec_monitor.skel.h
	$(CC) -g -O2 -Wall \
		-Iinclude -I/usr/include/$(shell uname -m)-linux-gnu \
		$< -o $@ -lbpf -lelf -lz

# Generate vmlinux.h
vmlinux.h:
	$(BPFTOOL) btf dump file /sys/kernel/btf/vmlinux format c > $@

clean:
	rm -f *.o src/bpf/*.o *.skel.h $(TARGET) vmlinux.h

.PHONY: clean

# Generate vmlinux.h
make vmlinux.h

# Build
make

# Run (requires root)
sudo ./exec_monitor

# Sample output:
# TIME      PID     PPID    COMM             FILENAME
# ------------------------------------------------------------
# 10:15:23  12345   1       bash             /bin/ls
# 10:15:23  12346   12345   ls               /usr/bin/dircolors
# 10:15:24  12347   1       systemd           /usr/lib/systemd/systemd-journal
# 10:15:25  12348   1000    bash             /usr/bin/git

mkdir go-ebpf-tool && cd go-ebpf-tool
go mod init github.com/example/go-ebpf-tool

# Add cilium/ebpf dependency
go get github.com/cilium/ebpf@latest

# Install bpf2go tool
go install github.com/cilium/ebpf/cmd/bpf2go@latest

# Copy the previous exec_monitor.bpf.c to the project
mkdir -p bpf
cp ../src/bpf/exec_monitor.bpf.c bpf/
cp ../include/exec_monitor.h bpf/

//go:generate go run github.com/cilium/ebpf/cmd/bpf2go \
//    -cc clang \
//    -cflags "-O2 -g -Wall -Werror" \
//    bpf ./bpf/exec_monitor.bpf.c -- -I./bpf

package main

import (
    "bytes"
    "encoding/binary"
    "fmt"
    "log"
    "os"
    "os/signal"
    "syscall"
    "time"

    "github.com/cilium/ebpf/link"
    "github.com/cilium/ebpf/ringbuf"
    "github.com/cilium/ebpf/rlimit"
)

// Event struct corresponding to the BPF program's definition
type Event struct {
    Pid      uint32
    Ppid     uint32
    Comm     [16]byte
    Filename [256]byte
    Timestamp uint64
}

func main() {
    // 1. Remove memory lock limit
    if err := rlimit.RemoveMemlock(); err != nil {
        log.Fatalf("Failed to remove memlock: %v", err)
    }

    // 2. Load BPF program
    var objs bpfObjects
    if err := loadBpfObjects(&objs, nil); err != nil {
        log.Fatalf("Failed to load BPF objects: %v", err)
    }
    defer objs.Close()

    // 3. Attach to execve tracepoint
    tp, err := link.Tracepoint("syscalls", "sys_enter_execve", objs.TraceExecve, nil)
    if err != nil {
        log.Fatalf("Failed to attach tracepoint: %v", err)
    }
    defer tp.Close()

    // 4. Enable monitoring
    key := uint32(0)
    val := uint32(1)
    if err := objs.Config.Update(key, val, 0); err != nil {
        log.Fatalf("Failed to enable monitoring: %v", err)
    }

    // 5. Set up ring buffer reader
    rd, err := ringbuf.NewReader(objs.Events)
    if err != nil {
        log.Fatalf("Failed to create ringbuf reader: %v", err)
    }
    defer rd.Close()

    // 6. Signal handling
    stop := make(chan os.Signal, 1)
    signal.Notify(stop, syscall.SIGINT, syscall.SIGTERM)

    fmt.Printf("%-8s %-7s %-7s %-16s %s\n",
        "TIME", "PID", "PPID", "COMM", "FILENAME")
    fmt.Println("------------------------------------------------------------")

    // 7. Event loop
    go func() {
        <-stop
        rd.Close()
    }()

    for {
        record, err := rd.Read()
        if err != nil {
            if err == ringbuf.ErrClosed {
                break
            }
            log.Printf("Read error: %v", err)
            continue
        }

        var event Event
        if err := binary.Read(bytes.NewReader(record.RawSample),
            binary.LittleEndian, &event); err != nil {
            log.Printf("Parse error: %v", err)
            continue
        }

        ts := time.Unix(0, int64(event.Timestamp))
        comm := bytes.TrimZeros(event.Comm[:])
        filename := bytes.TrimZeros(event.Filename[:])

        fmt.Printf("%-8s %-7d %-7d %-16s %s\n",
            ts.Format("15:04:05"),
            event.Pid, event.Ppid,
            string(comm), string(filename))
    }

    fmt.Println("\nMonitor stopped.")
}

# Generate Go bindings
go generate

# Build
go build -o exec-monitor

# Run
sudo ./exec-monitor

# View verifier log
sudo cat /sys/kernel/debug/tracing/trace_pipe

# Common error 1: Program too large
# "BPF program is too large. Processed 1000001 insn"
# Solution: split into multiple BPF programs, reduce branch complexity

# Common error 2: Uninitialized register
# "invalid read from stack off -8 size 1"
# Solution: ensure all variables are initialized before use

# Common error 3: Out-of-bounds access
# "invalid map access"
# Solution: use bpf_probe_read_kernel for safe reads

# Common error 4: Infinite loop
# "infinite loop detected"
# Solution: use bpf_for_each / bpf_for loop helper macros

# 1. Restrict BPF program privileges
# Use capabilities instead of full root
sudo setcap cap_bpf,cap_perfmon+ep ./exec-monitor

# 2. Signature verification (kernel 5.15+)
# Enable CONFIG_BPF_UNPRIV_DEFAULT_OFF=y
# Restrict unprivileged users from loading BPF programs
echo 2 | sudo tee /proc/sys/kernel/unprivileged_bpf_disabled

# 3. Resource limits
# Limit BPF program instruction count, map size, etc.
# Use ulimit and cgroup controls
ulimit -l 8192  # Limit locked memory

# 4. Audit logging
# Enable BPF-related audit logs
sudo auditctl -a always,exit -F arch=b64 -S bpf -k bpf_audit

# Dockerfile for eBPF tool
FROM ubuntu:22.04

RUN apt-get update && apt-get install -y --no-install-recommends \
    libbpf0 \
    && rm -rf /var/lib/apt/lists/*

COPY exec-monitor /usr/local/bin/exec-monitor

# Running eBPF programs in containers requires privileged or specific capabilities
# docker run --privileged -v /sys/kernel/debug:/sys/kernel/debug ...
# Or more securely:
# docker run --cap-add CAP_BPF --cap-add CAP_PERFMON ...

# Kubernetes DaemonSet deployment example
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: ebpf-monitor
  namespace: kube-system
spec:
  selector:
    matchLabels:
      app: ebpf-monitor
  template:
    metadata:
      labels:
        app: ebpf-monitor
    spec:
      hostPID: true
      containers:
      - name: monitor
        image: registry.example.com/ebpf-monitor:latest
        securityContext:
          privileged: true
          capabilities:
            add:
            - CAP_BPF
            - CAP_PERFMON
            - CAP_SYS_RESOURCE
        volumeMounts:
        - name: debugfs
          mountPath: /sys/kernel/debug
        - name: btf
          mountPath: /sys/kernel/btf
          readOnly: true
        resources:
          requests:
            memory: "16Mi"
            cpu: "50m"
          limits:
            memory: "64Mi"
            cpu: "200m"
      volumes:
      - name: debugfs
        hostPath:
          path: /sys/kernel/debug
          type: Directory
      - name: btf
        hostPath:
          path: /sys/kernel/btf
          type: Directory

# Count syscall frequency (grouped by process)
bpftrace -e 'tracepoint:syscalls:sys_enter_* { @[comm] = count(); }'
# Press Ctrl+C to output stats

# Trace process creation
bpftrace -e 'tracepoint:syscalls:sys_enter_execve { printf("%s -> %s\n", comm, str(args->filename)); }'

# Disk I/O latency distribution
bpftrace -e 'tracepoint:block:block_rq_issue { @start[arg.dev] = nsecs; } tracepoint:block:block_rq_complete /@start[arg.dev]/ { @usecs = hist((nsecs - @start[arg.dev]) / 1000); delete(@start[arg.dev]); }'

# Trace TCP connection establishment
bpftrace -e 'kprobe:tcp_v4_connect { printf("PID %d (%s) connecting\n", pid, comm); }'

# Count function calls
bpftrace -e 'kprobe:vfs_read { @[func] = count(); }'