Linux Performance Diagnostics in Depth — Hands-on with perf, eBPF, and FlameGraph

# Ubuntu/Debian
sudo apt install -y linux-perf linux-tools-$(uname -r)

# CentOS/RHEL
sudo yum install -y perf

# Check version
perf version

# Live view of CPU-hot functions system-wide (similar to top)
sudo perf top

# Specific process only (PID 1234)
sudo perf top -p 1234

# Include kernel symbols (when tracking kernel bottlenecks)
sudo perf top -g --call-graph dwarf

# Sample CPU system-wide for 30 seconds (99 Hz)
sudo perf record -F 99 -a -g -- sleep 30

# Sample a specific process for 30 seconds
sudo perf record -F 99 -p 1234 -g -- sleep 30

# Analyze results (TUI)
sudo perf report --stdio

# Analyze with call graph
sudo perf report -g graph --stdio | head -80

# Measure L1/L2/LLC cache misses
sudo perf stat -e cache-misses,cache-references,L1-dcache-loads,L1-dcache-load-misses \
  -p 1234 -- sleep 5

# Measure context switches and page faults
sudo perf stat -e context-switches,page-faults,cpu-migrations \
  -- ./my-program

# Measure branch misprediction rate
sudo perf stat -e branch-instructions,branch-misses -- ./my-program

git clone https://github.com/brendangregg/FlameGraph /opt/FlameGraph
export PATH=/opt/FlameGraph:$PATH

# 1. Sample
sudo perf record -F 99 -a -g -- sleep 30

# 2. Convert to text
sudo perf script > perf.out

# 3. Collapse stacks
/opt/FlameGraph/stackcollapse-perf.pl perf.out > perf.folded

# 4. Generate SVG
/opt/FlameGraph/flamegraph.pl perf.folded > flamegraph.svg

# Open in a browser
open flamegraph.svg          # macOS
xdg-open flamegraph.svg      # Linux

# Extract only stacks related to "nginx"
grep "nginx" perf.folded | /opt/FlameGraph/flamegraph.pl > nginx-flame.svg

# Summarize top CPU-time functions as text
sort perf.folded | uniq -c | sort -rn | head -20

# Pre-deploy sample
sudo perf record -F 99 -a -g -- sleep 30
sudo perf script | /opt/FlameGraph/stackcollapse-perf.pl > before.folded

# Post-deploy sample
sudo perf record -F 99 -a -g -- sleep 30
sudo perf script | /opt/FlameGraph/stackcollapse-perf.pl > after.folded

# Generate a differential SVG (red = increase, blue = decrease)
/opt/FlameGraph/difffolded.pl before.folded after.folded | \
  /opt/FlameGraph/flamegraph.pl > diff-flame.svg

# Ubuntu 22.04+
sudo apt install -y bpfcc-tools linux-headers-$(uname -r)

# Locate the tools
ls /usr/sbin/*-bpfcc 2>/dev/null || ls /usr/share/bcc/tools/

# Trace per-process disk I/O latency
sudo biolatency-bpfcc -D 10

# Trace slow filesystem operations (over 10 ms)
sudo fileslower-bpfcc 10

# Trace TCP connections (live)
sudo tcptracer-bpfcc

# TCP latency distribution (histogram)
sudo tcplife-bpfcc

# Trace process execution (execve)
sudo execsnoop-bpfcc

# Trace open files
sudo opensnoop-bpfcc -p 1234

# Processes with many context switches
sudo cpudist-bpfcc 5 1

# Trace memory allocations (when you suspect a malloc leak)
sudo memleak-bpfcc -p 1234

# CPU sampling with bcc profile (an alternative to perf)
sudo /usr/share/bcc/tools/profile -F 99 30 > bcc-profile.out

# Convert to FlameGraph
/opt/FlameGraph/flamegraph.pl bcc-profile.out > bcc-flame.svg

sudo apt install -y bpftrace

# Frequency of all syscall invocations (5 seconds)
sudo bpftrace -e 'tracepoint:syscalls:sys_enter_* { @[probe] = count(); } interval:s:5 { print(@); clear(@); exit(); }'

# Latency histogram for the read() syscall
sudo bpftrace -e '
tracepoint:syscalls:sys_enter_read { @ts[tid] = nsecs; }
tracepoint:syscalls:sys_exit_read  /@ts[tid]/ {
  @usecs = hist((nsecs - @ts[tid]) / 1000);
  delete(@ts[tid]);
}
interval:s:10 { print(@usecs); exit(); }'

# Size distribution per write() call for a specific process
sudo bpftrace -e '
tracepoint:syscalls:sys_enter_write /comm == "nginx"/ {
  @bytes = hist(args->count);
}
interval:s:10 { print(@bytes); exit(); }'

# TCP connection-setup latency
sudo bpftrace -e '
kprobe:tcp_v4_connect { @start[tid] = nsecs; }
kretprobe:tcp_v4_connect /@start[tid]/ {
  @ms = hist((nsecs - @start[tid]) / 1000000);
  delete(@start[tid]);
}
interval:s:10 { print(@ms); exit(); }'

# Disk I/O request size distribution
sudo bpftrace -e '
tracepoint:block:block_rq_issue {
  @bytes = hist(args->bytes);
}
interval:s:5 { print(@bytes); exit(); }'

# /usr/local/bin/slow-syscalls.bt
# Log syscalls that take longer than 100 ms

tracepoint:syscalls:sys_enter_* {
  @start[tid] = nsecs;
  @call[tid]  = probe;
}

tracepoint:syscalls:sys_exit_* /@start[tid]/ {
  $lat = (nsecs - @start[tid]) / 1000000;
  if ($lat > 100) {
    printf("%-20s %-30s %dms\n", comm, @call[tid], $lat);
  }
  delete(@start[tid]);
  delete(@call[tid]);
}

sudo bpftrace /usr/local/bin/slow-syscalls.bt

# Basic tracing (summary)
sudo strace -c -p 1234

# Trace specific syscalls only (file-related)
sudo strace -e trace=openat,read,write,close -p 1234

# Include timestamps and latency
sudo strace -T -tt -p 1234 2>&1 | head -50

# Trace network-related syscalls only
sudo strace -e trace=network -p 1234

# Follow child processes as well
sudo strace -f -e trace=execve ./my-program

# 1. Identify which process it is
top -b -n1 | head -20
ps aux --sort=-%cpu | head -10

# 2. Check CPU per thread for that PID
top -H -p 1234

# 3. Sample with perf for 30 seconds
sudo perf record -F 99 -p 1234 -g -- sleep 30

# 4. Generate a FlameGraph
sudo perf script | /opt/FlameGraph/stackcollapse-perf.pl | \
  /opt/FlameGraph/flamegraph.pl > /tmp/flame-$(date +%H%M).svg

# 5. Find the wide top functions in the FlameGraph → review that function's code

# 6. Additionally check call frequency of that function with bpftrace
sudo bpftrace -e 'uprobe:/usr/bin/myapp:slow_function { @calls = count(); } interval:s:5 { print(@calls); }'