Zombie Processes · OOM Killer — Diagnosis and Prevention

# List zombie processes
ps aux | grep -w Z
# USER PID %CPU %MEM ... STAT ... COMMAND
# www-data 1234 0.0 0.0 ... Z+  ... [nginx] <defunct>

# Count them
ps aux | awk '{print $8}' | grep Z | wc -l

# Check in top
top
# Tasks: 150 total, 1 running, 148 sleeping, 0 stopped, 1 zombie

# After identifying the zombie PID, find its parent
ps -o ppid= -p <zombie_PID>

# Check the parent process name
ps -p <parent_PID> -o comm=

# View as a tree
pstree -p | grep -B5 defunct

# Method 1: Gracefully restart the parent process (recommended)
# When the parent restarts, init(1) calls wait() and reaps the zombie
sudo systemctl restart nginx    # e.g. nginx worker zombie

# Method 2: Send SIGCHLD to the parent (force a wait() call)
kill -SIGCHLD <parent_PID>

# Method 3: Terminate the parent (init reaps the child zombies)
kill -TERM <parent_PID>

# Method 4: Last resort — force-kill the parent
kill -9 <parent_PID>

# Check whether the OOM Killer has fired
dmesg | grep -E "Out of memory|oom_kill|Killed process"
sudo journalctl -k | grep -i oom

# See which process was killed
dmesg | grep "Killed process"
# Killed process 1234 (myapp) total-vm:512000kB, anon-rss:450000kB

# Memory dump at the time of OOM
dmesg | grep -A30 "Out of memory"

# Check OOM score (-1000 to 1000; higher means killed first)
cat /proc/<PID>/oom_score

# Adjust oom_score_adj (-1000 to 1000)
# -1000 = fully excluded from the OOM Killer
# 0 = default
# +1000 = killed first

# Protect a critical process
echo -1000 | sudo tee /proc/$(pgrep postgres)/oom_score_adj

# Kill less important processes first
echo 500 | sudo tee /proc/$(pgrep node)/oom_score_adj

# Permanent setting for a systemd service
# /etc/systemd/system/myapp.service
# [Service]
# OOMScoreAdjust=-500

# Add a swap file (1GB)
sudo fallocate -l 1G /swapfile
sudo chmod 600 /swapfile
sudo mkswap /swapfile
sudo swapon /swapfile

# Make it persistent
echo '/swapfile none swap sw 0 0' | sudo tee -a /etc/fstab

# Adjust swappiness (0 = minimize swap, 100 = use aggressively)
sudo sysctl vm.swappiness=10
echo "vm.swappiness=10" | sudo tee -a /etc/sysctl.conf

# Memory limit on a systemd service
sudo systemctl edit myapp.service

# Add:
# [Service]
# MemoryMax=512M
# MemoryHigh=400M   # throttle above this

sudo systemctl daemon-reload
sudo systemctl restart myapp

# Check the current memory limit
sudo systemctl show myapp --property=MemoryMax

# Memory limit for a Docker container
docker run --memory=512m --memory-swap=512m myapp