Diagnosing I/O Bottlenecks — iostat · iotop · blktrace

vmstat 2 10
# procs  ----------memory---------- --swap-- -----io---- -system-- ------cpu-----
#  r  b   swpd   free   buff  cache   si  so    bi    bo   in   cs us sy id wa st
#  1  3      0  512M    20M   1.5G    0   0   8000  4000  500 1000  5  2  3 90  0
#                                                             ↑↑↑↑
# wa=90: CPU is waiting for I/O to complete — a clear I/O bottleneck

sudo apt install -y sysstat

# 2-second interval, 5 samples
iostat -xz 2 5

# Interpreting key columns
# %util: disk utilization (near 100% means saturated)
# await: average I/O response time (ms) — typical HDD: <20ms, SSD: <1ms
# r/s, w/s: read/write requests per second
# rkB/s, wkB/s: read/write throughput per second (KB)
# svctm: service time (await minus queue wait time)

# Specific devices only
iostat -x sda sdb 2

# Per-disk read/write ratio
iostat -d -k 2 | grep -v "^$"

sudo apt install -y iotop

# Interactive mode
sudo iotop

# Show only processes currently doing I/O (-o)
sudo iotop -o

# One-shot snapshot
sudo iotop -b -n 3 | head -20

# Keyboard shortcuts
# o: toggle active processes only
# p: toggle processes/threads
# a: toggle accumulated I/O

# Trace I/O for a specific process
pidstat -d -p 1234 2 5

# I/O for all processes
pidstat -d 2 5 | sort -k4 -rn | head -10

# Check open files (which files are being accessed)
sudo lsof -p 1234 | grep -E "REG|DIR"
sudo ls -la /proc/1234/fd

sudo apt install -y fio

# Sequential read performance
fio --name=seq-read --ioengine=libaio --iodepth=32   --rw=read --bs=128k --direct=1 --size=1G   --numjobs=1 --runtime=30 --filename=/tmp/fio-test

# Random read/write (IOPS measurement — SSD performance metric)
fio --name=rand-rw --ioengine=libaio --iodepth=32   --rw=randrw --bs=4k --direct=1 --size=1G   --numjobs=4 --runtime=30 --filename=/tmp/fio-test   --group_reporting

# Interpreting results
# READ: IOPS=50k, BW=200MiB/s (SSD is healthy)
# IOPS < 1000 indicates HDD-class performance

# Heavy DB writes → increase the buffer pool
# PostgreSQL: shared_buffers = 25% of total memory
# MySQL: innodb_buffer_pool_size = 50-70% of total memory

# Heavy log I/O → asynchronous commit
# PostgreSQL: synchronous_commit = off (risk of data loss)

# Check disk cache
free -h
# High buff/cache means the OS is using cache — this is normal

# Change the I/O scheduler (use none or mq-deadline for SSDs)
cat /sys/block/sda/queue/scheduler
echo mq-deadline | sudo tee /sys/block/sda/queue/scheduler