awk · sed in Practice — Log Parsing and Text Transformation

# Print fields by delimiter ($1 = first field)
awk '{print $1, $3}' access.log

# Specify delimiter (colon)
awk -F':' '{print $1}' /etc/passwd

# Conditional filter (rows where the 5th field is 200)
awk '$5 == 200 {print $0}' access.log

# Calculate a sum
awk '{sum += $NF} END {print "Total:", sum}' numbers.txt

# Aggregate by status code from Nginx access.log
awk '{print $9}' /var/log/nginx/access.log | sort | uniq -c | sort -rn

# Top 10 IPs by request count
awk '{print $1}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head -10

# Sum of response sizes (bytes)
awk '{sum += $10} END {printf "Total: %.2f MB\n", sum/1024/1024}' access.log

# Print 4xx and 5xx errors only
awk '$9 ~ /^[45]/' /var/log/nginx/access.log

# String substitution (first match)
sed 's/old/new/' file.txt

# Global substitution (g flag)
sed 's/old/new/g' file.txt

# Case-insensitive substitution
sed 's/old/new/gi' file.txt

# In-place file edit (-i)
sed -i 's/localhost/127.0.0.1/g' config.conf

# Delete a specific line (3rd line)
sed '3d' file.txt

# Remove empty lines
sed '/^$/d' file.txt

# Remove comment lines
sed '/^#/d' config.conf

# Change the port number
sed -i 's/^Port 22/Port 2222/' /etc/ssh/sshd_config

# Insert a new line after a matching line
sed -i '/^SELINUX=enforcing/a SELINUXTYPE=targeted' /etc/selinux/config

# Batch replace across multiple files
find . -name "*.conf" -exec sed -i 's/old_domain/new_domain/g' {} \;

# Use | instead of / as the delimiter (convenient when paths are involved)
sed 's|/old/path|/new/path|g' deploy.sh

# Process only a range of lines (lines 10–20)
sed -n '10,20p' large_file.log

# Print lines between two patterns
sed -n '/START/,/END/p' file.txt

# Parse with awk, then tidy the format with sed
awk -F',' '{print $1,$3}' data.csv | sed 's/ /\t/g'