Centralizing Logs with rsyslog — Remote Collection and Filtering

[remote servers] → rsyslog (UDP/TCP 514) → [central log server]
                                              ↓
                                    /var/log/ or DB/ES

sudo apt install -y rsyslog rsyslog-elasticsearch

# Check status
sudo systemctl status rsyslog

# Configuration files
/etc/rsyslog.conf                  # Main configuration
/etc/rsyslog.d/*.conf              # Drop-in configs (recommended for management)

# /etc/rsyslog.d/00-server.conf

# Enable UDP 514 reception
module(load="imudp")
input(type="imudp" port="514")

# TCP 514 reception (more reliable)
module(load="imtcp")
input(type="imtcp" port="514")

# Store logs in separate files per remote host
template(name="RemoteLog" type="string"
  string="/var/log/remote/%HOSTNAME%/%PROGRAMNAME%.log")

*.* ?RemoteLog

# Allow through firewall
sudo ufw allow 514/udp
sudo ufw allow 514/tcp

sudo systemctl restart rsyslog

# /etc/rsyslog.d/50-remote.conf

# Forward via UDP
*.* @192.168.1.100:514

# Forward via TCP (more reliable; use @@)
*.* @@192.168.1.100:514

# Use a queue (buffer locally and retry if the server is down)
action(type="omfwd"
  target="192.168.1.100"
  port="514"
  protocol="tcp"
  action.resumeRetryCount="100"
  queue.type="linkedList"
  queue.size="10000"
  queue.filename="remote-fwd"
  queue.maxDiskSpace="1g"
  queue.saveOnShutdown="on")

# Filters by priority (severity)
# auth.* → authentication logs such as SSH and sudo
# kern.* → kernel logs
# *.emerg → all emergency messages

# /etc/rsyslog.d/10-filters.conf

# Security logs to a separate file
auth,authpriv.*                /var/log/auth.log

# Errors and above to the central server
*.err                          @@192.168.1.100:514

# Logs from a specific program
:programname, isequal, "nginx" /var/log/nginx-syslog.log

# Messages containing a specific string
:msg, contains, "CRITICAL"    /var/log/critical.log

# Logs from a specific host
:fromhost-ip, isequal, "10.0.0.5" /var/log/specific-host.log

# Test with the logger command
logger -p local0.info "Test message from myapp"
logger -p auth.warning "Failed login attempt" -t myapp

# Using syslog from an app (Python)
import logging, logging.handlers
handler = logging.handlers.SysLogHandler(address='/dev/log')
handler.setFormatter(logging.Formatter('myapp: %(message)s'))
logging.getLogger().addHandler(handler)
logging.warning("Something went wrong")

# /etc/logrotate.d/remote-logs
/var/log/remote/*/*.log {
    daily
    rotate 30
    compress
    delaycompress
    missingok
    notifempty
    sharedscripts
    postrotate
        /usr/lib/rsyslog/rsyslog-rotate
    endscript
}

# Check configuration syntax
sudo rsyslogd -N1

# Reload configuration without a full restart
sudo kill -HUP $(cat /var/run/rsyslogd.pid)
sudo systemctl reload rsyslog

# Verify it is working
logger "test message"
tail -f /var/log/syslog