Octal   Symbolic   Meaning
 0       ---       no permissions
 1       --x       execute only
 2       -w-       write only
 4       r--       read only
 6       rw-       read + write
 7       rwx       read + write + execute

# Common patterns
chmod 600 ~/.ssh/id_rsa        # private key: owner read/write only
chmod 644 /etc/nginx/nginx.conf  # config: owner rw, others read
chmod 755 /usr/local/bin/script  # executable: owner rwx, others rx
chmod 700 /root/.gnupg           # directory: only owner can enter

Targets (grouping)         -> multi-user.target, network.target
  Services (.service)      -> long-running daemons, oneshot tasks
  Timers (.timer)          -> scheduled execution (replaces cron)
  Sockets (.socket)        -> socket-activated services
  Mounts (.mount)          -> filesystem mounts managed by systemd
  Paths (.path)            -> filesystem change triggers

#!/usr/bin/env bash
set -euo pipefail
# -e: exit on error
# -u: treat unset variables as errors
# -o pipefail: pipeline fails if any command in it fails

# Cleanup on exit - runs on success, error, and signals
TMPDIR_WORK=""
cleanup() {
    local exit_code=$?
    [[ -n "$TMPDIR_WORK" ]] && rm -rf "$TMPDIR_WORK"
    exit "$exit_code"
}
trap cleanup EXIT INT TERM

# Argument parsing with defaults and validation
usage() {
    echo "Usage: $0 [-e ENV] [-d] <target>"
    echo "  -e ENV   Environment (default: staging)"
    echo "  -d       Dry-run mode"
    exit 1
}

ENV="staging"
DRY_RUN=false

while getopts ":e:dh" opt; do
    case $opt in
        e) ENV="$OPTARG" ;;
        d) DRY_RUN=true ;;
        h) usage ;;
        :) echo "Option -$OPTARG requires an argument." >&2; usage ;;
        \?) echo "Unknown option: -$OPTARG" >&2; usage ;;
    esac
done
shift $((OPTIND - 1))

[[ $# -lt 1 ]] && { echo "Error: target required" >&2; usage; }
TARGET="$1"

# Use mktemp for safe temp directories
TMPDIR_WORK=$(mktemp -d)

# Log with timestamps
log() { echo "[$(date '+%Y-%m-%dT%H:%M:%S')] $*"; }
log "Starting deploy: env=$ENV target=$TARGET dry_run=$DRY_RUN"

# Dry-run wrapper
run() {
    if [[ "$DRY_RUN" == true ]]; then
        echo "[DRY-RUN] $*"
    else
        "$@"
    fi
}

run rsync -av --exclude='.git' "./" "deploy@${TARGET}:/opt/app/"
log "Deploy complete"

# /etc/systemd/system/db-backup.service
[Unit]
Description=Database backup
After=network-online.target postgresql.service
Wants=network-online.target
# Prevent starting if PostgreSQL is not running
Requires=postgresql.service

[Service]
Type=oneshot
User=backup
Group=backup
# Security hardening
NoNewPrivileges=true
ProtectSystem=strict
ProtectHome=true
ReadWritePaths=/var/backups/db
PrivateTmp=true

ExecStart=/usr/local/bin/db-backup.sh
StandardOutput=journal
StandardError=journal

# Retry on failure
Restart=on-failure
RestartSec=60

[Install]
WantedBy=multi-user.target

# /etc/systemd/system/db-backup.timer
[Unit]
Description=Run database backup daily at 02:00
Requires=db-backup.service

[Timer]
# Run at 02:00 every day
OnCalendar=*-*-* 02:00:00
# Run immediately if last run was missed (e.g., server was down)
Persistent=true
# Randomize start within 5 minutes to avoid thundering herd
RandomizedDelaySec=300

[Install]
WantedBy=timers.target

# Deploy and enable
sudo systemctl daemon-reload
sudo systemctl enable --now db-backup.timer

# Inspect
systemctl status db-backup.timer
systemctl list-timers db-backup.timer
journalctl -u db-backup.service -n 50

# /etc/ssh/sshd_config - production hardening

# Use SSH protocol 2 only (default in modern OpenSSH, make it explicit)
Protocol 2

# Disable root login - use a dedicated admin user with sudo
PermitRootLogin no

# Disable password authentication - key-based only
PasswordAuthentication no
ChallengeResponseAuthentication no
UsePAM yes

# Disable X11 forwarding unless needed
X11Forwarding no

# Limit login window to prevent slowloris-style attacks
LoginGraceTime 30
MaxAuthTries 4
MaxSessions 10

# Only allow specific groups to SSH
AllowGroups sshusers admins

# Restrict ciphers, MACs, and key exchange to modern algorithms
Ciphers chacha20-poly1305@openssh.com,aes256-gcm@openssh.com,aes128-gcm@openssh.com
MACs hmac-sha2-256-etm@openssh.com,hmac-sha2-512-etm@openssh.com
KexAlgorithms curve25519-sha256,curve25519-sha256@libssh.org

# Use privilege separation
UsePrivilegeSeparation sandbox

# Log at verbose level to capture key fingerprints on auth
LogLevel VERBOSE

# Set idle timeout: disconnect after 15 minutes of inactivity
ClientAliveInterval 300
ClientAliveCountMax 3

# Validate before restarting
sudo sshd -t

# Restart sshd (keep current session open until verified)
sudo systemctl restart sshd

# Verify from a NEW session before closing the old one
ssh -v user@host

# 1. Check interface state and IP assignment
ip addr show
ip link show

# 2. Check routing table
ip route show
# Expected: default route via gateway, local subnet route

# 3. Test gateway reachability
ping -c 4 $(ip route | awk '/default/ {print $3}')

# 4. Test DNS resolution
dig +short google.com @8.8.8.8    # direct to external resolver
resolvectl query google.com        # use system resolver (systemd-resolved)
cat /etc/resolv.conf               # check configured resolvers

# 5. Check listening ports and owning processes
ss -tulpn
# -t: TCP  -u: UDP  -l: listening  -p: process  -n: no name resolution

# 6. Test specific port connectivity
nc -zv 10.0.0.5 5432              # check if port is open
timeout 3 bash -c "</dev/tcp/10.0.0.5/5432" && echo open || echo closed

# 7. Trace the path
traceroute -n 8.8.8.8             # ICMP path tracing
mtr --report 8.8.8.8              # continuous path with stats (better than traceroute)

# 8. Capture traffic for deep inspection
# Capture all traffic on eth0 to/from a host on port 443
sudo tcpdump -i eth0 -n host 10.0.0.5 and port 443 -w /tmp/capture.pcap
# Quick view without saving
sudo tcpdump -i eth0 -n port 53   # watch DNS queries live

# --- ufw approach (recommended for most servers) ---

# Reset to defaults
sudo ufw --force reset
sudo ufw default deny incoming
sudo ufw default allow outgoing

# Allow SSH (do this BEFORE enabling to avoid lockout)
sudo ufw allow 22/tcp comment 'SSH'

# Web server
sudo ufw allow 80/tcp comment 'HTTP'
sudo ufw allow 443/tcp comment 'HTTPS'

# Allow specific source IP for admin access
sudo ufw allow from 192.168.1.0/24 to any port 5432 comment 'Postgres from internal'

# Enable and verify
sudo ufw --force enable
sudo ufw status verbose

# --- iptables approach for precise control ---

# Flush existing rules
iptables -F
iptables -X

# Default policies: drop everything
iptables -P INPUT DROP
iptables -P FORWARD DROP
iptables -P OUTPUT ACCEPT

# Allow loopback
iptables -A INPUT -i lo -j ACCEPT
iptables -A OUTPUT -o lo -j ACCEPT

# Allow established/related connections
iptables -A INPUT -m conntrack --ctstate ESTABLISHED,RELATED -j ACCEPT

# Allow SSH (rate-limit to prevent brute force)
iptables -A INPUT -p tcp --dport 22 -m conntrack --ctstate NEW \
    -m recent --set --name SSH --rsource
iptables -A INPUT -p tcp --dport 22 -m conntrack --ctstate NEW \
    -m recent --update --seconds 60 --hitcount 4 --name SSH --rsource -j DROP
iptables -A INPUT -p tcp --dport 22 -j ACCEPT

# Allow HTTP/HTTPS
iptables -A INPUT -p tcp -m multiport --dports 80,443 -j ACCEPT

# Save rules
iptables-save > /etc/iptables/rules.v4

# Check disk usage overview
df -hT
# -h: human readable  -T: show filesystem type

# Find large directories (top 10, depth-limited)
du -h --max-depth=2 /var | sort -rh | head -10

# Interactive disk usage explorer (install ncdu first)
ncdu /var/log

# Find large files
find /var -type f -size +100M -exec ls -lh {} \; 2>/dev/null | sort -k5 -rh

# Check journal size and truncate if needed
journalctl --disk-usage
sudo journalctl --vacuum-size=500M    # keep last 500MB
sudo journalctl --vacuum-time=30d     # keep last 30 days

# /etc/logrotate.d/myapp - custom log rotation
/var/log/myapp/*.log {
    daily
    rotate 14
    compress
    delaycompress
    missingok
    notifempty
    sharedscripts
    postrotate
        systemctl reload myapp 2>/dev/null || true
    endscript
}

# Test logrotate config without running it
logrotate --debug /etc/logrotate.d/myapp

# Force a rotation run
logrotate --force /etc/logrotate.d/myapp

# Overview: CPU, memory, load average
top -b -n 1 -o %CPU | head -20       # batch mode, sort by CPU
htop                                   # interactive, colored, tree view

# Find what a process is doing
pid=$(pgrep -x nginx | head -1)

# Open files and network connections
lsof -p "$pid"                        # all open files
lsof -p "$pid" -i                     # only network connections
lsof -i :8080                         # what process owns port 8080

# System calls (strace) - use when a process behaves unexpectedly
strace -p "$pid" -f -e trace=network  # network syscalls only
strace -p "$pid" -f -c                # count syscall frequency (summary)
strace -c cmd arg                     # profile syscalls of a new command

# Memory inspection
cat /proc/"$pid"/status | grep -E 'Vm|Threads'
cat /proc/"$pid"/smaps_rollup          # detailed memory breakdown

# Check zombie/defunct processes
ps aux | awk '$8 == "Z" {print}'

# Kill process tree (all children too)
kill -TERM -"$(ps -o pgid= -p "$pid" | tr -d ' ')"