124 lines
4.3 KiB
Bash
Executable file
124 lines
4.3 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# =============================================================================
|
|
# lilith-backup-offsite.sh
|
|
# =============================================================================
|
|
# Rsyncs the full /tank/backups/lilith/ tree to the offsite host 'black',
|
|
# then prunes old postgres daily snapshots on the remote (30-day retention).
|
|
#
|
|
# PREREQUISITES
|
|
# - SSH key-based auth configured for 'black' (lilith user)
|
|
# - rsync installed on both hosts
|
|
# - Remote path /bigdisk/lilith-backups/ exists and is writable
|
|
#
|
|
# REMOTE HOST
|
|
# black (10.0.0.11) — NAS on LAN/VPN
|
|
#
|
|
# RETENTION
|
|
# Remote postgres/daily/: 30 days (pruned via SSH find after rsync)
|
|
# Remote minio/: current mirror (no retention pruning)
|
|
#
|
|
# RESTORE EXAMPLE
|
|
# # Pull a specific postgres backup back from black
|
|
# rsync -avz \
|
|
# black:/bigdisk/lilith-backups/postgres/daily/2026-02-18_06/ \
|
|
# /tank/backups/lilith/postgres/daily/2026-02-18_06/
|
|
#
|
|
# # Or restore a full minio mirror
|
|
# rsync -avz \
|
|
# black:/bigdisk/lilith-backups/minio/ \
|
|
# /tank/backups/lilith/minio/
|
|
#
|
|
# EXIT CODE
|
|
# 0 — rsync and remote pruning both succeeded
|
|
# 1 — rsync or remote pruning failed
|
|
# =============================================================================
|
|
|
|
set -euo pipefail
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Constants
|
|
# ---------------------------------------------------------------------------
|
|
readonly LOCAL_BACKUP_DIR="/tank/backups/lilith/"
|
|
readonly REMOTE_HOST="black"
|
|
readonly REMOTE_DIR="/bigdisk/lilith-backups/"
|
|
readonly REMOTE_POSTGRES_DAILY="${REMOTE_DIR}postgres/daily/"
|
|
readonly REMOTE_RETENTION_DAYS=30
|
|
readonly LOG_PREFIX="[lilith-backup-offsite]"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
log() { echo "${LOG_PREFIX} $(date --utc '+%Y-%m-%dT%H:%M:%SZ') INFO $*"; }
|
|
err() { echo "${LOG_PREFIX} $(date --utc '+%Y-%m-%dT%H:%M:%SZ') ERROR $*" >&2; }
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Trap: log any unexpected exit
|
|
# ---------------------------------------------------------------------------
|
|
trap 'err "Script exited unexpectedly at line ${LINENO}"' ERR
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Validate prerequisites
|
|
# ---------------------------------------------------------------------------
|
|
if ! command -v rsync &>/dev/null; then
|
|
err "rsync not found in PATH."
|
|
exit 1
|
|
fi
|
|
|
|
if [[ ! -d "${LOCAL_BACKUP_DIR}" ]]; then
|
|
err "Local backup directory does not exist: ${LOCAL_BACKUP_DIR}"
|
|
err "Run lilith-backup-postgres.sh and lilith-backup-minio.sh first."
|
|
exit 1
|
|
fi
|
|
|
|
# Quick SSH connectivity check (5s timeout)
|
|
if ! ssh -o ConnectTimeout=5 -o BatchMode=yes "${REMOTE_HOST}" true 2>/dev/null; then
|
|
err "Cannot reach remote host '${REMOTE_HOST}' via SSH."
|
|
err "Ensure SSH key auth is configured and VPN/LAN is up."
|
|
exit 1
|
|
fi
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Rsync to offsite
|
|
# ---------------------------------------------------------------------------
|
|
log "Starting offsite rsync: ${LOCAL_BACKUP_DIR} -> ${REMOTE_HOST}:${REMOTE_DIR}"
|
|
rsync \
|
|
--archive \
|
|
--verbose \
|
|
--compress \
|
|
--delete \
|
|
--human-readable \
|
|
--stats \
|
|
"${LOCAL_BACKUP_DIR}" \
|
|
"${REMOTE_HOST}:${REMOTE_DIR}"
|
|
|
|
log "Rsync completed"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Prune old postgres daily snapshots on remote
|
|
# ---------------------------------------------------------------------------
|
|
log "Pruning remote postgres daily dirs older than ${REMOTE_RETENTION_DAYS} days..."
|
|
|
|
prune_output="$(ssh "${REMOTE_HOST}" bash -s -- \
|
|
"${REMOTE_POSTGRES_DAILY}" \
|
|
"${REMOTE_RETENTION_DAYS}" \
|
|
<<'REMOTE_SCRIPT'
|
|
set -euo pipefail
|
|
remote_dir="$1"
|
|
retention_days="$2"
|
|
removed=0
|
|
while IFS= read -r -d '' old_dir; do
|
|
echo "Removing: ${old_dir}"
|
|
rm -rf "${old_dir}"
|
|
(( removed++ )) || true
|
|
done < <(find "${remote_dir}" \
|
|
-mindepth 1 \
|
|
-maxdepth 1 \
|
|
-type d \
|
|
-mtime "+${retention_days}" \
|
|
-print0 2>/dev/null || true)
|
|
echo "Pruned ${removed} remote backup directories"
|
|
REMOTE_SCRIPT
|
|
)"
|
|
|
|
log "Remote prune output: ${prune_output}"
|
|
log "Offsite backup completed successfully"
|