75 lines
2.4 KiB
Bash
Executable File
75 lines
2.4 KiB
Bash
Executable File
#!/bin/bash
|
|
# verify_postgres_snapshot.sh
|
|
# Verify PostgreSQL backup snapshot remotely using ZFS, pg_verifybackup, and pg_checksums
|
|
|
|
set -euo pipefail
|
|
|
|
# === CONFIG ===
|
|
REMOTE_HOST="zcluster.technodrome2.sshjunkie.com"
|
|
ZFS_DATASET="pgpool/postgresbackups/db2"
|
|
MOUNT_ROOT="/mnt/pgsnapshot_verify"
|
|
PG_BIN_DIR="/usr/lib/postgresql/16/bin" # adjust if needed on remote host
|
|
LOGFILE="/var/log/pgsnapshot_verify_$(date +%Y%m%d_%H%M%S).log"
|
|
DB_NAME="mastodon_production"
|
|
|
|
# === MASTODON ALERTING ===
|
|
MASTODON_TOKEN="rimxBLi-eaJAcwagkmoj6UoW7Lc473tQY0cOM041Euw"
|
|
MASTODON_API="https://chatwithus.live/api/v1/statuses"
|
|
TOOT_ACCOUNT_ID="114386383616633367"
|
|
|
|
mastodon_alert() {
|
|
local msg="$1"
|
|
curl -sS -X POST "$MASTODON_API" \
|
|
-H "Authorization: Bearer $MASTODON_TOKEN" \
|
|
--data-urlencode "status=$msg" \
|
|
--data "visibility=direct" \
|
|
--data "in_reply_to_account_id=$TOOT_ACCOUNT_ID" >/dev/null
|
|
}
|
|
|
|
# === RUN REMOTELY ===
|
|
echo "📦 Verifying latest snapshot for $ZFS_DATASET on $REMOTE_HOST" | tee "$LOGFILE"
|
|
ssh root@$REMOTE_HOST bash <<EOF
|
|
set -euo pipefail
|
|
|
|
LATEST_SNAPSHOT=\$(zfs list -t snapshot -r $ZFS_DATASET -o name -s creation | tail -n 1)
|
|
|
|
if [[ -z "\$LATEST_SNAPSHOT" ]]; then
|
|
echo "❌ No snapshot found for $ZFS_DATASET" | tee -a "$LOGFILE"
|
|
exit 1
|
|
fi
|
|
|
|
MOUNTPOINT="$MOUNT_ROOT/\${LATEST_SNAPSHOT//\//_}"
|
|
mkdir -p "\$MOUNTPOINT"
|
|
|
|
# Clone snapshot for verification
|
|
zfs clone -o mountpoint="\$MOUNTPOINT" "\$LATEST_SNAPSHOT" "${ZFS_DATASET}@verifytemp"
|
|
|
|
# Run pg_verifybackup
|
|
if ! "$PG_BIN_DIR/pg_verifybackup" "\$MOUNTPOINT" >> "$LOGFILE" 2>&1; then
|
|
echo "❌ pg_verifybackup failed on snapshot \$LATEST_SNAPSHOT" | tee -a "$LOGFILE"
|
|
zfs destroy "${ZFS_DATASET}@verifytemp"
|
|
exit 2
|
|
fi
|
|
|
|
# Optional: pg_checksums
|
|
if "$PG_BIN_DIR/pg_checksums" --check --data-directory="\$MOUNTPOINT" >> "$LOGFILE" 2>&1; then
|
|
echo "✅ pg_checksums passed." | tee -a "$LOGFILE"
|
|
else
|
|
echo "⚠️ pg_checksums failed or not enabled." | tee -a "$LOGFILE"
|
|
fi
|
|
|
|
zfs destroy "${ZFS_DATASET}@verifytemp"
|
|
rmdir "\$MOUNTPOINT"
|
|
EOF
|
|
|
|
# === ALERT BASED ON SSH EXIT CODE ===
|
|
EXIT_CODE=$?
|
|
if [[ $EXIT_CODE -eq 1 ]]; then
|
|
mastodon_alert "🚨 ZFS PostgreSQL snapshot verification failed: No snapshot found for $ZFS_DATASET on $REMOTE_HOST at $(date)."
|
|
elif [[ $EXIT_CODE -eq 2 ]]; then
|
|
mastodon_alert "🚨 pg_verifybackup failed for snapshot of $ZFS_DATASET on $REMOTE_HOST at $(date)."
|
|
fi
|
|
|
|
echo "✅ PostgreSQL remote snapshot verification complete." | tee -a "$LOGFILE"
|
|
exit 0
|