93 lines
3.6 KiB
Bash
Executable File
93 lines
3.6 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# ---- CONFIG ----
|
|
PG_REMOTE_USER="postgres"
|
|
PG_REMOTE_HOST="cluster.db2.genesishostingtechnologies.com"
|
|
PG_REMOTE_PORT="5432"
|
|
PG_LOCAL_PORT="5432"
|
|
DUMP_DIR="/tmp/pgbackup_verify"
|
|
BACKUP_TARGET="root@thevault.bounceme.net:/mnt/backup3/pgdumps"
|
|
DBS=("mastodon_production" "giteaprod")
|
|
LOGFILE="$DUMP_DIR/verify_log_$(date +%Y%m%d_%H%M%S).txt"
|
|
mkdir -p "$DUMP_DIR"
|
|
|
|
# ==== Mastodon DM function ====
|
|
mastodon_alert() {
|
|
local msg="$1"
|
|
curl -sS -X POST "https://chatwithus.live/api/v1/statuses" \
|
|
-H "Authorization: Bearer rimxBLi-eaJAcwagkmoj6UoW7Lc473tQY0cOM041Euw" \
|
|
--data-urlencode "status=$msg" \
|
|
--data "visibility=direct" \
|
|
--data "in_reply_to_account_id=114386383616633367" >/dev/null
|
|
}
|
|
|
|
ALL_OK=true
|
|
UPLOAD_LIST=()
|
|
|
|
for DB in "${DBS[@]}"; do
|
|
echo "=== [$(date)] Dumping $DB from $PG_REMOTE_HOST ===" | tee -a "$LOGFILE"
|
|
DUMPFILE="$DUMP_DIR/${DB}_$(date +%Y%m%d_%H%M%S).sql"
|
|
|
|
# Dump from remote
|
|
pg_dump -h "$PG_REMOTE_HOST" -p "$PG_REMOTE_PORT" -U "$PG_REMOTE_USER" -d "$DB" > "$DUMPFILE"
|
|
if [ $? -ne 0 ]; then
|
|
echo "[FAIL] Failed to dump $DB! Skipping upload." | tee -a "$LOGFILE"
|
|
ALL_OK=false
|
|
mastodon_alert "🚨 Database backup/verify FAILED: Could not dump $DB from $PG_REMOTE_HOST on $(hostname) at $(date). See log: $LOGFILE"
|
|
continue
|
|
fi
|
|
|
|
# Restore/verify on Krang
|
|
TESTDB="verify_${DB}_$RANDOM"
|
|
echo "Creating test database $TESTDB" | tee -a "$LOGFILE"
|
|
sudo -u postgres createdb -p "$PG_LOCAL_PORT" "$TESTDB"
|
|
if [ $? -ne 0 ]; then
|
|
echo "[FAIL] Failed to create $TESTDB!" | tee -a "$LOGFILE"
|
|
ALL_OK=false
|
|
mastodon_alert "🚨 Database backup/verify FAILED: Could not create test DB $TESTDB on $(hostname) at $(date). See log: $LOGFILE"
|
|
continue
|
|
fi
|
|
|
|
echo "Restoring to $TESTDB" | tee -a "$LOGFILE"
|
|
sudo -u postgres psql -p "$PG_LOCAL_PORT" -d "$TESTDB" < "$DUMPFILE"
|
|
if [ $? -ne 0 ]; then
|
|
echo "[FAIL] Restore failed for $DB!" | tee -a "$LOGFILE"
|
|
sudo -u postgres dropdb -p "$PG_LOCAL_PORT" "$TESTDB"
|
|
ALL_OK=false
|
|
mastodon_alert "🚨 Database backup/verify FAILED: Restore failed for $DB on $(hostname) at $(date). See log: $LOGFILE"
|
|
continue
|
|
fi
|
|
|
|
# Quick table listing for sanity
|
|
sudo -u postgres psql -p "$PG_LOCAL_PORT" -d "$TESTDB" -c "\dt" | tee -a "$LOGFILE"
|
|
if [ $? -eq 0 ]; then
|
|
echo "[PASS] $DB: Dump and restore OK." | tee -a "$LOGFILE"
|
|
UPLOAD_LIST+=("$DUMPFILE")
|
|
else
|
|
echo "[FAIL] $DB: Test query failed!" | tee -a "$LOGFILE"
|
|
ALL_OK=false
|
|
mastodon_alert "🚨 Database backup/verify FAILED: Test query failed for $DB on $(hostname) at $(date). See log: $LOGFILE"
|
|
fi
|
|
|
|
sudo -u postgres dropdb -p "$PG_LOCAL_PORT" "$TESTDB"
|
|
echo "Cleaned up $TESTDB" | tee -a "$LOGFILE"
|
|
echo "" | tee -a "$LOGFILE"
|
|
done
|
|
|
|
if $ALL_OK && [ "${#UPLOAD_LIST[@]}" -eq "${#DBS[@]}" ]; then
|
|
echo "All dumps verified, sending to $BACKUP_TARGET" | tee -a "$LOGFILE"
|
|
scp "${UPLOAD_LIST[@]}" "$BACKUP_TARGET"
|
|
if [ $? -eq 0 ]; then
|
|
echo "Uploads successful." | tee -a "$LOGFILE"
|
|
rm -f "${UPLOAD_LIST[@]}"
|
|
else
|
|
echo "Upload failed!" | tee -a "$LOGFILE"
|
|
mastodon_alert "🚨 Database backup/verify FAILED: Upload to $BACKUP_TARGET failed on $(hostname) at $(date). See log: $LOGFILE"
|
|
fi
|
|
else
|
|
echo "Not all backups verified! Nothing uploaded." | tee -a "$LOGFILE"
|
|
mastodon_alert "🚨 Database backup/verify FAILED: One or more DBs failed verification on $(hostname) at $(date). See log: $LOGFILE"
|
|
fi
|
|
|
|
echo "DONE. Log: $LOGFILE"
|