#!/bin/bash # ---- CONFIG ---- PG_REMOTE_USER="postgres" PG_REMOTE_HOST="cluster.db2.genesishostingtechnologies.com" PG_REMOTE_PORT="5432" PG_LOCAL_PORT="5432" DUMP_DIR="/tmp/pgbackup_verify" BACKUP_TARGET="root@backup.sshjunkie.com:/mnt/backup/pgdumps" CC_TARGET="doc@clustercontrol.sshjunkie.com:/home/doc/backups" DBS=("mastodon_production" "giteaprod" "hostingtootdb" "radiotootdb") LOGFILE="$DUMP_DIR/verify_log_$(date +%Y%m%d_%H%M%S).txt" mkdir -p "$DUMP_DIR" # ==== Mastodon DM function ==== mastodon_alert() { local msg="$1" curl -sS -X POST "https://chatwithus.live/api/v1/statuses" \ -H "Authorization: Bearer rimxBLi-eaJAcwagkmoj6UoW7Lc473tQY0cOM041Euw" \ --data-urlencode "status=$msg" \ --data "visibility=direct" \ --data "in_reply_to_account_id=114386383616633367" >/dev/null } ALL_OK=true UPLOAD_LIST=() for DB in "${DBS[@]}"; do echo "=== [$(date)] Dumping $DB from $PG_REMOTE_HOST ===" | tee -a "$LOGFILE" DUMPFILE="$DUMP_DIR/${DB}_$(date +%Y%m%d_%H%M%S).sql" # Dump from remote pg_dump -h "$PG_REMOTE_HOST" -p "$PG_REMOTE_PORT" -U "$PG_REMOTE_USER" -d "$DB" > "$DUMPFILE" if [ $? -ne 0 ]; then echo "[FAIL] Failed to dump $DB! Skipping upload." | tee -a "$LOGFILE" ALL_OK=false mastodon_alert "🚨 Database backup/verify FAILED: Could not dump $DB from $PG_REMOTE_HOST on $(hostname) at $(date). See log: $LOGFILE" continue fi # Restore/verify on Krang TESTDB="verify_${DB}_$RANDOM" echo "Creating test database $TESTDB" | tee -a "$LOGFILE" sudo -u postgres createdb -p "$PG_LOCAL_PORT" "$TESTDB" if [ $? -ne 0 ]; then echo "[FAIL] Failed to create $TESTDB!" | tee -a "$LOGFILE" ALL_OK=false mastodon_alert "🚨 Database backup/verify FAILED: Could not create test DB $TESTDB on $(hostname) at $(date). See log: $LOGFILE" continue fi echo "Restoring to $TESTDB" | tee -a "$LOGFILE" sudo -u postgres psql -p "$PG_LOCAL_PORT" -d "$TESTDB" < "$DUMPFILE" if [ $? -ne 0 ]; then echo "[FAIL] Restore failed for $DB!" | tee -a "$LOGFILE" sudo -u postgres dropdb -p "$PG_LOCAL_PORT" "$TESTDB" ALL_OK=false mastodon_alert "🚨 Database backup/verify FAILED: Restore failed for $DB on $(hostname) at $(date). See log: $LOGFILE" continue fi # Quick table listing for sanity sudo -u postgres psql -p "$PG_LOCAL_PORT" -d "$TESTDB" -c "\dt" | tee -a "$LOGFILE" if [ $? -eq 0 ]; then echo "[PASS] $DB: Dump and restore OK." | tee -a "$LOGFILE" UPLOAD_LIST+=("$DUMPFILE") else echo "[FAIL] $DB: Test query failed!" | tee -a "$LOGFILE" ALL_OK=false mastodon_alert "🚨 Database backup/verify FAILED: Test query failed for $DB on $(hostname) at $(date). See log: $LOGFILE" fi sudo -u postgres dropdb -p "$PG_LOCAL_PORT" "$TESTDB" echo "Cleaned up $TESTDB" | tee -a "$LOGFILE" echo "" | tee -a "$LOGFILE" done if $ALL_OK && [ "${#UPLOAD_LIST[@]}" -eq "${#DBS[@]}" ]; then echo "All dumps verified, sending to $BACKUP_TARGET" | tee -a "$LOGFILE" scp "${UPLOAD_LIST[@]}" "$BACKUP_TARGET" if [ $? -eq 0 ]; then echo "Uploads to thevault successful." | tee -a "$LOGFILE" # --NEW: Also upload to ClusterControl controller echo "Uploading to ClusterControl controller at $CC_TARGET" | tee -a "$LOGFILE" scp "${UPLOAD_LIST[@]}" "$CC_TARGET" if [ $? -eq 0 ]; then echo "Uploads to ClusterControl successful." | tee -a "$LOGFILE" rm -f "${UPLOAD_LIST[@]}" else echo "[WARN] Upload to ClusterControl controller failed!" | tee -a "$LOGFILE" mastodon_alert "⚠️ Database backup verified, but upload to ClusterControl at $CC_TARGET failed on $(hostname) at $(date). See log: $LOGFILE" fi else echo "[FAIL] Upload to thevault failed!" | tee -a "$LOGFILE" mastodon_alert "🚨 Database backup/verify FAILED: Upload to $BACKUP_TARGET failed on $(hostname) at $(date). See log: $LOGFILE" fi else echo "Not all backups verified! Nothing uploaded." | tee -a "$LOGFILE" mastodon_alert "🚨 Database backup/verify FAILED: One or more DBs failed verification on $(hostname) at $(date). See log: $LOGFILE" fi echo "DONE. Log: $LOGFILE"