diff --git a/miscellaneous/dbv.sh b/miscellaneous/dbv.sh new file mode 100755 index 0000000..22bf099 --- /dev/null +++ b/miscellaneous/dbv.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +# ---- CONFIG ---- +PG_REMOTE_USER="postgres" +PG_REMOTE_HOST="cluster.db2.genesishostingtechnologies.com" +PG_REMOTE_PORT="5432" +PG_LOCAL_PORT="5432" +DUMP_DIR="/tmp/pgbackup_verify" +BACKUP_TARGET="root@thevault.bounceme.net:/mnt/backup3/pgdumps" +DBS=("mastodon_production" "giteaprod") +LOGFILE="$DUMP_DIR/verify_log_$(date +%Y%m%d_%H%M%S).txt" +mkdir -p "$DUMP_DIR" + +# ==== Mastodon DM function ==== +mastodon_alert() { + local msg="$1" + curl -sS -X POST "https://chatwithus.live/api/v1/statuses" \ + -H "Authorization: Bearer rimxBLi-eaJAcwagkmoj6UoW7Lc473tQY0cOM041Euw" \ + --data-urlencode "status=$msg" \ + --data "visibility=direct" \ + --data "in_reply_to_account_id=114386383616633367" >/dev/null +} + +ALL_OK=true +UPLOAD_LIST=() + +for DB in "${DBS[@]}"; do + echo "=== [$(date)] Dumping $DB from $PG_REMOTE_HOST ===" | tee -a "$LOGFILE" + DUMPFILE="$DUMP_DIR/${DB}_$(date +%Y%m%d_%H%M%S).sql" + + # Dump from remote + pg_dump -h "$PG_REMOTE_HOST" -p "$PG_REMOTE_PORT" -U "$PG_REMOTE_USER" -d "$DB" > "$DUMPFILE" + if [ $? -ne 0 ]; then + echo "[FAIL] Failed to dump $DB! Skipping upload." | tee -a "$LOGFILE" + ALL_OK=false + mastodon_alert "🚨 Database backup/verify FAILED: Could not dump $DB from $PG_REMOTE_HOST on $(hostname) at $(date). See log: $LOGFILE" + continue + fi + + # Restore/verify on Krang + TESTDB="verify_${DB}_$RANDOM" + echo "Creating test database $TESTDB" | tee -a "$LOGFILE" + sudo -u postgres createdb -p "$PG_LOCAL_PORT" "$TESTDB" + if [ $? -ne 0 ]; then + echo "[FAIL] Failed to create $TESTDB!" | tee -a "$LOGFILE" + ALL_OK=false + mastodon_alert "🚨 Database backup/verify FAILED: Could not create test DB $TESTDB on $(hostname) at $(date). See log: $LOGFILE" + continue + fi + + echo "Restoring to $TESTDB" | tee -a "$LOGFILE" + sudo -u postgres psql -p "$PG_LOCAL_PORT" -d "$TESTDB" < "$DUMPFILE" + if [ $? -ne 0 ]; then + echo "[FAIL] Restore failed for $DB!" | tee -a "$LOGFILE" + sudo -u postgres dropdb -p "$PG_LOCAL_PORT" "$TESTDB" + ALL_OK=false + mastodon_alert "🚨 Database backup/verify FAILED: Restore failed for $DB on $(hostname) at $(date). See log: $LOGFILE" + continue + fi + + # Quick table listing for sanity + sudo -u postgres psql -p "$PG_LOCAL_PORT" -d "$TESTDB" -c "\dt" | tee -a "$LOGFILE" + if [ $? -eq 0 ]; then + echo "[PASS] $DB: Dump and restore OK." | tee -a "$LOGFILE" + UPLOAD_LIST+=("$DUMPFILE") + else + echo "[FAIL] $DB: Test query failed!" | tee -a "$LOGFILE" + ALL_OK=false + mastodon_alert "🚨 Database backup/verify FAILED: Test query failed for $DB on $(hostname) at $(date). See log: $LOGFILE" + fi + + sudo -u postgres dropdb -p "$PG_LOCAL_PORT" "$TESTDB" + echo "Cleaned up $TESTDB" | tee -a "$LOGFILE" + echo "" | tee -a "$LOGFILE" +done + +if $ALL_OK && [ "${#UPLOAD_LIST[@]}" -eq "${#DBS[@]}" ]; then + echo "All dumps verified, sending to $BACKUP_TARGET" | tee -a "$LOGFILE" + scp "${UPLOAD_LIST[@]}" "$BACKUP_TARGET" + if [ $? -eq 0 ]; then + echo "Uploads successful." | tee -a "$LOGFILE" + rm -f "${UPLOAD_LIST[@]}" + else + echo "Upload failed!" | tee -a "$LOGFILE" + mastodon_alert "🚨 Database backup/verify FAILED: Upload to $BACKUP_TARGET failed on $(hostname) at $(date). See log: $LOGFILE" + fi +else + echo "Not all backups verified! Nothing uploaded." | tee -a "$LOGFILE" + mastodon_alert "🚨 Database backup/verify FAILED: One or more DBs failed verification on $(hostname) at $(date). See log: $LOGFILE" +fi + +echo "DONE. Log: $LOGFILE"