bin-there-done-that/miscellaneous/dbv1.sh

#!/bin/bash

# ---- CONFIG ----
PG_REMOTE_USER="postgres"
PG_REMOTE_HOST="cluster.db2.genesishostingtechnologies.com"
PG_REMOTE_PORT="5432"
PG_LOCAL_PORT="5432"
DUMP_DIR="/tmp/pgbackup_verify"
BACKUP_TARGET="root@backup.sshjunkie.com:/mnt/backup/pgdumps"
CC_TARGET="doc@clustercontrol.sshjunkie.com:/home/doc/backups"
DBS=("mastodon_production" "giteaprod" "hostingtootdb" "radiotootdb")
LOGFILE="$DUMP_DIR/verify_log_$(date +%Y%m%d_%H%M%S).txt"
mkdir -p "$DUMP_DIR"

# ==== Mastodon DM function ====
mastodon_alert() {
    local msg="$1"
    curl -sS -X POST "https://chatwithus.live/api/v1/statuses" \
      -H "Authorization: Bearer rimxBLi-eaJAcwagkmoj6UoW7Lc473tQY0cOM041Euw" \
      --data-urlencode "status=$msg" \
      --data "visibility=direct" \
      --data "in_reply_to_account_id=114386383616633367" >/dev/null
}

ALL_OK=true
UPLOAD_LIST=()

for DB in "${DBS[@]}"; do
    echo "=== [$(date)] Dumping $DB from $PG_REMOTE_HOST ===" | tee -a "$LOGFILE"
    DUMPFILE="$DUMP_DIR/${DB}_$(date +%Y%m%d_%H%M%S).sql"

    # Dump from remote
    pg_dump -h "$PG_REMOTE_HOST" -p "$PG_REMOTE_PORT" -U "$PG_REMOTE_USER" -d "$DB" > "$DUMPFILE"
    if [ $? -ne 0 ]; then
        echo "[FAIL] Failed to dump $DB! Skipping upload." | tee -a "$LOGFILE"
        ALL_OK=false
        mastodon_alert "🚨 Database backup/verify FAILED: Could not dump $DB from $PG_REMOTE_HOST on $(hostname) at $(date). See log: $LOGFILE"
        continue
    fi

    # Generate checksum for the dump file
    CHECKSUM_FILE="$DUMPFILE.sha256"
    sha256sum "$DUMPFILE" > "$CHECKSUM_FILE"

    # Restore/verify on Krang
    TESTDB="verify_${DB}_$RANDOM"
    echo "Creating test database $TESTDB" | tee -a "$LOGFILE"
    sudo -u postgres createdb -p "$PG_LOCAL_PORT" "$TESTDB"
    if [ $? -ne 0 ]; then
        echo "[FAIL] Failed to create $TESTDB!" | tee -a "$LOGFILE"
        ALL_OK=false
        mastodon_alert "🚨 Database backup/verify FAILED: Could not create test DB $TESTDB on $(hostname) at $(date). See log: $LOGFILE"
        continue
    fi

    # Restore the dump
    echo "Restoring to $TESTDB" | tee -a "$LOGFILE"
    sudo -u postgres psql -p "$PG_LOCAL_PORT" -d "$TESTDB" < "$DUMPFILE"
    if [ $? -ne 0 ]; then
        echo "[FAIL] Restore failed for $DB!" | tee -a "$LOGFILE"
        sudo -u postgres dropdb -p "$PG_LOCAL_PORT" "$TESTDB"
        ALL_OK=false
        mastodon_alert "🚨 Database backup/verify FAILED: Restore failed for $DB on $(hostname) at $(date). See log: $LOGFILE"
        continue
    fi

    # Sanity check: check row count of a critical table (e.g., 'users' table)
    ROW_COUNT=$(sudo -u postgres psql -p "$PG_LOCAL_PORT" -d "$TESTDB" -t -c "SELECT count(*) FROM users;")
    echo "Row count for 'users' table in $TESTDB: $ROW_COUNT" | tee -a "$LOGFILE"

    if [ "$ROW_COUNT" -le 0 ]; then
        echo "[FAIL] No rows in 'users' table after restore!" | tee -a "$LOGFILE"
        ALL_OK=false
        mastodon_alert "🚨 Database backup/verify FAILED: No rows found in 'users' table after restore for $DB on $(hostname) at $(date). See log: $LOGFILE"
        continue
    else
        echo "[PASS] Row count OK for 'users' table in $DB." | tee -a "$LOGFILE"
    fi

    # Verify checksum
    RESTORED_CHECKSUM_FILE="$DUMPFILE.sha256"
    if ! cmp -s "$CHECKSUM_FILE" "$RESTORED_CHECKSUM_FILE"; then
        echo "[FAIL] Checksum mismatch for $DB!" | tee -a "$LOGFILE"
        ALL_OK=false
        mastodon_alert "🚨 Database backup/verify FAILED: Checksum mismatch for $DB on $(hostname) at $(date). See log: $LOGFILE"
        continue
    else
        echo "[PASS] Checksum verified for $DB." | tee -a "$LOGFILE"
    fi

    # Quick table listing for sanity
    sudo -u postgres psql -p "$PG_LOCAL_PORT" -d "$TESTDB" -c "\dt" | tee -a "$LOGFILE"
    if [ $? -eq 0 ]; then
        echo "[PASS] $DB: Dump and restore OK." | tee -a "$LOGFILE"
        UPLOAD_LIST+=("$DUMPFILE")
    else
        echo "[FAIL] $DB: Test query failed!" | tee -a "$LOGFILE"
        ALL_OK=false
        mastodon_alert "🚨 Database backup/verify FAILED: Test query failed for $DB on $(hostname) at $(date). See log: $LOGFILE"
    fi

    sudo -u postgres dropdb -p "$PG_LOCAL_PORT" "$TESTDB"
    echo "Cleaned up $TESTDB" | tee -a "$LOGFILE"
    echo "" | tee -a "$LOGFILE"
done

if $ALL_OK && [ "${#UPLOAD_LIST[@]}" -eq "${#DBS[@]}" ]; then
    echo "All dumps verified, sending to $BACKUP_TARGET" | tee -a "$LOGFILE"
    scp "${UPLOAD_LIST[@]}" "$BACKUP_TARGET"
    if [ $? -eq 0 ]; then
        echo "Uploads to thevault successful." | tee -a "$LOGFILE"
        # --NEW: Also upload to ClusterControl controller
        echo "Uploading to ClusterControl controller at $CC_TARGET" | tee -a "$LOGFILE"
        scp "${UPLOAD_LIST[@]}" "$CC_TARGET"
        if [ $? -eq 0 ]; then
            echo "Uploads to ClusterControl successful." | tee -a "$LOGFILE"
            rm -f "${UPLOAD_LIST[@]}"
        else
            echo "[WARN] Upload to ClusterControl controller failed!" | tee -a "$LOGFILE"
            mastodon_alert "⚠️ Database backup verified, but upload to ClusterControl at $CC_TARGET failed on $(hostname) at $(date). See log: $LOGFILE"
        fi
    else
        echo "[FAIL] Upload to thevault failed!" | tee -a "$LOGFILE"
        mastodon_alert "🚨 Database backup/verify FAILED: Upload to $BACKUP_TARGET failed on $(hostname) at $(date). See log: $LOGFILE"
    fi
else
    echo "Not all backups verified! Nothing uploaded." | tee -a "$LOGFILE"
    mastodon_alert "🚨 Database backup/verify FAILED: One or more DBs failed verification on $(hostname) at $(date). See log: $LOGFILE"
fi

echo "DONE. Log: $LOGFILE"