#!/bin/bash
# verify_postgres_snapshot.sh
# Verify PostgreSQL backup snapshot remotely using ZFS, pg_verifybackup, and pg_checksums

set -euo pipefail

# === CONFIG ===
REMOTE_HOST="zcluster.technodrome2.sshjunkie.com"
ZFS_DATASET="pgpool/postgresbackups/db2"
MOUNT_ROOT="/mnt/pgsnapshot_verify"
PG_BIN_DIR="/usr/lib/postgresql/16/bin"  # adjust if needed on remote host
LOGFILE="/var/log/pgsnapshot_verify_$(date +%Y%m%d_%H%M%S).log"
DB_NAME="mastodon_production"

# === MASTODON ALERTING ===
MASTODON_TOKEN="rimxBLi-eaJAcwagkmoj6UoW7Lc473tQY0cOM041Euw"
MASTODON_API="https://chatwithus.live/api/v1/statuses"
TOOT_ACCOUNT_ID="114386383616633367"

mastodon_alert() {
  local msg="$1"
  curl -sS -X POST "$MASTODON_API" \
    -H "Authorization: Bearer $MASTODON_TOKEN" \
    --data-urlencode "status=$msg" \
    --data "visibility=direct" \
    --data "in_reply_to_account_id=$TOOT_ACCOUNT_ID" >/dev/null
}

# === RUN REMOTELY ===
echo "📦 Verifying latest snapshot for $ZFS_DATASET on $REMOTE_HOST" | tee "$LOGFILE"
ssh root@$REMOTE_HOST bash <<EOF
set -euo pipefail

LATEST_SNAPSHOT=\$(zfs list -t snapshot -r $ZFS_DATASET -o name -s creation | tail -n 1)

if [[ -z "\$LATEST_SNAPSHOT" ]]; then
  echo "❌ No snapshot found for $ZFS_DATASET" | tee -a "$LOGFILE"
  exit 1
fi

MOUNTPOINT="$MOUNT_ROOT/\${LATEST_SNAPSHOT//\//_}"
mkdir -p "\$MOUNTPOINT"

# Clone snapshot for verification
zfs clone -o mountpoint="\$MOUNTPOINT" "\$LATEST_SNAPSHOT" "${ZFS_DATASET}@verifytemp"

# Run pg_verifybackup
if ! "$PG_BIN_DIR/pg_verifybackup" "\$MOUNTPOINT" >> "$LOGFILE" 2>&1; then
  echo "❌ pg_verifybackup failed on snapshot \$LATEST_SNAPSHOT" | tee -a "$LOGFILE"
  zfs destroy "${ZFS_DATASET}@verifytemp"
  exit 2
fi

# Optional: pg_checksums
if "$PG_BIN_DIR/pg_checksums" --check --data-directory="\$MOUNTPOINT" >> "$LOGFILE" 2>&1; then
  echo "✅ pg_checksums passed." | tee -a "$LOGFILE"
else
  echo "⚠️  pg_checksums failed or not enabled." | tee -a "$LOGFILE"
fi

zfs destroy "${ZFS_DATASET}@verifytemp"
rmdir "\$MOUNTPOINT"
EOF

# === ALERT BASED ON SSH EXIT CODE ===
EXIT_CODE=$?
if [[ $EXIT_CODE -eq 1 ]]; then
  mastodon_alert "🚨 ZFS PostgreSQL snapshot verification failed: No snapshot found for $ZFS_DATASET on $REMOTE_HOST at $(date)."
elif [[ $EXIT_CODE -eq 2 ]]; then
  mastodon_alert "🚨 pg_verifybackup failed for snapshot of $ZFS_DATASET on $REMOTE_HOST at $(date)."
fi

echo "✅ PostgreSQL remote snapshot verification complete." | tee -a "$LOGFILE"
exit 0