#!/bin/bash

SOURCE_DIR="/tmp/db2_backup/"
DEST_DIR="thevault:/nexus/postgresql/db2/"
DATE=$(date +%Y%m%d%H%M)
PG_USER="postgres"
SOURCE_SERVER="zcluster.technodrome2.sshjunkie.com"  # Source server (database server)
SOURCE_REMOTE="technodrome2"  # rclone remote for source server (configured earlier)
DEST_REMOTE="thevault"        # rclone remote for destination server (The Vault)

# Ensure necessary tools are available
command -v pg_basebackup >/dev/null 2>&1 || { echo "pg_basebackup not found, exiting."; exit 1; }
command -v rclone >/dev/null 2>&1 || { echo "rclone not found, exiting."; exit 1; }
command -v ssh >/dev/null 2>&1 || { echo "ssh not found, exiting."; exit 1; }

# Step 1: SSH into the database server and run pg_basebackup
echo "Starting pg_basebackup for db2 on $SOURCE_SERVER..."

# Ensure the directory exists on the source server
ssh doc@$SOURCE_SERVER "sudo mkdir -p $SOURCE_DIR$DATE"  # Create the directory if it doesn't exist

# Run pg_basebackup
ssh doc@$SOURCE_SERVER "pg_basebackup -h localhost -D $SOURCE_DIR$DATE -U $PG_USER -Ft -z -P"

# Check if pg_basebackup was successful
if [ $? -eq 0 ]; then
    echo "pg_basebackup completed successfully. Verifying backup directory on $SOURCE_SERVER..."

    # Verify that the backup directory exists
    ssh doc@$SOURCE_SERVER "ls -l $SOURCE_DIR$DATE"

    # If the directory exists, proceed with rclone transfer
    if [ $? -eq 0 ]; then
        echo "Backup directory exists. Proceeding to rclone transfer..."

        # Step 2: Use rclone to copy the backup from the source server to The Vault
        retry=0
        max_retries=3
        while ! rclone copy $SOURCE_REMOTE:$SOURCE_DIR$DATE/ $DEST_REMOTE:$DEST_DIR --progress --checksum; do
            if [ $retry -ge $max_retries ]; then
                echo "Rclone transfer failed after $max_retries attempts."
                exit 1
            fi
            retry=$((retry+1))
            echo "Retrying rclone transfer... attempt $retry"
            sleep 5
        done

        # Check if rclone was successful
        if [ $? -eq 0 ]; then
            echo "Rclone transfer completed successfully. Proceeding to snapshot..."

            # Step 3: Create a ZFS snapshot on The Vault
            ssh root@thevault.sshjunkie.com "sudo zfs snapshot nexus/postgresql/db2@$DATE"

            # Verify snapshot creation on The Vault
            ssh root@thevault.sshjunkie.com "zfs list -t snapshot | grep nexus/postgresql/db2@$DATE" >/dev/null 2>&1
            if [ $? -eq 0 ]; then
                echo "ZFS snapshot nexus/postgresql/db2@$DATE created successfully on The Vault."
            else
                echo "Snapshot creation failed on The Vault."
                exit 1
            fi
        else
            echo "Error during rclone transfer. Backup not transferred."
            exit 1
        fi
    else
        echo "Backup directory not found on $SOURCE_SERVER. Aborting transfer."
        exit 1
    fi
else
    echo "Error during pg_basebackup. Backup not created."
    exit 1
fi

# Clean up after success
ssh doc@$SOURCE_SERVER "sudo rm -rf $SOURCE_DIR$DATE"
echo "Backup and snapshot process completed successfully."