diff --git a/documents/disklabels/baboon.md b/documents/disklabels/baboon.md new file mode 100644 index 0000000..5f450b8 --- /dev/null +++ b/documents/disklabels/baboon.md @@ -0,0 +1,10 @@ +🟩 **Detailed /dev/disk/by-id/ Partitions (Live Ones)** +====================================================== + +```shell +lrwxrwxrwx 1 root root 10 May 29 08:53 ata-Timetec_30TT253X2-256GB_PL211014YSA256G0116-part1 -> ../../sda1 +lrwxrwxrwx 1 root root 10 May 29 08:53 ata-Timetec_30TT253X2-256GB_PL211014YSA256G0116-part2 -> ../../sda2 +lrwxrwxrwx 1 root root 10 May 29 08:53 ata-Timetec_30TT253X2-256GB_PL211014YSA256G0116-part3 -> ../../sda3 +lrwxrwxrwx 1 root root 10 May 29 08:53 ata-Timetec_30TT253X2-256GB_PL220302YSA256G0877-part1 -> ../../sdb1 +lrwxrwxrwx 1 root root 10 May 29 08:53 scsi-35000cca291d4b870-part1 -> ../../sdd1 +lrwxrwxrwx 1 root root 10 May 29 08:53 scsi-35000cca2b093888c-part1 -> ../../sdc1 diff --git a/genesis-tools b/genesis-tools new file mode 120000 index 0000000..4802bda --- /dev/null +++ b/genesis-tools @@ -0,0 +1 @@ +/deadbeef/genesis-tools \ No newline at end of file diff --git a/miscellaneous/bash/bin/arecordalertfailzero.sh b/miscellaneous/bash/bin/arecordalertfailzero.sh index a4ae65c..fadeba8 100755 --- a/miscellaneous/bash/bin/arecordalertfailzero.sh +++ b/miscellaneous/bash/bin/arecordalertfailzero.sh @@ -13,7 +13,6 @@ CRITICAL_HOSTS=( "zcluster.technodrome1.sshjunkie.com" "zcluster.technodrome2.sshjunkie.com" "krang.core.sshjunkie.com" - "clustercontrol.sshjunkie.com" "tt.themediahub.org" "toot.themediahub.org" "chatwithus.live" diff --git a/miscellaneous/bash/bin/brains_backup.sh b/miscellaneous/bash/bin/brains_backup.sh new file mode 100755 index 0000000..fe25dd5 --- /dev/null +++ b/miscellaneous/bash/bin/brains_backup.sh @@ -0,0 +1,105 @@ +#!/bin/bash +set -euo pipefail + +# === SSH Hosts === +SHREDDER_HOST="doc@shredder.sshjunkie.com" +PORTAL_HOST="root@portal.genesishostingtechnologies.com" +DA_HOST="root@da.genesishostingtechnologies.com" +VAULT_HOST="root@thevault.bounceme.net" + +# === Telegram Setup === +TG_TOKEN="7277705363:AAGSw5Pmcbf7IsSyZKMqU6PJ4VsVwdKLRH0" +TG_CHAT_ID="1559582356" + +send_telegram() { + local message="$1" + curl -s -X POST "https://api.telegram.org/bot$TG_TOKEN/sendMessage" \ + -d "chat_id=$TG_CHAT_ID&text=$message" +} + +# === Local Staging Area on Krang === +STAGING_DATASET="/deadbeef/staging" +DATE=$(date +%Y%m%d%H%M%S) +BACKUP_DIR="$STAGING_DATASET/brains-$DATE" +mkdir -p "$BACKUP_DIR" + +send_telegram "🧠 Starting centralized backup from Krang to $BACKUP_DIR..." + +# === Sanity Checks on TheVault === +send_telegram "πŸ§ͺ Sanity check: ensuring TheVault datasets are mounted..." +ssh -o BatchMode=yes $VAULT_HOST "zfs mount -a && ls /backups/azuracast /backups/krang /backups/directadmin" && \ +send_telegram "βœ… TheVault mountpoints verified!" || \ +send_telegram "❌ TheVault mountpoint sanity check FAILED!" + +# === Sanity Checks on Shredder === +send_telegram "πŸ§ͺ Sanity check: ensuring Shredder datasets are mounted..." +ssh -o BatchMode=yes $SHREDDER_HOST "zfs mount -a && ls /assets/splmedia /assets/azuracast /assets/pokbackups /assets/splshows" && \ +send_telegram "βœ… Shredder mountpoints verified!" || \ +send_telegram "❌ Shredder mountpoint sanity check FAILED!" + +# === Helper Function for Steps === +run_backup_step() { + local description="$1" + shift + send_telegram "πŸ”„ $description" + if "$@"; then + send_telegram "βœ… $description complete!" + else + send_telegram "❌ $description FAILED!" + fi +} + +# === 1️⃣ Sync SPL and Shredder Data === +run_backup_step "Syncing SPL and Shredder data" \ + rsync -avz -e "ssh -o BatchMode=yes" $SHREDDER_HOST:/mnt/spl/ "$BACKUP_DIR/splmedia/" && \ + rsync -avz -e "ssh -o BatchMode=yes" $SHREDDER_HOST:/mnt/spl/ "$BACKUP_DIR/splassets/" + +# === 2️⃣ Backup Krang's Configs === +run_backup_step "Backing up Krang host configs" \ + rsync -avz /etc/pve "$BACKUP_DIR/krang-pve/" && \ + rsync -avz /etc/network/interfaces "$BACKUP_DIR/krang-net/" && \ + rsync -avz /etc/ssh "$BACKUP_DIR/krang-ssh/" + +# === 3️⃣ Backup AzuraCast DB from Portal === +run_backup_step "Backing up AzuraCast DB from Portal" \ + ssh -o BatchMode=yes $PORTAL_HOST "pg_dump -U postgres azuracast" > "$BACKUP_DIR/databases/azuracast.sql" + +# === 4️⃣ Backup AzuraCast Configs === +run_backup_step "Backing up AzuraCast configs from Portal" \ + rsync -avz -e "ssh -o BatchMode=yes" $PORTAL_HOST:/var/azuracast/.env "$BACKUP_DIR/azuracast/" && \ + rsync -avz -e "ssh -o BatchMode=yes" $PORTAL_HOST:/var/azuracast/docker-compose.yml "$BACKUP_DIR/azuracast/" && \ + rsync -avz -e "ssh -o BatchMode=yes" $PORTAL_HOST:/var/azuracast/stations "$BACKUP_DIR/azuracast/stations" + +# === 5️⃣ Sync AzuraCast Media from Portal to Shredder, then to Krang === +run_backup_step "Syncing AzuraCast media from Portal to Shredder" \ + ssh -o BatchMode=yes $PORTAL_HOST "rsync -avz /mnt/azuracast1/ doc@shredder.sshjunkie.com:/assets/azuracast/" + +run_backup_step "Backing up AzuraCast media from Shredder to Krang" \ + rsync -avz -e "ssh -o BatchMode=yes" $SHREDDER_HOST:/assets/azuracast/ "$BACKUP_DIR/azuracast-media/" + +# === 6️⃣ Backup DirectAdmin Configs === +run_backup_step "Backing up DirectAdmin from Portal" \ + rsync -avz -e "ssh -o BatchMode=yes" $DA_HOST:/usr/local/directadmin/data/admin/ "$BACKUP_DIR/directadmin/" + +# === 7️⃣ Push to TheVault === +send_telegram "πŸ”„ Pushing backups to TheVault datasets..." + +# AzuraCast Configs & DB +rsync -avz -e "ssh -o BatchMode=yes" "$BACKUP_DIR/azuracast/" $VAULT_HOST:/backups/azuracast/configs/$DATE/ +rsync -avz -e "ssh -o BatchMode=yes" "$BACKUP_DIR/databases/" $VAULT_HOST:/backups/azuracast/configs/$DATE/ + +# AzuraCast Media +rsync -avz -e "ssh -o BatchMode=yes" "$BACKUP_DIR/azuracast-media/" $VAULT_HOST:/backups/azuracast/$DATE/ + +# DirectAdmin +rsync -avz -e "ssh -o BatchMode=yes" "$BACKUP_DIR/directadmin/" $VAULT_HOST:/backups/directadmin/$DATE/ + +# Krang +rsync -avz -e "ssh -o BatchMode=yes" "$BACKUP_DIR/krang-*/" $VAULT_HOST:/backups/krang/$DATE/ + +# SPL +rsync -avz -e "ssh -o BatchMode=yes" "$BACKUP_DIR/splmedia/" $VAULT_HOST:/backups/splmedia/$DATE/ +rsync -avz -e "ssh -o BatchMode=yes" "$BACKUP_DIR/splassets/" $VAULT_HOST:/backups/splassets/$DATE/ + +send_telegram "πŸŽ‰ FULL SYSTEM BACKUP COMPLETED & MIRRORED TO VAULT!" +exit 0 diff --git a/miscellaneous/bash/bin/brains_backup2.sh b/miscellaneous/bash/bin/brains_backup2.sh new file mode 100755 index 0000000..d34e56e --- /dev/null +++ b/miscellaneous/bash/bin/brains_backup2.sh @@ -0,0 +1,69 @@ +#!/bin/bash +set -euo pipefail + +# === SSH Hosts === +SHREDDER_HOST="doc@shredder.sshjunkie.com" +PORTAL_HOST="root@portal.genesishostingtechnologies.com" +DA_HOST="root@da.genesishostingtechnologies.com" +VAULT_HOST="root@thevault.bounceme.net" +DATE=$(date +%Y%m%d%H%M%S) + +# === Telegram Setup === +TG_TOKEN="7277705363:AAGSw5Pmcbf7IsSyZKMqU6PJ4VsVwdKLRH0" +TG_CHAT_ID="1559582356" + +send_telegram() { + local message="$1" + curl -s -X POST "https://api.telegram.org/bot$TG_TOKEN/sendMessage" \ + -d "chat_id=$TG_CHAT_ID&text=$message" +} + +send_telegram "🧠 Starting V2 Direct-to-Vault Backup Orchestration..." + +# === Sanity Checks on TheVault and Shredder Mounts === +send_telegram "πŸ§ͺ Sanity check: vault & shredder mounts..." +#ssh -o BatchMode=yes $VAULT_HOST "zfs mount -a && ls /backups/azuracast /backups/krang /backups/directadmin" || send_telegram "❌ Vault sanity check FAILED!" +#ssh -o BatchMode=yes $SHREDDER_HOST " +# zfs mount assets/splmedia && +# zfs mount assets/azuracast && +# zfs mount assets/splshows && +# ls /assets/splmedia /assets/azuracast /assets/splshows +#" + +# === 1️⃣ Direct SPL data: Shredder β†’ TheVault === +send_telegram "πŸ”„ Syncing SPL data directly from Shredder to TheVault..." +ssh -o BatchMode=yes $SHREDDER_HOST "rsync -avz /mnt/spl/splmedia/ $VAULT_HOST:/backups/splmedia/$DATE/" +ssh -o BatchMode=yes $SHREDDER_HOST "rsync -avz /mnt/spl/splassets/ $VAULT_HOST:/backups/splassets/$DATE/" +ssh -o BatchMode=yes $SHREDDER_HOST "rsync -avz /mnt/spl/splshows/ $VAULT_HOST:/backups/splshows/$DATE/" +send_telegram "βœ… SPL data sync complete!" + +# === 2️⃣ Direct AzuraCast media: Shredder β†’ TheVault === +send_telegram "πŸ”„ Syncing AzuraCast media directly from Shredder to TheVault..." +ssh -o BatchMode=yes $SHREDDER_HOST "rsync -avz /mnt/shredder.sshjunkie.com/azuracast/ $VAULT_HOST:/backups/azuracast/$DATE/" +send_telegram "βœ… AzuraCast media sync complete!" + +# === 3️⃣ Direct AzuraCast configs: Portal β†’ TheVault === +send_telegram "πŸ”„ Syncing AzuraCast configs from Portal to TheVault..." +ssh -o BatchMode=yes $PORTAL_HOST "rsync -avz /var/azuracast/.env /var/azuracast/docker-compose.yml /var/azuracast/stations $VAULT_HOST:/backups/azuracast/configs/$DATE/" +send_telegram "βœ… AzuraCast configs sync complete!" + +# === 4️⃣ Direct AzuraCast DB dump: Portal β†’ TheVault === +send_telegram "πŸ”„ Dumping and pushing AzuraCast DB from Portal to TheVault..." +ssh -o BatchMode=yes $PORTAL_HOST "pg_dump -U postgres azuracast | ssh $VAULT_HOST 'cat > /backups/azuracast/configs/$DATE/azuracast.sql'" +send_telegram "βœ… AzuraCast DB push complete!" + +# === 5️⃣ Direct DirectAdmin backup: Portal β†’ TheVault === +send_telegram "πŸ”„ Syncing DirectAdmin configs from Portal to TheVault..." +ssh -o BatchMode=yes $PORTAL_HOST "rsync -avz /usr/local/directadmin/data/admin/ $VAULT_HOST:/backups/directadmin/$DATE/" +send_telegram "βœ… DirectAdmin sync complete!" + +# === 6️⃣ Krang's Proxmox configs: Krang β†’ TheVault === +send_telegram "πŸ”„ Syncing Krang configs to TheVault..." +rsync -avz /etc/pve $VAULT_HOST:/backups/krang/$DATE/pve/ +rsync -avz /etc/network/interfaces $VAULT_HOST:/backups/krang/$DATE/network/ +rsync -avz /etc/ssh $VAULT_HOST:/backups/krang/$DATE/ssh/ +send_telegram "βœ… Krang configs push complete!" + +# === πŸŽ‰ All done! === +send_telegram "πŸŽ‰ V2 Direct-to-Vault Backup COMPLETED!" +exit 0 diff --git a/miscellaneous/bash/bin/snapshot_send_to_vault.sh b/miscellaneous/bash/bin/snapshot_send_to_vault.sh new file mode 100755 index 0000000..387aad1 --- /dev/null +++ b/miscellaneous/bash/bin/snapshot_send_to_vault.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# snapshot_and_send_to_vault.sh +# Create a ZFS snapshot of /deadbeef/genesis-tools and send it to the vault + +set -euo pipefail + +# βš™οΈ Config +POOL="deadbeef" +DATASET="genesis-tools" +REMOTE_USER="root" +REMOTE_HOST="thevault.bounceme.net" +REMOTE_DATASET="backups/krang" + +# πŸ—“οΈ Create snapshot name +DATE=$(date +%F) +SNAPSHOT_NAME="${DATE}" + +echo "πŸ”§ Creating snapshot ${POOL}/${DATASET}@${SNAPSHOT_NAME}..." +sudo zfs snapshot ${POOL}/${DATASET}@${SNAPSHOT_NAME} + +echo "πŸš€ Sending snapshot to ${REMOTE_HOST}..." +sudo zfs send ${POOL}/${DATASET}@${SNAPSHOT_NAME} | \ + ssh ${REMOTE_USER}@${REMOTE_HOST} sudo zfs receive -F ${REMOTE_DATASET} + +echo "βœ… Snapshot ${SNAPSHOT_NAME} replicated to ${REMOTE_HOST}:${REMOTE_DATASET}" + +echo "πŸŽ‰ All done!" diff --git a/miscellaneous/bash/bin/sync_everything_v3.sh b/miscellaneous/bash/bin/sync_everything_v3.sh index 0aea422..6cfdca4 100755 --- a/miscellaneous/bash/bin/sync_everything_v3.sh +++ b/miscellaneous/bash/bin/sync_everything_v3.sh @@ -3,7 +3,7 @@ # GenesisSync: SPL Remote Sync Runner (Krang Orchestrated) # Krang orchestrates sync by SSHing into Shredder, where the SPL shares live. -SHREDDER_HOST="shredder" +SHREDDER_HOST="shredder.sshjunkie.com" SHREDDER_USER="doc" REMOTE_SCRIPT="/tmp/genesis_sync_remote.sh" LOGFILE="/home/doc/genesis_sync_spl.log" diff --git a/miscellaneous/bash/bin/watchman.sh b/miscellaneous/bash/bin/watchman.sh new file mode 100755 index 0000000..a07e331 --- /dev/null +++ b/miscellaneous/bash/bin/watchman.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +#set -e +export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +# === Enable Full Debug Logging === +exec >> /home/doc/healthchecks/watchman.log 2>&1 +set -x # Print each command as it’s run +export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +echo "[$DATE] Watchman script executed" >> /var/log/watchman_cron.log + +# === Config === +PRIMARY_IP="38.102.127.168" # Main TeamTalk server +BACKUP_IP="172.238.63.162" # Backup TeamTalk server +CF_ZONE_ID="c5099d42caa2d9763227267c597cb758" +CF_RECORD_ID="7001484a25f0fe5c323845b6695f7544" +CF_API_TOKEN="lCz1kH6nBZPJL0EWrNI-xEDwfR0oOLpg05fq6M81" +THRESHOLD_LATENCY=150 +THRESHOLD_LOSS=5 +BOT_TOKEN="123456:ABC-DEF1234ghIkl-zyx57W2v1u123ew11" +CHAT_ID="987654321" +DNS_NAME="tt.themediahub.org" + +LOG_FILE="/home/doc/healthchecks/watchman.log" +DATE="$(date '+%Y-%m-%d %H:%M:%S')" + +# === Current DNS IP === +CURRENT_IP=$(/usr/bin/dig +short "$DNS_NAME" | grep -Eo '([0-9]{1,3}\.){3}[0-9]{1,3}' | head -1) +echo "[$DATE] Current IP: $CURRENT_IP" +# === Check Primary Server Health === +echo "[$DATE] πŸ”Ž Checking ping to $PRIMARY_IP..." +PING_OUTPUT=$(/bin/ping -c 4 "$PRIMARY_IP" || echo "Ping failed") +LATENCY=$(echo "$PING_OUTPUT" | tail -1 | /usr/bin/awk -F '/' '{print $5}') +echo "[$DATE] Ping output: $PING_OUTPUT" +LOSS=$(echo "$PING_OUTPUT" | /bin/grep -oP '\d+(?=% packet loss)') +echo "[$DATE] Parsed latency: $LATENCY, loss: $LOSS" +echo "[$DATE] Ping output: $PING_OUTPUT" +echo "[$DATE] Parsed latency: $LATENCY, loss: $LOSS" +echo "[$DATE] Current DNS IP: $CURRENT_IP" + +if [[ -z "$LATENCY" || "$LOSS" -ge "$THRESHOLD_LOSS" || ( -n "$LATENCY" && "$(echo "$LATENCY > $THRESHOLD_LATENCY" | bc)" -eq 1 ) ]]; then + if [[ "$CURRENT_IP" != "$BACKUP_IP" ]]; then + echo "[$DATE] 🚨 Primary down! Switching DNS to backup IP ($BACKUP_IP)..." + MESSAGE="🚨 ALERT: Primary TeamTalk ($PRIMARY_IP) down. Loss: ${LOSS}%, Latency: ${LATENCY}ms. Switching to backup: $BACKUP_IP" + curl -v -s -X POST "https://api.telegram.org/bot${BOT_TOKEN}/sendMessage" \ + -d "chat_id=${CHAT_ID}" -d "text=${MESSAGE}" + + echo "[$DATE] πŸ”„ Sending DNS switch request to Cloudflare..." + API_RESPONSE=$(curl -v -s -X PUT "https://api.cloudflare.com/client/v4/zones/${CF_ZONE_ID}/dns_records/${CF_RECORD_ID}" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" \ + -H "Content-Type: application/json" \ + --data "{\"type\":\"A\",\"name\":\"${DNS_NAME}\",\"content\":\"${BACKUP_IP}\",\"ttl\":60,\"proxied\":false}") + echo "[$DATE] Cloudflare API response: $API_RESPONSE" + echo "[$DATE] βœ… DNS switched to backup." + else + echo "[$DATE] πŸ”„ Primary down, but already on backup. No DNS change needed." + fi +else + if [[ "$CURRENT_IP" != "$PRIMARY_IP" ]]; then + echo "[$DATE] βœ… Primary healthy! Switching DNS back to primary IP ($PRIMARY_IP)..." + MESSAGE="βœ… Primary TeamTalk ($PRIMARY_IP) back online. Loss: ${LOSS}%, Latency: ${LATENCY}ms. Switching DNS back to primary." + curl -v -s -X POST "https://api.telegram.org/bot${BOT_TOKEN}/sendMessage" \ + -d "chat_id=${CHAT_ID}" -d "text=${MESSAGE}" + + echo "[$DATE] πŸ”„ Sending DNS switch back to Cloudflare..." + API_RESPONSE=$(curl -v -s -X PUT "https://api.cloudflare.com/client/v4/zones/${CF_ZONE_ID}/dns_records/${CF_RECORD_ID}" \ + -H "Authorization: Bearer ${CF_API_TOKEN}" \ + -H "Content-Type: application/json" \ + --data "{\"type\":\"A\",\"name\":\"${DNS_NAME}\",\"content\":\"${PRIMARY_IP}\",\"ttl\":60,\"proxied\":false}") + echo "[$DATE] Cloudflare API response: $API_RESPONSE" + echo "[$DATE] βœ… DNS switched back to primary." + else + echo "[$DATE] βœ… Primary healthy, already using primary IP. No DNS change needed." + fi +fi