Auto-commit from giteapush.sh at 2025-06-04 12:15:45

2025-06-04 12:15:45 -04:00 · 2025-06-04 12:15:45 -04:00 · 352b9c1467
commit 352b9c1467
parent 468c0b1955
8 changed files with 287 additions and 2 deletions
--- a/documents/disklabels/baboon.md
+++ b/documents/disklabels/baboon.md
@ -0,0 +1,10 @@
+🟩 **Detailed /dev/disk/by-id/ Partitions (Live Ones)**
+======================================================
+
+```shell
+lrwxrwxrwx 1 root root 10 May 29 08:53 ata-Timetec_30TT253X2-256GB_PL211014YSA256G0116-part1 -> ../../sda1
+lrwxrwxrwx 1 root root 10 May 29 08:53 ata-Timetec_30TT253X2-256GB_PL211014YSA256G0116-part2 -> ../../sda2
+lrwxrwxrwx 1 root root 10 May 29 08:53 ata-Timetec_30TT253X2-256GB_PL211014YSA256G0116-part3 -> ../../sda3
+lrwxrwxrwx 1 root root 10 May 29 08:53 ata-Timetec_30TT253X2-256GB_PL220302YSA256G0877-part1 -> ../../sdb1
+lrwxrwxrwx 1 root root 10 May 29 08:53 scsi-35000cca291d4b870-part1 -> ../../sdd1
+lrwxrwxrwx 1 root root 10 May 29 08:53 scsi-35000cca2b093888c-part1 -> ../../sdc1
--- a/1
+++ b/1
@ -0,0 +1 @@
+/deadbeef/genesis-tools
--- a/miscellaneous/bash/bin/arecordalertfailzero.sh
+++ b/miscellaneous/bash/bin/arecordalertfailzero.sh
@ -13,7 +13,6 @@ CRITICAL_HOSTS=(
  "zcluster.technodrome1.sshjunkie.com"
  "zcluster.technodrome2.sshjunkie.com"
  "krang.core.sshjunkie.com"
-  "clustercontrol.sshjunkie.com"
  "tt.themediahub.org"
  "toot.themediahub.org"
  "chatwithus.live"
--- a/miscellaneous/bash/bin/brains_backup.sh
+++ b/miscellaneous/bash/bin/brains_backup.sh
@ -0,0 +1,105 @@
+#!/bin/bash
+set -euo pipefail
+
+# === SSH Hosts ===
+SHREDDER_HOST="doc@shredder.sshjunkie.com"
+PORTAL_HOST="root@portal.genesishostingtechnologies.com"
+DA_HOST="root@da.genesishostingtechnologies.com"
+VAULT_HOST="root@thevault.bounceme.net"
+
+# === Telegram Setup ===
+TG_TOKEN="7277705363:AAGSw5Pmcbf7IsSyZKMqU6PJ4VsVwdKLRH0"
+TG_CHAT_ID="1559582356"
+
+send_telegram() {
+  local message="$1"
+  curl -s -X POST "https://api.telegram.org/bot$TG_TOKEN/sendMessage" \
+       -d "chat_id=$TG_CHAT_ID&text=$message"
+}
+
+# === Local Staging Area on Krang ===
+STAGING_DATASET="/deadbeef/staging"
+DATE=$(date +%Y%m%d%H%M%S)
+BACKUP_DIR="$STAGING_DATASET/brains-$DATE"
+mkdir -p "$BACKUP_DIR"
+
+send_telegram "🧠 Starting centralized backup from Krang to $BACKUP_DIR..."
+
+# === Sanity Checks on TheVault ===
+send_telegram "🧪 Sanity check: ensuring TheVault datasets are mounted..."
+ssh -o BatchMode=yes $VAULT_HOST "zfs mount -a && ls /backups/azuracast /backups/krang /backups/directadmin" && \
+send_telegram "✅ TheVault mountpoints verified!" || \
+send_telegram "❌ TheVault mountpoint sanity check FAILED!"
+
+# === Sanity Checks on Shredder ===
+send_telegram "🧪 Sanity check: ensuring Shredder datasets are mounted..."
+ssh -o BatchMode=yes $SHREDDER_HOST "zfs mount -a && ls /assets/splmedia /assets/azuracast /assets/pokbackups /assets/splshows" && \
+send_telegram "✅ Shredder mountpoints verified!" || \
+send_telegram "❌ Shredder mountpoint sanity check FAILED!"
+
+# === Helper Function for Steps ===
+run_backup_step() {
+  local description="$1"
+  shift
+  send_telegram "🔄 $description"
+  if "$@"; then
+    send_telegram "✅ $description complete!"
+  else
+    send_telegram "❌ $description FAILED!"
+  fi
+}
+
+# === 1️⃣ Sync SPL and Shredder Data ===
+run_backup_step "Syncing SPL and Shredder data" \
+  rsync -avz -e "ssh -o BatchMode=yes" $SHREDDER_HOST:/mnt/spl/ "$BACKUP_DIR/splmedia/" && \
+  rsync -avz -e "ssh -o BatchMode=yes" $SHREDDER_HOST:/mnt/spl/ "$BACKUP_DIR/splassets/"
+
+# === 2️⃣ Backup Krang's Configs ===
+run_backup_step "Backing up Krang host configs" \
+  rsync -avz /etc/pve "$BACKUP_DIR/krang-pve/" && \
+  rsync -avz /etc/network/interfaces "$BACKUP_DIR/krang-net/" && \
+  rsync -avz /etc/ssh "$BACKUP_DIR/krang-ssh/"
+
+# === 3️⃣ Backup AzuraCast DB from Portal ===
+run_backup_step "Backing up AzuraCast DB from Portal" \
+  ssh -o BatchMode=yes $PORTAL_HOST "pg_dump -U postgres azuracast" > "$BACKUP_DIR/databases/azuracast.sql"
+
+# === 4️⃣ Backup AzuraCast Configs ===
+run_backup_step "Backing up AzuraCast configs from Portal" \
+  rsync -avz -e "ssh -o BatchMode=yes" $PORTAL_HOST:/var/azuracast/.env "$BACKUP_DIR/azuracast/" && \
+  rsync -avz -e "ssh -o BatchMode=yes" $PORTAL_HOST:/var/azuracast/docker-compose.yml "$BACKUP_DIR/azuracast/" && \
+  rsync -avz -e "ssh -o BatchMode=yes" $PORTAL_HOST:/var/azuracast/stations "$BACKUP_DIR/azuracast/stations"
+
+# === 5️⃣ Sync AzuraCast Media from Portal to Shredder, then to Krang ===
+run_backup_step "Syncing AzuraCast media from Portal to Shredder" \
+  ssh -o BatchMode=yes $PORTAL_HOST "rsync -avz /mnt/azuracast1/ doc@shredder.sshjunkie.com:/assets/azuracast/"
+
+run_backup_step "Backing up AzuraCast media from Shredder to Krang" \
+  rsync -avz -e "ssh -o BatchMode=yes" $SHREDDER_HOST:/assets/azuracast/ "$BACKUP_DIR/azuracast-media/"
+
+# === 6️⃣ Backup DirectAdmin Configs ===
+run_backup_step "Backing up DirectAdmin from Portal" \
+  rsync -avz -e "ssh -o BatchMode=yes" $DA_HOST:/usr/local/directadmin/data/admin/ "$BACKUP_DIR/directadmin/"
+
+# === 7️⃣ Push to TheVault ===
+send_telegram "🔄 Pushing backups to TheVault datasets..."
+
+# AzuraCast Configs & DB
+rsync -avz -e "ssh -o BatchMode=yes" "$BACKUP_DIR/azuracast/" $VAULT_HOST:/backups/azuracast/configs/$DATE/
+rsync -avz -e "ssh -o BatchMode=yes" "$BACKUP_DIR/databases/" $VAULT_HOST:/backups/azuracast/configs/$DATE/
+
+# AzuraCast Media
+rsync -avz -e "ssh -o BatchMode=yes" "$BACKUP_DIR/azuracast-media/" $VAULT_HOST:/backups/azuracast/$DATE/
+
+# DirectAdmin
+rsync -avz -e "ssh -o BatchMode=yes" "$BACKUP_DIR/directadmin/" $VAULT_HOST:/backups/directadmin/$DATE/
+
+# Krang
+rsync -avz -e "ssh -o BatchMode=yes" "$BACKUP_DIR/krang-*/" $VAULT_HOST:/backups/krang/$DATE/
+
+# SPL
+rsync -avz -e "ssh -o BatchMode=yes" "$BACKUP_DIR/splmedia/" $VAULT_HOST:/backups/splmedia/$DATE/
+rsync -avz -e "ssh -o BatchMode=yes" "$BACKUP_DIR/splassets/" $VAULT_HOST:/backups/splassets/$DATE/
+
+send_telegram "🎉 FULL SYSTEM BACKUP COMPLETED & MIRRORED TO VAULT!"
+exit 0
--- a/miscellaneous/bash/bin/brains_backup2.sh
+++ b/miscellaneous/bash/bin/brains_backup2.sh
@ -0,0 +1,69 @@
+#!/bin/bash
+set -euo pipefail
+
+# === SSH Hosts ===
+SHREDDER_HOST="doc@shredder.sshjunkie.com"
+PORTAL_HOST="root@portal.genesishostingtechnologies.com"
+DA_HOST="root@da.genesishostingtechnologies.com"
+VAULT_HOST="root@thevault.bounceme.net"
+DATE=$(date +%Y%m%d%H%M%S)
+
+# === Telegram Setup ===
+TG_TOKEN="7277705363:AAGSw5Pmcbf7IsSyZKMqU6PJ4VsVwdKLRH0"
+TG_CHAT_ID="1559582356"
+
+send_telegram() {
+  local message="$1"
+  curl -s -X POST "https://api.telegram.org/bot$TG_TOKEN/sendMessage" \
+       -d "chat_id=$TG_CHAT_ID&text=$message"
+}
+
+send_telegram "🧠 Starting V2 Direct-to-Vault Backup Orchestration..."
+
+# === Sanity Checks on TheVault and Shredder Mounts ===
+send_telegram "🧪 Sanity check: vault & shredder mounts..."
+#ssh -o BatchMode=yes $VAULT_HOST "zfs mount -a && ls /backups/azuracast /backups/krang /backups/directadmin" || send_telegram "❌ Vault sanity check FAILED!"
+#ssh -o BatchMode=yes $SHREDDER_HOST "
+#  zfs mount assets/splmedia &&
+#  zfs mount assets/azuracast &&
+#  zfs mount assets/splshows &&
+#  ls /assets/splmedia /assets/azuracast /assets/splshows
+#"
+
+# === 1️⃣ Direct SPL data: Shredder → TheVault ===
+send_telegram "🔄 Syncing SPL data directly from Shredder to TheVault..."
+ssh -o BatchMode=yes $SHREDDER_HOST "rsync -avz /mnt/spl/splmedia/ $VAULT_HOST:/backups/splmedia/$DATE/"
+ssh -o BatchMode=yes $SHREDDER_HOST "rsync -avz /mnt/spl/splassets/ $VAULT_HOST:/backups/splassets/$DATE/"
+ssh -o BatchMode=yes $SHREDDER_HOST "rsync -avz /mnt/spl/splshows/ $VAULT_HOST:/backups/splshows/$DATE/"
+send_telegram "✅ SPL data sync complete!"
+
+# === 2️⃣ Direct AzuraCast media: Shredder → TheVault ===
+send_telegram "🔄 Syncing AzuraCast media directly from Shredder to TheVault..."
+ssh -o BatchMode=yes $SHREDDER_HOST "rsync -avz /mnt/shredder.sshjunkie.com/azuracast/ $VAULT_HOST:/backups/azuracast/$DATE/"
+send_telegram "✅ AzuraCast media sync complete!"
+
+# === 3️⃣ Direct AzuraCast configs: Portal → TheVault ===
+send_telegram "🔄 Syncing AzuraCast configs from Portal to TheVault..."
+ssh -o BatchMode=yes $PORTAL_HOST "rsync -avz /var/azuracast/.env /var/azuracast/docker-compose.yml /var/azuracast/stations $VAULT_HOST:/backups/azuracast/configs/$DATE/"
+send_telegram "✅ AzuraCast configs sync complete!"
+
+# === 4️⃣ Direct AzuraCast DB dump: Portal → TheVault ===
+send_telegram "🔄 Dumping and pushing AzuraCast DB from Portal to TheVault..."
+ssh -o BatchMode=yes $PORTAL_HOST "pg_dump -U postgres azuracast | ssh $VAULT_HOST 'cat > /backups/azuracast/configs/$DATE/azuracast.sql'"
+send_telegram "✅ AzuraCast DB push complete!"
+
+# === 5️⃣ Direct DirectAdmin backup: Portal → TheVault ===
+send_telegram "🔄 Syncing DirectAdmin configs from Portal to TheVault..."
+ssh -o BatchMode=yes $PORTAL_HOST "rsync -avz /usr/local/directadmin/data/admin/ $VAULT_HOST:/backups/directadmin/$DATE/"
+send_telegram "✅ DirectAdmin sync complete!"
+
+# === 6️⃣ Krang's Proxmox configs: Krang → TheVault ===
+send_telegram "🔄 Syncing Krang configs to TheVault..."
+rsync -avz /etc/pve $VAULT_HOST:/backups/krang/$DATE/pve/
+rsync -avz /etc/network/interfaces $VAULT_HOST:/backups/krang/$DATE/network/
+rsync -avz /etc/ssh $VAULT_HOST:/backups/krang/$DATE/ssh/
+send_telegram "✅ Krang configs push complete!"
+
+# === 🎉 All done! ===
+send_telegram "🎉 V2 Direct-to-Vault Backup COMPLETED!"
+exit 0
--- a/miscellaneous/bash/bin/snapshot_send_to_vault.sh
+++ b/miscellaneous/bash/bin/snapshot_send_to_vault.sh
@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+
+# snapshot_and_send_to_vault.sh
+# Create a ZFS snapshot of /deadbeef/genesis-tools and send it to the vault
+
+set -euo pipefail
+
+# ⚙️ Config
+POOL="deadbeef"
+DATASET="genesis-tools"
+REMOTE_USER="root"
+REMOTE_HOST="thevault.bounceme.net"
+REMOTE_DATASET="backups/krang"
+
+# 🗓️ Create snapshot name
+DATE=$(date +%F)
+SNAPSHOT_NAME="${DATE}"
+
+echo "🔧 Creating snapshot ${POOL}/${DATASET}@${SNAPSHOT_NAME}..."
+sudo zfs snapshot ${POOL}/${DATASET}@${SNAPSHOT_NAME}
+
+echo "🚀 Sending snapshot to ${REMOTE_HOST}..."
+sudo zfs send ${POOL}/${DATASET}@${SNAPSHOT_NAME} | \
+  ssh ${REMOTE_USER}@${REMOTE_HOST} sudo zfs receive -F ${REMOTE_DATASET}
+
+echo "✅ Snapshot ${SNAPSHOT_NAME} replicated to ${REMOTE_HOST}:${REMOTE_DATASET}"
+
+echo "🎉 All done!"
--- a/miscellaneous/bash/bin/sync_everything_v3.sh
+++ b/miscellaneous/bash/bin/sync_everything_v3.sh
@ -3,7 +3,7 @@
 # GenesisSync: SPL Remote Sync Runner (Krang Orchestrated)
 # Krang orchestrates sync by SSHing into Shredder, where the SPL shares live.

-SHREDDER_HOST="shredder"
+SHREDDER_HOST="shredder.sshjunkie.com"
 SHREDDER_USER="doc"
 REMOTE_SCRIPT="/tmp/genesis_sync_remote.sh"
 LOGFILE="/home/doc/genesis_sync_spl.log"
--- a/miscellaneous/bash/bin/watchman.sh
+++ b/miscellaneous/bash/bin/watchman.sh
@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+#set -e
+export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+# === Enable Full Debug Logging ===
+exec >> /home/doc/healthchecks/watchman.log 2>&1
+set -x  # Print each command as it’s run
+export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+echo "[$DATE] Watchman script executed" >> /var/log/watchman_cron.log
+
+# === Config ===
+PRIMARY_IP="38.102.127.168"           # Main TeamTalk server
+BACKUP_IP="172.238.63.162"            # Backup TeamTalk server
+CF_ZONE_ID="c5099d42caa2d9763227267c597cb758"
+CF_RECORD_ID="7001484a25f0fe5c323845b6695f7544"
+CF_API_TOKEN="lCz1kH6nBZPJL0EWrNI-xEDwfR0oOLpg05fq6M81"
+THRESHOLD_LATENCY=150
+THRESHOLD_LOSS=5
+BOT_TOKEN="123456:ABC-DEF1234ghIkl-zyx57W2v1u123ew11"
+CHAT_ID="987654321"
+DNS_NAME="tt.themediahub.org"
+
+LOG_FILE="/home/doc/healthchecks/watchman.log"
+DATE="$(date '+%Y-%m-%d %H:%M:%S')"
+
+# === Current DNS IP ===
+CURRENT_IP=$(/usr/bin/dig +short "$DNS_NAME" | grep -Eo '([0-9]{1,3}\.){3}[0-9]{1,3}' | head -1)
+echo "[$DATE] Current IP: $CURRENT_IP"
+# === Check Primary Server Health ===
+echo "[$DATE] 🔎 Checking ping to $PRIMARY_IP..."
+PING_OUTPUT=$(/bin/ping -c 4 "$PRIMARY_IP" || echo "Ping failed")
+LATENCY=$(echo "$PING_OUTPUT" | tail -1 | /usr/bin/awk -F '/' '{print $5}')
+echo "[$DATE] Ping output: $PING_OUTPUT"
+LOSS=$(echo "$PING_OUTPUT" | /bin/grep -oP '\d+(?=% packet loss)')
+echo "[$DATE] Parsed latency: $LATENCY, loss: $LOSS"
+echo "[$DATE] Ping output: $PING_OUTPUT"
+echo "[$DATE] Parsed latency: $LATENCY, loss: $LOSS"
+echo "[$DATE] Current DNS IP: $CURRENT_IP"
+
+if [[ -z "$LATENCY" || "$LOSS" -ge "$THRESHOLD_LOSS" || ( -n "$LATENCY" && "$(echo "$LATENCY > $THRESHOLD_LATENCY" | bc)" -eq 1 ) ]]; then
+  if [[ "$CURRENT_IP" != "$BACKUP_IP" ]]; then
+    echo "[$DATE] 🚨 Primary down! Switching DNS to backup IP ($BACKUP_IP)..."
+    MESSAGE="🚨 ALERT: Primary TeamTalk ($PRIMARY_IP) down. Loss: ${LOSS}%, Latency: ${LATENCY}ms. Switching to backup: $BACKUP_IP"
+    curl -v -s -X POST "https://api.telegram.org/bot${BOT_TOKEN}/sendMessage" \
+      -d "chat_id=${CHAT_ID}" -d "text=${MESSAGE}"
+
+    echo "[$DATE] 🔄 Sending DNS switch request to Cloudflare..."
+    API_RESPONSE=$(curl -v -s -X PUT "https://api.cloudflare.com/client/v4/zones/${CF_ZONE_ID}/dns_records/${CF_RECORD_ID}" \
+      -H "Authorization: Bearer ${CF_API_TOKEN}" \
+      -H "Content-Type: application/json" \
+      --data "{\"type\":\"A\",\"name\":\"${DNS_NAME}\",\"content\":\"${BACKUP_IP}\",\"ttl\":60,\"proxied\":false}")
+    echo "[$DATE] Cloudflare API response: $API_RESPONSE"
+    echo "[$DATE] ✅ DNS switched to backup."
+  else
+    echo "[$DATE] 🔄 Primary down, but already on backup. No DNS change needed."
+  fi
+else
+  if [[ "$CURRENT_IP" != "$PRIMARY_IP" ]]; then
+    echo "[$DATE] ✅ Primary healthy! Switching DNS back to primary IP ($PRIMARY_IP)..."
+    MESSAGE="✅ Primary TeamTalk ($PRIMARY_IP) back online. Loss: ${LOSS}%, Latency: ${LATENCY}ms. Switching DNS back to primary."
+    curl -v -s -X POST "https://api.telegram.org/bot${BOT_TOKEN}/sendMessage" \
+      -d "chat_id=${CHAT_ID}" -d "text=${MESSAGE}"
+
+    echo "[$DATE] 🔄 Sending DNS switch back to Cloudflare..."
+    API_RESPONSE=$(curl -v -s -X PUT "https://api.cloudflare.com/client/v4/zones/${CF_ZONE_ID}/dns_records/${CF_RECORD_ID}" \
+      -H "Authorization: Bearer ${CF_API_TOKEN}" \
+      -H "Content-Type: application/json" \
+      --data "{\"type\":\"A\",\"name\":\"${DNS_NAME}\",\"content\":\"${PRIMARY_IP}\",\"ttl\":60,\"proxied\":false}")
+    echo "[$DATE] Cloudflare API response: $API_RESPONSE"
+    echo "[$DATE] ✅ DNS switched back to primary."
+  else
+    echo "[$DATE] ✅ Primary healthy, already using primary IP. No DNS change needed."
+  fi
+fi