Auto-commit from giteapush.sh at 2025-05-08 15:36:35

2025-05-08 15:36:35 -04:00 · 2025-05-08 15:36:35 -04:00 · 21d02c69cc
commit 21d02c69cc
parent 191da79d05
9 changed files with 499 additions and 1 deletions
--- a/genesishostingmd/pmgenesisiorealignment.md
+++ b/genesishostingmd/pmgenesisiorealignment.md
@ -0,0 +1,83 @@
+# Postmortem: Genesis I/O Realignment
+
+**Date:** May 8, 2025  
+**Author:** Doc  
+**Systems Involved:** minioraid5, shredder, chatwithus.live, zcluster.technodrome1/2, thevault  
+**Scope:** Local-first mirroring, permission normalization, MinIO transition
+
+---
+
+## 🎯 Objective
+
+To realign the Genesis file flow architecture by:
+
+- Making local block storage the **primary source** of truth for AzuraCast and Genesis buckets
+- Transitioning FTP uploads to target local storage instead of MinIO directly
+- Establishing **two-way mirroring** between local paths and MinIO buckets
+- Correcting inherited permission issues across `/mnt/raid5` using `find + chmod`
+- Preserving MinIO buckets as **backup mirrors**, not primary data stores
+
+---
+
+## 🔧 Work Performed
+
+### ✅ Infrastructure changes:
+- Deployed block storage volume to Linode Mastodon instance
+- Mirrored MinIO buckets (`genesisassets`, `genesislibrary`, `azuracast`) to local paths
+- Configured cron-based `mc mirror` jobs:
+  - Local ➜ MinIO: every 5 minutes with `--overwrite --remove`
+  - MinIO ➜ Local: nightly pull, no `--remove`
+
+### ✅ FTP Pipeline Adjustments:
+- Users now upload to `/mnt/spl/ftp/uploads` (local)
+- Permissions set so only admins access full `/mnt/spl/ftp`
+- FTP directory structure created for SPL automation
+
+### ✅ System Tuning:
+- Set `vm.swappiness=10` on all nodes
+- Apache disabled where not in use
+- Daily health checks via `pull_health_everywhere.sh`
+- Krang Telegram alerts deployed for cleanup and system state
+
+---
+
+## 🧠 Observations
+
+- **High load** on `minioraid5` during `mc mirror` and `chmod` overlap
+  - Load ~6.5 due to concurrent I/O pressure
+  - `chmod` stuck in `D` state (I/O wait) while `mc` dominated disk queues
+  - Resolved after `mc` completion — `chmod` resumed and completed
+
+- **MinIO buckets were temporarily inaccessible** due to permissions accidentally inherited by FTP group
+  - Resolved by recursively resetting permissions on `/mnt/raid5`
+
+- **Krang telemetry** verified:
+  - Mastodon swap usage rising under asset load
+  - All nodes had Apache disabled or dormant
+  - Health alerts triggered on high swap or load
+
+---
+
+## ✅ Outcome
+
+- Full Genesis and AzuraCast data now reside locally with resilient S3 mirrors
+- Mastodon running on block storage, no longer dependent on MinIO latency
+- FTP integration with SPL directory trees complete
+- Cleanup script successfully deployed across all nodes via Krang
+- Daily health reports operational with alerts for high swap/load
+
+---
+
+## 🔁 Recommendations
+
+- Consider adding snapshot-based ZFS backups for `/mnt/raid5`
+- Build `verify_mirror.sh` to detect drift between MinIO and local storage
+- Auto-trigger `chmod` only after `mc mirror` finishes
+- Monitor long-running background jobs with Krang watchdogs
+
+---
+
+**Signed,**  
+Doc  
+Genesis Hosting Technologies
+
--- a/miscellaneous/bash/mastodon_status-check.sh
+++ b/miscellaneous/bash/mastodon_status-check.sh
@ -0,0 +1,69 @@
+#!/bin/bash
+
+echo "Step 0: Starting script..."
+
+# Load token from ~/.mastodon-token or environment
+TOKEN_FILE="$HOME/.mastodon-token"
+if [ -f "$TOKEN_FILE" ]; then
+  export MASTO_TOKEN=$(cat "$TOKEN_FILE")
+fi
+
+if [ -z "$MASTO_TOKEN" ]; then
+  echo "❌ No Mastodon access token found. Set \$MASTO_TOKEN or create ~/.mastodon-token"
+  exit 1
+fi
+
+echo "Step 1: Token loaded."
+
+TMPFILE=$(mktemp)
+MASTO_API="https://chatwithus.live/api/v1/statuses"
+
+SERVICES=(
+  "Genesis Radio|https://genesis-radio.net"
+  "Mastodon|https://chatwithus.live"
+  "MinIO|https://console.sshjunkie.com"
+  "AzuraCast|portal.genesishostingtechnologies.com/login"
+  "TeamTalk|tcp://tt.themediahub.org:10442"
+  "DirectAdmin|https://da.genesishostingtechnologies.com"
+)
+
+echo "[Status Check @ $(date -u '+%H:%M %Z')]" > "$TMPFILE"
+
+for service in "${SERVICES[@]}"; do
+  IFS="|" read -r NAME URL <<< "$service"
+
+  if [[ $URL == tcp://* ]]; then
+    # Handle TCP port check
+    HOSTPORT=${URL#tcp://}
+    HOST=${HOSTPORT%%:*}
+    PORT=${HOSTPORT##*:}
+    echo "Checking TCP: $NAME on $HOST:$PORT"
+    timeout 5 bash -c "</dev/tcp/$HOST/$PORT" &>/dev/null
+  else
+    # Handle HTTP(S) check
+    echo "Checking HTTP: $NAME -> $URL"
+    curl -s --head --fail --max-time 5 "$URL" >/dev/null
+  fi
+
+  if [ $? -eq 0 ]; then
+    echo "✅ $NAME: Online" >> "$TMPFILE"
+  else
+    echo "❌ $NAME: Offline" >> "$TMPFILE"
+  fi
+done
+
+echo "Step 2: Results collected."
+cat "$TMPFILE"
+
+# Convert newlines to URL-encoded format
+POST_BODY=$(sed ':a;N;$!ba;s/\n/%0A/g' "$TMPFILE")
+
+echo "Step 3: Posting to Mastodon..."
+
+curl -s -X POST "$MASTO_API" \
+  -H "Authorization: Bearer $MASTO_TOKEN" \
+  -d "status=$POST_BODY"
+
+echo "Step 4: Done."
+
+rm -f "$TMPFILE"
--- a/miscellaneous/bash/pull_health_everywhere
+++ b/miscellaneous/bash/pull_health_everywhere
@ -0,0 +1,74 @@
+#!/bin/bash
+
+# === CONFIG ===
+REMOTE_USER="doc"
+BOT_TOKEN="8178867489:AAH0VjN7VnZSCIWasSz_y97iBLLjPJA751k"
+CHAT_ID="1559582356"
+TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S')
+LOGFILE="$HOME/krang-logs/health-$(date '+%Y%m%d-%H%M').log"
+
+# Thresholds
+SWAP_LIMIT_MB=512
+LOAD_LIMIT=4.0
+
+mkdir -p "$HOME/krang-logs"
+
+SERVERS=(
+  thevault.sshjunkie.com
+  zcluster.technodrome1.sshjunkie.com
+  zcluster.technodrome2.sshjunkie.com
+  shredder.sshjunkie.com
+  chatwithus.live
+)
+
+SUMMARY="📡 Krang System Health Report - $TIMESTAMP\n\n"
+
+for HOST in "${SERVERS[@]}"; do
+  echo "🔍 Collecting from $HOST..."
+
+  DATA=$(ssh "$REMOTE_USER@$HOST" bash -s << 'EOF'
+HOST=$(hostname)
+MEM=$(free -h | awk '/Mem:/ {print $4 " free"}')
+SWAP_RAW=$(free -m | awk '/Swap:/ {print $3}')
+SWAP="$SWAP_RAW Mi used"
+DISK=$(df -h / | awk 'NR==2 {print $4 " free"}')
+LOAD=$(uptime | awk -F'load average:' '{print $2}' | cut -d, -f1 | xargs)
+APACHE=$(systemctl is-active apache2 2>/dev/null || systemctl is-active httpd 2>/dev/null)
+[ "$APACHE" = "active" ] && APACHE_STATUS="✅ Apache running" || APACHE_STATUS="❌ Apache not running"
+
+echo "$HOST|$MEM|$SWAP_RAW|$SWAP|$DISK|$LOAD|$APACHE_STATUS"
+EOF
+)
+
+  IFS='|' read -r H MEM SWAP_MB SWAP_HUMAN DISK LOAD1 APACHE_STATUS <<< "$DATA"
+
+  ALERTS=""
+  if (( SWAP_MB > SWAP_LIMIT_MB )); then
+    ALERTS+="⚠️ HIGH SWAP ($SWAP_HUMAN)\n"
+  fi
+
+  LOAD_INT=$(awk "BEGIN {print ($LOAD1 > $LOAD_LIMIT) ? 1 : 0}")
+  if [ "$LOAD_INT" -eq 1 ]; then
+    ALERTS+="⚠️ HIGH LOAD ($LOAD1)\n"
+  fi
+
+  ALERTS_MSG=""
+  [ -n "$ALERTS" ] && ALERTS_MSG="🚨 ALERTS:\n$ALERTS"
+
+  SUMMARY+="🖥️ $H
+• Mem: $MEM
+• Swap: $SWAP_HUMAN
+• Disk: $DISK
+• Load: $LOAD1
+• $APACHE_STATUS
+$ALERTS_MSG
+\n"
+done
+
+# Log to file
+echo -e "$SUMMARY" > "$LOGFILE"
+
+# Send to Telegram
+curl -s -X POST https://api.telegram.org/bot$BOT_TOKEN/sendMessage \
+  -d chat_id="$CHAT_ID" \
+  -d text="$SUMMARY"
--- a/miscellaneous/bash/watchdog.sh
+++ b/miscellaneous/bash/watchdog.sh
@ -0,0 +1,39 @@
+#!/bin/bash
+
+# === CONFIG ===
+WATCH_STRING="find /mnt/raid5 -type d -exec chmod o+X {} \\;"  # Adjust if needed
+CHECK_INTERVAL=60  # seconds
+BOT_TOKEN="8178867489:AAH0VjN7VnZSCIWasSz_y97iBLLjPJA751k"
+CHAT_ID="1559582356"
+HOST=$(hostname)
+LOGFILE="$HOME/krang-logs/chmod_watchdog_$(date '+%Y%m%d-%H%M').log"
+mkdir -p "$HOME/krang-logs"
+
+# === FIND TARGET PID ===
+PID=$(pgrep -f "$WATCH_STRING")
+
+if [ -z "$PID" ]; then
+  echo "❌ No matching chmod process found." | tee -a "$LOGFILE"
+  exit 1
+fi
+
+echo "👁️ Watching PID $PID for chmod job on $HOST..." | tee -a "$LOGFILE"
+
+# === MONITOR LOOP ===
+while kill -0 "$PID" 2>/dev/null; do
+  echo "⏳ [$HOST] chmod PID $PID still running..." >> "$LOGFILE"
+  sleep "$CHECK_INTERVAL"
+done
+
+# === COMPLETE ===
+MSG="✅ [$HOST] chmod finished on /mnt/raid5
+Time: $(date '+%Y-%m-%d %H:%M:%S')
+PID: $PID
+Watchdog confirmed completion."
+
+echo -e "$MSG" | tee -a "$LOGFILE"
+
+curl -s -X POST https://api.telegram.org/bot$BOT_TOKEN/sendMessage \
+  -d chat_id="$CHAT_ID" \
+  -d text="$MSG"
+
--- a/miscellaneous/dotheneedfuleverywhere.sh
+++ b/miscellaneous/dotheneedfuleverywhere.sh
@ -0,0 +1,49 @@
+#!/bin/bash
+
+# === CONFIG ===
+SCRIPT_PATH="/usr/local/bin/do_the_needful.sh"
+REMOTE_USER="doc"
+BOT_TOKEN="8178867489:AAH0VjN7VnZSCIWasSz_y97iBLLjPJA751k"
+CHAT_ID="1559582356"
+TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S')
+
+SERVERS=(
+  thevault.sshjunkie.com
+  zcluster.technodrome1.sshjunkie.com
+  zcluster.technodrome2.sshjunkie.com
+  shredder.sshjunkie.com
+  chatwithus.live
+)
+
+SUMMARY="🤖 Krang Deployment Report - $TIMESTAMP\n\n"
+FAILURES=0
+
+for HOST in "${SERVERS[@]}"; do
+  echo "🚀 Deploying to $HOST..."
+
+  # Upload script to temp location
+  scp "$SCRIPT_PATH" "$REMOTE_USER@$HOST:/tmp/do_the_needful.sh"
+  if [ $? -ne 0 ]; then
+    SUMMARY+="❌ $HOST: SCP failed\n"
+    ((FAILURES++))
+    continue
+  fi
+
+  # Move into place and execute
+  ssh "$REMOTE_USER@$HOST" "sudo install -m 755 /tmp/do_the_needful.sh $SCRIPT_PATH && sudo $SCRIPT_PATH"
+  if [ $? -ne 0 ]; then
+    SUMMARY+="❌ $HOST: sudo execution failed\n"
+    ((FAILURES++))
+  else
+    SUMMARY+="✅ $HOST: cleaned successfully\n"
+  fi
+
+  echo "----------------------------------"
+done
+
+# === Send Telegram Summary ===
+FINAL_STATUS="🚨 Some hosts failed." && [ "$FAILURES" -eq 0 ] && FINAL_STATUS="✅ All hosts completed."
+
+curl -s -X POST https://api.telegram.org/bot$BOT_TOKEN/sendMessage \
+  -d chat_id="$CHAT_ID" \
+  -d text="$FINAL_STATUS\n\n$SUMMARY"
--- a/miscellaneous/fixsudoerseverywhere.sh
+++ b/miscellaneous/fixsudoerseverywhere.sh
@ -0,0 +1,33 @@
+#!/bin/bash
+
+# === CONFIG ===
+REMOTE_USER="doc"
+SERVERS=(
+  thevault.sshjunkie.com
+  zcluster.technodrome1.sshjunkie.com
+  zcluster.technodrome2.sshjunkie.com
+  shredder.sshjunkie.com
+  chatwithus.live
+)
+
+SUDO_LINE="doc ALL=(ALL) NOPASSWD:ALL"
+
+# === Execution ===
+for HOST in "${SERVERS[@]}"; do
+  echo "🔧 Fixing sudoers on $HOST..."
+
+  ssh "$REMOTE_USER@$HOST" "sudo bash -c '
+    cp /etc/sudoers /etc/sudoers.bak_krang &&
+    grep -q \"$SUDO_LINE\" /etc/sudoers ||
+    echo \"$SUDO_LINE\" >> /etc/sudoers &&
+    visudo -c >/dev/null
+  '"
+
+  if ssh "$REMOTE_USER@$HOST" "sudo -n true"; then
+    echo "✅ $HOST: sudo access confirmed"
+  else
+    echo "❌ $HOST: sudo access STILL broken"
+  fi
+
+  echo "----------------------------------"
+done
--- a/postmortem/pmgenesisiorealignment.md
+++ b/postmortem/pmgenesisiorealignment.md
@ -0,0 +1,83 @@
+# Postmortem: Genesis I/O Realignment
+
+**Date:** May 8, 2025  
+**Author:** Doc  
+**Systems Involved:** minioraid5, shredder, chatwithus.live, zcluster.technodrome1/2, thevault  
+**Scope:** Local-first mirroring, permission normalization, MinIO transition
+
+---
+
+## 🎯 Objective
+
+To realign the Genesis file flow architecture by:
+
+- Making local block storage the **primary source** of truth for AzuraCast and Genesis buckets
+- Transitioning FTP uploads to target local storage instead of MinIO directly
+- Establishing **two-way mirroring** between local paths and MinIO buckets
+- Correcting inherited permission issues across `/mnt/raid5` using `find + chmod`
+- Preserving MinIO buckets as **backup mirrors**, not primary data stores
+
+---
+
+## 🔧 Work Performed
+
+### ✅ Infrastructure changes:
+- Deployed block storage volume to Linode Mastodon instance
+- Mirrored MinIO buckets (`genesisassets`, `genesislibrary`, `azuracast`) to local paths
+- Configured cron-based `mc mirror` jobs:
+  - Local ➜ MinIO: every 5 minutes with `--overwrite --remove`
+  - MinIO ➜ Local: nightly pull, no `--remove`
+
+### ✅ FTP Pipeline Adjustments:
+- Users now upload to `/mnt/spl/ftp/uploads` (local)
+- Permissions set so only admins access full `/mnt/spl/ftp`
+- FTP directory structure created for SPL automation
+
+### ✅ System Tuning:
+- Set `vm.swappiness=10` on all nodes
+- Apache disabled where not in use
+- Daily health checks via `pull_health_everywhere.sh`
+- Krang Telegram alerts deployed for cleanup and system state
+
+---
+
+## 🧠 Observations
+
+- **High load** on `minioraid5` during `mc mirror` and `chmod` overlap
+  - Load ~6.5 due to concurrent I/O pressure
+  - `chmod` stuck in `D` state (I/O wait) while `mc` dominated disk queues
+  - Resolved after `mc` completion — `chmod` resumed and completed
+
+- **MinIO buckets were temporarily inaccessible** due to permissions accidentally inherited by FTP group
+  - Resolved by recursively resetting permissions on `/mnt/raid5`
+
+- **Krang telemetry** verified:
+  - Mastodon swap usage rising under asset load
+  - All nodes had Apache disabled or dormant
+  - Health alerts triggered on high swap or load
+
+---
+
+## ✅ Outcome
+
+- Full Genesis and AzuraCast data now reside locally with resilient S3 mirrors
+- Mastodon running on block storage, no longer dependent on MinIO latency
+- FTP integration with SPL directory trees complete
+- Cleanup script successfully deployed across all nodes via Krang
+- Daily health reports operational with alerts for high swap/load
+
+---
+
+## 🔁 Recommendations
+
+- Consider adding snapshot-based ZFS backups for `/mnt/raid5`
+- Build `verify_mirror.sh` to detect drift between MinIO and local storage
+- Auto-trigger `chmod` only after `mc mirror` finishes
+- Monitor long-running background jobs with Krang watchdogs
+
+---
+
+**Signed,**  
+Doc  
+Genesis Hosting Technologies
+
--- a/radiotoot/live.py
+++ b/radiotoot/live.py
@ -0,0 +1,68 @@
+import requests
+import time
+from mastodon import Mastodon
+
+# === Config ===
+MASTODON_BASE_URL = "https://chatwithus.live"
+MASTODON_ACCESS_TOKEN = "07w3Emdw-cv_TncysrNU8Ed_sHJhwtnvKmnLqKlHmKA"
+ICECAST_STATUS_URL = "http://cast3.my-control-panel.com:7454/status-json.xsl"
+LIVE_MOUNTPOINT = "/live"
+CHECK_INTERVAL = 30  # seconds
+LIVE_MIN_INTERVAL = 600  # 10 minutes
+
+mastodon = Mastodon(
+    access_token=MASTODON_ACCESS_TOKEN,
+    api_base_url=MASTODON_BASE_URL
+)
+
+last_title_posted = None
+last_post_time = 0
+
+def get_live_stream_title():
+    try:
+        r = requests.get(ICECAST_STATUS_URL, timeout=5)
+        r.raise_for_status()
+        data = r.json()
+        sources = data.get("icestats", {}).get("source", [])
+
+        if isinstance(sources, dict):
+            sources = [sources]
+
+        for source in sources:
+            listenurl = source.get("listenurl", "")
+            title = source.get("title") or source.get("server_name")
+            title = title.strip() if title else None
+            listeners = int(source.get("listeners", 0))
+
+            print(f"[DEBUG] {listenurl=} {title=} {listeners=}")  # Keep for troubleshooting
+
+            if LIVE_MOUNTPOINT in listenurl and title and listeners > 0:
+                return title
+    except Exception as e:
+        print(f"[ERROR] Icecast fetch failed: {e}")
+    return None
+
+def main():
+    global last_title_posted, last_post_time
+    print("🎙️ Watching /live only. Toots only when DJs are on deck.")
+
+    while True:
+        now = time.time()
+        title = get_live_stream_title()
+
+        if title and title != last_title_posted and (now - last_post_time) > LIVE_MIN_INTERVAL:
+            toot_msg = f"🔴 Live now on Genesis Radio: {title}! Tune in: http://stream.genesis-radio.net:7454/stream"
+            try:
+                mastodon.status_post(toot_msg, visibility='public')
+                print(f"[TOOTED] {toot_msg}")
+                last_title_posted = title
+                last_post_time = now
+            except Exception as e:
+                print(f"[ERROR] Mastodon post failed: {e}")
+        else:
+            print("🔍 No new live DJ activity.")
+
+        time.sleep(CHECK_INTERVAL)
+
+if __name__ == "__main__":
+    main()
--- a/recordtheshow/show_schedule.json
+++ b/recordtheshow/show_schedule.json
@ -40,7 +40,7 @@
        ]
    },
    "au": {
-        "recording": true,
+        "recording": false,
        "duration": 18000,
        "schedule": [
            {