diff --git a/genesishostingmd/pmgenesisiorealignment.md b/genesishostingmd/pmgenesisiorealignment.md new file mode 100644 index 0000000..8789bca --- /dev/null +++ b/genesishostingmd/pmgenesisiorealignment.md @@ -0,0 +1,83 @@ +# Postmortem: Genesis I/O Realignment + +**Date:** May 8, 2025 +**Author:** Doc +**Systems Involved:** minioraid5, shredder, chatwithus.live, zcluster.technodrome1/2, thevault +**Scope:** Local-first mirroring, permission normalization, MinIO transition + +--- + +## 🎯 Objective + +To realign the Genesis file flow architecture by: + +- Making local block storage the **primary source** of truth for AzuraCast and Genesis buckets +- Transitioning FTP uploads to target local storage instead of MinIO directly +- Establishing **two-way mirroring** between local paths and MinIO buckets +- Correcting inherited permission issues across `/mnt/raid5` using `find + chmod` +- Preserving MinIO buckets as **backup mirrors**, not primary data stores + +--- + +## 🔧 Work Performed + +### ✅ Infrastructure changes: +- Deployed block storage volume to Linode Mastodon instance +- Mirrored MinIO buckets (`genesisassets`, `genesislibrary`, `azuracast`) to local paths +- Configured cron-based `mc mirror` jobs: + - Local ➜ MinIO: every 5 minutes with `--overwrite --remove` + - MinIO ➜ Local: nightly pull, no `--remove` + +### ✅ FTP Pipeline Adjustments: +- Users now upload to `/mnt/spl/ftp/uploads` (local) +- Permissions set so only admins access full `/mnt/spl/ftp` +- FTP directory structure created for SPL automation + +### ✅ System Tuning: +- Set `vm.swappiness=10` on all nodes +- Apache disabled where not in use +- Daily health checks via `pull_health_everywhere.sh` +- Krang Telegram alerts deployed for cleanup and system state + +--- + +## 🧠 Observations + +- **High load** on `minioraid5` during `mc mirror` and `chmod` overlap + - Load ~6.5 due to concurrent I/O pressure + - `chmod` stuck in `D` state (I/O wait) while `mc` dominated disk queues + - Resolved after `mc` completion — `chmod` resumed and completed + +- **MinIO buckets were temporarily inaccessible** due to permissions accidentally inherited by FTP group + - Resolved by recursively resetting permissions on `/mnt/raid5` + +- **Krang telemetry** verified: + - Mastodon swap usage rising under asset load + - All nodes had Apache disabled or dormant + - Health alerts triggered on high swap or load + +--- + +## ✅ Outcome + +- Full Genesis and AzuraCast data now reside locally with resilient S3 mirrors +- Mastodon running on block storage, no longer dependent on MinIO latency +- FTP integration with SPL directory trees complete +- Cleanup script successfully deployed across all nodes via Krang +- Daily health reports operational with alerts for high swap/load + +--- + +## 🔁 Recommendations + +- Consider adding snapshot-based ZFS backups for `/mnt/raid5` +- Build `verify_mirror.sh` to detect drift between MinIO and local storage +- Auto-trigger `chmod` only after `mc mirror` finishes +- Monitor long-running background jobs with Krang watchdogs + +--- + +**Signed,** +Doc +Genesis Hosting Technologies + diff --git a/miscellaneous/bash/mastodon_status-check.sh b/miscellaneous/bash/mastodon_status-check.sh new file mode 100755 index 0000000..046ab83 --- /dev/null +++ b/miscellaneous/bash/mastodon_status-check.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +echo "Step 0: Starting script..." + +# Load token from ~/.mastodon-token or environment +TOKEN_FILE="$HOME/.mastodon-token" +if [ -f "$TOKEN_FILE" ]; then + export MASTO_TOKEN=$(cat "$TOKEN_FILE") +fi + +if [ -z "$MASTO_TOKEN" ]; then + echo "❌ No Mastodon access token found. Set \$MASTO_TOKEN or create ~/.mastodon-token" + exit 1 +fi + +echo "Step 1: Token loaded." + +TMPFILE=$(mktemp) +MASTO_API="https://chatwithus.live/api/v1/statuses" + +SERVICES=( + "Genesis Radio|https://genesis-radio.net" + "Mastodon|https://chatwithus.live" + "MinIO|https://console.sshjunkie.com" + "AzuraCast|portal.genesishostingtechnologies.com/login" + "TeamTalk|tcp://tt.themediahub.org:10442" + "DirectAdmin|https://da.genesishostingtechnologies.com" +) + +echo "[Status Check @ $(date -u '+%H:%M %Z')]" > "$TMPFILE" + +for service in "${SERVICES[@]}"; do + IFS="|" read -r NAME URL <<< "$service" + + if [[ $URL == tcp://* ]]; then + # Handle TCP port check + HOSTPORT=${URL#tcp://} + HOST=${HOSTPORT%%:*} + PORT=${HOSTPORT##*:} + echo "Checking TCP: $NAME on $HOST:$PORT" + timeout 5 bash -c "/dev/null + else + # Handle HTTP(S) check + echo "Checking HTTP: $NAME -> $URL" + curl -s --head --fail --max-time 5 "$URL" >/dev/null + fi + + if [ $? -eq 0 ]; then + echo "✅ $NAME: Online" >> "$TMPFILE" + else + echo "❌ $NAME: Offline" >> "$TMPFILE" + fi +done + +echo "Step 2: Results collected." +cat "$TMPFILE" + +# Convert newlines to URL-encoded format +POST_BODY=$(sed ':a;N;$!ba;s/\n/%0A/g' "$TMPFILE") + +echo "Step 3: Posting to Mastodon..." + +curl -s -X POST "$MASTO_API" \ + -H "Authorization: Bearer $MASTO_TOKEN" \ + -d "status=$POST_BODY" + +echo "Step 4: Done." + +rm -f "$TMPFILE" diff --git a/miscellaneous/bash/pull_health_everywhere b/miscellaneous/bash/pull_health_everywhere new file mode 100755 index 0000000..f66ff1c --- /dev/null +++ b/miscellaneous/bash/pull_health_everywhere @@ -0,0 +1,74 @@ +#!/bin/bash + +# === CONFIG === +REMOTE_USER="doc" +BOT_TOKEN="8178867489:AAH0VjN7VnZSCIWasSz_y97iBLLjPJA751k" +CHAT_ID="1559582356" +TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S') +LOGFILE="$HOME/krang-logs/health-$(date '+%Y%m%d-%H%M').log" + +# Thresholds +SWAP_LIMIT_MB=512 +LOAD_LIMIT=4.0 + +mkdir -p "$HOME/krang-logs" + +SERVERS=( + thevault.sshjunkie.com + zcluster.technodrome1.sshjunkie.com + zcluster.technodrome2.sshjunkie.com + shredder.sshjunkie.com + chatwithus.live +) + +SUMMARY="📡 Krang System Health Report - $TIMESTAMP\n\n" + +for HOST in "${SERVERS[@]}"; do + echo "🔍 Collecting from $HOST..." + + DATA=$(ssh "$REMOTE_USER@$HOST" bash -s << 'EOF' +HOST=$(hostname) +MEM=$(free -h | awk '/Mem:/ {print $4 " free"}') +SWAP_RAW=$(free -m | awk '/Swap:/ {print $3}') +SWAP="$SWAP_RAW Mi used" +DISK=$(df -h / | awk 'NR==2 {print $4 " free"}') +LOAD=$(uptime | awk -F'load average:' '{print $2}' | cut -d, -f1 | xargs) +APACHE=$(systemctl is-active apache2 2>/dev/null || systemctl is-active httpd 2>/dev/null) +[ "$APACHE" = "active" ] && APACHE_STATUS="✅ Apache running" || APACHE_STATUS="❌ Apache not running" + +echo "$HOST|$MEM|$SWAP_RAW|$SWAP|$DISK|$LOAD|$APACHE_STATUS" +EOF +) + + IFS='|' read -r H MEM SWAP_MB SWAP_HUMAN DISK LOAD1 APACHE_STATUS <<< "$DATA" + + ALERTS="" + if (( SWAP_MB > SWAP_LIMIT_MB )); then + ALERTS+="⚠️ HIGH SWAP ($SWAP_HUMAN)\n" + fi + + LOAD_INT=$(awk "BEGIN {print ($LOAD1 > $LOAD_LIMIT) ? 1 : 0}") + if [ "$LOAD_INT" -eq 1 ]; then + ALERTS+="⚠️ HIGH LOAD ($LOAD1)\n" + fi + + ALERTS_MSG="" + [ -n "$ALERTS" ] && ALERTS_MSG="🚨 ALERTS:\n$ALERTS" + + SUMMARY+="🖥️ $H +• Mem: $MEM +• Swap: $SWAP_HUMAN +• Disk: $DISK +• Load: $LOAD1 +• $APACHE_STATUS +$ALERTS_MSG +\n" +done + +# Log to file +echo -e "$SUMMARY" > "$LOGFILE" + +# Send to Telegram +curl -s -X POST https://api.telegram.org/bot$BOT_TOKEN/sendMessage \ + -d chat_id="$CHAT_ID" \ + -d text="$SUMMARY" diff --git a/miscellaneous/bash/watchdog.sh b/miscellaneous/bash/watchdog.sh new file mode 100755 index 0000000..9c6848d --- /dev/null +++ b/miscellaneous/bash/watchdog.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# === CONFIG === +WATCH_STRING="find /mnt/raid5 -type d -exec chmod o+X {} \\;" # Adjust if needed +CHECK_INTERVAL=60 # seconds +BOT_TOKEN="8178867489:AAH0VjN7VnZSCIWasSz_y97iBLLjPJA751k" +CHAT_ID="1559582356" +HOST=$(hostname) +LOGFILE="$HOME/krang-logs/chmod_watchdog_$(date '+%Y%m%d-%H%M').log" +mkdir -p "$HOME/krang-logs" + +# === FIND TARGET PID === +PID=$(pgrep -f "$WATCH_STRING") + +if [ -z "$PID" ]; then + echo "❌ No matching chmod process found." | tee -a "$LOGFILE" + exit 1 +fi + +echo "👁️ Watching PID $PID for chmod job on $HOST..." | tee -a "$LOGFILE" + +# === MONITOR LOOP === +while kill -0 "$PID" 2>/dev/null; do + echo "⏳ [$HOST] chmod PID $PID still running..." >> "$LOGFILE" + sleep "$CHECK_INTERVAL" +done + +# === COMPLETE === +MSG="✅ [$HOST] chmod finished on /mnt/raid5 +Time: $(date '+%Y-%m-%d %H:%M:%S') +PID: $PID +Watchdog confirmed completion." + +echo -e "$MSG" | tee -a "$LOGFILE" + +curl -s -X POST https://api.telegram.org/bot$BOT_TOKEN/sendMessage \ + -d chat_id="$CHAT_ID" \ + -d text="$MSG" + diff --git a/miscellaneous/dotheneedfuleverywhere.sh b/miscellaneous/dotheneedfuleverywhere.sh new file mode 100755 index 0000000..7926b78 --- /dev/null +++ b/miscellaneous/dotheneedfuleverywhere.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +# === CONFIG === +SCRIPT_PATH="/usr/local/bin/do_the_needful.sh" +REMOTE_USER="doc" +BOT_TOKEN="8178867489:AAH0VjN7VnZSCIWasSz_y97iBLLjPJA751k" +CHAT_ID="1559582356" +TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S') + +SERVERS=( + thevault.sshjunkie.com + zcluster.technodrome1.sshjunkie.com + zcluster.technodrome2.sshjunkie.com + shredder.sshjunkie.com + chatwithus.live +) + +SUMMARY="🤖 Krang Deployment Report - $TIMESTAMP\n\n" +FAILURES=0 + +for HOST in "${SERVERS[@]}"; do + echo "🚀 Deploying to $HOST..." + + # Upload script to temp location + scp "$SCRIPT_PATH" "$REMOTE_USER@$HOST:/tmp/do_the_needful.sh" + if [ $? -ne 0 ]; then + SUMMARY+="❌ $HOST: SCP failed\n" + ((FAILURES++)) + continue + fi + + # Move into place and execute + ssh "$REMOTE_USER@$HOST" "sudo install -m 755 /tmp/do_the_needful.sh $SCRIPT_PATH && sudo $SCRIPT_PATH" + if [ $? -ne 0 ]; then + SUMMARY+="❌ $HOST: sudo execution failed\n" + ((FAILURES++)) + else + SUMMARY+="✅ $HOST: cleaned successfully\n" + fi + + echo "----------------------------------" +done + +# === Send Telegram Summary === +FINAL_STATUS="🚨 Some hosts failed." && [ "$FAILURES" -eq 0 ] && FINAL_STATUS="✅ All hosts completed." + +curl -s -X POST https://api.telegram.org/bot$BOT_TOKEN/sendMessage \ + -d chat_id="$CHAT_ID" \ + -d text="$FINAL_STATUS\n\n$SUMMARY" diff --git a/miscellaneous/fixsudoerseverywhere.sh b/miscellaneous/fixsudoerseverywhere.sh new file mode 100755 index 0000000..acf282e --- /dev/null +++ b/miscellaneous/fixsudoerseverywhere.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# === CONFIG === +REMOTE_USER="doc" +SERVERS=( + thevault.sshjunkie.com + zcluster.technodrome1.sshjunkie.com + zcluster.technodrome2.sshjunkie.com + shredder.sshjunkie.com + chatwithus.live +) + +SUDO_LINE="doc ALL=(ALL) NOPASSWD:ALL" + +# === Execution === +for HOST in "${SERVERS[@]}"; do + echo "🔧 Fixing sudoers on $HOST..." + + ssh "$REMOTE_USER@$HOST" "sudo bash -c ' + cp /etc/sudoers /etc/sudoers.bak_krang && + grep -q \"$SUDO_LINE\" /etc/sudoers || + echo \"$SUDO_LINE\" >> /etc/sudoers && + visudo -c >/dev/null + '" + + if ssh "$REMOTE_USER@$HOST" "sudo -n true"; then + echo "✅ $HOST: sudo access confirmed" + else + echo "❌ $HOST: sudo access STILL broken" + fi + + echo "----------------------------------" +done diff --git a/postmortem/pmgenesisiorealignment.md b/postmortem/pmgenesisiorealignment.md new file mode 100644 index 0000000..8789bca --- /dev/null +++ b/postmortem/pmgenesisiorealignment.md @@ -0,0 +1,83 @@ +# Postmortem: Genesis I/O Realignment + +**Date:** May 8, 2025 +**Author:** Doc +**Systems Involved:** minioraid5, shredder, chatwithus.live, zcluster.technodrome1/2, thevault +**Scope:** Local-first mirroring, permission normalization, MinIO transition + +--- + +## 🎯 Objective + +To realign the Genesis file flow architecture by: + +- Making local block storage the **primary source** of truth for AzuraCast and Genesis buckets +- Transitioning FTP uploads to target local storage instead of MinIO directly +- Establishing **two-way mirroring** between local paths and MinIO buckets +- Correcting inherited permission issues across `/mnt/raid5` using `find + chmod` +- Preserving MinIO buckets as **backup mirrors**, not primary data stores + +--- + +## 🔧 Work Performed + +### ✅ Infrastructure changes: +- Deployed block storage volume to Linode Mastodon instance +- Mirrored MinIO buckets (`genesisassets`, `genesislibrary`, `azuracast`) to local paths +- Configured cron-based `mc mirror` jobs: + - Local ➜ MinIO: every 5 minutes with `--overwrite --remove` + - MinIO ➜ Local: nightly pull, no `--remove` + +### ✅ FTP Pipeline Adjustments: +- Users now upload to `/mnt/spl/ftp/uploads` (local) +- Permissions set so only admins access full `/mnt/spl/ftp` +- FTP directory structure created for SPL automation + +### ✅ System Tuning: +- Set `vm.swappiness=10` on all nodes +- Apache disabled where not in use +- Daily health checks via `pull_health_everywhere.sh` +- Krang Telegram alerts deployed for cleanup and system state + +--- + +## 🧠 Observations + +- **High load** on `minioraid5` during `mc mirror` and `chmod` overlap + - Load ~6.5 due to concurrent I/O pressure + - `chmod` stuck in `D` state (I/O wait) while `mc` dominated disk queues + - Resolved after `mc` completion — `chmod` resumed and completed + +- **MinIO buckets were temporarily inaccessible** due to permissions accidentally inherited by FTP group + - Resolved by recursively resetting permissions on `/mnt/raid5` + +- **Krang telemetry** verified: + - Mastodon swap usage rising under asset load + - All nodes had Apache disabled or dormant + - Health alerts triggered on high swap or load + +--- + +## ✅ Outcome + +- Full Genesis and AzuraCast data now reside locally with resilient S3 mirrors +- Mastodon running on block storage, no longer dependent on MinIO latency +- FTP integration with SPL directory trees complete +- Cleanup script successfully deployed across all nodes via Krang +- Daily health reports operational with alerts for high swap/load + +--- + +## 🔁 Recommendations + +- Consider adding snapshot-based ZFS backups for `/mnt/raid5` +- Build `verify_mirror.sh` to detect drift between MinIO and local storage +- Auto-trigger `chmod` only after `mc mirror` finishes +- Monitor long-running background jobs with Krang watchdogs + +--- + +**Signed,** +Doc +Genesis Hosting Technologies + diff --git a/radiotoot/live.py b/radiotoot/live.py new file mode 100644 index 0000000..3754a4d --- /dev/null +++ b/radiotoot/live.py @@ -0,0 +1,68 @@ +import requests +import time +from mastodon import Mastodon + +# === Config === +MASTODON_BASE_URL = "https://chatwithus.live" +MASTODON_ACCESS_TOKEN = "07w3Emdw-cv_TncysrNU8Ed_sHJhwtnvKmnLqKlHmKA" +ICECAST_STATUS_URL = "http://cast3.my-control-panel.com:7454/status-json.xsl" +LIVE_MOUNTPOINT = "/live" +CHECK_INTERVAL = 30 # seconds +LIVE_MIN_INTERVAL = 600 # 10 minutes + +mastodon = Mastodon( + access_token=MASTODON_ACCESS_TOKEN, + api_base_url=MASTODON_BASE_URL +) + +last_title_posted = None +last_post_time = 0 + +def get_live_stream_title(): + try: + r = requests.get(ICECAST_STATUS_URL, timeout=5) + r.raise_for_status() + data = r.json() + sources = data.get("icestats", {}).get("source", []) + + if isinstance(sources, dict): + sources = [sources] + + for source in sources: + listenurl = source.get("listenurl", "") + title = source.get("title") or source.get("server_name") + title = title.strip() if title else None + listeners = int(source.get("listeners", 0)) + + print(f"[DEBUG] {listenurl=} {title=} {listeners=}") # Keep for troubleshooting + + if LIVE_MOUNTPOINT in listenurl and title and listeners > 0: + return title + except Exception as e: + print(f"[ERROR] Icecast fetch failed: {e}") + return None + +def main(): + global last_title_posted, last_post_time + print("🎙️ Watching /live only. Toots only when DJs are on deck.") + + while True: + now = time.time() + title = get_live_stream_title() + + if title and title != last_title_posted and (now - last_post_time) > LIVE_MIN_INTERVAL: + toot_msg = f"🔴 Live now on Genesis Radio: {title}! Tune in: http://stream.genesis-radio.net:7454/stream" + try: + mastodon.status_post(toot_msg, visibility='public') + print(f"[TOOTED] {toot_msg}") + last_title_posted = title + last_post_time = now + except Exception as e: + print(f"[ERROR] Mastodon post failed: {e}") + else: + print("🔍 No new live DJ activity.") + + time.sleep(CHECK_INTERVAL) + +if __name__ == "__main__": + main() diff --git a/recordtheshow/show_schedule.json b/recordtheshow/show_schedule.json index 359103f..21daa6e 100755 --- a/recordtheshow/show_schedule.json +++ b/recordtheshow/show_schedule.json @@ -40,7 +40,7 @@ ] }, "au": { - "recording": true, + "recording": false, "duration": 18000, "schedule": [ {