doctator/bin-there-done-that
doctator/bin-there-done-that
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Auto-commit from giteapush.sh at 2025-05-02 10:52:38

Browse Source
This commit is contained in:
DocTator 2025-05-02 10:52:38 -04:00
parent fc66fc65c9
commit 30c8caf850
11 changed files with 13021 additions and 237 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hostingtoot/.env Normal file
View File

@ -0,0 +1,6 @@
GNU nano 7.2 .env
DB_USER=
DB_PASSWORD=
DB_NAME=
DB_HOST_PRIMARY=
MASTODON_ACCESS_TOKEN=

6
miscellaneous/bash/dbv1.sh
View File

@ -1,14 +1,14 @@
#!/bin/bash #!/bin/bash
# ---- CONFIG ---- # ---- CONFIG ----
PG_REMOTE_USER="postgres" PG_REMOTE_USER="doc"
PG_REMOTE_HOST="cluster.db2.genesishostingtechnologies.com" PG_REMOTE_HOST="zcluster.technodrome2.sshjunkie.com"
PG_REMOTE_PORT="5432" PG_REMOTE_PORT="5432"
PG_LOCAL_PORT="5432" PG_LOCAL_PORT="5432"
DUMP_DIR="/tmp/pgbackup_verify" DUMP_DIR="/tmp/pgbackup_verify"
BACKUP_TARGET="root@backup.sshjunkie.com:/mnt/backup/pgdumps" BACKUP_TARGET="root@backup.sshjunkie.com:/mnt/backup/pgdumps"
CC_TARGET="doc@clustercontrol.sshjunkie.com:/home/doc/backups" CC_TARGET="doc@clustercontrol.sshjunkie.com:/home/doc/backups"
DBS=("mastodon_production" "giteaprod" "hostingtootdb" "radiotootdb") DBS=("mastodon_production" "hostingtootdb" "radiotootdb")
LOGFILE="$DUMP_DIR/verify_log_$(date +%Y%m%d_%H%M%S).txt" LOGFILE="$DUMP_DIR/verify_log_$(date +%Y%m%d_%H%M%S).txt"
mkdir -p "$DUMP_DIR" mkdir -p "$DUMP_DIR"

74
miscellaneous/bash/dbv2.sh Executable file
View File

@ -0,0 +1,74 @@
#!/bin/bash
# verify_postgres_snapshot.sh
# Verify PostgreSQL backup snapshot remotely using ZFS, pg_verifybackup, and pg_checksums
set -euo pipefail
# === CONFIG ===
REMOTE_HOST="zcluster.technodrome2.sshjunkie.com"
ZFS_DATASET="pgpool/postgresbackups/db2"
MOUNT_ROOT="/mnt/pgsnapshot_verify"
PG_BIN_DIR="/usr/lib/postgresql/16/bin" # adjust if needed on remote host
LOGFILE="/var/log/pgsnapshot_verify_$(date +%Y%m%d_%H%M%S).log"
DB_NAME="mastodon_production"
# === MASTODON ALERTING ===
MASTODON_TOKEN="rimxBLi-eaJAcwagkmoj6UoW7Lc473tQY0cOM041Euw"
MASTODON_API="https://chatwithus.live/api/v1/statuses"
TOOT_ACCOUNT_ID="114386383616633367"
mastodon_alert() {
local msg="$1"
curl -sS -X POST "$MASTODON_API" \
-H "Authorization: Bearer $MASTODON_TOKEN" \
--data-urlencode "status=$msg" \
--data "visibility=direct" \
--data "in_reply_to_account_id=$TOOT_ACCOUNT_ID" >/dev/null
}
# === RUN REMOTELY ===
echo "📦 Verifying latest snapshot for $ZFS_DATASET on $REMOTE_HOST" | tee "$LOGFILE"
ssh root@$REMOTE_HOST bash <<EOF
set -euo pipefail
LATEST_SNAPSHOT=\$(zfs list -t snapshot -r $ZFS_DATASET -o name -s creation | tail -n 1)
if [[ -z "\$LATEST_SNAPSHOT" ]]; then
echo "❌ No snapshot found for $ZFS_DATASET" | tee -a "$LOGFILE"
exit 1
fi
MOUNTPOINT="$MOUNT_ROOT/\${LATEST_SNAPSHOT//\//_}"
mkdir -p "\$MOUNTPOINT"
# Clone snapshot for verification
zfs clone -o mountpoint="\$MOUNTPOINT" "\$LATEST_SNAPSHOT" "${ZFS_DATASET}@verifytemp"
# Run pg_verifybackup
if ! "$PG_BIN_DIR/pg_verifybackup" "\$MOUNTPOINT" >> "$LOGFILE" 2>&1; then
echo "❌ pg_verifybackup failed on snapshot \$LATEST_SNAPSHOT" | tee -a "$LOGFILE"
zfs destroy "${ZFS_DATASET}@verifytemp"
exit 2
fi
# Optional: pg_checksums
if "$PG_BIN_DIR/pg_checksums" --check --data-directory="\$MOUNTPOINT" >> "$LOGFILE" 2>&1; then
echo "✅ pg_checksums passed." | tee -a "$LOGFILE"
else
echo "⚠️ pg_checksums failed or not enabled." | tee -a "$LOGFILE"
fi
zfs destroy "${ZFS_DATASET}@verifytemp"
rmdir "\$MOUNTPOINT"
EOF
# === ALERT BASED ON SSH EXIT CODE ===
EXIT_CODE=$?
if [[ $EXIT_CODE -eq 1 ]]; then
mastodon_alert "🚨 ZFS PostgreSQL snapshot verification failed: No snapshot found for $ZFS_DATASET on $REMOTE_HOST at $(date)."
elif [[ $EXIT_CODE -eq 2 ]]; then
mastodon_alert "🚨 pg_verifybackup failed for snapshot of $ZFS_DATASET on $REMOTE_HOST at $(date)."
fi
echo "✅ PostgreSQL remote snapshot verification complete." | tee -a "$LOGFILE"
exit 0

2
miscellaneous/bash/malips.sh
View File

@ -4,7 +4,7 @@
SNORT_LOG="/var/log/snort/snort.alert.fast" SNORT_LOG="/var/log/snort/snort.alert.fast"
# Database connection details # Database connection details
DB_HOST="38.102.127.174" DB_HOST="zcluster.technodrome1.sshjunkie.com"
DB_USER="ipblocks_user" DB_USER="ipblocks_user"
DB_PASS="rusty2281" DB_PASS="rusty2281"
DB_NAME="ipblocks" DB_NAME="ipblocks"

6
miscellaneous/bash/prompt_prod.sh Executable file
View File

@ -0,0 +1,6 @@
#!/bin/bash
# /etc/profile.d/prompt_env.sh for Production
if [[ $- == *i* ]]; then
export PS1="\[\e[1;31m\][prod] \u@\h:\w\\$ \[\e[0m\]"
fi

56
miscellaneous/bash/rsync_zfs_sync_helper.sh Normal file
View File

@ -0,0 +1,56 @@
#!/bin/bash
# sync_to_vault.sh
# Rsync + ZFS sanity tool with built-in slash wisdom
set -euo pipefail
# === CONFIG ===
VAULT_HOST="thevault.sshjunkie.com"
BASE_TARGET="/nexus/miniodata/assets"
# === USAGE ===
if [[ $# -lt 2 ]]; then
echo "Usage: $0 <source_dir> <bucket_name>"
echo "Example: $0 /mnt/backup3/tempshit/genesisassets/ genesisassets-secure"
exit 1
fi
SRC="$1"
BUCKET="$2"
DST="${BASE_TARGET}/${BUCKET}/"
# === WISDOM ===
echo "🧘 Trailing slashes, my friend. — John Northrup"
echo
if [[ "$SRC" != */ ]]; then
echo "⚠️ Warning: Source path does not end in a slash."
echo " You may be copying the folder itself instead of its contents."
echo " You probably want: ${SRC}/"
echo
fi
# === VERIFY SOURCE ===
if [[ ! -d "$SRC" ]]; then
echo "❌ Source directory does not exist: $SRC"
exit 1
fi
# === CREATE ZFS DATASET ON REMOTE IF MISSING ===
echo "🔍 Ensuring dataset exists on $VAULT_HOST..."
ssh root@$VAULT_HOST "zfs list nexus/miniodata/assets/$BUCKET" >/dev/null 2>&1 || {
echo "📁 Creating dataset nexus/miniodata/assets/$BUCKET on $VAULT_HOST"
ssh root@$VAULT_HOST "zfs create nexus/miniodata/assets/$BUCKET"
}
# === RSYNC ===
echo "🚀 Starting rsync from $SRC to $VAULT_HOST:$DST"
rsync -avhP "$SRC" root@$VAULT_HOST:"$DST"
# === SNAPSHOT ===
SNAPNAME="rsync_$(date +%Y%m%d_%H%M%S)"
echo "📸 Creating post-sync snapshot: $SNAPNAME"
ssh root@$VAULT_HOST "zfs snapshot nexus/miniodata/assets/$BUCKET@$SNAPNAME"
# === DONE ===
echo "✅ Sync and snapshot complete: $BUCKET@$SNAPNAME"

11725
miscellaneous/cron_backup.log
View File

File diff suppressed because it is too large Load Diff

861
miscellaneous/dbcheck.log
View File

@ -1220,3 +1220,864 @@ Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"} Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"} Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"} Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 10:45:16 ⚠️
⚡ 2 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 62 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
✅ Genesis Radio Healthcheck 2025-05-01 11:00:15: All systems normal.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 11:15:17 ⚠️
⚡ 3 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
✅ Genesis Radio Healthcheck 2025-05-01 11:30:14: All systems normal.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
✅ Genesis Radio Healthcheck 2025-05-01 11:45:15: All systems normal.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
✅ Genesis Radio Healthcheck 2025-05-01 12:00:13: All systems normal.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
✅ Genesis Radio Healthcheck 2025-05-01 12:15:19: All systems normal.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
✅ Genesis Radio Healthcheck 2025-05-01 12:30:14: All systems normal.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 12:45:20 ⚠️
⚡ 1 warnings found:
- 💥 [db2] WARNING: Replication lag is 428 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 13:00:18 ⚠️
⚡ 1 warnings found:
- 💥 [db2] WARNING: Replication lag is 1325 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 13:15:17 ⚠️
⚡ 2 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 2224 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 13:30:17 ⚠️
⚡ 2 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 3124 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 13:45:18 ⚠️
⚡ 1 warnings found:
- 💥 [db2] WARNING: Replication lag is 4025 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 14:00:19 ⚠️
⚡ 1 warnings found:
- 💥 [db2] WARNING: Replication lag is 4926 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 14:15:16 ⚠️
⚡ 1 warnings found:
- 💥 [db2] WARNING: Replication lag is 5823 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 14:30:16 ⚠️
⚡ 1 warnings found:
- 💥 [db2] WARNING: Replication lag is 6723 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 14:45:16 ⚠️
⚡ 1 warnings found:
- 💥 [db2] WARNING: Replication lag is 7623 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 15:00:17 ⚠️
⚡ 1 warnings found:
- 💥 [db2] WARNING: Replication lag is 8524 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 15:15:17 ⚠️
⚡ 1 warnings found:
- 💥 [db2] WARNING: Replication lag is 9425 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 15:30:18 ⚠️
⚡ 1 warnings found:
- 💥 [db2] WARNING: Replication lag is 10325 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 15:45:19 ⚠️
⚡ 2 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 11227 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 16:00:15 ⚠️
⚡ 1 warnings found:
- 💥 [db2] WARNING: Replication lag is 12122 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 16:15:17 ⚠️
⚡ 1 warnings found:
- 💥 [db2] WARNING: Replication lag is 13024 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 16:30:18 ⚠️
⚡ 2 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 13925 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 16:45:23 ⚠️
⚡ 2 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 14831 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 17:00:15 ⚠️
⚡ 1 warnings found:
- 💥 [db2] WARNING: Replication lag is 15723 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 17:15:43 ⚠️
⚡ 1 warnings found:
- 💥 [db2] WARNING: Replication lag is 16650 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 17:30:18 ⚠️
⚡ 1 warnings found:
- 💥 [db2] WARNING: Replication lag is 17525 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 17:45:23 ⚠️
⚡ 1 warnings found:
- 💥 [db2] WARNING: Replication lag is 18431 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 18:00:15 ⚠️
⚡ 1 warnings found:
- 💥 [db2] WARNING: Replication lag is 19323 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 18:15:21 ⚠️
⚡ 1 warnings found:
- 💥 [db2] WARNING: Replication lag is 20228 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 18:30:17 ⚠️
⚡ 3 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 21124 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 18:45:17 ⚠️
⚡ 2 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 22024 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 19:00:18 ⚠️
⚡ 1 warnings found:
- 💥 [db2] WARNING: Replication lag is 22925 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 19:15:23 ⚠️
⚡ 3 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 23830 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 19:30:16 ⚠️
⚡ 1 warnings found:
- 💥 [db2] WARNING: Replication lag is 24723 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 19:45:26 ⚠️
⚡ 2 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 25633 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 20:00:13 ⚠️
⚡ 1 warnings found:
- 💥 [db2] WARNING: Replication lag is 26520 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 20:15:13 ⚠️
⚡ 2 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 27420 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 20:30:13 ⚠️
⚡ 1 warnings found:
- 💥 [db2] WARNING: Replication lag is 28321 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 20:45:22 ⚠️
⚡ 3 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 29229 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 21:00:23 ⚠️
⚡ 2 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 30131 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 21:15:16 ⚠️
⚡ 3 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 31023 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 21:30:19 ⚠️
⚡ 2 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 31926 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 21:45:26 ⚠️
⚡ 2 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 32834 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 22:00:17 ⚠️
⚡ 2 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 33724 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 22:15:18 ⚠️
⚡ 2 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 34625 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 22:30:16 ⚠️
⚡ 2 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 35523 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 22:45:24 ⚠️
⚡ 1 warnings found:
- 💥 [db2] WARNING: Replication lag is 36432 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 23:00:12 ⚠️
⚡ 1 warnings found:
- 💥 [db2] WARNING: Replication lag is 37320 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 23:15:19 ⚠️
⚡ 3 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 38226 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 23:30:16 ⚠️
⚡ 7 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 39123 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-01 23:45:16 ⚠️
⚡ 4 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 40023 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-02 00:00:13 ⚠️
⚡ 2 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 40920 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-02 00:15:17 ⚠️
⚡ 4 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 41824 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-02 00:30:14 ⚠️
⚡ 5 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 42721 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
Exception (client): Error reading SSH protocol banner
Traceback (most recent call last):
File "/home/doc/dbcheck/lib/python3.12/site-packages/paramiko/transport.py", line 2369, in _check_banner
buf = self.packetizer.readline(timeout)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/doc/dbcheck/lib/python3.12/site-packages/paramiko/packet.py", line 395, in readline
buf += self._read_timeout(timeout)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/doc/dbcheck/lib/python3.12/site-packages/paramiko/packet.py", line 665, in _read_timeout
raise EOFError()
EOFError
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/doc/dbcheck/lib/python3.12/site-packages/paramiko/transport.py", line 2185, in run
self._check_banner()
File "/home/doc/dbcheck/lib/python3.12/site-packages/paramiko/transport.py", line 2373, in _check_banner
raise SSHException(
paramiko.ssh_exception.SSHException: Error reading SSH protocol banner
Exception (client): Error reading SSH protocol banner[Errno 104] Connection reset by peer
Traceback (most recent call last):
File "/home/doc/dbcheck/lib/python3.12/site-packages/paramiko/transport.py", line 2369, in _check_banner
buf = self.packetizer.readline(timeout)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/doc/dbcheck/lib/python3.12/site-packages/paramiko/packet.py", line 395, in readline
buf += self._read_timeout(timeout)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/doc/dbcheck/lib/python3.12/site-packages/paramiko/packet.py", line 663, in _read_timeout
x = self.__socket.recv(128)
^^^^^^^^^^^^^^^^^^^^^^^
ConnectionResetError: [Errno 104] Connection reset by peer
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/doc/dbcheck/lib/python3.12/site-packages/paramiko/transport.py", line 2185, in run
self._check_banner()
File "/home/doc/dbcheck/lib/python3.12/site-packages/paramiko/transport.py", line 2373, in _check_banner
raise SSHException(
paramiko.ssh_exception.SSHException: Error reading SSH protocol banner[Errno 104] Connection reset by peer
Exception (client): Error reading SSH protocol banner
Traceback (most recent call last):
File "/home/doc/dbcheck/lib/python3.12/site-packages/paramiko/transport.py", line 2369, in _check_banner
buf = self.packetizer.readline(timeout)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/doc/dbcheck/lib/python3.12/site-packages/paramiko/packet.py", line 395, in readline
buf += self._read_timeout(timeout)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/doc/dbcheck/lib/python3.12/site-packages/paramiko/packet.py", line 665, in _read_timeout
raise EOFError()
EOFError
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/doc/dbcheck/lib/python3.12/site-packages/paramiko/transport.py", line 2185, in run
self._check_banner()
File "/home/doc/dbcheck/lib/python3.12/site-packages/paramiko/transport.py", line 2373, in _check_banner
raise SSHException(
paramiko.ssh_exception.SSHException: Error reading SSH protocol banner
Exception (client): Error reading SSH protocol banner
Traceback (most recent call last):
File "/home/doc/dbcheck/lib/python3.12/site-packages/paramiko/transport.py", line 2369, in _check_banner
buf = self.packetizer.readline(timeout)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/doc/dbcheck/lib/python3.12/site-packages/paramiko/packet.py", line 395, in readline
buf += self._read_timeout(timeout)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/doc/dbcheck/lib/python3.12/site-packages/paramiko/packet.py", line 665, in _read_timeout
raise EOFError()
EOFError
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/doc/dbcheck/lib/python3.12/site-packages/paramiko/transport.py", line 2185, in run
self._check_banner()
File "/home/doc/dbcheck/lib/python3.12/site-packages/paramiko/transport.py", line 2373, in _check_banner
raise SSHException(
paramiko.ssh_exception.SSHException: Error reading SSH protocol banner
Exception (client): Error reading SSH protocol banner
Traceback (most recent call last):
File "/home/doc/dbcheck/lib/python3.12/site-packages/paramiko/transport.py", line 2369, in _check_banner
buf = self.packetizer.readline(timeout)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/doc/dbcheck/lib/python3.12/site-packages/paramiko/packet.py", line 395, in readline
buf += self._read_timeout(timeout)
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/doc/dbcheck/lib/python3.12/site-packages/paramiko/packet.py", line 665, in _read_timeout
raise EOFError()
EOFError
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/home/doc/dbcheck/lib/python3.12/site-packages/paramiko/transport.py", line 2185, in run
self._check_banner()
File "/home/doc/dbcheck/lib/python3.12/site-packages/paramiko/transport.py", line 2373, in _check_banner
raise SSHException(
paramiko.ssh_exception.SSHException: Error reading SSH protocol banner
⚠️ Genesis Radio Warning Healthcheck 2025-05-02 00:45:15 ⚠️
⚡ 13 warnings found:
- ⚠️ [shredder] ERROR: Could not read log /var/log/syslog: Error reading SSH protocol banner
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [db1] ERROR: Could not read log /var/log/nginx/error.log: Error reading SSH protocol banner
- 💥 [db2] WARNING: Replication lag is 43622 seconds.
- ⚠️ [db2] ERROR: Could not read log /var/log/syslog: Error reading SSH protocol banner
- ⚠️ [db2] ERROR: Could not read log /var/log/nginx/error.log: Error reading SSH protocol banner
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-02 01:00:12 ⚠️
⚡ 5 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 44520 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-02 01:15:24 ⚠️
⚡ 2 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 45431 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-02 01:30:15 ⚠️
⚡ 3 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 46322 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-02 01:45:18 ⚠️
⚡ 3 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 47226 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-02 02:00:09 ⚠️
⚡ 3 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 48117 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-02 02:15:10 ⚠️
⚡ 4 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 49017 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-02 02:30:10 ⚠️
⚡ 3 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 49917 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-02 02:45:11 ⚠️
⚡ 5 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 50818 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-02 03:00:14 ⚠️
⚡ 9 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 51722 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-02 03:15:14 ⚠️
⚡ 1 warnings found:
- 💥 [db2] WARNING: Replication lag is 52622 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-02 03:30:13 ⚠️
⚡ 2 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 53521 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-02 03:45:12 ⚠️
⚡ 4 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 54419 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-02 04:00:15 ⚠️
⚡ 2 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 55322 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-02 04:15:13 ⚠️
⚡ 2 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 56220 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-02 04:30:08 ⚠️
⚡ 4 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 57116 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-02 04:45:18 ⚠️
⚡ 2 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 58025 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-02 05:00:20 ⚠️
⚡ 3 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 58928 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-02 05:15:18 ⚠️
⚡ 4 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 59826 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-02 05:30:22 ⚠️
⚡ 4 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 60729 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-02 05:45:22 ⚠️
⚡ 2 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 61630 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-02 06:00:18 ⚠️
⚡ 1 warnings found:
- 💥 [db2] WARNING: Replication lag is 62526 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-02 06:15:19 ⚠️
⚡ 6 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 63426 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-02 06:30:23 ⚠️
⚡ 5 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db2] WARNING: Replication lag is 64330 seconds.
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-02 06:46:10 ⚠️
⚡ 7 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db1] ERROR: Replication check failed: connection to server at "cluster.db1.genesishostingtechnologies.com" (38.102.127.174), port 5432 failed: No route to host
Is the server running on that host and accepting TCP/IP connections?
- ⚠️ [db1] ERROR: Could not read log /var/log/syslog: [Errno None] Unable to connect to port 22 on 38.102.127.174
- ⚠️ [db1] ERROR: Could not read log /var/log/nginx/error.log: [Errno None] Unable to connect to port 22 on 38.102.127.174
- 💥 [db2] ERROR: Replication check failed: connection to server at "cluster.db2.genesishostingtechnologies.com" (38.102.127.169), port 5432 failed: No route to host
Is the server running on that host and accepting TCP/IP connections?
- ⚠️ [db2] ERROR: Could not read log /var/log/syslog: [Errno None] Unable to connect to port 22 on 38.102.127.169
- ⚠️ [db2] ERROR: Could not read log /var/log/nginx/error.log: [Errno None] Unable to connect to port 22 on 38.102.127.169
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-02 07:00:54 ⚠️
⚡ 11 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db1] ERROR: Replication check failed: connection to server at "cluster.db1.genesishostingtechnologies.com" (38.102.127.174), port 5432 failed: No route to host
Is the server running on that host and accepting TCP/IP connections?
- ⚠️ [db1] ERROR: Could not read log /var/log/syslog: [Errno None] Unable to connect to port 22 on 38.102.127.174
- ⚠️ [db1] ERROR: Could not read log /var/log/nginx/error.log: [Errno None] Unable to connect to port 22 on 38.102.127.174
- 💥 [db2] ERROR: Replication check failed: connection to server at "cluster.db2.genesishostingtechnologies.com" (38.102.127.169), port 5432 failed: No route to host
Is the server running on that host and accepting TCP/IP connections?
- ⚠️ [db2] ERROR: Could not read log /var/log/syslog: [Errno None] Unable to connect to port 22 on 38.102.127.169
- ⚠️ [db2] ERROR: Could not read log /var/log/nginx/error.log: [Errno None] Unable to connect to port 22 on 38.102.127.169
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-02 07:15:53 ⚠️
⚡ 6 warnings found:
- 💥 [db1] ERROR: Replication check failed: connection to server at "cluster.db1.genesishostingtechnologies.com" (38.102.127.174), port 5432 failed: No route to host
Is the server running on that host and accepting TCP/IP connections?
- ⚠️ [db1] ERROR: Could not read log /var/log/syslog: [Errno None] Unable to connect to port 22 on 38.102.127.174
- ⚠️ [db1] ERROR: Could not read log /var/log/nginx/error.log: [Errno None] Unable to connect to port 22 on 38.102.127.174
- 💥 [db2] ERROR: Replication check failed: connection to server at "cluster.db2.genesishostingtechnologies.com" (38.102.127.169), port 5432 failed: No route to host
Is the server running on that host and accepting TCP/IP connections?
- ⚠️ [db2] ERROR: Could not read log /var/log/syslog: [Errno None] Unable to connect to port 22 on 38.102.127.169
- ⚠️ [db2] ERROR: Could not read log /var/log/nginx/error.log: [Errno None] Unable to connect to port 22 on 38.102.127.169
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-02 07:30:47 ⚠️
⚡ 8 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db1] ERROR: Replication check failed: connection to server at "cluster.db1.genesishostingtechnologies.com" (38.102.127.174), port 5432 failed: No route to host
Is the server running on that host and accepting TCP/IP connections?
- ⚠️ [db1] ERROR: Could not read log /var/log/syslog: [Errno None] Unable to connect to port 22 on 38.102.127.174
- ⚠️ [db1] ERROR: Could not read log /var/log/nginx/error.log: [Errno None] Unable to connect to port 22 on 38.102.127.174
- 💥 [db2] ERROR: Replication check failed: connection to server at "cluster.db2.genesishostingtechnologies.com" (38.102.127.169), port 5432 failed: No route to host
Is the server running on that host and accepting TCP/IP connections?
- ⚠️ [db2] ERROR: Could not read log /var/log/syslog: [Errno None] Unable to connect to port 22 on 38.102.127.169
- ⚠️ [db2] ERROR: Could not read log /var/log/nginx/error.log: [Errno None] Unable to connect to port 22 on 38.102.127.169
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-02 07:45:49 ⚠️
⚡ 9 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db1] ERROR: Replication check failed: connection to server at "cluster.db1.genesishostingtechnologies.com" (38.102.127.174), port 5432 failed: No route to host
Is the server running on that host and accepting TCP/IP connections?
- ⚠️ [db1] ERROR: Could not read log /var/log/syslog: [Errno None] Unable to connect to port 22 on 38.102.127.174
- ⚠️ [db1] ERROR: Could not read log /var/log/nginx/error.log: [Errno None] Unable to connect to port 22 on 38.102.127.174
- 💥 [db2] ERROR: Replication check failed: connection to server at "cluster.db2.genesishostingtechnologies.com" (38.102.127.169), port 5432 failed: No route to host
Is the server running on that host and accepting TCP/IP connections?
- ⚠️ [db2] ERROR: Could not read log /var/log/syslog: [Errno None] Unable to connect to port 22 on 38.102.127.169
- ⚠️ [db2] ERROR: Could not read log /var/log/nginx/error.log: [Errno None] Unable to connect to port 22 on 38.102.127.169
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-02 08:00:50 ⚠️
⚡ 8 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db1] ERROR: Replication check failed: connection to server at "cluster.db1.genesishostingtechnologies.com" (38.102.127.174), port 5432 failed: No route to host
Is the server running on that host and accepting TCP/IP connections?
- ⚠️ [db1] ERROR: Could not read log /var/log/syslog: [Errno None] Unable to connect to port 22 on 38.102.127.174
- ⚠️ [db1] ERROR: Could not read log /var/log/nginx/error.log: [Errno None] Unable to connect to port 22 on 38.102.127.174
- 💥 [db2] ERROR: Replication check failed: connection to server at "cluster.db2.genesishostingtechnologies.com" (38.102.127.169), port 5432 failed: No route to host
Is the server running on that host and accepting TCP/IP connections?
- ⚠️ [db2] ERROR: Could not read log /var/log/syslog: [Errno None] Unable to connect to port 22 on 38.102.127.169
- ⚠️ [db2] ERROR: Could not read log /var/log/nginx/error.log: [Errno None] Unable to connect to port 22 on 38.102.127.169
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-02 08:15:46 ⚠️
⚡ 38 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db1] ERROR: Replication check failed: connection to server at "cluster.db1.genesishostingtechnologies.com" (38.102.127.174), port 5432 failed: No route to host
Is the server running on that host and accepting TCP/IP connections?
- ⚠️ [db1] ERROR: Could not read log /var/log/syslog: [Errno None] Unable to connect to port 22 on 38.102.127.174
- ⚠️ [db1] ERROR: Could not read log /var/log/nginx/error.log: [Errno None] Unable to connect to port 22 on 38.102.127.174
- 💥 [db2] ERROR: Replication check failed: connection to server at "cluster.db2.genesishostingtechnologies.com" (38.102.127.169), port 5432 failed: No route to host
Is the server running on that host and accepting TCP/IP connections?
- ⚠️ [db2] ERROR: Could not read log /var/log/syslog: [Errno None] Unable to connect to port 22 on 38.102.127.169
- ⚠️ [db2] ERROR: Could not read log /var/log/nginx/error.log: [Errno None] Unable to connect to port 22 on 38.102.127.169
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
⚠️ Genesis Radio Warning Healthcheck 2025-05-02 08:30:48 ⚠️
⚡ 7 warnings found:
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
- 💥 [db1] ERROR: Replication check failed: connection to server at "cluster.db1.genesishostingtechnologies.com" (38.102.127.174), port 5432 failed: No route to host
Is the server running on that host and accepting TCP/IP connections?
- ⚠️ [db1] ERROR: Could not read log /var/log/syslog: [Errno None] Unable to connect to port 22 on 38.102.127.174
- ⚠️ [db1] ERROR: Could not read log /var/log/nginx/error.log: [Errno None] Unable to connect to port 22 on 38.102.127.174
- 💥 [db2] ERROR: Replication check failed: connection to server at "cluster.db2.genesishostingtechnologies.com" (38.102.127.169), port 5432 failed: No route to host
Is the server running on that host and accepting TCP/IP connections?
- ⚠️ [db2] ERROR: Could not read log /var/log/syslog: [Errno None] Unable to connect to port 22 on 38.102.127.169
- ⚠️ [db2] ERROR: Could not read log /var/log/nginx/error.log: [Errno None] Unable to connect to port 22 on 38.102.127.169
Failed to send Mastodon DM (attempt 1): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 2): {"error":"The access token is invalid"}
Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"}
/home/doc/dbcheck/bin/python: can't open file '/home/doc/genesis-tools/miscellaneous/python/dbcheck1.py': [Errno 2] No such file or directory
/home/doc/dbcheck/bin/python: can't open file '/home/doc/genesis-tools/miscellaneous/python/dbcheck1.py': [Errno 2] No such file or directory
/home/doc/dbcheck/bin/python: can't open file '/home/doc/genesis-tools/miscellaneous/python/dbcheck1.py': [Errno 2] No such file or directory
/home/doc/dbcheck/bin/python: can't open file '/home/doc/genesis-tools/miscellaneous/python/dbcheck1.py': [Errno 2] No such file or directory
/home/doc/dbcheck/bin/python: can't open file '/home/doc/genesis-tools/miscellaneous/python/dbcheck1.py': [Errno 2] No such file or directory
/home/doc/dbcheck/bin/python: can't open file '/home/doc/genesis-tools/miscellaneous/python/dbcheck1.py': [Errno 2] No such file or directory
/home/doc/dbcheck/bin/python: can't open file '/home/doc/genesis-tools/miscellaneous/python/dbcheck1.py': [Errno 2] No such file or directory
/home/doc/dbcheck/bin/python: can't open file '/home/doc/genesis-tools/miscellaneous/python/dbcheck1.py': [Errno 2] No such file or directory
/home/doc/dbcheck/bin/python: can't open file '/home/doc/genesis-tools/miscellaneous/python/dbcheck1.py': [Errno 2] No such file or directory

233
miscellaneous/python/dbcheck1.py
View File

@ -1,233 +0,0 @@
import os
import requests
import datetime
import paramiko
import time
import psycopg2
# ==== CONFIG ====
MASTODON_INSTANCE = "https://chatwithus.live"
MASTODON_TOKEN = ""
MASTODON_USER_ID = ""
HEALTHCHECK_HTML = "/var/www/html/healthcheck.html"
DISK_WARN_THRESHOLD = 10
INODE_WARN_THRESHOLD = 10
LOG_FILES = ["/var/log/syslog", "/var/log/nginx/error.log"]
LOG_PATTERNS = ["ERROR", "FATAL", "disk full", "out of memory"]
SUPPRESSED_PATTERNS = ["SomeKnownHarmlessMastodonError"]
NODES = [
{"name": "shredder", "host": "38.102.127.171", "ssh_user": "doc", "services": ["minio.service"], "disks": ["/", "/mnt/raid5"], "db": False, "raid": True},
{"name": "mastodon", "host": "chatwithus.live", "ssh_user": "root", "services": ["nginx", "mastodon-web"], "disks": ["/"], "db": False, "raid": False},
{"name": "db1", "host": "cluster.db1.genesishostingtechnologies.com", "ssh_user": "doc", "services": ["postgresql@16-main.service"], "disks": ["/", "/var/lib/postgresql"], "db": True, "raid": False},
{"name": "db2", "host": "cluster.db2.genesishostingtechnologies.com", "ssh_user": "doc", "services": ["postgresql@16-main.service"], "disks": ["/", "/var/lib/postgresql"], "db": True, "raid": False}
]
# ==== Mastodon DM ====
def mastodon_dm(message, retries=3):
url = f"{MASTODON_INSTANCE}/api/v1/statuses"
headers = {"Authorization": f"Bearer {MASTODON_TOKEN}"}
payload = {"status": message, "visibility": "direct", "in_reply_to_account_id": MASTODON_USER_ID}
for attempt in range(retries):
resp = requests.post(url, headers=headers, data=payload)
if resp.status_code == 200:
return
print(f"Failed to send Mastodon DM (attempt {attempt+1}): {resp.text}")
time.sleep(5)
# ==== SSH Helper ====
def ssh_command(host, user, cmd):
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(hostname=host, username=user, timeout=10)
stdin, stdout, stderr = ssh.exec_command(cmd)
out = stdout.read().decode().strip()
ssh.close()
return out
# ==== Health Check Helpers ====
def choose_emoji(line):
if "RAID" in line:
return "🧨"
if "disk" in line.lower():
return "📈"
if "rclone" in line.lower():
return "🐢"
if "Service" in line:
return "🛑"
if "Replication" in line:
return "💥"
return "⚠️"
def check_remote_disk(host, user, path, node_name):
try:
cmd = f"df --output=pcent {path} | tail -1 | tr -dc '0-9'"
out = ssh_command(host, user, cmd)
if not out:
return f"[{node_name}] ERROR: Disk {path} not found or could not check disk usage."
percent = int(out)
if percent > (100 - DISK_WARN_THRESHOLD):
return f"[{node_name}] WARNING: Only {100 - percent}% disk free on {path}."
except Exception as e:
return f"[{node_name}] ERROR: Disk check failed: {e}"
return None
def check_remote_service(host, user, service, node_name):
try:
cmd = f"systemctl is-active {service}"
out = ssh_command(host, user, cmd)
if out.strip() != "active":
return f"[{node_name}] CRITICAL: Service {service} not running!"
except Exception as e:
return f"[{node_name}] ERROR: Service check failed: {e}"
return None
def check_replication(host, node_name):
try:
conn = psycopg2.connect(host=host, dbname="postgres", user="postgres", connect_timeout=5)
cur = conn.cursor()
cur.execute("SELECT pg_is_in_recovery();")
is_replica = cur.fetchone()[0]
if is_replica:
cur.execute("SELECT EXTRACT(EPOCH FROM (now() - pg_last_xact_replay_timestamp()))::INT;")
lag = cur.fetchone()[0]
if lag is None:
return f"[{node_name}] CRITICAL: Standby not streaming! Replication down."
elif lag >= 60:
return f"[{node_name}] WARNING: Replication lag is {lag} seconds."
cur.close()
conn.close()
except Exception as e:
return f"[{node_name}] ERROR: Replication check failed: {e}"
return None
def check_remote_raid_md0(host, user, node_name):
try:
mdstat = ssh_command(host, user, "cat /proc/mdstat")
lines = mdstat.splitlines()
status = None
inside_md0 = False
for line in lines:
if line.startswith("md0"):
inside_md0 = True
elif inside_md0:
if "[" in line and "]" in line:
status = line[line.index("["):line.index("]")+1]
break
if line.strip() == "" or ":" in line:
break
if status is None:
return f"[{node_name}] CRITICAL: /dev/md0 RAID status string not found!"
if "_" in status:
return f"[{node_name}] WARNING: /dev/md0 RAID degraded! Status: {status}"
except Exception as e:
return f"[{node_name}] ERROR: RAID check failed: {e}"
return None
def check_remote_logs(host, user, node_name):
alerts = []
for log in LOG_FILES:
cmd = f"tail -500 {log}"
try:
out = ssh_command(host, user, cmd)
lines = out.split("\n")
for pattern in LOG_PATTERNS:
for line in lines:
if pattern in line and not any(suppress in line for suppress in SUPPRESSED_PATTERNS):
alerts.append(f"[{node_name}] WARNING: Pattern '{pattern}' in {log}")
except Exception as e:
alerts.append(f"[{node_name}] ERROR: Could not read log {log}: {e}")
return alerts
# ==== Main Routine ====
def main():
critical_problems = []
warning_problems = []
node_status = {}
for node in NODES:
status = " Healthy"
for disk in node["disks"]:
res = check_remote_disk(node["host"], node["ssh_user"], disk, node["name"])
if res:
if "CRITICAL" in res:
critical_problems.append(res)
status = "🚨 Critical"
elif "WARNING" in res and status != "🚨 Critical":
warning_problems.append(res)
status = "Warning"
for svc in node["services"]:
res = check_remote_service(node["host"], node["ssh_user"], svc, node["name"])
if res:
if "CRITICAL" in res:
critical_problems.append(res)
status = "🚨 Critical"
elif "WARNING" in res and status != "🚨 Critical":
warning_problems.append(res)
status = "⚠️ Warning"
if node.get("db"):
res = check_replication(node["host"], node["name"])
if res:
if "CRITICAL" in res:
critical_problems.append(res)
status = " Critical"
else:
warning_problems.append(res)
if status != " Critical":
status = " Warning"
if node.get("raid", False):
res = check_remote_raid_md0(node["host"], node["ssh_user"], node["name"])
if res:
if "CRITICAL" in res:
critical_problems.append(res)
status = "Critical"
else:
warning_problems.append(res)
if status != "Critical":
status = "Warning"
logs = check_remote_logs(node["host"], node["ssh_user"], node["name"])
for log_alert in logs:
warning_problems.append(log_alert)
if status != "Critical":
status = "Warning"
node_status[node["name"]] = status
now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
if critical_problems:
formatted = "\n".join(f"- {choose_emoji(p)} {p}" for p in critical_problems)
msg = f"🚨 Genesis Radio Critical Healthcheck {now} 🚨\n⚡ {len(critical_problems)} critical issues found:\n{formatted}"
print(msg)
mastodon_dm(msg)
if warning_problems:
formatted = "\n".join(f"- {choose_emoji(p)} {p}" for p in warning_problems)
msg = f"⚠️ Genesis Radio Warning Healthcheck {now} ⚠️\n⚡ {len(warning_problems)} warnings found:\n{formatted}"
print(msg)
mastodon_dm(msg)
if not critical_problems and not warning_problems:
msg = f"✅ Genesis Radio Healthcheck {now}: All systems normal."
print(msg)
mastodon_dm(msg)
# Write dashboard
with open(HEALTHCHECK_HTML, "w") as f:
f.write("<html><head><title>Genesis Radio Healthcheck</title><meta http-equiv='refresh' content='60'></head><body>")
f.write(f"<h1>Genesis Radio System Health</h1>")
f.write(f"<p>Last Checked: {now}</p>")
f.write("<table border='1' cellpadding='5' style='border-collapse: collapse;'><tr><th>System</th><th>Status</th></tr>")
for node, status in node_status.items():
color = 'green' if 'Healthy' in status else ('orange' if 'Warning' in status else 'red')
f.write(f"<tr><td>{node}</td><td style='color:{color};'>{status}</td></tr>")
f.write("</table></body></html>")
if __name__ == "__main__":
main()

284
miscellaneous/python/dbcheck3.py Normal file
View File

@ -0,0 +1,284 @@
import subprocess
import os
import requests
import datetime
import paramiko
import time
import psycopg2
# ==== CONFIG ====
MASTODON_INSTANCE = "https://chatwithus.live"
MASTODON_TOKEN = "rimxBLi-eaJAcwagkmoj6UoW7Lc473tQY0cOM041Euw"
MASTODON_USER_ID = "114386383616633367"
HEALTHCHECK_HTML = "/var/www/html/healthcheck.html"
SUDO_PASSWORD = 'rusty2281'
DISK_WARN_THRESHOLD = 10
INODE_WARN_THRESHOLD = 10
LOG_FILES = ["/var/log/syslog", "/var/log/nginx/error.log"]
LOG_PATTERNS = ["ERROR", "FATAL", "disk full", "out of memory"]
SUPPRESSED_PATTERNS = ["SomeKnownHarmlessMastodonError"]
NODES = [
{"name": "shredder", "host": "38.102.127.172", "ssh_user": "doc", "services": ["minio.service"], "disks": ["/", "/mnt/raid5"], "db": False, "raid": True},
{"name": "mastodon", "host": "chatwithus.live", "ssh_user": "root", "services": ["nginx", "mastodon-web"], "disks": ["/"], "db": False, "raid": False},
{"name": "db1", "host": "zcluster.technodrome1.sshjunkie.com", "ssh_user": "doc", "services": ["postgresql@16-main.service"], "disks": ["/", "/var/lib/postgresql"], "db": True, "raid": True},
{"name": "db2", "host": "zcluster.technodrome1.sshjunkie.com", "ssh_user": "doc", "services": ["postgresql@16-main.service"], "disks": ["/", "/var/lib/postgresql"], "db": True, "raid": True}
]
def mastodon_dm(message, retries=3):
url = f"{MASTODON_INSTANCE}/api/v1/statuses"
headers = {"Authorization": f"Bearer {MASTODON_TOKEN}"}
payload = {"status": message, "visibility": "direct", "in_reply_to_account_id": MASTODON_USER_ID}
for attempt in range(retries):
try:
resp = requests.post(url, headers=headers, data=payload, timeout=10)
if resp.status_code == 200:
return
print(f"Failed to send Mastodon DM (attempt {attempt+1}): {resp.text}")
except Exception as e:
print(f"Exception during Mastodon DM (attempt {attempt+1}): {e}")
time.sleep(5)
def ssh_command(host, user, cmd):
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(hostname=host, username=user, key_filename=os.path.expanduser("~/.ssh/genesis_healthcheck"), timeout=10)
stdin, stdout, stderr = ssh.exec_command(cmd)
out = stdout.read().decode().strip()
err = stderr.read().decode().strip()
if not out or "permission denied" in err.lower() or "sudo:" in err.lower():
sudo_cmd = f"echo '{SUDO_PASSWORD}' | sudo -S {cmd}"
stdin, stdout, stderr = ssh.exec_command(sudo_cmd)
out = stdout.read().decode().strip()
err = stderr.read().decode().strip()
ssh.close()
return out if out else err
def choose_emoji(line):
if "RAID" in line:
if "disk" in line.lower():
return "📈"
if "rclone" in line.lower():
return "🐢"
if "Service" in line:
return "🚩"
if "Replication" in line:
return "💥"
return "⚠️"
def check_remote_disk(host, user, path, node_name):
try:
cmd = f"df --output=pcent {path} | tail -1 | tr -dc '0-9'"
out = ssh_command(host, user, cmd)
if not out:
return f"[{node_name}] ERROR: Disk {path} not found or could not check disk usage."
percent = int(out)
if percent > (100 - DISK_WARN_THRESHOLD):
return f"[{node_name}] WARNING: Only {100 - percent}% disk free on {path}."
except Exception as e:
return f"[{node_name}] ERROR: Disk check failed: {e}"
return None
def check_remote_service(host, user, service, node_name):
try:
cmd = f"systemctl is-active {service}"
out = ssh_command(host, user, cmd)
if out.strip() != "active":
return f"[{node_name}] CRITICAL: Service {service} not running!"
except Exception as e:
return f"[{node_name}] ERROR: Service check failed: {e}"
return None
def check_replication(host, node_name):
try:
conn = psycopg2.connect(host=host, dbname="postgres", user="postgres", connect_timeout=5)
cur = conn.cursor()
cur.execute("SELECT pg_is_in_recovery();")
is_replica = cur.fetchone()[0]
if is_replica:
cur.execute("SELECT EXTRACT(EPOCH FROM (now() - pg_last_xact_replay_timestamp()))::INT;")
lag = cur.fetchone()[0]
if lag is None:
return f"[{node_name}] CRITICAL: Standby not streaming! Replication down."
elif lag >= 60:
return f"[{node_name}] WARNING: Replication lag is {lag} seconds."
cur.close()
conn.close()
except Exception as e:
return f"[{node_name}] ERROR: Replication check failed: {e}"
return None
def check_remote_raid_md0(host, user, node_name):
alerts = []
try:
pools_output = ssh_command(host, user, "zpool list -H -o name")
pool_names = pools_output.strip().splitlines()
for pool in pool_names:
health_cmd = f"zpool status -x {pool}"
health_out = ssh_command(host, user, health_cmd)
if f"pool '{pool}' is healthy" not in health_out.lower():
alerts.append(f"[{node_name}] WARNING: ZFS pool '{pool}' is not healthy: {health_out.strip()}")
snap_cmd = f"zfs list -t snapshot -o name,creation -s creation -H -r {pool}"
snap_out = ssh_command(host, user, snap_cmd)
if not snap_out.strip():
alerts.append(f"[{node_name}] WARNING: No snapshots found in ZFS pool '{pool}'")
else:
last_snap = snap_out.strip().splitlines()[-1]
snap_parts = last_snap.split("\t")
if len(snap_parts) == 2:
snap_time_str = snap_parts[1].strip()
snap_time = datetime.datetime.strptime(snap_time_str, "%a %b %d %H:%M %Y")
delta = datetime.datetime.now() - snap_time
if delta.total_seconds() > 86400:
alerts.append(f"[{node_name}] WARNING: Last snapshot on pool '{pool}' is older than 24h: {snap_time_str}")
except Exception as e:
alerts.append(f"[{node_name}] ERROR: ZFS RAID check failed: {e}")
try:
mdstat = ssh_command(host, user, "cat /proc/mdstat")
lines = mdstat.splitlines()
status = None
inside_md0 = False
for line in lines:
if line.startswith("md0"):
inside_md0 = True
elif inside_md0:
if "[" in line and "]" in line:
status = line[line.index("["):line.index("]")+1]
break
if line.strip() == "" or ":" in line:
break
if status is None:
alerts.append(f"[{node_name}] CRITICAL: /dev/md0 RAID status string not found!")
elif "_" in status:
alerts.append(f"[{node_name}] WARNING: /dev/md0 RAID degraded! Status: {status}")
except Exception as fallback_e:
alerts.append(f"[{node_name}] ERROR: RAID check failed (ZFS+mdstat): {e}; {fallback_e}")
return "\n".join(alerts) if alerts else None
def check_remote_logs(host, user, node_name):
alerts = []
for log in LOG_FILES:
cmd = f"tail -500 {log}"
try:
out = ssh_command(host, user, cmd)
lines = out.split("\n")
for pattern in LOG_PATTERNS:
for line in lines:
if pattern in line and not any(suppress in line for suppress in SUPPRESSED_PATTERNS):
alerts.append(f"[{node_name}] WARNING: Pattern '{pattern}' in {log}")
except Exception as e:
alerts.append(f"[{node_name}] ERROR: Could not read log {log}: {e}")
return alerts
def check_postgres_snapshot():
try:
result = subprocess.run(
["sudo", "-S", "bash", "-c", "/root/genesis-tools/miscellaneous/bash/dbv2.sh"],
input=f"{SUDO_PASSWORD}\n",
capture_output=True,
text=True
)
if result.returncode != 0:
mastodon_dm(f"🚨 Snapshot verification failed:\nstdout:\n{result.stdout.strip()}\nstderr:\n{result.stderr.strip()}")
except Exception as e:
mastodon_dm(f"🚨 Snapshot verification failed to run: {e}")
def main():
critical_problems = []
warning_problems = []
node_status = {}
for node in NODES:
status = "Healthy"
raid_info = "All pools healthy"
for disk in node["disks"]:
disk_res = check_remote_disk(node["host"], node["ssh_user"], disk, node["name"])
if disk_res:
if "CRITICAL" in disk_res:
critical_problems.append(disk_res)
status = "Critical"
elif "WARNING" in disk_res and status != "Critical":
warning_problems.append(disk_res)
status = "Warning"
for svc in node["services"]:
svc_res = check_remote_service(node["host"], node["ssh_user"], svc, node["name"])
if svc_res:
if "CRITICAL" in svc_res:
critical_problems.append(svc_res)
status = "Critical"
elif "WARNING" in svc_res and status != "Critical":
warning_problems.append(svc_res)
status = "Warning"
if node.get("db"):
rep_res = check_replication(node["host"], node["name"])
if rep_res:
if "CRITICAL" in rep_res:
critical_problems.append(rep_res)
status = "Critical"
else:
warning_problems.append(rep_res)
if status != "Critical":
status = "Warning"
if node.get("raid", False):
raid_res = check_remote_raid_md0(node["host"], node["ssh_user"], node["name"])
if raid_res:
if "CRITICAL" in raid_res:
critical_problems.append(raid_res)
status = "Critical"
else:
warning_problems.append(raid_res)
if status != "Critical":
status = "Warning"
raid_info = raid_res
logs = check_remote_logs(node["host"], node["ssh_user"], node["name"])
for log_alert in logs:
warning_problems.append(log_alert)
if status != "Critical":
status = "Warning"
node_status[node["name"]] = (status, raid_info)
now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
if critical_problems:
formatted = "\n".join(f"- {choose_emoji(p)} {p}" for p in critical_problems)
msg = f"🚨 Genesis Radio Critical Healthcheck {now} 🚨\n⚡ {len(critical_problems)} critical issues found:\n{formatted}"
print(msg)
mastodon_dm(msg)
if warning_problems:
formatted = "\n".join(f"- {choose_emoji(p)} {p}" for p in warning_problems)
msg = f"⚠️ Genesis Radio Warning Healthcheck {now} ⚠️\n⚡ {len(warning_problems)} warnings found:\n{formatted}"
print(msg)
mastodon_dm(msg)
if not critical_problems and not warning_problems:
msg = f"✅ Genesis Radio Healthcheck {now}: All systems normal."
print(msg)
mastodon_dm(msg)
with open(HEALTHCHECK_HTML, "w") as f:
f.write("<html><head><title>Genesis Radio Healthcheck</title><meta http-equiv='refresh' content='60'></head><body>")
f.write(f"<h1>Genesis Radio System Health</h1>")
f.write(f"<p>Last Checked: {now}</p>")
f.write("<table border='1' cellpadding='5' style='border-collapse: collapse;'><tr><th>System</th><th>Status</th><th>ZFS Details</th></tr>")
for node, (status, zfs_info) in node_status.items():
color = 'green' if 'Healthy' in status else ('orange' if 'Warning' in status else 'red')
f.write(f"<tr><td>{node}</td><td style='color:{color};'>{status}</td><td><pre style='white-space: pre-wrap; font-size: small;'>{zfs_info}</pre></td></tr>")
f.write("</table></body></html>")
check_postgres_snapshot()
if __name__ == "__main__":
main()

5
radiotoot/.env Normal file
View File

@ -0,0 +1,5 @@
DB_USER=radiotootuser
DB_PASSWORD=rusty2281
DB_NAME=radiotootdb
DB_HOST_PRIMARY=zcluster.technodrome1.sshjunkie.com
MASTODON_ACCESS_TOKEN=07w3Emdw-cv_TncysrNU8Ed_sHJhwtnvKmnLqKlHmKA