diff --git a/archive.html/2025-05-02/healthcheck_2025-05-02 14-14-12.html b/archive.html/2025-05-02/healthcheck_2025-05-02 14-14-12.html new file mode 100644 index 0000000..6069eac --- /dev/null +++ b/archive.html/2025-05-02/healthcheck_2025-05-02 14-14-12.html @@ -0,0 +1,4 @@ +Genesis Radio Healthcheck

Genesis Radio System Health

Last Checked: 2025-05-02 14:14:12

SystemStatusZFS Details
shredderWarning
[shredder] ERROR: ZFS RAID check failed: Authentication failed.
+[shredder] ERROR: RAID check failed (ZFS+mdstat): Authentication failed.; Authentication failed.
mastodonWarning
All pools healthy
db1Warning
[db1] ERROR: ZFS RAID check failed: Authentication failed.
+[db1] ERROR: RAID check failed (ZFS+mdstat): Authentication failed.; Authentication failed.
db2Warning
[db2] ERROR: ZFS RAID check failed: Authentication failed.
+[db2] ERROR: RAID check failed (ZFS+mdstat): Authentication failed.; Authentication failed.
\ No newline at end of file diff --git a/archive.html/2025-05-02/healthcheck_2025-05-02 14-15-26.html b/archive.html/2025-05-02/healthcheck_2025-05-02 14-15-26.html new file mode 100644 index 0000000..215a9cf --- /dev/null +++ b/archive.html/2025-05-02/healthcheck_2025-05-02 14-15-26.html @@ -0,0 +1,4 @@ +Genesis Radio Healthcheck

Genesis Radio System Health

Last Checked: 2025-05-02 14:15:26

SystemStatusZFS Details
shredderWarning
[shredder] ERROR: ZFS RAID check failed: Authentication failed.
+[shredder] ERROR: RAID check failed (ZFS+mdstat): Authentication failed.; Authentication failed.
mastodonWarning
All pools healthy
db1Warning
[db1] ERROR: ZFS RAID check failed: Authentication failed.
+[db1] ERROR: RAID check failed (ZFS+mdstat): Authentication failed.; Authentication failed.
db2Warning
[db2] ERROR: ZFS RAID check failed: Authentication failed.
+[db2] ERROR: RAID check failed (ZFS+mdstat): Authentication failed.; Authentication failed.
\ No newline at end of file diff --git a/archive.html/2025-05-02/healthcheck_2025-05-02 14-18-37.html b/archive.html/2025-05-02/healthcheck_2025-05-02 14-18-37.html new file mode 100644 index 0000000..255fc6c --- /dev/null +++ b/archive.html/2025-05-02/healthcheck_2025-05-02 14-18-37.html @@ -0,0 +1,2 @@ +Genesis Radio Healthcheck

Genesis Radio System Health

Last Checked: 2025-05-02 14:18:37

SystemStatusZFS Details
shredderWarning
[shredder] WARNING: ZFS pool 'nexus' is not healthy: pool 'nexus' is healthy
mastodonWarning
All pools healthy
db1Warning
[db1] WARNING: ZFS pool 'pgpool' is not healthy: pool 'pgpool' is healthy
db2Critical
[db2] WARNING: ZFS pool 'pgpool' is not healthy: pool 'pgpool' is healthy
+[db2] WARNING: No snapshots found in ZFS pool 'pgpool'
\ No newline at end of file diff --git a/archive.html/2025-05-02/healthcheck_2025-05-02 14-19-40.html b/archive.html/2025-05-02/healthcheck_2025-05-02 14-19-40.html new file mode 100644 index 0000000..9307200 --- /dev/null +++ b/archive.html/2025-05-02/healthcheck_2025-05-02 14-19-40.html @@ -0,0 +1,2 @@ +Genesis Radio Healthcheck

Genesis Radio System Health

Last Checked: 2025-05-02 14:19:40

SystemStatusZFS Details
shredderWarning
[shredder] WARNING: ZFS pool 'nexus' is not healthy: pool 'nexus' is healthy
mastodonWarning
All pools healthy
db1Warning
[db1] WARNING: ZFS pool 'pgpool' is not healthy: pool 'pgpool' is healthy
db2Critical
[db2] WARNING: ZFS pool 'pgpool' is not healthy: pool 'pgpool' is healthy
+[db2] WARNING: No snapshots found in ZFS pool 'pgpool'
\ No newline at end of file diff --git a/archive.html/2025-05-02/healthcheck_2025-05-02 14-23-18.html b/archive.html/2025-05-02/healthcheck_2025-05-02 14-23-18.html new file mode 100644 index 0000000..9e86a89 --- /dev/null +++ b/archive.html/2025-05-02/healthcheck_2025-05-02 14-23-18.html @@ -0,0 +1,2 @@ +Genesis Radio Healthcheck

Genesis Radio System Health

Last Checked: 2025-05-02 14:23:18

SystemStatusZFS Details
shredderWarning
[shredder] WARNING: ZFS pool 'nexus' is not healthy: pool 'nexus' is healthy
mastodonWarning
All pools healthy
db1Warning
[db1] WARNING: ZFS pool 'pgpool' is not healthy: pool 'pgpool' is healthy
db2Critical
[db2] WARNING: ZFS pool 'pgpool' is not healthy: pool 'pgpool' is healthy
+[db2] WARNING: No snapshots found in ZFS pool 'pgpool'
\ No newline at end of file diff --git a/archive.html/2025-05-02/healthcheck_2025-05-02 14-25-55.html b/archive.html/2025-05-02/healthcheck_2025-05-02 14-25-55.html new file mode 100644 index 0000000..98df44f --- /dev/null +++ b/archive.html/2025-05-02/healthcheck_2025-05-02 14-25-55.html @@ -0,0 +1,2 @@ +Genesis Radio Healthcheck

Genesis Radio System Health

Last Checked: 2025-05-02 14:25:55

SystemStatusZFS Details
shredderCritical
[shredder] WARNING: ZFS pool 'nexus' is not healthy: pool 'nexus' is healthy
mastodonWarning
All pools healthy
db1Critical
[db1] WARNING: ZFS pool 'pgpool' is not healthy: pool 'pgpool' is healthy
db2Critical
[db2] WARNING: ZFS pool 'pgpool' is not healthy: pool 'pgpool' is healthy
+[db2] WARNING: No snapshots found in ZFS pool 'pgpool'
\ No newline at end of file diff --git a/archive.html/2025-05-02/healthcheck_2025-05-02 14-27-51.html b/archive.html/2025-05-02/healthcheck_2025-05-02 14-27-51.html new file mode 100644 index 0000000..a91e48a --- /dev/null +++ b/archive.html/2025-05-02/healthcheck_2025-05-02 14-27-51.html @@ -0,0 +1,2 @@ +Genesis Radio Healthcheck

Genesis Radio System Health

Last Checked: 2025-05-02 14:27:51

SystemStatusZFS Details
shredderCritical
[shredder] WARNING: ZFS pool 'nexus' is not healthy: pool 'nexus' is healthy
mastodonWarning
All pools healthy
db1Critical
[db1] WARNING: ZFS pool 'pgpool' is not healthy: pool 'pgpool' is healthy
db2Critical
[db2] WARNING: ZFS pool 'pgpool' is not healthy: pool 'pgpool' is healthy
+[db2] WARNING: No snapshots found in ZFS pool 'pgpool'
\ No newline at end of file diff --git a/archive.html/2025-05-02/healthcheck_2025-05-02 14-29-37.html b/archive.html/2025-05-02/healthcheck_2025-05-02 14-29-37.html new file mode 100644 index 0000000..5656050 --- /dev/null +++ b/archive.html/2025-05-02/healthcheck_2025-05-02 14-29-37.html @@ -0,0 +1,2 @@ +Genesis Radio Healthcheck

Genesis Radio System Health

Last Checked: 2025-05-02 14:29:37

SystemStatusZFS Details
shredderWarning
[shredder] WARNING: ZFS pool 'nexus' is not healthy: pool 'nexus' is healthy
mastodonWarning
All pools healthy
db1Warning
[db1] WARNING: ZFS pool 'pgpool' is not healthy: pool 'pgpool' is healthy
db2Warning
[db2] WARNING: ZFS pool 'pgpool' is not healthy: pool 'pgpool' is healthy
+[db2] WARNING: No snapshots found in ZFS pool 'pgpool'
\ No newline at end of file diff --git a/archive.html/2025-05-02/healthcheck_2025-05-02 14-30-53.html b/archive.html/2025-05-02/healthcheck_2025-05-02 14-30-53.html new file mode 100644 index 0000000..67d1c0e --- /dev/null +++ b/archive.html/2025-05-02/healthcheck_2025-05-02 14-30-53.html @@ -0,0 +1,2 @@ +Genesis Radio Healthcheck

Genesis Radio System Health

Last Checked: 2025-05-02 14:30:53

SystemStatusZFS Details
shredderWarning
[shredder] WARNING: ZFS pool 'nexus' is not healthy: pool 'nexus' is healthy
mastodonWarning
All pools healthy
db1Warning
[db1] WARNING: ZFS pool 'pgpool' is not healthy: pool 'pgpool' is healthy
db2Warning
[db2] WARNING: ZFS pool 'pgpool' is not healthy: pool 'pgpool' is healthy
+[db2] WARNING: No snapshots found in ZFS pool 'pgpool'
\ No newline at end of file diff --git a/archive.html/2025-05-02/healthcheck_2025-05-02 14-34-37.html b/archive.html/2025-05-02/healthcheck_2025-05-02 14-34-37.html new file mode 100644 index 0000000..dd60310 --- /dev/null +++ b/archive.html/2025-05-02/healthcheck_2025-05-02 14-34-37.html @@ -0,0 +1 @@ +Genesis Radio Healthcheck

Genesis Radio System Health

Last Checked: 2025-05-02 14:34:37

SystemStatusZFS Details
shredderHealthy
All pools healthy
mastodonWarning
All pools healthy
db1Healthy
All pools healthy
db2Healthy
All pools healthy
\ No newline at end of file diff --git a/archive.html/2025-05-02/healthcheck_2025-05-02 14-35-19.html b/archive.html/2025-05-02/healthcheck_2025-05-02 14-35-19.html new file mode 100644 index 0000000..eb27f13 --- /dev/null +++ b/archive.html/2025-05-02/healthcheck_2025-05-02 14-35-19.html @@ -0,0 +1 @@ +Genesis Radio Healthcheck

Genesis Radio System Health

Last Checked: 2025-05-02 14:35:19

SystemStatusZFS Details
shredderHealthy
All pools healthy
mastodonCritical
All pools healthy
db1Healthy
All pools healthy
db2Healthy
All pools healthy
\ No newline at end of file diff --git a/archive.html/2025-05-02/healthcheck_2025-05-02 14-36-34.html b/archive.html/2025-05-02/healthcheck_2025-05-02 14-36-34.html new file mode 100644 index 0000000..480373b --- /dev/null +++ b/archive.html/2025-05-02/healthcheck_2025-05-02 14-36-34.html @@ -0,0 +1 @@ +Genesis Radio Healthcheck

Genesis Radio System Health

Last Checked: 2025-05-02 14:36:34

SystemStatusZFS Details
shredderHealthy
All pools healthy
mastodonCritical
All pools healthy
db1Healthy
All pools healthy
db2Healthy
All pools healthy
\ No newline at end of file diff --git a/archive.html/2025-05-02/healthcheck_2025-05-02 14-39-33.html b/archive.html/2025-05-02/healthcheck_2025-05-02 14-39-33.html new file mode 100644 index 0000000..4d5ff6c --- /dev/null +++ b/archive.html/2025-05-02/healthcheck_2025-05-02 14-39-33.html @@ -0,0 +1 @@ +Genesis Radio Healthcheck

Genesis Radio System Health

Last Checked: 2025-05-02 14:39:33

SystemStatusZFS Details
shredderHealthy
All pools healthy
mastodonCritical
All pools healthy
db1Healthy
All pools healthy
db2Healthy
All pools healthy
\ No newline at end of file diff --git a/archive.html/2025-05-02/healthcheck_2025-05-02 14-40-39.html b/archive.html/2025-05-02/healthcheck_2025-05-02 14-40-39.html new file mode 100644 index 0000000..207dede --- /dev/null +++ b/archive.html/2025-05-02/healthcheck_2025-05-02 14-40-39.html @@ -0,0 +1 @@ +Genesis Radio Healthcheck

Genesis Radio System Health

Last Checked: 2025-05-02 14:40:39

SystemStatusZFS Details
shredderHealthy
All pools healthy
mastodonCritical
All pools healthy
db1Healthy
All pools healthy
db2Healthy
All pools healthy
\ No newline at end of file diff --git a/archive.html/2025-05-02/healthcheck_2025-05-02 14-42-35.html b/archive.html/2025-05-02/healthcheck_2025-05-02 14-42-35.html new file mode 100644 index 0000000..294ee93 --- /dev/null +++ b/archive.html/2025-05-02/healthcheck_2025-05-02 14-42-35.html @@ -0,0 +1 @@ +Genesis Radio Healthcheck

Genesis Radio System Health

Last Checked: 2025-05-02 14:42:35

SystemStatusZFS Details
shredderHealthy
All pools healthy
mastodonCritical
All pools healthy
db1Healthy
All pools healthy
db2Healthy
All pools healthy
\ No newline at end of file diff --git a/archive.html/2025-05-02/healthcheck_2025-05-02 14-43-10.html b/archive.html/2025-05-02/healthcheck_2025-05-02 14-43-10.html new file mode 100644 index 0000000..42f7b54 --- /dev/null +++ b/archive.html/2025-05-02/healthcheck_2025-05-02 14-43-10.html @@ -0,0 +1 @@ +Genesis Radio Healthcheck

Genesis Radio System Health

Last Checked: 2025-05-02 14:43:10

SystemStatusZFS Details
shredderHealthy
All pools healthy
mastodonCritical
All pools healthy
db1Healthy
All pools healthy
db2Healthy
All pools healthy
\ No newline at end of file diff --git a/archive.html/2025-05-02/healthcheck_2025-05-02 14-43-24.html b/archive.html/2025-05-02/healthcheck_2025-05-02 14-43-24.html new file mode 100644 index 0000000..46a4992 --- /dev/null +++ b/archive.html/2025-05-02/healthcheck_2025-05-02 14-43-24.html @@ -0,0 +1 @@ +Genesis Radio Healthcheck

Genesis Radio System Health

Last Checked: 2025-05-02 14:43:24

SystemStatusZFS Details
shredderHealthy
All pools healthy
mastodonCritical
All pools healthy
db1Healthy
All pools healthy
db2Healthy
All pools healthy
\ No newline at end of file diff --git a/archive.html/cleanup.log b/archive.html/cleanup.log new file mode 100644 index 0000000..e69de29 diff --git a/archivecontrol/recordit2.py b/archivecontrol/recordit2.py index ca9add1..4b94009 100644 --- a/archivecontrol/recordit2.py +++ b/archivecontrol/recordit2.py @@ -4,7 +4,7 @@ from mastodon import Mastodon # === Configuration === -ROOT_DIR = r"/mnt/genesisarchives-secure" # Update this path to where your shows live +ROOT_DIR = r"/mnt/archives" # Update this path to where your shows live ALLOWED_EXTENSIONS = {".mp3", ".wav", ".flac", ".m4a"} BANNER_FILENAMES = ["banner.jpg", "banner.png", "banner.jpeg"] diff --git a/genesishostingmd/backups/dr/assets-mastodon-bucket.md b/genesishostingmd/backups/dr/assets-mastodon-bucket.md new file mode 100644 index 0000000..6a36a15 --- /dev/null +++ b/genesishostingmd/backups/dr/assets-mastodon-bucket.md @@ -0,0 +1,45 @@ +## 2025-05-02 22:24:25 – MinIO Bucket Access Configuration for Mastodon + +**Bucket**: `assets-mastodon` +**Server**: `shredderv2` +**User**: `genesisuser` +**Permissions**: Read / Write / Delete +**Policy Name**: `assets-mastodon-rw-policy` + +### Commands Executed: + +```bash +mc alias set localminio http://localhost:9000 genesisadmin MutationXv3! + +cat > assets_mastodon_rw_policy.json < assets_mastodon_rw_policy.json < **RAID protects against hardware failure — not human error, data corruption, or bad automation.** + +--- + +## 🔍 What Happened + +- `shredderv1` uses RAID5 for media storage. +- The dataset `nexus/miniodata` (housing `genesisassets`, `genesislibrary`, etc.) was accidentally destroyed. +- **No disks failed.** The failure was logical, not physical. + +--- + +## 🔥 Impact + +- StationPlaylist (SPL) lost access to the Genesis media library. +- MinIO bucket data was instantly inaccessible. +- Temporary outage and scrambling to reconfigure mounts, media, and streaming. + +--- + +## ✅ Recovery + +Thanks to our disaster recovery stack: + +- Nightly **rsync backups** were synced to **The Vault** (backup server). +- **ZFS snapshots** existed on The Vault for the affected datasets. +- We restored the latest snapshot **from The Vault back to Shredder**, effectively reversing the loss. +- No data corruption occurred; sync validation showed dataset integrity. + +--- + +## 🎓 Takeaway + +This is a live demonstration of why: + +- **RAID is not a backup** +- **Snapshots without off-host replication** are not enough +- **Real backups must be off-server and regularly tested** + +--- + +## 🔐 Current Protection Measures + +- Production data (`genesisassets`, `genesislibrary`) now replicated nightly to The Vault via `rsync`. +- ZFS snapshots are validated daily via a **dry-run restore validator**. +- Telegram alerts notify success/failure of backup verification jobs. +- Future goal: full ZFS storage on all production servers for native snapshot support. + +--- + +## 🧠 Lessons Learned + +- Always assume you'll delete the wrong thing eventually. +- Snapshots are amazing — **if** they're somewhere else. +- Automated restore testing should be part of every backup pipeline. + diff --git a/masto_restart.log b/masto_restart.log index 51a88ac..e5708a8 100644 --- a/masto_restart.log +++ b/masto_restart.log @@ -39,3 +39,25 @@ All services restarted. [✓] All tasks complete. Logged out of root@chatwithus.live. [2025-05-01 08:03:47] === Mastodon Restart Complete === +[2025-05-03 20:31:04] === Mastodon Restart Initiated === +[*] Posting 2-minute warning to Mastodon... +[✓] 2-minute warning posted. +[*] Posting 1-minute warning to Mastodon... +[✓] 1-minute warning posted. +[*] Connecting to root@chatwithus.live to restart Mastodon services... +Restarting mastodon-web... +Warning: The unit file, source configuration file or drop-ins of mastodon-web.service changed on disk. Run 'systemctl daemon-reload' to reload units. +Restarting mastodon-sidekiq... +Warning: The unit file, source configuration file or drop-ins of mastodon-sidekiq.service changed on disk. Run 'systemctl daemon-reload' to reload units. +Restarting mastodon-streaming... +Warning: The unit file, source configuration file or drop-ins of mastodon-streaming.service changed on disk. Run 'systemctl daemon-reload' to reload units. +All services restarted. +[*] Waiting for Mastodon to come back online... +[+] Mastodon is back online. +[*] Posting final status to Mastodon... +[✓] Final status posted. +[*] Sending Telegram alert... +[✓] Telegram alert sent. +[✓] All tasks complete. Logged out of root@chatwithus.live. +[2025-05-03 20:31:04] === Mastodon Restart Complete === + diff --git a/miscellaneous/bash/sync-trigger.sh b/miscellaneous/bash/sync-trigger.sh new file mode 100755 index 0000000..b1c30f3 --- /dev/null +++ b/miscellaneous/bash/sync-trigger.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# === CONFIG === +REMOTE_HOST="shredder.sshjunkie.com" +REMOTE_USER="doc" +REMOTE_SCRIPT="/home/doc/sync.sh" +LOG_TAG="[Krang → SPL Sync]" + +# === Mastodon Alert Settings === +MASTODON_INSTANCE="https://chatwithus.live" +ACCESS_TOKEN="07w3Emdw-cv_TncysrNU8Ed_sHJhwtnvKmnLqKlHmKA" +TOOT_VISIBILITY="unlisted" + +# === Telegram Settings === +TELEGRAM_BOT_TOKEN="8178867489:AAH0VjN7VnZSCIWasSz_y97iBLLjPJA751k" +TELEGRAM_CHAT_ID="1559582356" + +# === Execution === +echo "$LOG_TAG Triggering remote sync..." +OUTPUT=$(ssh ${REMOTE_USER}@${REMOTE_HOST} "${REMOTE_SCRIPT}" 2>&1) + +if echo "$OUTPUT" | grep -q "All syncs finished"; then + echo "$LOG_TAG ✅ Sync complete." + + # Mastodon alert + curl -s -X POST "$MASTODON_INSTANCE/api/v1/statuses" \ + -H "Authorization: Bearer $ACCESS_TOKEN" \ + -d "status=✅ SPL Sync completed successfully via Krang" \ + -d "visibility=$TOOT_VISIBILITY" >/dev/null + + # Telegram alert + curl -s -X POST "https://api.telegram.org/bot$TELEGRAM_BOT_TOKEN/sendMessage" \ + -d "chat_id=$TELEGRAM_CHAT_ID" \ + -d "text=✅ SPL Sync completed successfully from Krang." >/dev/null +else + echo "$LOG_TAG ❌ Sync may have failed. Check logs." + + # Failure alerts + curl -s -X POST "$MASTODON_INSTANCE/api/v1/statuses" \ + -H "Authorization: Bearer $ACCESS_TOKEN" \ + -d "status=❌ SPL Sync failed from Krang. Check logs." \ + -d "visibility=$TOOT_VISIBILITY" >/dev/null + + curl -s -X POST "https://api.telegram.org/bot$TELEGRAM_BOT_TOKEN/sendMessage" \ + -d "chat_id=$TELEGRAM_CHAT_ID" \ + -d "text=❌ SPL Sync failed from Krang. Manual check needed." >/dev/null +fi + diff --git a/miscellaneous/cron_backup.log b/miscellaneous/cron_backup.log index 348abed..6eb1093 100644 --- a/miscellaneous/cron_backup.log +++ b/miscellaneous/cron_backup.log @@ -645788,3 +645788,1477 @@ pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (3 [FAIL] Failed to dump radiotootdb! Skipping upload. Not all backups verified! Nothing uploaded. DONE. Log: /tmp/pgbackup_verify/verify_log_20250502_100001.txt +📂 Ensuring remote path exists... +=== [Fri May 2 11:00:01 AM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +Password: +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +=== [Fri May 2 11:00:02 AM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Fri May 2 11:00:02 AM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250502_110001.txt +📂 Ensuring remote path exists... +=== [Fri May 2 12:00:01 PM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +=== [Fri May 2 12:00:02 PM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Fri May 2 12:00:04 PM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250502_120001.txt +📂 Ensuring remote path exists... +=== [Fri May 2 01:00:01 PM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +=== [Fri May 2 01:00:02 PM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Fri May 2 01:00:03 PM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250502_130001.txt +📂 Ensuring remote path exists... +=== [Fri May 2 02:00:01 PM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +=== [Fri May 2 02:00:03 PM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Fri May 2 02:00:04 PM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250502_140001.txt +📂 Ensuring remote path exists... +=== [Fri May 2 03:00:01 PM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +=== [Fri May 2 03:00:02 PM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Fri May 2 03:00:02 PM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250502_150001.txt +📂 Ensuring remote path exists... +=== [Fri May 2 04:00:01 PM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +Password: +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +=== [Fri May 2 04:00:02 PM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Fri May 2 04:00:04 PM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250502_160001.txt +📂 Ensuring remote path exists... +=== [Fri May 2 05:00:01 PM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +=== [Fri May 2 05:00:02 PM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Fri May 2 05:00:03 PM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250502_170001.txt +📂 Ensuring remote path exists... +=== [Fri May 2 06:00:01 PM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +=== [Fri May 2 06:00:03 PM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Fri May 2 06:00:03 PM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250502_180001.txt +📂 Ensuring remote path exists... +=== [Fri May 2 07:00:01 PM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +=== [Fri May 2 07:00:01 PM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Fri May 2 07:00:02 PM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250502_190001.txt +📂 Ensuring remote path exists... +=== [Fri May 2 08:00:01 PM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +Password: +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +=== [Fri May 2 08:00:04 PM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Fri May 2 08:00:06 PM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250502_200001.txt +📂 Ensuring remote path exists... +=== [Fri May 2 09:00:01 PM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +=== [Fri May 2 09:00:02 PM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Fri May 2 09:00:03 PM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250502_210001.txt +📂 Ensuring remote path exists... +=== [Fri May 2 10:00:01 PM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +=== [Fri May 2 10:00:02 PM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Fri May 2 10:00:04 PM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250502_220001.txt +📂 Ensuring remote path exists... +=== [Fri May 2 11:00:01 PM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +=== [Fri May 2 11:00:02 PM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Fri May 2 11:00:03 PM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250502_230001.txt +📂 Ensuring remote path exists... +=== [Sat May 3 12:00:01 AM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +Password: +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +=== [Sat May 3 12:00:02 AM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Sat May 3 12:00:03 AM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250503_000001.txt +📂 Ensuring remote path exists... +=== [Sat May 3 01:00:01 AM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +=== [Sat May 3 01:00:02 AM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Sat May 3 01:00:04 AM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250503_010001.txt +📂 Ensuring remote path exists... +=== [Sat May 3 02:00:01 AM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +=== [Sat May 3 02:00:02 AM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Sat May 3 02:00:03 AM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250503_020001.txt +📂 Ensuring remote path exists... +=== [Sat May 3 03:00:01 AM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +=== [Sat May 3 03:00:04 AM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Sat May 3 03:00:05 AM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250503_030001.txt +📂 Ensuring remote path exists... +=== [Sat May 3 04:00:01 AM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +Password: +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +=== [Sat May 3 04:00:03 AM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Sat May 3 04:00:04 AM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250503_040001.txt +📂 Ensuring remote path exists... +=== [Sat May 3 05:00:01 AM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +Password: +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +=== [Sat May 3 05:00:03 AM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Sat May 3 05:00:05 AM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250503_050001.txt +📂 Ensuring remote path exists... +=== [Sat May 3 06:00:01 AM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +=== [Sat May 3 06:00:02 AM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Sat May 3 06:00:02 AM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250503_060001.txt +📂 Ensuring remote path exists... +=== [Sat May 3 07:00:01 AM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +=== [Sat May 3 07:00:02 AM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Sat May 3 07:00:03 AM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250503_070001.txt +📂 Ensuring remote path exists... +=== [Sat May 3 08:00:01 AM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +=== [Sat May 3 08:00:02 AM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Sat May 3 08:00:02 AM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250503_080001.txt +📂 Ensuring remote path exists... +=== [Sat May 3 09:00:01 AM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +=== [Sat May 3 09:00:02 AM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Sat May 3 09:00:04 AM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250503_090001.txt +📂 Ensuring remote path exists... +=== [Sat May 3 10:00:01 AM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +Password: +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +=== [Sat May 3 10:00:03 AM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Sat May 3 10:00:04 AM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250503_100001.txt +📂 Ensuring remote path exists... +=== [Sat May 3 11:00:01 AM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +=== [Sat May 3 11:00:02 AM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Sat May 3 11:00:03 AM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250503_110001.txt +📂 Ensuring remote path exists... +=== [Sat May 3 12:00:01 PM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +Password: +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +=== [Sat May 3 12:00:03 PM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Sat May 3 12:00:03 PM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250503_120001.txt +📂 Ensuring remote path exists... +=== [Sat May 3 01:00:01 PM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +=== [Sat May 3 01:00:02 PM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Sat May 3 01:00:03 PM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250503_130001.txt +📂 Ensuring remote path exists... +=== [Sat May 3 02:00:01 PM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +Password: +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +=== [Sat May 3 02:00:02 PM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Sat May 3 02:00:03 PM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250503_140001.txt +=== [Sat May 3 03:00:01 PM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +📂 Ensuring remote path exists... +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +=== [Sat May 3 03:00:03 PM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Sat May 3 03:00:03 PM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250503_150001.txt +📂 Ensuring remote path exists... +=== [Sat May 3 04:00:01 PM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +=== [Sat May 3 04:00:05 PM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Sat May 3 04:00:08 PM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250503_160001.txt +📂 Ensuring remote path exists... +=== [Sat May 3 05:00:01 PM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +Password: +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +=== [Sat May 3 05:00:04 PM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Sat May 3 05:00:10 PM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250503_170001.txt +📂 Ensuring remote path exists... +=== [Sat May 3 06:00:01 PM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +=== [Sat May 3 06:00:05 PM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Sat May 3 06:00:12 PM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250503_180001.txt +📂 Ensuring remote path exists... +=== [Sat May 3 07:00:01 PM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +=== [Sat May 3 07:00:02 PM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Sat May 3 07:00:07 PM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250503_190001.txt +📂 Ensuring remote path exists... +=== [Sat May 3 08:00:01 PM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +=== [Sat May 3 08:00:04 PM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Sat May 3 08:00:10 PM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250503_200001.txt +📂 Ensuring remote path exists... +=== [Sat May 3 09:00:01 PM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +Password: +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +=== [Sat May 3 09:00:02 PM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Sat May 3 09:00:05 PM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250503_210001.txt +📂 Ensuring remote path exists... +=== [Sat May 3 10:00:01 PM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +=== [Sat May 3 10:00:02 PM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Sat May 3 10:00:02 PM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250503_220001.txt +📂 Ensuring remote path exists... +=== [Sat May 3 11:00:01 PM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +Password: +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +=== [Sat May 3 11:00:02 PM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Sat May 3 11:00:03 PM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250503_230001.txt +📂 Ensuring remote path exists... +=== [Sun May 4 12:00:01 AM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +=== [Sun May 4 12:00:03 AM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Sun May 4 12:00:05 AM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250504_000001.txt +📂 Ensuring remote path exists... +=== [Sun May 4 01:00:01 AM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +=== [Sun May 4 01:00:02 AM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Sun May 4 01:00:03 AM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250504_010001.txt +📂 Ensuring remote path exists... +=== [Sun May 4 02:00:01 AM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +=== [Sun May 4 02:00:02 AM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Sun May 4 02:00:03 AM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250504_020001.txt +📂 Ensuring remote path exists... +=== [Sun May 4 03:00:01 AM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +=== [Sun May 4 03:00:02 AM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Sun May 4 03:00:04 AM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250504_030001.txt +📂 Ensuring remote path exists... +=== [Sun May 4 04:00:01 AM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +=== [Sun May 4 04:00:02 AM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Sun May 4 04:00:02 AM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250504_040001.txt +📂 Ensuring remote path exists... +=== [Sun May 4 05:00:01 AM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +Password: +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +=== [Sun May 4 05:00:03 AM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Sun May 4 05:00:04 AM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250504_050001.txt +📂 Ensuring remote path exists... +=== [Sun May 4 06:00:01 AM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +=== [Sun May 4 06:00:03 AM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Sun May 4 06:00:05 AM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250504_060001.txt +📂 Ensuring remote path exists... +=== [Sun May 4 07:00:01 AM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +=== [Sun May 4 07:00:01 AM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Sun May 4 07:00:02 AM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250504_070001.txt +📂 Ensuring remote path exists... +=== [Sun May 4 08:00:01 AM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +=== [Sun May 4 08:00:01 AM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Sun May 4 08:00:04 AM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250504_080001.txt +=== [Sun May 4 09:00:01 AM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +📂 Ensuring remote path exists... +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +curl: (28) Failed to connect to chatwithus.live port 443 after 132975 ms: Couldn't connect to server +=== [Sun May 4 09:02:16 AM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +curl: (28) Failed to connect to chatwithus.live port 443 after 133951 ms: Couldn't connect to server +=== [Sun May 4 09:04:31 AM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +curl: (28) Failed to connect to chatwithus.live port 443 after 134983 ms: Couldn't connect to server +Not all backups verified! Nothing uploaded. +curl: (28) Failed to connect to chatwithus.live port 443 after 135145 ms: Couldn't connect to server +DONE. Log: /tmp/pgbackup_verify/verify_log_20250504_090001.txt +📂 Ensuring remote path exists... +=== [Sun May 4 10:00:01 AM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +=== [Sun May 4 10:00:03 AM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Sun May 4 10:00:03 AM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250504_100001.txt +📂 Ensuring remote path exists... +=== [Sun May 4 11:00:01 AM EDT 2025] Dumping mastodon_production from zcluster.technodrome2.sshjunkie.com === +Password: +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +@ WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! @ +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ +IT IS POSSIBLE THAT SOMEONE IS DOING SOMETHING NASTY! +Someone could be eavesdropping on you right now (man-in-the-middle attack)! +It is also possible that a host key has just been changed. +The fingerprint for the ED25519 key sent by the remote host is +SHA256:HB1SqCcc/yQnct/gUYBuuI6xVfvPMKVxAw0rcFU9Gy4. +Please contact your system administrator. +Add correct host key in /home/doc/.ssh/known_hosts to get rid of this message. +Offending ECDSA key in /home/doc/.ssh/known_hosts:32 + remove with: + ssh-keygen -f '/home/doc/.ssh/known_hosts' -R 'backup.sshjunkie.com' +Host key for backup.sshjunkie.com has changed and you have requested strict checking. +Host key verification failed. +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump mastodon_production! Skipping upload. +=== [Sun May 4 11:00:04 AM EDT 2025] Dumping hostingtootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump hostingtootdb! Skipping upload. +=== [Sun May 4 11:00:06 AM EDT 2025] Dumping radiotootdb from zcluster.technodrome2.sshjunkie.com === +Password: +pg_dump: error: connection to server at "zcluster.technodrome2.sshjunkie.com" (38.102.127.166), port 5432 failed: fe_sendauth: no password supplied +[FAIL] Failed to dump radiotootdb! Skipping upload. +Not all backups verified! Nothing uploaded. +DONE. Log: /tmp/pgbackup_verify/verify_log_20250504_110001.txt diff --git a/miscellaneous/dbcheck.log b/miscellaneous/dbcheck.log index 52306fe..8230b22 100644 --- a/miscellaneous/dbcheck.log +++ b/miscellaneous/dbcheck.log @@ -2081,3 +2081,1474 @@ Failed to send Mastodon DM (attempt 3): {"error":"The access token is invalid"} /home/doc/dbcheck/bin/python: can't open file '/home/doc/genesis-tools/miscellaneous/python/dbcheck1.py': [Errno 2] No such file or directory /home/doc/dbcheck/bin/python: can't open file '/home/doc/genesis-tools/miscellaneous/python/dbcheck1.py': [Errno 2] No such file or directory /home/doc/dbcheck/bin/python: can't open file '/home/doc/genesis-tools/miscellaneous/python/dbcheck1.py': [Errno 2] No such file or directory +⚠️ Genesis Radio Warning Healthcheck 2025-05-02 11:00:10 ⚠️ +⚡ 2 warnings found: +- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog +- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog +⚠️ Genesis Radio Warning Healthcheck 2025-05-02 11:15:10 ⚠️ +⚡ 2 warnings found: +- ⚠️ [mastodon] ERROR: Could not read log /var/log/syslog: Authentication failed. +- ⚠️ [mastodon] ERROR: Could not read log /var/log/nginx/error.log: Authentication failed. +⚠️ Genesis Radio Warning Healthcheck 2025-05-02 11:30:11 ⚠️ +⚡ 2 warnings found: +- ⚠️ [mastodon] ERROR: Could not read log /var/log/syslog: Authentication failed. +- ⚠️ [mastodon] ERROR: Could not read log /var/log/nginx/error.log: Authentication failed. +⚠️ Genesis Radio Warning Healthcheck 2025-05-02 11:45:09 ⚠️ +⚡ 2 warnings found: +- ⚠️ [mastodon] ERROR: Could not read log /var/log/syslog: Authentication failed. +- ⚠️ [mastodon] ERROR: Could not read log /var/log/nginx/error.log: Authentication failed. +⚠️ Genesis Radio Warning Healthcheck 2025-05-02 12:00:10 ⚠️ +⚡ 2 warnings found: +- ⚠️ [mastodon] ERROR: Could not read log /var/log/syslog: Authentication failed. +- ⚠️ [mastodon] ERROR: Could not read log /var/log/nginx/error.log: Authentication failed. +⚠️ Genesis Radio Warning Healthcheck 2025-05-02 12:15:10 ⚠️ +⚡ 2 warnings found: +- ⚠️ [mastodon] ERROR: Could not read log /var/log/syslog: Authentication failed. +- ⚠️ [mastodon] ERROR: Could not read log /var/log/nginx/error.log: Authentication failed. +⚠️ Genesis Radio Warning Healthcheck 2025-05-02 12:30:08 ⚠️ +⚡ 2 warnings found: +- ⚠️ [mastodon] ERROR: Could not read log /var/log/syslog: Authentication failed. +- ⚠️ [mastodon] ERROR: Could not read log /var/log/nginx/error.log: Authentication failed. +⚠️ Genesis Radio Warning Healthcheck 2025-05-02 12:45:10 ⚠️ +⚡ 2 warnings found: +- ⚠️ [mastodon] ERROR: Could not read log /var/log/syslog: Authentication failed. +- ⚠️ [mastodon] ERROR: Could not read log /var/log/nginx/error.log: Authentication failed. +⚠️ Genesis Radio Warning Healthcheck 2025-05-02 13:00:08 ⚠️ +⚡ 2 warnings found: +- ⚠️ [mastodon] ERROR: Could not read log /var/log/syslog: Authentication failed. +- ⚠️ [mastodon] ERROR: Could not read log /var/log/nginx/error.log: Authentication failed. +⚠️ Genesis Radio Warning Healthcheck 2025-05-02 13:15:09 ⚠️ +⚡ 2 warnings found: +- ⚠️ [mastodon] ERROR: Could not read log /var/log/syslog: Authentication failed. +- ⚠️ [mastodon] ERROR: Could not read log /var/log/nginx/error.log: Authentication failed. +⚠️ Genesis Radio Warning Healthcheck 2025-05-02 13:30:09 ⚠️ +⚡ 2 warnings found: +- ⚠️ [mastodon] ERROR: Could not read log /var/log/syslog: Authentication failed. +- ⚠️ [mastodon] ERROR: Could not read log /var/log/nginx/error.log: Authentication failed. +⚠️ Genesis Radio Warning Healthcheck 2025-05-02 13:45:09 ⚠️ +⚡ 2 warnings found: +- ⚠️ [mastodon] ERROR: Could not read log /var/log/syslog: Authentication failed. +- ⚠️ [mastodon] ERROR: Could not read log /var/log/nginx/error.log: Authentication failed. +⚠️ Genesis Radio Warning Healthcheck 2025-05-02 14:00:09 ⚠️ +⚡ 2 warnings found: +- ⚠️ [mastodon] ERROR: Could not read log /var/log/syslog: Authentication failed. +- ⚠️ [mastodon] ERROR: Could not read log /var/log/nginx/error.log: Authentication failed. +⚠️ Genesis Radio Warning Healthcheck 2025-05-02 14:15:09 ⚠️ +⚡ 2 warnings found: +- ⚠️ [mastodon] ERROR: Could not read log /var/log/syslog: Authentication failed. +- ⚠️ [mastodon] ERROR: Could not read log /var/log/nginx/error.log: Authentication failed. +⚠️ Genesis Radio Warning Healthcheck 2025-05-02 14:30:08 ⚠️ +⚡ 2 warnings found: +- ⚠️ [mastodon] ERROR: Could not read log /var/log/syslog: Authentication failed. +- ⚠️ [mastodon] ERROR: Could not read log /var/log/nginx/error.log: Authentication failed. +⚠️ Genesis Radio Warning Healthcheck 2025-05-02 14:45:09 ⚠️ +⚡ 1 warnings found: +- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog +✅ Genesis Radio Healthcheck 2025-05-02 15:00:09: All systems normal. +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] CRITICAL: Service minio.service not running! (pgrep 'minio' found nothing) +[shredder] WARNING: Pattern 'FATAL' found in /var/log/syslog +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] WARNING: Pattern 'FATAL' found in /var/log/syslog +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] WARNING: Pattern 'FATAL' found in /var/log/syslog +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] WARNING: Pattern 'FATAL' found in /var/log/syslog +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] WARNING: Pattern 'FATAL' found in /var/log/syslog +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] WARNING: Pattern 'FATAL' found in /var/log/syslog +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] WARNING: Pattern 'FATAL' found in /var/log/syslog +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] WARNING: Pattern 'FATAL' found in /var/log/syslog +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] WARNING: Pattern 'FATAL' found in /var/log/syslog +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] WARNING: Pattern 'FATAL' found in /var/log/syslog +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] WARNING: Pattern 'FATAL' found in /var/log/syslog +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] WARNING: Pattern 'FATAL' found in /var/log/syslog +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] WARNING: Pattern 'FATAL' found in /var/log/syslog +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] WARNING: Pattern 'FATAL' found in /var/log/syslog +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] WARNING: Pattern 'FATAL' found in /var/log/syslog +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] WARNING: Pattern 'FATAL' found in /var/log/syslog +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] WARNING: Pattern 'FATAL' found in /var/log/syslog +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] WARNING: Pattern 'FATAL' found in /var/log/syslog +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] WARNING: Pattern 'FATAL' found in /var/log/syslog +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] WARNING: Pattern 'FATAL' found in /var/log/syslog +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] WARNING: Pattern 'FATAL' found in /var/log/syslog +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] WARNING: Pattern 'FATAL' found in /var/log/syslog +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] WARNING: Pattern 'FATAL' found in /var/log/syslog +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] WARNING: Pattern 'FATAL' found in /var/log/syslog +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] WARNING: Pattern 'FATAL' found in /var/log/syslog +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] WARNING: Pattern 'FATAL' found in /var/log/syslog +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] WARNING: Pattern 'FATAL' found in /var/log/syslog +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] WARNING: Pattern 'FATAL' found in /var/log/syslog +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] WARNING: Pattern 'FATAL' found in /var/log/syslog +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] WARNING: Pattern 'FATAL' found in /var/log/syslog +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] WARNING: Pattern 'FATAL' found in /var/log/syslog +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] WARNING: Pattern 'FATAL' found in /var/log/syslog +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] WARNING: Pattern 'FATAL' found in /var/log/syslog +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] WARNING: Pattern 'FATAL' found in /var/log/syslog +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] WARNING: Pattern 'FATAL' found in /var/log/syslog +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] WARNING: Pattern 'FATAL' found in /var/log/syslog +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] WARNING: Pattern 'FATAL' found in /var/log/syslog +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] WARNING: Pattern 'FATAL' found in /var/log/syslog +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] WARNING: Pattern 'FATAL' found in /var/log/syslog +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] WARNING: Pattern 'FATAL' found in /var/log/syslog +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] WARNING: Pattern 'FATAL' found in /var/log/syslog +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] CRITICAL: Service minio.service not running! (pgrep 'minio' found nothing) +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] CRITICAL: Service minio.service not running! (pgrep 'minio' found nothing) +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] CRITICAL: Service minio.service not running! (pgrep 'minio' found nothing) +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: timed out' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: [Errno None] Unable to connect to port 22 on 38.102.127.172' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Exception (client): Error reading SSH protocol banner +Traceback (most recent call last): + File "/home/doc/dbcheck/lib/python3.12/site-packages/paramiko/transport.py", line 2369, in _check_banner + buf = self.packetizer.readline(timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/doc/dbcheck/lib/python3.12/site-packages/paramiko/packet.py", line 395, in readline + buf += self._read_timeout(timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/doc/dbcheck/lib/python3.12/site-packages/paramiko/packet.py", line 665, in _read_timeout + raise EOFError() +EOFError + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/home/doc/dbcheck/lib/python3.12/site-packages/paramiko/transport.py", line 2185, in run + self._check_banner() + File "/home/doc/dbcheck/lib/python3.12/site-packages/paramiko/transport.py", line 2373, in _check_banner + raise SSHException( +paramiko.ssh_exception.SSHException: Error reading SSH protocol banner + +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +[db1] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Error reading SSH protocol banner' +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Exception (client): Error reading SSH protocol banner +Traceback (most recent call last): + File "/home/doc/dbcheck/lib/python3.12/site-packages/paramiko/transport.py", line 2369, in _check_banner + buf = self.packetizer.readline(timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/doc/dbcheck/lib/python3.12/site-packages/paramiko/packet.py", line 395, in readline + buf += self._read_timeout(timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/doc/dbcheck/lib/python3.12/site-packages/paramiko/packet.py", line 665, in _read_timeout + raise EOFError() +EOFError + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/home/doc/dbcheck/lib/python3.12/site-packages/paramiko/transport.py", line 2185, in run + self._check_banner() + File "/home/doc/dbcheck/lib/python3.12/site-packages/paramiko/transport.py", line 2373, in _check_banner + raise SSHException( +paramiko.ssh_exception.SSHException: Error reading SSH protocol banner + +Exception (client): Error reading SSH protocol banner +Traceback (most recent call last): + File "/home/doc/dbcheck/lib/python3.12/site-packages/paramiko/transport.py", line 2369, in _check_banner + buf = self.packetizer.readline(timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/doc/dbcheck/lib/python3.12/site-packages/paramiko/packet.py", line 395, in readline + buf += self._read_timeout(timeout) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/home/doc/dbcheck/lib/python3.12/site-packages/paramiko/packet.py", line 665, in _read_timeout + raise EOFError() +EOFError + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/home/doc/dbcheck/lib/python3.12/site-packages/paramiko/transport.py", line 2185, in run + self._check_banner() + File "/home/doc/dbcheck/lib/python3.12/site-packages/paramiko/transport.py", line 2373, in _check_banner + raise SSHException( +paramiko.ssh_exception.SSHException: Error reading SSH protocol banner + +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Error reading SSH protocol banner' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] WARNING: Pattern 'ERROR' found in /var/log/syslog +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: timed out' +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Error sending Mastodon DM: HTTPSConnectionPool(host='chatwithus.live', port=443): Max retries exceeded with url: /api/v1/statuses (Caused by ConnectTimeoutError(, 'Connection to chatwithus.live timed out. (connect timeout=None)')) +Error sending Mastodon DM: HTTPSConnectionPool(host='chatwithus.live', port=443): Max retries exceeded with url: /api/v1/statuses (Caused by ConnectTimeoutError(, 'Connection to chatwithus.live timed out. (connect timeout=None)')) +Error sending Mastodon DM: HTTPSConnectionPool(host='chatwithus.live', port=443): Max retries exceeded with url: /api/v1/statuses (Caused by ConnectTimeoutError(, 'Connection to chatwithus.live timed out. (connect timeout=None)')) +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: timed out' +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Error sending Mastodon DM: HTTPSConnectionPool(host='chatwithus.live', port=443): Max retries exceeded with url: /api/v1/statuses (Caused by ConnectTimeoutError(, 'Connection to chatwithus.live timed out. (connect timeout=None)')) +Error sending Mastodon DM: HTTPSConnectionPool(host='chatwithus.live', port=443): Max retries exceeded with url: /api/v1/statuses (Caused by ConnectTimeoutError(, 'Connection to chatwithus.live timed out. (connect timeout=None)')) +Error sending Mastodon DM: HTTPSConnectionPool(host='chatwithus.live', port=443): Max retries exceeded with url: /api/v1/statuses (Caused by ConnectTimeoutError(, 'Connection to chatwithus.live timed out. (connect timeout=None)')) +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: timed out' +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Error sending Mastodon DM: HTTPSConnectionPool(host='chatwithus.live', port=443): Max retries exceeded with url: /api/v1/statuses (Caused by ConnectTimeoutError(, 'Connection to chatwithus.live timed out. (connect timeout=None)')) +Error sending Mastodon DM: HTTPSConnectionPool(host='chatwithus.live', port=443): Max retries exceeded with url: /api/v1/statuses (Caused by ConnectTimeoutError(, 'Connection to chatwithus.live timed out. (connect timeout=None)')) +Error sending Mastodon DM: HTTPSConnectionPool(host='chatwithus.live', port=443): Max retries exceeded with url: /api/v1/statuses (Caused by ConnectTimeoutError(, 'Connection to chatwithus.live timed out. (connect timeout=None)')) +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: timed out' +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Error sending Mastodon DM: HTTPSConnectionPool(host='chatwithus.live', port=443): Max retries exceeded with url: /api/v1/statuses (Caused by ConnectTimeoutError(, 'Connection to chatwithus.live timed out. (connect timeout=None)')) +Failed to send Mastodon DM (attempt 2): + + + + +This page is not correct + - Mastodon Glitch Edition + + + + + + +
+
+Mastodon Glitch Edition +
+
+

We're sorry, but something went wrong on our end. +

+
+
+ + + +Failed to send Mastodon DM (attempt 3): + + + + +This page is not correct + - Mastodon Glitch Edition + + + + + + +
+
+Mastodon Glitch Edition +
+
+

We're sorry, but something went wrong on our end. +

+
+
+ + + +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: mastodon @ chatwithus.live +[mastodon] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: timed out' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: [Errno None] Unable to connect to port 22 on 38.102.127.172' +Checking node: mastodon @ chatwithus.live +[mastodon] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com +Genesis Healthcheck started... +Checking node: shredder @ 38.102.127.172 +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: [Errno None] Unable to connect to port 22 on 38.102.127.172' +[shredder] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: [Errno None] Unable to connect to port 22 on 38.102.127.172' +Checking node: mastodon @ chatwithus.live +[mastodon] ERROR: Disk check failed: invalid literal for int() with base 10: 'SSH error: Authentication failed.' +Checking node: db1 @ zcluster.technodrome1.sshjunkie.com +Checking node: db2 @ zcluster.technodrome2.sshjunkie.com diff --git a/miscellaneous/python/dbcheck4.py b/miscellaneous/python/dbcheck4.py new file mode 100644 index 0000000..0af2276 --- /dev/null +++ b/miscellaneous/python/dbcheck4.py @@ -0,0 +1,165 @@ +import subprocess +import os +import requests +import datetime +import paramiko +import time +import psycopg2 + +# ==== CONFIG ==== +MASTODON_INSTANCE = "https://chatwithus.live" +MASTODON_TOKEN = "rimxBLi-eaJAcwagkmoj6UoW7Lc473tQY0cOM041Euw" +MASTODON_USER_ID = "114386383616633367" +HEALTHCHECK_HTML = "/var/www/html/healthcheck.html" +ARCHIVE_DIR = "/home/doc/genesis-tools/archive.html" + +DISK_WARN_THRESHOLD = 10 +LOG_FILES = ["/var/log/syslog", "/var/log/nginx/error.log"] +LOG_PATTERNS = ["ERROR", "FATAL", "disk full", "out of memory"] +SUPPRESSED_PATTERNS = ["SomeKnownHarmlessMastodonError"] + +NODES = [ + {"name": "shredder", "host": "38.102.127.172", "ssh_user": "doc", "services": ["minio.service"], "disks": ["/", "/mnt/miniodata"], "db": False, "raid": True}, + {"name": "mastodon", "host": "chatwithus.live", "ssh_user": "root", "services": ["nginx", "mastodon-web"], "disks": ["/"], "db": False, "raid": False}, + {"name": "db1", "host": "zcluster.technodrome1.sshjunkie.com", "ssh_user": "doc", "services": ["postgresql@16-main.service"], "disks": ["/", "/var/lib/postgresql"], "db": True, "raid": True}, + {"name": "db2", "host": "zcluster.technodrome2.sshjunkie.com", "ssh_user": "doc", "services": ["postgresql@16-main.service"], "disks": ["/", "/var/lib/postgresql"], "db": True, "raid": True} +] + +# ==== Mastodon DM ==== +def mastodon_dm(message, retries=3): + url = f"{MASTODON_INSTANCE}/api/v1/statuses" + headers = {"Authorization": f"Bearer {MASTODON_TOKEN}"} + payload = {"status": message, "visibility": "direct", "in_reply_to_account_id": MASTODON_USER_ID} + for attempt in range(retries): + try: + resp = requests.post(url, headers=headers, data=payload) + if resp.status_code == 200: + return + print(f"Failed to send Mastodon DM (attempt {attempt+1}): {resp.text}") + except Exception as e: + print(f"Error sending Mastodon DM: {e}") + time.sleep(5) + +# ==== SSH Helper ==== +def ssh_command(host, user, cmd): + try: + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect(hostname=host, username=user, timeout=10) + stdin, stdout, stderr = ssh.exec_command(cmd) + out = stdout.read().decode().strip() + err = stderr.read().decode().strip() + if "Authentication failed" in out or "permission denied" in out.lower() or "permission denied" in err.lower(): + stdin, stdout, stderr = ssh.exec_command(f"sudo {cmd}") + out = stdout.read().decode().strip() + ssh.close() + return out + except Exception as e: + return f"SSH error: {e}" + +SERVICE_PROCESS_MAP = { + "minio.service": "minio", + "postgresql@16-main.service": "postgres", + "mastodon-web": "puma" +} + +def check_remote_service(host, user, service, node_name): + try: + process_name = SERVICE_PROCESS_MAP.get(service, service) + cmd = f"pgrep -f '{process_name}'" + out = ssh_command(host, user, cmd) + if not out.strip(): + return f"[{node_name}] CRITICAL: Service {service} not running! (pgrep '{process_name}' found nothing)" + except Exception as e: + return f"[{node_name}] ERROR: Service check failed: {e}" + return None + +def choose_emoji(line): + if "RAID" in line: + if "disk" in line.lower(): + return "\U0001F4C8" + if "rclone" in line.lower(): + return "\U0001F422" + if "Service" in line: + return "\U0001F6D1" + if "Replication" in line: + return "\U0001F4A5" + return "\u26A0\uFE0F" + +def check_remote_disk(host, user, path, node_name): + try: + cmd = f"df --output=pcent {path} | tail -1 | tr -dc '0-9'" + out = ssh_command(host, user, cmd) + if not out: + return f"[{node_name}] ERROR: Disk {path} not found or could not check disk usage." + percent = int(out) + if percent > (100 - DISK_WARN_THRESHOLD): + return f"[{node_name}] WARNING: Only {100 - percent}% disk free on {path}." + except Exception as e: + return f"[{node_name}] ERROR: Disk check failed: {e}" + return None + +def check_remote_logs(host, user, node_name): + alerts = [] + for log in LOG_FILES: + cmd = f"tail -500 {log}" + try: + out = ssh_command(host, user, cmd) + lines = out.split("\n") + for pattern in LOG_PATTERNS: + if any(pattern in line and not any(s in line for s in SUPPRESSED_PATTERNS) for line in lines): + alerts.append(f"[{node_name}] WARNING: Pattern '{pattern}' found in {log}") + except Exception as e: + alerts.append(f"[{node_name}] ERROR: Could not read log {log}: {e}") + return alerts + +# === MAIN CALL === +if __name__ == "__main__": + print("Genesis Healthcheck started...") + critical_problems = [] + warning_problems = [] + + for node in NODES: + print(f"Checking node: {node['name']} @ {node['host']}") + for disk in node['disks']: + result = check_remote_disk(node['host'], node['ssh_user'], disk, node['name']) + if result: + print(result) + if "CRITICAL" in result: + critical_problems.append(result) + else: + warning_problems.append(result) + + for service in node['services']: + result = check_remote_service(node['host'], node['ssh_user'], service, node['name']) + if result: + print(result) + if "CRITICAL" in result: + critical_problems.append(result) + else: + warning_problems.append(result) + + logs = check_remote_logs(node['host'], node['ssh_user'], node['name']) + for log in logs: + print(log) + warning_problems.append(log) + + now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + if critical_problems: + msg_lines = [ + f"🚨 Genesis Radio Critical Healthcheck {now} 🚨", + f"⚡ {len(critical_problems)} critical issues found:" + ] + msg_lines.extend(f"- {choose_emoji(p)} {p}" for p in critical_problems) + mastodon_dm("\n".join(msg_lines)) + + if warning_problems: + msg_lines = [ + f"⚠️ Genesis Radio Warning Healthcheck {now} ⚠️", + f"⚡ {len(warning_problems)} warnings found:" + ] + msg_lines.extend(f"- {choose_emoji(p)} {p}" for p in warning_problems) + mastodon_dm("\n".join(msg_lines)) + + if not critical_problems and not warning_problems: + mastodon_dm(f"✅ Genesis Radio Healthcheck {now}: All systems normal.") diff --git a/miscellaneous/python/snappysnaps.py b/miscellaneous/python/snappysnaps.py new file mode 100644 index 0000000..949a2a7 --- /dev/null +++ b/miscellaneous/python/snappysnaps.py @@ -0,0 +1,81 @@ +import os +import paramiko +import requests +from datetime import datetime +from collections import defaultdict + +# === CONFIG === +NODES = { + "shredderv2": {"host": "shredderv2.sshjunkie.com", "user": "doc"}, + "thevault": {"host": "209.209.9.128", "user": "doc"}, + "technodrome1": {"host": "38.102.127.165", "user": "doc"}, + "technodrome2": {"host": "38.102.127.166", "user": "doc"}, +} + +SSH_KEY = "~/.ssh/genesis_healthcheck" +ZFS_CMD = "zfs list -t snapshot -o name,creation -s creation" + +# === TELEGRAM CONFIG === +TELEGRAM_BOT_TOKEN="8178867489:AAH0VjN7VnZSCIWasSz_y97iBLLjPJA751k" +TELEGRAM_CHAT_ID="1559582356" + +def send_to_telegram(message): + url = f"https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/sendMessage" + data = { + "chat_id": TELEGRAM_CHAT_ID, + "text": message, + "parse_mode": "Markdown" + } + try: + requests.post(url, data=data, timeout=10) + except Exception as e: + print(f"Telegram send failed: {e}") + +def ssh_run(host, user, cmd): + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect(hostname=host, username=user, key_filename=os.path.expanduser(SSH_KEY), timeout=10) + stdin, stdout, stderr = ssh.exec_command(cmd) + out = stdout.read().decode().strip() + ssh.close() + return out + +def parse_snapshot_output(output): + lines = output.strip().split("\n")[1:] # skip header + snaps = defaultdict(list) + + for line in lines: + try: + name, creation = line.strip().split(None, 1) + dataset, _ = name.split("@") + dt = datetime.strptime(creation.strip(), "%a %b %d %H:%M %Y") + snaps[dataset].append(dt) + except Exception: + continue + return snaps + +def summarize(node_name, dataset_snaps): + if not dataset_snaps: + send_to_telegram(f"⚠️ *No snapshots found* on `{node_name}`") + return + + msg = [f"📦 *ZFS Snapshot Report — {node_name}*"] + for dataset, times in sorted(dataset_snaps.items()): + age = (datetime.now() - max(times)).total_seconds() / 3600 + msg.append( + f"*{dataset}*\n" + f"- Count: {len(times)}\n" + f"- Oldest: {min(times)}\n" + f"- Newest: {max(times)}\n" + f"- Recent age: {age:.2f}h\n" + ) + send_to_telegram("\n".join(msg)) + +if __name__ == "__main__": + for name, node in NODES.items(): + try: + output = ssh_run(node["host"], node["user"], ZFS_CMD) + data = parse_snapshot_output(output) + summarize(name.upper(), data) + except Exception as e: + send_to_telegram(f"❌ Failed to check `{name}`: {e}")