Auto-commit from giteapush.sh at 2025-05-04 11:54:40
This commit is contained in:
parent
30c8caf850
commit
872dc70044
@ -0,0 +1,4 @@
|
|||||||
|
<html><head><title>Genesis Radio Healthcheck</title><meta http-equiv='refresh' content='60'></head><body><h1>Genesis Radio System Health</h1><p>Last Checked: 2025-05-02 14:14:12</p><table border='1' cellpadding='5'><tr><th>System</th><th>Status</th><th>ZFS Details</th></tr><tr><td>shredder</td><td style='color:orange;'>Warning</td><td><pre>[shredder] ERROR: ZFS RAID check failed: Authentication failed.
|
||||||
|
[shredder] ERROR: RAID check failed (ZFS+mdstat): Authentication failed.; Authentication failed.</pre></td></tr><tr><td>mastodon</td><td style='color:orange;'>Warning</td><td><pre>All pools healthy</pre></td></tr><tr><td>db1</td><td style='color:orange;'>Warning</td><td><pre>[db1] ERROR: ZFS RAID check failed: Authentication failed.
|
||||||
|
[db1] ERROR: RAID check failed (ZFS+mdstat): Authentication failed.; Authentication failed.</pre></td></tr><tr><td>db2</td><td style='color:orange;'>Warning</td><td><pre>[db2] ERROR: ZFS RAID check failed: Authentication failed.
|
||||||
|
[db2] ERROR: RAID check failed (ZFS+mdstat): Authentication failed.; Authentication failed.</pre></td></tr></table></body></html>
|
||||||
@ -0,0 +1,4 @@
|
|||||||
|
<html><head><title>Genesis Radio Healthcheck</title><meta http-equiv='refresh' content='60'></head><body><h1>Genesis Radio System Health</h1><p>Last Checked: 2025-05-02 14:15:26</p><table border='1' cellpadding='5'><tr><th>System</th><th>Status</th><th>ZFS Details</th></tr><tr><td>shredder</td><td style='color:orange;'>Warning</td><td><pre>[shredder] ERROR: ZFS RAID check failed: Authentication failed.
|
||||||
|
[shredder] ERROR: RAID check failed (ZFS+mdstat): Authentication failed.; Authentication failed.</pre></td></tr><tr><td>mastodon</td><td style='color:orange;'>Warning</td><td><pre>All pools healthy</pre></td></tr><tr><td>db1</td><td style='color:orange;'>Warning</td><td><pre>[db1] ERROR: ZFS RAID check failed: Authentication failed.
|
||||||
|
[db1] ERROR: RAID check failed (ZFS+mdstat): Authentication failed.; Authentication failed.</pre></td></tr><tr><td>db2</td><td style='color:orange;'>Warning</td><td><pre>[db2] ERROR: ZFS RAID check failed: Authentication failed.
|
||||||
|
[db2] ERROR: RAID check failed (ZFS+mdstat): Authentication failed.; Authentication failed.</pre></td></tr></table></body></html>
|
||||||
@ -0,0 +1,2 @@
|
|||||||
|
<html><head><title>Genesis Radio Healthcheck</title><meta http-equiv='refresh' content='60'></head><body><h1>Genesis Radio System Health</h1><p>Last Checked: 2025-05-02 14:18:37</p><table border='1' cellpadding='5'><tr><th>System</th><th>Status</th><th>ZFS Details</th></tr><tr><td>shredder</td><td style='color:orange;'>Warning</td><td><pre>[shredder] WARNING: ZFS pool 'nexus' is not healthy: pool 'nexus' is healthy</pre></td></tr><tr><td>mastodon</td><td style='color:orange;'>Warning</td><td><pre>All pools healthy</pre></td></tr><tr><td>db1</td><td style='color:orange;'>Warning</td><td><pre>[db1] WARNING: ZFS pool 'pgpool' is not healthy: pool 'pgpool' is healthy</pre></td></tr><tr><td>db2</td><td style='color:red;'>Critical</td><td><pre>[db2] WARNING: ZFS pool 'pgpool' is not healthy: pool 'pgpool' is healthy
|
||||||
|
[db2] WARNING: No snapshots found in ZFS pool 'pgpool'</pre></td></tr></table></body></html>
|
||||||
@ -0,0 +1,2 @@
|
|||||||
|
<html><head><title>Genesis Radio Healthcheck</title><meta http-equiv='refresh' content='60'></head><body><h1>Genesis Radio System Health</h1><p>Last Checked: 2025-05-02 14:19:40</p><table border='1' cellpadding='5'><tr><th>System</th><th>Status</th><th>ZFS Details</th></tr><tr><td>shredder</td><td style='color:orange;'>Warning</td><td><pre>[shredder] WARNING: ZFS pool 'nexus' is not healthy: pool 'nexus' is healthy</pre></td></tr><tr><td>mastodon</td><td style='color:orange;'>Warning</td><td><pre>All pools healthy</pre></td></tr><tr><td>db1</td><td style='color:orange;'>Warning</td><td><pre>[db1] WARNING: ZFS pool 'pgpool' is not healthy: pool 'pgpool' is healthy</pre></td></tr><tr><td>db2</td><td style='color:red;'>Critical</td><td><pre>[db2] WARNING: ZFS pool 'pgpool' is not healthy: pool 'pgpool' is healthy
|
||||||
|
[db2] WARNING: No snapshots found in ZFS pool 'pgpool'</pre></td></tr></table></body></html>
|
||||||
@ -0,0 +1,2 @@
|
|||||||
|
<html><head><title>Genesis Radio Healthcheck</title><meta http-equiv='refresh' content='60'></head><body><h1>Genesis Radio System Health</h1><p>Last Checked: 2025-05-02 14:23:18</p><table border='1' cellpadding='5'><tr><th>System</th><th>Status</th><th>ZFS Details</th></tr><tr><td>shredder</td><td style='color:orange;'>Warning</td><td><pre>[shredder] WARNING: ZFS pool 'nexus' is not healthy: pool 'nexus' is healthy</pre></td></tr><tr><td>mastodon</td><td style='color:orange;'>Warning</td><td><pre>All pools healthy</pre></td></tr><tr><td>db1</td><td style='color:orange;'>Warning</td><td><pre>[db1] WARNING: ZFS pool 'pgpool' is not healthy: pool 'pgpool' is healthy</pre></td></tr><tr><td>db2</td><td style='color:red;'>Critical</td><td><pre>[db2] WARNING: ZFS pool 'pgpool' is not healthy: pool 'pgpool' is healthy
|
||||||
|
[db2] WARNING: No snapshots found in ZFS pool 'pgpool'</pre></td></tr></table></body></html>
|
||||||
@ -0,0 +1,2 @@
|
|||||||
|
<html><head><title>Genesis Radio Healthcheck</title><meta http-equiv='refresh' content='60'></head><body><h1>Genesis Radio System Health</h1><p>Last Checked: 2025-05-02 14:25:55</p><table border='1' cellpadding='5'><tr><th>System</th><th>Status</th><th>ZFS Details</th></tr><tr><td>shredder</td><td style='color:red;'>Critical</td><td><pre>[shredder] WARNING: ZFS pool 'nexus' is not healthy: pool 'nexus' is healthy</pre></td></tr><tr><td>mastodon</td><td style='color:orange;'>Warning</td><td><pre>All pools healthy</pre></td></tr><tr><td>db1</td><td style='color:red;'>Critical</td><td><pre>[db1] WARNING: ZFS pool 'pgpool' is not healthy: pool 'pgpool' is healthy</pre></td></tr><tr><td>db2</td><td style='color:red;'>Critical</td><td><pre>[db2] WARNING: ZFS pool 'pgpool' is not healthy: pool 'pgpool' is healthy
|
||||||
|
[db2] WARNING: No snapshots found in ZFS pool 'pgpool'</pre></td></tr></table></body></html>
|
||||||
@ -0,0 +1,2 @@
|
|||||||
|
<html><head><title>Genesis Radio Healthcheck</title><meta http-equiv='refresh' content='60'></head><body><h1>Genesis Radio System Health</h1><p>Last Checked: 2025-05-02 14:27:51</p><table border='1' cellpadding='5'><tr><th>System</th><th>Status</th><th>ZFS Details</th></tr><tr><td>shredder</td><td style='color:red;'>Critical</td><td><pre>[shredder] WARNING: ZFS pool 'nexus' is not healthy: pool 'nexus' is healthy</pre></td></tr><tr><td>mastodon</td><td style='color:orange;'>Warning</td><td><pre>All pools healthy</pre></td></tr><tr><td>db1</td><td style='color:red;'>Critical</td><td><pre>[db1] WARNING: ZFS pool 'pgpool' is not healthy: pool 'pgpool' is healthy</pre></td></tr><tr><td>db2</td><td style='color:red;'>Critical</td><td><pre>[db2] WARNING: ZFS pool 'pgpool' is not healthy: pool 'pgpool' is healthy
|
||||||
|
[db2] WARNING: No snapshots found in ZFS pool 'pgpool'</pre></td></tr></table></body></html>
|
||||||
@ -0,0 +1,2 @@
|
|||||||
|
<html><head><title>Genesis Radio Healthcheck</title><meta http-equiv='refresh' content='60'></head><body><h1>Genesis Radio System Health</h1><p>Last Checked: 2025-05-02 14:29:37</p><table border='1' cellpadding='5'><tr><th>System</th><th>Status</th><th>ZFS Details</th></tr><tr><td>shredder</td><td style='color:orange;'>Warning</td><td><pre>[shredder] WARNING: ZFS pool 'nexus' is not healthy: pool 'nexus' is healthy</pre></td></tr><tr><td>mastodon</td><td style='color:orange;'>Warning</td><td><pre>All pools healthy</pre></td></tr><tr><td>db1</td><td style='color:orange;'>Warning</td><td><pre>[db1] WARNING: ZFS pool 'pgpool' is not healthy: pool 'pgpool' is healthy</pre></td></tr><tr><td>db2</td><td style='color:orange;'>Warning</td><td><pre>[db2] WARNING: ZFS pool 'pgpool' is not healthy: pool 'pgpool' is healthy
|
||||||
|
[db2] WARNING: No snapshots found in ZFS pool 'pgpool'</pre></td></tr></table></body></html>
|
||||||
@ -0,0 +1,2 @@
|
|||||||
|
<html><head><title>Genesis Radio Healthcheck</title><meta http-equiv='refresh' content='60'></head><body><h1>Genesis Radio System Health</h1><p>Last Checked: 2025-05-02 14:30:53</p><table border='1' cellpadding='5'><tr><th>System</th><th>Status</th><th>ZFS Details</th></tr><tr><td>shredder</td><td style='color:orange;'>Warning</td><td><pre>[shredder] WARNING: ZFS pool 'nexus' is not healthy: pool 'nexus' is healthy</pre></td></tr><tr><td>mastodon</td><td style='color:orange;'>Warning</td><td><pre>All pools healthy</pre></td></tr><tr><td>db1</td><td style='color:orange;'>Warning</td><td><pre>[db1] WARNING: ZFS pool 'pgpool' is not healthy: pool 'pgpool' is healthy</pre></td></tr><tr><td>db2</td><td style='color:orange;'>Warning</td><td><pre>[db2] WARNING: ZFS pool 'pgpool' is not healthy: pool 'pgpool' is healthy
|
||||||
|
[db2] WARNING: No snapshots found in ZFS pool 'pgpool'</pre></td></tr></table></body></html>
|
||||||
@ -0,0 +1 @@
|
|||||||
|
<html><head><title>Genesis Radio Healthcheck</title><meta http-equiv='refresh' content='60'></head><body><h1>Genesis Radio System Health</h1><p>Last Checked: 2025-05-02 14:34:37</p><table border='1' cellpadding='5'><tr><th>System</th><th>Status</th><th>ZFS Details</th></tr><tr><td>shredder</td><td style='color:green;'>Healthy</td><td><pre>All pools healthy</pre></td></tr><tr><td>mastodon</td><td style='color:orange;'>Warning</td><td><pre>All pools healthy</pre></td></tr><tr><td>db1</td><td style='color:green;'>Healthy</td><td><pre>All pools healthy</pre></td></tr><tr><td>db2</td><td style='color:green;'>Healthy</td><td><pre>All pools healthy</pre></td></tr></table></body></html>
|
||||||
@ -0,0 +1 @@
|
|||||||
|
<html><head><title>Genesis Radio Healthcheck</title><meta http-equiv='refresh' content='60'></head><body><h1>Genesis Radio System Health</h1><p>Last Checked: 2025-05-02 14:35:19</p><table border='1' cellpadding='5'><tr><th>System</th><th>Status</th><th>ZFS Details</th></tr><tr><td>shredder</td><td style='color:green;'>Healthy</td><td><pre>All pools healthy</pre></td></tr><tr><td>mastodon</td><td style='color:red;'>Critical</td><td><pre>All pools healthy</pre></td></tr><tr><td>db1</td><td style='color:green;'>Healthy</td><td><pre>All pools healthy</pre></td></tr><tr><td>db2</td><td style='color:green;'>Healthy</td><td><pre>All pools healthy</pre></td></tr></table></body></html>
|
||||||
@ -0,0 +1 @@
|
|||||||
|
<html><head><title>Genesis Radio Healthcheck</title><meta http-equiv='refresh' content='60'></head><body><h1>Genesis Radio System Health</h1><p>Last Checked: 2025-05-02 14:36:34</p><table border='1' cellpadding='5'><tr><th>System</th><th>Status</th><th>ZFS Details</th></tr><tr><td>shredder</td><td style='color:green;'>Healthy</td><td><pre>All pools healthy</pre></td></tr><tr><td>mastodon</td><td style='color:red;'>Critical</td><td><pre>All pools healthy</pre></td></tr><tr><td>db1</td><td style='color:green;'>Healthy</td><td><pre>All pools healthy</pre></td></tr><tr><td>db2</td><td style='color:green;'>Healthy</td><td><pre>All pools healthy</pre></td></tr></table></body></html>
|
||||||
@ -0,0 +1 @@
|
|||||||
|
<html><head><title>Genesis Radio Healthcheck</title><meta http-equiv='refresh' content='60'></head><body><h1>Genesis Radio System Health</h1><p>Last Checked: 2025-05-02 14:39:33</p><table border='1' cellpadding='5'><tr><th>System</th><th>Status</th><th>ZFS Details</th></tr><tr><td>shredder</td><td style='color:green;'>Healthy</td><td><pre>All pools healthy</pre></td></tr><tr><td>mastodon</td><td style='color:red;'>Critical</td><td><pre>All pools healthy</pre></td></tr><tr><td>db1</td><td style='color:green;'>Healthy</td><td><pre>All pools healthy</pre></td></tr><tr><td>db2</td><td style='color:green;'>Healthy</td><td><pre>All pools healthy</pre></td></tr></table></body></html>
|
||||||
@ -0,0 +1 @@
|
|||||||
|
<html><head><title>Genesis Radio Healthcheck</title><meta http-equiv='refresh' content='60'></head><body><h1>Genesis Radio System Health</h1><p>Last Checked: 2025-05-02 14:40:39</p><table border='1' cellpadding='5'><tr><th>System</th><th>Status</th><th>ZFS Details</th></tr><tr><td>shredder</td><td style='color:green;'>Healthy</td><td><pre>All pools healthy</pre></td></tr><tr><td>mastodon</td><td style='color:red;'>Critical</td><td><pre>All pools healthy</pre></td></tr><tr><td>db1</td><td style='color:green;'>Healthy</td><td><pre>All pools healthy</pre></td></tr><tr><td>db2</td><td style='color:green;'>Healthy</td><td><pre>All pools healthy</pre></td></tr></table></body></html>
|
||||||
@ -0,0 +1 @@
|
|||||||
|
<html><head><title>Genesis Radio Healthcheck</title><meta http-equiv='refresh' content='60'></head><body><h1>Genesis Radio System Health</h1><p>Last Checked: 2025-05-02 14:42:35</p><table border='1' cellpadding='5'><tr><th>System</th><th>Status</th><th>ZFS Details</th></tr><tr><td>shredder</td><td style='color:green;'>Healthy</td><td><pre>All pools healthy</pre></td></tr><tr><td>mastodon</td><td style='color:red;'>Critical</td><td><pre>All pools healthy</pre></td></tr><tr><td>db1</td><td style='color:green;'>Healthy</td><td><pre>All pools healthy</pre></td></tr><tr><td>db2</td><td style='color:green;'>Healthy</td><td><pre>All pools healthy</pre></td></tr></table></body></html>
|
||||||
@ -0,0 +1 @@
|
|||||||
|
<html><head><title>Genesis Radio Healthcheck</title><meta http-equiv='refresh' content='60'></head><body><h1>Genesis Radio System Health</h1><p>Last Checked: 2025-05-02 14:43:10</p><table border='1' cellpadding='5'><tr><th>System</th><th>Status</th><th>ZFS Details</th></tr><tr><td>shredder</td><td style='color:green;'>Healthy</td><td><pre>All pools healthy</pre></td></tr><tr><td>mastodon</td><td style='color:red;'>Critical</td><td><pre>All pools healthy</pre></td></tr><tr><td>db1</td><td style='color:green;'>Healthy</td><td><pre>All pools healthy</pre></td></tr><tr><td>db2</td><td style='color:green;'>Healthy</td><td><pre>All pools healthy</pre></td></tr></table></body></html>
|
||||||
@ -0,0 +1 @@
|
|||||||
|
<html><head><title>Genesis Radio Healthcheck</title><meta http-equiv='refresh' content='60'></head><body><h1>Genesis Radio System Health</h1><p>Last Checked: 2025-05-02 14:43:24</p><table border='1' cellpadding='5'><tr><th>System</th><th>Status</th><th>ZFS Details</th></tr><tr><td>shredder</td><td style='color:green;'>Healthy</td><td><pre>All pools healthy</pre></td></tr><tr><td>mastodon</td><td style='color:red;'>Critical</td><td><pre>All pools healthy</pre></td></tr><tr><td>db1</td><td style='color:green;'>Healthy</td><td><pre>All pools healthy</pre></td></tr><tr><td>db2</td><td style='color:green;'>Healthy</td><td><pre>All pools healthy</pre></td></tr></table></body></html>
|
||||||
0
archive.html/cleanup.log
Normal file
0
archive.html/cleanup.log
Normal file
@ -4,7 +4,7 @@ from mastodon import Mastodon
|
|||||||
|
|
||||||
# === Configuration ===
|
# === Configuration ===
|
||||||
|
|
||||||
ROOT_DIR = r"/mnt/genesisarchives-secure" # Update this path to where your shows live
|
ROOT_DIR = r"/mnt/archives" # Update this path to where your shows live
|
||||||
ALLOWED_EXTENSIONS = {".mp3", ".wav", ".flac", ".m4a"}
|
ALLOWED_EXTENSIONS = {".mp3", ".wav", ".flac", ".m4a"}
|
||||||
BANNER_FILENAMES = ["banner.jpg", "banner.png", "banner.jpeg"]
|
BANNER_FILENAMES = ["banner.jpg", "banner.png", "banner.jpeg"]
|
||||||
|
|
||||||
|
|||||||
45
genesishostingmd/backups/dr/assets-mastodon-bucket.md
Normal file
45
genesishostingmd/backups/dr/assets-mastodon-bucket.md
Normal file
@ -0,0 +1,45 @@
|
|||||||
|
## 2025-05-02 22:24:25 – MinIO Bucket Access Configuration for Mastodon
|
||||||
|
|
||||||
|
**Bucket**: `assets-mastodon`
|
||||||
|
**Server**: `shredderv2`
|
||||||
|
**User**: `genesisuser`
|
||||||
|
**Permissions**: Read / Write / Delete
|
||||||
|
**Policy Name**: `assets-mastodon-rw-policy`
|
||||||
|
|
||||||
|
### Commands Executed:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
mc alias set localminio http://localhost:9000 genesisadmin MutationXv3!
|
||||||
|
|
||||||
|
cat > assets_mastodon_rw_policy.json <<EOF
|
||||||
|
{
|
||||||
|
"Version": "2012-10-17",
|
||||||
|
"Statement": [
|
||||||
|
{
|
||||||
|
"Action": [
|
||||||
|
"s3:GetBucketLocation",
|
||||||
|
"s3:ListBucket"
|
||||||
|
],
|
||||||
|
"Effect": "Allow",
|
||||||
|
"Resource": "arn:aws:s3:::assets-mastodon"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"Action": [
|
||||||
|
"s3:PutObject",
|
||||||
|
"s3:GetObject",
|
||||||
|
"s3:DeleteObject"
|
||||||
|
],
|
||||||
|
"Effect": "Allow",
|
||||||
|
"Resource": "arn:aws:s3:::assets-mastodon/*"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
EOF
|
||||||
|
|
||||||
|
mc admin policy add localminio assets-mastodon-rw-policy assets_mastodon_rw_policy.json
|
||||||
|
mc admin policy set localminio assets-mastodon-rw-policy user=genesisuser
|
||||||
|
```
|
||||||
|
|
||||||
|
### Outcome:
|
||||||
|
|
||||||
|
User `genesisuser` now has full authenticated access to `assets-mastodon` on `shredderv2`'s MinIO.
|
||||||
93
genesishostingmd/backups/dr/assets_azuracast.md
Normal file
93
genesishostingmd/backups/dr/assets_azuracast.md
Normal file
@ -0,0 +1,93 @@
|
|||||||
|
## 2025-05-02 22:24:25 – MinIO Bucket Access Configuration for Mastodon
|
||||||
|
|
||||||
|
**Bucket**: `assets-mastodon`
|
||||||
|
**Server**: `shredderv2`
|
||||||
|
**User**: `genesisuser`
|
||||||
|
**Permissions**: Read / Write / Delete
|
||||||
|
**Policy Name**: `assets-mastodon-rw-policy`
|
||||||
|
|
||||||
|
### Commands Executed:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
mc alias set localminio http://localhost:9000 genesisadmin MutationXv3!
|
||||||
|
|
||||||
|
cat > assets_mastodon_rw_policy.json <<EOF
|
||||||
|
{
|
||||||
|
"Version": "2012-10-17",
|
||||||
|
"Statement": [
|
||||||
|
{
|
||||||
|
"Action": [
|
||||||
|
"s3:GetBucketLocation",
|
||||||
|
"s3:ListBucket"
|
||||||
|
],
|
||||||
|
"Effect": "Allow",
|
||||||
|
"Resource": "arn:aws:s3:::assets-mastodon"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"Action": [
|
||||||
|
"s3:PutObject",
|
||||||
|
"s3:GetObject",
|
||||||
|
"s3:DeleteObject"
|
||||||
|
],
|
||||||
|
"Effect": "Allow",
|
||||||
|
"Resource": "arn:aws:s3:::assets-mastodon/*"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
EOF
|
||||||
|
|
||||||
|
mc admin policy add localminio assets-mastodon-rw-policy assets_mastodon_rw_policy.json
|
||||||
|
mc admin policy set localminio assets-mastodon-rw-policy user=genesisuser
|
||||||
|
```
|
||||||
|
|
||||||
|
### Outcome:
|
||||||
|
|
||||||
|
User `genesisuser` now has full authenticated access to `assets-mastodon` on `shredderv2`'s MinIO.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2025-05-02 22:43:00 – MinIO Transfer Log: AzuraCast Assets
|
||||||
|
|
||||||
|
**Source**: `thevault:/nexus/miniodata/assets_azuracast`
|
||||||
|
**Destination**: `shredderv2 MinIO` bucket `assets-azuracast`
|
||||||
|
|
||||||
|
### Transfer Method:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
rclone sync thevault:/nexus/miniodata/assets_azuracast localminio:assets-azuracast \
|
||||||
|
--progress \
|
||||||
|
--transfers=8 \
|
||||||
|
--checkers=8 \
|
||||||
|
--s3-chunk-size=64M \
|
||||||
|
--s3-upload-concurrency=4 \
|
||||||
|
--s3-acl=private \
|
||||||
|
--s3-storage-class=STANDARD
|
||||||
|
```
|
||||||
|
|
||||||
|
### Outcome:
|
||||||
|
|
||||||
|
Data from AzuraCast backup (`assets_azuracast`) successfully synchronized to MinIO bucket `assets-azuracast` on `shredderv2`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2025-05-02 23:05:00 – MinIO Transfer Log: Mastodon Assets
|
||||||
|
|
||||||
|
**Source**: `thevault:/nexus/miniodata/assets_mastodon`
|
||||||
|
**Destination**: `shredderv2 MinIO` bucket `assets-mastodon`
|
||||||
|
|
||||||
|
### Transfer Method:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
rclone sync thevault:/nexus/miniodata/assets_mastodon localminio:assets-mastodon \
|
||||||
|
--progress \
|
||||||
|
--transfers=8 \
|
||||||
|
--checkers=8 \
|
||||||
|
--s3-chunk-size=64M \
|
||||||
|
--s3-upload-concurrency=4 \
|
||||||
|
--s3-acl=private \
|
||||||
|
--s3-storage-class=STANDARD
|
||||||
|
```
|
||||||
|
|
||||||
|
### Outcome:
|
||||||
|
|
||||||
|
Assets from `assets_mastodon` replicated to `assets-mastodon` bucket on `shredderv2`. No impact to production (`shredderv1`) during sync.
|
||||||
64
genesishostingmd/disrec/zfsdestroycasestudy.md
Normal file
64
genesishostingmd/disrec/zfsdestroycasestudy.md
Normal file
@ -0,0 +1,64 @@
|
|||||||
|
# 📛 Case Study: Why RAID Is Not a Backup
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
On May 4, 2025, we experienced a production data loss incident involving the `nexus` dataset on `shredderv1`, a Linux RAID5 server. Despite no hardware failure, critical files were lost due to an unintended command affecting live data.
|
||||||
|
|
||||||
|
This incident serves as a clear, real-world illustration of the maxim:
|
||||||
|
|
||||||
|
> **RAID protects against hardware failure — not human error, data corruption, or bad automation.**
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🔍 What Happened
|
||||||
|
|
||||||
|
- `shredderv1` uses RAID5 for media storage.
|
||||||
|
- The dataset `nexus/miniodata` (housing `genesisassets`, `genesislibrary`, etc.) was accidentally destroyed.
|
||||||
|
- **No disks failed.** The failure was logical, not physical.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🔥 Impact
|
||||||
|
|
||||||
|
- StationPlaylist (SPL) lost access to the Genesis media library.
|
||||||
|
- MinIO bucket data was instantly inaccessible.
|
||||||
|
- Temporary outage and scrambling to reconfigure mounts, media, and streaming.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## ✅ Recovery
|
||||||
|
|
||||||
|
Thanks to our disaster recovery stack:
|
||||||
|
|
||||||
|
- Nightly **rsync backups** were synced to **The Vault** (backup server).
|
||||||
|
- **ZFS snapshots** existed on The Vault for the affected datasets.
|
||||||
|
- We restored the latest snapshot **from The Vault back to Shredder**, effectively reversing the loss.
|
||||||
|
- No data corruption occurred; sync validation showed dataset integrity.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🎓 Takeaway
|
||||||
|
|
||||||
|
This is a live demonstration of why:
|
||||||
|
|
||||||
|
- **RAID is not a backup**
|
||||||
|
- **Snapshots without off-host replication** are not enough
|
||||||
|
- **Real backups must be off-server and regularly tested**
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🔐 Current Protection Measures
|
||||||
|
|
||||||
|
- Production data (`genesisassets`, `genesislibrary`) now replicated nightly to The Vault via `rsync`.
|
||||||
|
- ZFS snapshots are validated daily via a **dry-run restore validator**.
|
||||||
|
- Telegram alerts notify success/failure of backup verification jobs.
|
||||||
|
- Future goal: full ZFS storage on all production servers for native snapshot support.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🧠 Lessons Learned
|
||||||
|
|
||||||
|
- Always assume you'll delete the wrong thing eventually.
|
||||||
|
- Snapshots are amazing — **if** they're somewhere else.
|
||||||
|
- Automated restore testing should be part of every backup pipeline.
|
||||||
|
|
||||||
@ -39,3 +39,25 @@ All services restarted.
|
|||||||
[✓] All tasks complete. Logged out of root@chatwithus.live.
|
[✓] All tasks complete. Logged out of root@chatwithus.live.
|
||||||
[2025-05-01 08:03:47] === Mastodon Restart Complete ===
|
[2025-05-01 08:03:47] === Mastodon Restart Complete ===
|
||||||
|
|
||||||
|
[2025-05-03 20:31:04] === Mastodon Restart Initiated ===
|
||||||
|
[*] Posting 2-minute warning to Mastodon...
|
||||||
|
[✓] 2-minute warning posted.
|
||||||
|
[*] Posting 1-minute warning to Mastodon...
|
||||||
|
[✓] 1-minute warning posted.
|
||||||
|
[*] Connecting to root@chatwithus.live to restart Mastodon services...
|
||||||
|
Restarting mastodon-web...
|
||||||
|
Warning: The unit file, source configuration file or drop-ins of mastodon-web.service changed on disk. Run 'systemctl daemon-reload' to reload units.
|
||||||
|
Restarting mastodon-sidekiq...
|
||||||
|
Warning: The unit file, source configuration file or drop-ins of mastodon-sidekiq.service changed on disk. Run 'systemctl daemon-reload' to reload units.
|
||||||
|
Restarting mastodon-streaming...
|
||||||
|
Warning: The unit file, source configuration file or drop-ins of mastodon-streaming.service changed on disk. Run 'systemctl daemon-reload' to reload units.
|
||||||
|
All services restarted.
|
||||||
|
[*] Waiting for Mastodon to come back online...
|
||||||
|
[+] Mastodon is back online.
|
||||||
|
[*] Posting final status to Mastodon...
|
||||||
|
[✓] Final status posted.
|
||||||
|
[*] Sending Telegram alert...
|
||||||
|
[✓] Telegram alert sent.
|
||||||
|
[✓] All tasks complete. Logged out of root@chatwithus.live.
|
||||||
|
[2025-05-03 20:31:04] === Mastodon Restart Complete ===
|
||||||
|
|
||||||
|
|||||||
48
miscellaneous/bash/sync-trigger.sh
Executable file
48
miscellaneous/bash/sync-trigger.sh
Executable file
@ -0,0 +1,48 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
# === CONFIG ===
|
||||||
|
REMOTE_HOST="shredder.sshjunkie.com"
|
||||||
|
REMOTE_USER="doc"
|
||||||
|
REMOTE_SCRIPT="/home/doc/sync.sh"
|
||||||
|
LOG_TAG="[Krang → SPL Sync]"
|
||||||
|
|
||||||
|
# === Mastodon Alert Settings ===
|
||||||
|
MASTODON_INSTANCE="https://chatwithus.live"
|
||||||
|
ACCESS_TOKEN="07w3Emdw-cv_TncysrNU8Ed_sHJhwtnvKmnLqKlHmKA"
|
||||||
|
TOOT_VISIBILITY="unlisted"
|
||||||
|
|
||||||
|
# === Telegram Settings ===
|
||||||
|
TELEGRAM_BOT_TOKEN="8178867489:AAH0VjN7VnZSCIWasSz_y97iBLLjPJA751k"
|
||||||
|
TELEGRAM_CHAT_ID="1559582356"
|
||||||
|
|
||||||
|
# === Execution ===
|
||||||
|
echo "$LOG_TAG Triggering remote sync..."
|
||||||
|
OUTPUT=$(ssh ${REMOTE_USER}@${REMOTE_HOST} "${REMOTE_SCRIPT}" 2>&1)
|
||||||
|
|
||||||
|
if echo "$OUTPUT" | grep -q "All syncs finished"; then
|
||||||
|
echo "$LOG_TAG ✅ Sync complete."
|
||||||
|
|
||||||
|
# Mastodon alert
|
||||||
|
curl -s -X POST "$MASTODON_INSTANCE/api/v1/statuses" \
|
||||||
|
-H "Authorization: Bearer $ACCESS_TOKEN" \
|
||||||
|
-d "status=✅ SPL Sync completed successfully via Krang" \
|
||||||
|
-d "visibility=$TOOT_VISIBILITY" >/dev/null
|
||||||
|
|
||||||
|
# Telegram alert
|
||||||
|
curl -s -X POST "https://api.telegram.org/bot$TELEGRAM_BOT_TOKEN/sendMessage" \
|
||||||
|
-d "chat_id=$TELEGRAM_CHAT_ID" \
|
||||||
|
-d "text=✅ SPL Sync completed successfully from Krang." >/dev/null
|
||||||
|
else
|
||||||
|
echo "$LOG_TAG ❌ Sync may have failed. Check logs."
|
||||||
|
|
||||||
|
# Failure alerts
|
||||||
|
curl -s -X POST "$MASTODON_INSTANCE/api/v1/statuses" \
|
||||||
|
-H "Authorization: Bearer $ACCESS_TOKEN" \
|
||||||
|
-d "status=❌ SPL Sync failed from Krang. Check logs." \
|
||||||
|
-d "visibility=$TOOT_VISIBILITY" >/dev/null
|
||||||
|
|
||||||
|
curl -s -X POST "https://api.telegram.org/bot$TELEGRAM_BOT_TOKEN/sendMessage" \
|
||||||
|
-d "chat_id=$TELEGRAM_CHAT_ID" \
|
||||||
|
-d "text=❌ SPL Sync failed from Krang. Manual check needed." >/dev/null
|
||||||
|
fi
|
||||||
|
|
||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
165
miscellaneous/python/dbcheck4.py
Normal file
165
miscellaneous/python/dbcheck4.py
Normal file
@ -0,0 +1,165 @@
|
|||||||
|
import subprocess
|
||||||
|
import os
|
||||||
|
import requests
|
||||||
|
import datetime
|
||||||
|
import paramiko
|
||||||
|
import time
|
||||||
|
import psycopg2
|
||||||
|
|
||||||
|
# ==== CONFIG ====
|
||||||
|
MASTODON_INSTANCE = "https://chatwithus.live"
|
||||||
|
MASTODON_TOKEN = "rimxBLi-eaJAcwagkmoj6UoW7Lc473tQY0cOM041Euw"
|
||||||
|
MASTODON_USER_ID = "114386383616633367"
|
||||||
|
HEALTHCHECK_HTML = "/var/www/html/healthcheck.html"
|
||||||
|
ARCHIVE_DIR = "/home/doc/genesis-tools/archive.html"
|
||||||
|
|
||||||
|
DISK_WARN_THRESHOLD = 10
|
||||||
|
LOG_FILES = ["/var/log/syslog", "/var/log/nginx/error.log"]
|
||||||
|
LOG_PATTERNS = ["ERROR", "FATAL", "disk full", "out of memory"]
|
||||||
|
SUPPRESSED_PATTERNS = ["SomeKnownHarmlessMastodonError"]
|
||||||
|
|
||||||
|
NODES = [
|
||||||
|
{"name": "shredder", "host": "38.102.127.172", "ssh_user": "doc", "services": ["minio.service"], "disks": ["/", "/mnt/miniodata"], "db": False, "raid": True},
|
||||||
|
{"name": "mastodon", "host": "chatwithus.live", "ssh_user": "root", "services": ["nginx", "mastodon-web"], "disks": ["/"], "db": False, "raid": False},
|
||||||
|
{"name": "db1", "host": "zcluster.technodrome1.sshjunkie.com", "ssh_user": "doc", "services": ["postgresql@16-main.service"], "disks": ["/", "/var/lib/postgresql"], "db": True, "raid": True},
|
||||||
|
{"name": "db2", "host": "zcluster.technodrome2.sshjunkie.com", "ssh_user": "doc", "services": ["postgresql@16-main.service"], "disks": ["/", "/var/lib/postgresql"], "db": True, "raid": True}
|
||||||
|
]
|
||||||
|
|
||||||
|
# ==== Mastodon DM ====
|
||||||
|
def mastodon_dm(message, retries=3):
|
||||||
|
url = f"{MASTODON_INSTANCE}/api/v1/statuses"
|
||||||
|
headers = {"Authorization": f"Bearer {MASTODON_TOKEN}"}
|
||||||
|
payload = {"status": message, "visibility": "direct", "in_reply_to_account_id": MASTODON_USER_ID}
|
||||||
|
for attempt in range(retries):
|
||||||
|
try:
|
||||||
|
resp = requests.post(url, headers=headers, data=payload)
|
||||||
|
if resp.status_code == 200:
|
||||||
|
return
|
||||||
|
print(f"Failed to send Mastodon DM (attempt {attempt+1}): {resp.text}")
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Error sending Mastodon DM: {e}")
|
||||||
|
time.sleep(5)
|
||||||
|
|
||||||
|
# ==== SSH Helper ====
|
||||||
|
def ssh_command(host, user, cmd):
|
||||||
|
try:
|
||||||
|
ssh = paramiko.SSHClient()
|
||||||
|
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||||
|
ssh.connect(hostname=host, username=user, timeout=10)
|
||||||
|
stdin, stdout, stderr = ssh.exec_command(cmd)
|
||||||
|
out = stdout.read().decode().strip()
|
||||||
|
err = stderr.read().decode().strip()
|
||||||
|
if "Authentication failed" in out or "permission denied" in out.lower() or "permission denied" in err.lower():
|
||||||
|
stdin, stdout, stderr = ssh.exec_command(f"sudo {cmd}")
|
||||||
|
out = stdout.read().decode().strip()
|
||||||
|
ssh.close()
|
||||||
|
return out
|
||||||
|
except Exception as e:
|
||||||
|
return f"SSH error: {e}"
|
||||||
|
|
||||||
|
SERVICE_PROCESS_MAP = {
|
||||||
|
"minio.service": "minio",
|
||||||
|
"postgresql@16-main.service": "postgres",
|
||||||
|
"mastodon-web": "puma"
|
||||||
|
}
|
||||||
|
|
||||||
|
def check_remote_service(host, user, service, node_name):
|
||||||
|
try:
|
||||||
|
process_name = SERVICE_PROCESS_MAP.get(service, service)
|
||||||
|
cmd = f"pgrep -f '{process_name}'"
|
||||||
|
out = ssh_command(host, user, cmd)
|
||||||
|
if not out.strip():
|
||||||
|
return f"[{node_name}] CRITICAL: Service {service} not running! (pgrep '{process_name}' found nothing)"
|
||||||
|
except Exception as e:
|
||||||
|
return f"[{node_name}] ERROR: Service check failed: {e}"
|
||||||
|
return None
|
||||||
|
|
||||||
|
def choose_emoji(line):
|
||||||
|
if "RAID" in line:
|
||||||
|
if "disk" in line.lower():
|
||||||
|
return "\U0001F4C8"
|
||||||
|
if "rclone" in line.lower():
|
||||||
|
return "\U0001F422"
|
||||||
|
if "Service" in line:
|
||||||
|
return "\U0001F6D1"
|
||||||
|
if "Replication" in line:
|
||||||
|
return "\U0001F4A5"
|
||||||
|
return "\u26A0\uFE0F"
|
||||||
|
|
||||||
|
def check_remote_disk(host, user, path, node_name):
|
||||||
|
try:
|
||||||
|
cmd = f"df --output=pcent {path} | tail -1 | tr -dc '0-9'"
|
||||||
|
out = ssh_command(host, user, cmd)
|
||||||
|
if not out:
|
||||||
|
return f"[{node_name}] ERROR: Disk {path} not found or could not check disk usage."
|
||||||
|
percent = int(out)
|
||||||
|
if percent > (100 - DISK_WARN_THRESHOLD):
|
||||||
|
return f"[{node_name}] WARNING: Only {100 - percent}% disk free on {path}."
|
||||||
|
except Exception as e:
|
||||||
|
return f"[{node_name}] ERROR: Disk check failed: {e}"
|
||||||
|
return None
|
||||||
|
|
||||||
|
def check_remote_logs(host, user, node_name):
|
||||||
|
alerts = []
|
||||||
|
for log in LOG_FILES:
|
||||||
|
cmd = f"tail -500 {log}"
|
||||||
|
try:
|
||||||
|
out = ssh_command(host, user, cmd)
|
||||||
|
lines = out.split("\n")
|
||||||
|
for pattern in LOG_PATTERNS:
|
||||||
|
if any(pattern in line and not any(s in line for s in SUPPRESSED_PATTERNS) for line in lines):
|
||||||
|
alerts.append(f"[{node_name}] WARNING: Pattern '{pattern}' found in {log}")
|
||||||
|
except Exception as e:
|
||||||
|
alerts.append(f"[{node_name}] ERROR: Could not read log {log}: {e}")
|
||||||
|
return alerts
|
||||||
|
|
||||||
|
# === MAIN CALL ===
|
||||||
|
if __name__ == "__main__":
|
||||||
|
print("Genesis Healthcheck started...")
|
||||||
|
critical_problems = []
|
||||||
|
warning_problems = []
|
||||||
|
|
||||||
|
for node in NODES:
|
||||||
|
print(f"Checking node: {node['name']} @ {node['host']}")
|
||||||
|
for disk in node['disks']:
|
||||||
|
result = check_remote_disk(node['host'], node['ssh_user'], disk, node['name'])
|
||||||
|
if result:
|
||||||
|
print(result)
|
||||||
|
if "CRITICAL" in result:
|
||||||
|
critical_problems.append(result)
|
||||||
|
else:
|
||||||
|
warning_problems.append(result)
|
||||||
|
|
||||||
|
for service in node['services']:
|
||||||
|
result = check_remote_service(node['host'], node['ssh_user'], service, node['name'])
|
||||||
|
if result:
|
||||||
|
print(result)
|
||||||
|
if "CRITICAL" in result:
|
||||||
|
critical_problems.append(result)
|
||||||
|
else:
|
||||||
|
warning_problems.append(result)
|
||||||
|
|
||||||
|
logs = check_remote_logs(node['host'], node['ssh_user'], node['name'])
|
||||||
|
for log in logs:
|
||||||
|
print(log)
|
||||||
|
warning_problems.append(log)
|
||||||
|
|
||||||
|
now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||||
|
if critical_problems:
|
||||||
|
msg_lines = [
|
||||||
|
f"🚨 Genesis Radio Critical Healthcheck {now} 🚨",
|
||||||
|
f"⚡ {len(critical_problems)} critical issues found:"
|
||||||
|
]
|
||||||
|
msg_lines.extend(f"- {choose_emoji(p)} {p}" for p in critical_problems)
|
||||||
|
mastodon_dm("\n".join(msg_lines))
|
||||||
|
|
||||||
|
if warning_problems:
|
||||||
|
msg_lines = [
|
||||||
|
f"⚠️ Genesis Radio Warning Healthcheck {now} ⚠️",
|
||||||
|
f"⚡ {len(warning_problems)} warnings found:"
|
||||||
|
]
|
||||||
|
msg_lines.extend(f"- {choose_emoji(p)} {p}" for p in warning_problems)
|
||||||
|
mastodon_dm("\n".join(msg_lines))
|
||||||
|
|
||||||
|
if not critical_problems and not warning_problems:
|
||||||
|
mastodon_dm(f"✅ Genesis Radio Healthcheck {now}: All systems normal.")
|
||||||
81
miscellaneous/python/snappysnaps.py
Normal file
81
miscellaneous/python/snappysnaps.py
Normal file
@ -0,0 +1,81 @@
|
|||||||
|
import os
|
||||||
|
import paramiko
|
||||||
|
import requests
|
||||||
|
from datetime import datetime
|
||||||
|
from collections import defaultdict
|
||||||
|
|
||||||
|
# === CONFIG ===
|
||||||
|
NODES = {
|
||||||
|
"shredderv2": {"host": "shredderv2.sshjunkie.com", "user": "doc"},
|
||||||
|
"thevault": {"host": "209.209.9.128", "user": "doc"},
|
||||||
|
"technodrome1": {"host": "38.102.127.165", "user": "doc"},
|
||||||
|
"technodrome2": {"host": "38.102.127.166", "user": "doc"},
|
||||||
|
}
|
||||||
|
|
||||||
|
SSH_KEY = "~/.ssh/genesis_healthcheck"
|
||||||
|
ZFS_CMD = "zfs list -t snapshot -o name,creation -s creation"
|
||||||
|
|
||||||
|
# === TELEGRAM CONFIG ===
|
||||||
|
TELEGRAM_BOT_TOKEN="8178867489:AAH0VjN7VnZSCIWasSz_y97iBLLjPJA751k"
|
||||||
|
TELEGRAM_CHAT_ID="1559582356"
|
||||||
|
|
||||||
|
def send_to_telegram(message):
|
||||||
|
url = f"https://api.telegram.org/bot{TELEGRAM_BOT_TOKEN}/sendMessage"
|
||||||
|
data = {
|
||||||
|
"chat_id": TELEGRAM_CHAT_ID,
|
||||||
|
"text": message,
|
||||||
|
"parse_mode": "Markdown"
|
||||||
|
}
|
||||||
|
try:
|
||||||
|
requests.post(url, data=data, timeout=10)
|
||||||
|
except Exception as e:
|
||||||
|
print(f"Telegram send failed: {e}")
|
||||||
|
|
||||||
|
def ssh_run(host, user, cmd):
|
||||||
|
ssh = paramiko.SSHClient()
|
||||||
|
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
||||||
|
ssh.connect(hostname=host, username=user, key_filename=os.path.expanduser(SSH_KEY), timeout=10)
|
||||||
|
stdin, stdout, stderr = ssh.exec_command(cmd)
|
||||||
|
out = stdout.read().decode().strip()
|
||||||
|
ssh.close()
|
||||||
|
return out
|
||||||
|
|
||||||
|
def parse_snapshot_output(output):
|
||||||
|
lines = output.strip().split("\n")[1:] # skip header
|
||||||
|
snaps = defaultdict(list)
|
||||||
|
|
||||||
|
for line in lines:
|
||||||
|
try:
|
||||||
|
name, creation = line.strip().split(None, 1)
|
||||||
|
dataset, _ = name.split("@")
|
||||||
|
dt = datetime.strptime(creation.strip(), "%a %b %d %H:%M %Y")
|
||||||
|
snaps[dataset].append(dt)
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
return snaps
|
||||||
|
|
||||||
|
def summarize(node_name, dataset_snaps):
|
||||||
|
if not dataset_snaps:
|
||||||
|
send_to_telegram(f"⚠️ *No snapshots found* on `{node_name}`")
|
||||||
|
return
|
||||||
|
|
||||||
|
msg = [f"📦 *ZFS Snapshot Report — {node_name}*"]
|
||||||
|
for dataset, times in sorted(dataset_snaps.items()):
|
||||||
|
age = (datetime.now() - max(times)).total_seconds() / 3600
|
||||||
|
msg.append(
|
||||||
|
f"*{dataset}*\n"
|
||||||
|
f"- Count: {len(times)}\n"
|
||||||
|
f"- Oldest: {min(times)}\n"
|
||||||
|
f"- Newest: {max(times)}\n"
|
||||||
|
f"- Recent age: {age:.2f}h\n"
|
||||||
|
)
|
||||||
|
send_to_telegram("\n".join(msg))
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
for name, node in NODES.items():
|
||||||
|
try:
|
||||||
|
output = ssh_run(node["host"], node["user"], ZFS_CMD)
|
||||||
|
data = parse_snapshot_output(output)
|
||||||
|
summarize(name.upper(), data)
|
||||||
|
except Exception as e:
|
||||||
|
send_to_telegram(f"❌ Failed to check `{name}`: {e}")
|
||||||
Loading…
x
Reference in New Issue
Block a user