Compare commits
No commits in common. "839eb81301a49022b6f5a60ec2a2aa453b1080c9" and "046d5204ead25e563e3198f2b7732057df932618" have entirely different histories.
839eb81301
...
046d5204ea
2
OPS.md
2
OPS.md
@ -1,7 +1,7 @@
|
|||||||
# 🚀 Genesis Radio - Healthcheck Response Runbook
|
# 🚀 Genesis Radio - Healthcheck Response Runbook
|
||||||
|
|
||||||
## Purpose
|
## Purpose
|
||||||
When an alert fires (Critical or Warning), this guide tells you what to do so that **any team member** can react quickly, even if the admin is not available.
|
When an alert fires (Critical or Warning), this guide tells you what to do so that **any team member** (like Brice) can react quickly, even if the admin is not available.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
File diff suppressed because it is too large
Load Diff
@ -402,36 +402,3 @@ NameError: name 'check_remote_disk' is not defined. Did you mean: 'check_remote_
|
|||||||
⚠️ Genesis Radio Warning Healthcheck 2025-04-28 15:15:11 ⚠️
|
⚠️ Genesis Radio Warning Healthcheck 2025-04-28 15:15:11 ⚠️
|
||||||
⚡ 1 warnings found:
|
⚡ 1 warnings found:
|
||||||
- 💥 [db2] WARNING: Replication lag is 105 seconds.
|
- 💥 [db2] WARNING: Replication lag is 105 seconds.
|
||||||
✅ Genesis Radio Healthcheck 2025-04-28 15:30:13: All systems normal.
|
|
||||||
⚠️ Genesis Radio Warning Healthcheck 2025-04-28 15:45:13 ⚠️
|
|
||||||
⚡ 1 warnings found:
|
|
||||||
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
|
|
||||||
✅ Genesis Radio Healthcheck 2025-04-28 16:00:12: All systems normal.
|
|
||||||
⚠️ Genesis Radio Warning Healthcheck 2025-04-28 16:15:11 ⚠️
|
|
||||||
⚡ 3 warnings found:
|
|
||||||
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
|
|
||||||
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
|
|
||||||
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
|
|
||||||
✅ Genesis Radio Healthcheck 2025-04-28 16:30:11: All systems normal.
|
|
||||||
⚠️ Genesis Radio Warning Healthcheck 2025-04-28 16:45:11 ⚠️
|
|
||||||
⚡ 1 warnings found:
|
|
||||||
- 💥 [db2] WARNING: Replication lag is 88 seconds.
|
|
||||||
✅ Genesis Radio Healthcheck 2025-04-28 17:00:10: All systems normal.
|
|
||||||
✅ Genesis Radio Healthcheck 2025-04-28 17:15:13: All systems normal.
|
|
||||||
⚠️ Genesis Radio Warning Healthcheck 2025-04-28 17:30:14 ⚠️
|
|
||||||
⚡ 2 warnings found:
|
|
||||||
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
|
|
||||||
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
|
|
||||||
⚠️ Genesis Radio Warning Healthcheck 2025-04-28 17:45:15 ⚠️
|
|
||||||
⚡ 1 warnings found:
|
|
||||||
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
|
|
||||||
✅ Genesis Radio Healthcheck 2025-04-28 18:00:13: All systems normal.
|
|
||||||
✅ Genesis Radio Healthcheck 2025-04-28 18:15:12: All systems normal.
|
|
||||||
✅ Genesis Radio Healthcheck 2025-04-28 18:30:11: All systems normal.
|
|
||||||
✅ Genesis Radio Healthcheck 2025-04-28 18:45:16: All systems normal.
|
|
||||||
✅ Genesis Radio Healthcheck 2025-04-28 19:00:15: All systems normal.
|
|
||||||
⚠️ Genesis Radio Warning Healthcheck 2025-04-28 19:15:12 ⚠️
|
|
||||||
⚡ 1 warnings found:
|
|
||||||
- 💥 [db2] WARNING: Replication lag is 101 seconds.
|
|
||||||
✅ Genesis Radio Healthcheck 2025-04-28 19:30:12: All systems normal.
|
|
||||||
✅ Genesis Radio Healthcheck 2025-04-28 19:45:14: All systems normal.
|
|
||||||
|
@ -1,27 +0,0 @@
|
|||||||
# Rotate Genesis Mount Guardian Logs
|
|
||||||
# Author: You
|
|
||||||
|
|
||||||
$logfile = "C:\genesis_rclone_mount.log"
|
|
||||||
$archiveFolder = "C:\log_archive"
|
|
||||||
$maxSizeMB = 5
|
|
||||||
|
|
||||||
# Create archive folder if missing
|
|
||||||
if (!(Test-Path $archiveFolder)) {
|
|
||||||
New-Item -ItemType Directory -Path $archiveFolder
|
|
||||||
}
|
|
||||||
|
|
||||||
# Check if logfile exists and size
|
|
||||||
if (Test-Path $logfile) {
|
|
||||||
$fileSizeMB = (Get-Item $logfile).Length / 1MB
|
|
||||||
|
|
||||||
if ($fileSizeMB -ge $maxSizeMB) {
|
|
||||||
$timestamp = Get-Date -Format "yyyyMMdd_HHmmss"
|
|
||||||
$archivePath = Join-Path $archiveFolder "genesis_rclone_mount_$timestamp.log"
|
|
||||||
Move-Item $logfile $archivePath
|
|
||||||
Write-Output "? Rotated log to $archivePath"
|
|
||||||
} else {
|
|
||||||
Write-Output "?? Log size is fine ($([math]::Round($fileSizeMB,2)) MB). No rotation needed."
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
Write-Output "?? Log file not found. Nothing to rotate."
|
|
||||||
}
|
|
@ -1,89 +0,0 @@
|
|||||||
# Genesis Radio Rclone Mount Guardian - Windows PowerShell Edition
|
|
||||||
# Author: Doc
|
|
||||||
|
|
||||||
$logfile = "C:\genesis_rclone_mount.log"
|
|
||||||
|
|
||||||
$assets_remote = "genesisassets:genesisassets"
|
|
||||||
$library_remote = "genesislibrary:genesislibrary"
|
|
||||||
|
|
||||||
$assets_drive = "Q:\"
|
|
||||||
$library_drive = "R:\"
|
|
||||||
|
|
||||||
$asset_cache_L = "L:\assetcache"
|
|
||||||
$asset_cache_X = "X:\cache"
|
|
||||||
|
|
||||||
$rclone_opts = "--vfs-cache-mode writes --vfs-cache-max-size 3T --vfs-cache-max-age 48h --vfs-read-ahead 1G --buffer-size 1G --no-traverse --rc --rc-addr :5572"
|
|
||||||
|
|
||||||
# Mastodon settings
|
|
||||||
$mastodonInstance = "https://chatwithus.live"
|
|
||||||
$mastodonToken = "rimxBLi-eaJAcwagkmoj6UoW7Lc473tQY0cOM041Euw"
|
|
||||||
$mastodonUserID = "114386383616633367"
|
|
||||||
|
|
||||||
function Log {
|
|
||||||
$timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss"
|
|
||||||
Add-Content $logfile "$timestamp : $args"
|
|
||||||
}
|
|
||||||
|
|
||||||
function Send-Mastodon-DM {
|
|
||||||
param (
|
|
||||||
[string]$message
|
|
||||||
)
|
|
||||||
|
|
||||||
$headers = @{
|
|
||||||
"Authorization" = "Bearer $mastodonToken"
|
|
||||||
}
|
|
||||||
|
|
||||||
$body = @{
|
|
||||||
"status" = $message
|
|
||||||
"visibility" = "direct"
|
|
||||||
"in_reply_to_account_id" = $mastodonUserID
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
Invoke-RestMethod -Uri "$mastodonInstance/api/v1/statuses" -Headers $headers -Method Post -Body $body
|
|
||||||
Log "? Mastodon DM sent: $message"
|
|
||||||
}
|
|
||||||
catch {
|
|
||||||
Log "? Failed to send Mastodon DM: $_"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function Ensure-Mount {
|
|
||||||
param (
|
|
||||||
[string]$DriveLetter,
|
|
||||||
[string]$Remote
|
|
||||||
)
|
|
||||||
|
|
||||||
if (Test-Path $DriveLetter) {
|
|
||||||
Log "$DriveLetter already mounted."
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
Log "$DriveLetter is NOT mounted. Attempting to mount $Remote."
|
|
||||||
|
|
||||||
Start-Process rclone -ArgumentList @(
|
|
||||||
"mount", "$Remote", "$DriveLetter",
|
|
||||||
"--cache-dir=$asset_cache_L",
|
|
||||||
"--cache-dir=$asset_cache_X",
|
|
||||||
$rclone_opts
|
|
||||||
) -WindowStyle Hidden
|
|
||||||
|
|
||||||
Start-Sleep -Seconds 5
|
|
||||||
|
|
||||||
if (Test-Path $DriveLetter) {
|
|
||||||
Log "? Successfully mounted $DriveLetter"
|
|
||||||
}
|
|
||||||
else {
|
|
||||||
Log "? Failed to mount $DriveLetter"
|
|
||||||
Send-Mastodon-DM "?? Genesis Radio Ops: Failed to mount $DriveLetter after recovery attempt!"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
# === Main ===
|
|
||||||
|
|
||||||
Log "===== Genesis Radio Rclone Mount Guardian starting ====="
|
|
||||||
|
|
||||||
Ensure-Mount -DriveLetter $assets_drive -Remote $assets_remote
|
|
||||||
Ensure-Mount -DriveLetter $library_drive -Remote $library_remote
|
|
||||||
|
|
||||||
Log "===== Genesis Radio Rclone Mount Guardian finished ====="
|
|
Loading…
x
Reference in New Issue
Block a user