Compare commits
No commits in common. "839eb81301a49022b6f5a60ec2a2aa453b1080c9" and "046d5204ead25e563e3198f2b7732057df932618" have entirely different histories.
839eb81301
...
046d5204ea
2
OPS.md
2
OPS.md
@ -1,7 +1,7 @@
|
||||
# 🚀 Genesis Radio - Healthcheck Response Runbook
|
||||
|
||||
## Purpose
|
||||
When an alert fires (Critical or Warning), this guide tells you what to do so that **any team member** can react quickly, even if the admin is not available.
|
||||
When an alert fires (Critical or Warning), this guide tells you what to do so that **any team member** (like Brice) can react quickly, even if the admin is not available.
|
||||
|
||||
---
|
||||
|
||||
|
File diff suppressed because it is too large
Load Diff
@ -402,36 +402,3 @@ NameError: name 'check_remote_disk' is not defined. Did you mean: 'check_remote_
|
||||
⚠️ Genesis Radio Warning Healthcheck 2025-04-28 15:15:11 ⚠️
|
||||
⚡ 1 warnings found:
|
||||
- 💥 [db2] WARNING: Replication lag is 105 seconds.
|
||||
✅ Genesis Radio Healthcheck 2025-04-28 15:30:13: All systems normal.
|
||||
⚠️ Genesis Radio Warning Healthcheck 2025-04-28 15:45:13 ⚠️
|
||||
⚡ 1 warnings found:
|
||||
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
|
||||
✅ Genesis Radio Healthcheck 2025-04-28 16:00:12: All systems normal.
|
||||
⚠️ Genesis Radio Warning Healthcheck 2025-04-28 16:15:11 ⚠️
|
||||
⚡ 3 warnings found:
|
||||
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
|
||||
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
|
||||
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
|
||||
✅ Genesis Radio Healthcheck 2025-04-28 16:30:11: All systems normal.
|
||||
⚠️ Genesis Radio Warning Healthcheck 2025-04-28 16:45:11 ⚠️
|
||||
⚡ 1 warnings found:
|
||||
- 💥 [db2] WARNING: Replication lag is 88 seconds.
|
||||
✅ Genesis Radio Healthcheck 2025-04-28 17:00:10: All systems normal.
|
||||
✅ Genesis Radio Healthcheck 2025-04-28 17:15:13: All systems normal.
|
||||
⚠️ Genesis Radio Warning Healthcheck 2025-04-28 17:30:14 ⚠️
|
||||
⚡ 2 warnings found:
|
||||
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
|
||||
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
|
||||
⚠️ Genesis Radio Warning Healthcheck 2025-04-28 17:45:15 ⚠️
|
||||
⚡ 1 warnings found:
|
||||
- ⚠️ [mastodon] WARNING: Pattern 'ERROR' in /var/log/syslog
|
||||
✅ Genesis Radio Healthcheck 2025-04-28 18:00:13: All systems normal.
|
||||
✅ Genesis Radio Healthcheck 2025-04-28 18:15:12: All systems normal.
|
||||
✅ Genesis Radio Healthcheck 2025-04-28 18:30:11: All systems normal.
|
||||
✅ Genesis Radio Healthcheck 2025-04-28 18:45:16: All systems normal.
|
||||
✅ Genesis Radio Healthcheck 2025-04-28 19:00:15: All systems normal.
|
||||
⚠️ Genesis Radio Warning Healthcheck 2025-04-28 19:15:12 ⚠️
|
||||
⚡ 1 warnings found:
|
||||
- 💥 [db2] WARNING: Replication lag is 101 seconds.
|
||||
✅ Genesis Radio Healthcheck 2025-04-28 19:30:12: All systems normal.
|
||||
✅ Genesis Radio Healthcheck 2025-04-28 19:45:14: All systems normal.
|
||||
|
@ -1,27 +0,0 @@
|
||||
# Rotate Genesis Mount Guardian Logs
|
||||
# Author: You
|
||||
|
||||
$logfile = "C:\genesis_rclone_mount.log"
|
||||
$archiveFolder = "C:\log_archive"
|
||||
$maxSizeMB = 5
|
||||
|
||||
# Create archive folder if missing
|
||||
if (!(Test-Path $archiveFolder)) {
|
||||
New-Item -ItemType Directory -Path $archiveFolder
|
||||
}
|
||||
|
||||
# Check if logfile exists and size
|
||||
if (Test-Path $logfile) {
|
||||
$fileSizeMB = (Get-Item $logfile).Length / 1MB
|
||||
|
||||
if ($fileSizeMB -ge $maxSizeMB) {
|
||||
$timestamp = Get-Date -Format "yyyyMMdd_HHmmss"
|
||||
$archivePath = Join-Path $archiveFolder "genesis_rclone_mount_$timestamp.log"
|
||||
Move-Item $logfile $archivePath
|
||||
Write-Output "? Rotated log to $archivePath"
|
||||
} else {
|
||||
Write-Output "?? Log size is fine ($([math]::Round($fileSizeMB,2)) MB). No rotation needed."
|
||||
}
|
||||
} else {
|
||||
Write-Output "?? Log file not found. Nothing to rotate."
|
||||
}
|
@ -1,89 +0,0 @@
|
||||
# Genesis Radio Rclone Mount Guardian - Windows PowerShell Edition
|
||||
# Author: Doc
|
||||
|
||||
$logfile = "C:\genesis_rclone_mount.log"
|
||||
|
||||
$assets_remote = "genesisassets:genesisassets"
|
||||
$library_remote = "genesislibrary:genesislibrary"
|
||||
|
||||
$assets_drive = "Q:\"
|
||||
$library_drive = "R:\"
|
||||
|
||||
$asset_cache_L = "L:\assetcache"
|
||||
$asset_cache_X = "X:\cache"
|
||||
|
||||
$rclone_opts = "--vfs-cache-mode writes --vfs-cache-max-size 3T --vfs-cache-max-age 48h --vfs-read-ahead 1G --buffer-size 1G --no-traverse --rc --rc-addr :5572"
|
||||
|
||||
# Mastodon settings
|
||||
$mastodonInstance = "https://chatwithus.live"
|
||||
$mastodonToken = "rimxBLi-eaJAcwagkmoj6UoW7Lc473tQY0cOM041Euw"
|
||||
$mastodonUserID = "114386383616633367"
|
||||
|
||||
function Log {
|
||||
$timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss"
|
||||
Add-Content $logfile "$timestamp : $args"
|
||||
}
|
||||
|
||||
function Send-Mastodon-DM {
|
||||
param (
|
||||
[string]$message
|
||||
)
|
||||
|
||||
$headers = @{
|
||||
"Authorization" = "Bearer $mastodonToken"
|
||||
}
|
||||
|
||||
$body = @{
|
||||
"status" = $message
|
||||
"visibility" = "direct"
|
||||
"in_reply_to_account_id" = $mastodonUserID
|
||||
}
|
||||
|
||||
try {
|
||||
Invoke-RestMethod -Uri "$mastodonInstance/api/v1/statuses" -Headers $headers -Method Post -Body $body
|
||||
Log "? Mastodon DM sent: $message"
|
||||
}
|
||||
catch {
|
||||
Log "? Failed to send Mastodon DM: $_"
|
||||
}
|
||||
}
|
||||
|
||||
function Ensure-Mount {
|
||||
param (
|
||||
[string]$DriveLetter,
|
||||
[string]$Remote
|
||||
)
|
||||
|
||||
if (Test-Path $DriveLetter) {
|
||||
Log "$DriveLetter already mounted."
|
||||
}
|
||||
else {
|
||||
Log "$DriveLetter is NOT mounted. Attempting to mount $Remote."
|
||||
|
||||
Start-Process rclone -ArgumentList @(
|
||||
"mount", "$Remote", "$DriveLetter",
|
||||
"--cache-dir=$asset_cache_L",
|
||||
"--cache-dir=$asset_cache_X",
|
||||
$rclone_opts
|
||||
) -WindowStyle Hidden
|
||||
|
||||
Start-Sleep -Seconds 5
|
||||
|
||||
if (Test-Path $DriveLetter) {
|
||||
Log "? Successfully mounted $DriveLetter"
|
||||
}
|
||||
else {
|
||||
Log "? Failed to mount $DriveLetter"
|
||||
Send-Mastodon-DM "?? Genesis Radio Ops: Failed to mount $DriveLetter after recovery attempt!"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# === Main ===
|
||||
|
||||
Log "===== Genesis Radio Rclone Mount Guardian starting ====="
|
||||
|
||||
Ensure-Mount -DriveLetter $assets_drive -Remote $assets_remote
|
||||
Ensure-Mount -DriveLetter $library_drive -Remote $library_remote
|
||||
|
||||
Log "===== Genesis Radio Rclone Mount Guardian finished ====="
|
Loading…
x
Reference in New Issue
Block a user