Auto commit from /home/doc/genesis-tools

This commit is contained in:
DocTator 2025-04-28 14:56:27 -04:00
parent 94b425146a
commit 4e4f7d797a

View File

@ -23,7 +23,7 @@ NODES = [
{"name": "db2", "host": "cluster.db2.genesishostingtechnologies.com", "ssh_user": "doc", "services": ["postgresql@16-main.service"], "disks": ["/", "/var/lib/postgresql"], "type": "remote", "db": True, "raid": False} {"name": "db2", "host": "cluster.db2.genesishostingtechnologies.com", "ssh_user": "doc", "services": ["postgresql@16-main.service"], "disks": ["/", "/var/lib/postgresql"], "type": "remote", "db": True, "raid": False}
] ]
# ==== Mastodon DM function with retry ==== # ==== Mastodon DM with retry ====
def mastodon_dm(message, retries=3): def mastodon_dm(message, retries=3):
url = f"{MASTODON_INSTANCE}/api/v1/statuses" url = f"{MASTODON_INSTANCE}/api/v1/statuses"
headers = {"Authorization": f"Bearer {MASTODON_TOKEN}"} headers = {"Authorization": f"Bearer {MASTODON_TOKEN}"}
@ -59,7 +59,7 @@ def choose_emoji(line):
return "💥" return "💥"
return "⚠️" return "⚠️"
# ==== Check rclone health ==== # ==== Health checks ====
def check_rclone_health(node): def check_rclone_health(node):
try: try:
result = ssh_command(node["host"], node["ssh_user"], "rclone rc vfs/stats") result = ssh_command(node["host"], node["ssh_user"], "rclone rc vfs/stats")
@ -73,7 +73,6 @@ def check_rclone_health(node):
return ("critical", f"[{node['name']}] ERROR: Could not check rclone health: {str(e)}") return ("critical", f"[{node['name']}] ERROR: Could not check rclone health: {str(e)}")
return None return None
# ==== Remote log scan ====
def check_remote_logs(host, user, node_name): def check_remote_logs(host, user, node_name):
alerts = [] alerts = []
for log in LOG_FILES: for log in LOG_FILES:
@ -89,7 +88,7 @@ def check_remote_logs(host, user, node_name):
alerts.append(f"[{node_name}] ERROR: Could not read log {log}: {e}") alerts.append(f"[{node_name}] ERROR: Could not read log {log}: {e}")
return alerts return alerts
# ==== Main Routine ==== # ==== Main routine ====
def main(): def main():
critical_problems = [] critical_problems = []
warning_problems = [] warning_problems = []
@ -178,8 +177,7 @@ def main():
f.write("<html><head><title>Genesis Radio Healthcheck</title><meta http-equiv='refresh' content='60'></head><body>") f.write("<html><head><title>Genesis Radio Healthcheck</title><meta http-equiv='refresh' content='60'></head><body>")
f.write(f"<h1>Genesis Radio System Health</h1>") f.write(f"<h1>Genesis Radio System Health</h1>")
f.write(f"<p>Last Checked: {now}</p>") f.write(f"<p>Last Checked: {now}</p>")
f.write("<table border='1' cellpadding='5' style='border-collapse: collapse;'> f.write("<table border='1' cellpadding='5' style='border-collapse: collapse;'><tr><th>System</th><th>Status</th></tr>")
<tr><th>System</th><th>Status</th></tr>")
for node, status in node_status.items(): for node, status in node_status.items():
color = 'green' if 'Healthy' in status else ('orange' if 'Warning' in status else 'red') color = 'green' if 'Healthy' in status else ('orange' if 'Warning' in status else 'red')
f.write(f"<tr><td>{node}</td><td style='color:{color};'>{status}</td></tr>") f.write(f"<tr><td>{node}</td><td style='color:{color};'>{status}</td></tr>")