From 070442cb42e92aa7a30df815ab1e81fa678a11c9 Mon Sep 17 00:00:00 2001 From: DocTator Date: Wed, 23 Apr 2025 16:02:27 -0400 Subject: [PATCH] Auto commit from /home/doc/genesis-tools --- miscellaneous/dbcheck1.py | 251 ++++++++++++++++++++++++++++++++++++++ miscellaneous/malips.sh | 36 +++--- 2 files changed, 270 insertions(+), 17 deletions(-) create mode 100644 miscellaneous/dbcheck1.py diff --git a/miscellaneous/dbcheck1.py b/miscellaneous/dbcheck1.py new file mode 100644 index 0000000..6f34921 --- /dev/null +++ b/miscellaneous/dbcheck1.py @@ -0,0 +1,251 @@ +import os +import requests +import datetime +import paramiko + +# ==== CONFIG ==== +MASTODON_INSTANCE = "https://chatwithus.live" +MASTODON_TOKEN = "rimxBLi-eaJAcwagkmoj6UoW7Lc473tQY0cOM041Euw" +MASTODON_USER_ID = "114386383616633367" + +DISK_WARN_THRESHOLD = 10 # percent free +INODE_WARN_THRESHOLD = 10 # percent free +LOG_FILES = ["/var/log/syslog", "/var/log/nginx/error.log"] +LOG_PATTERNS = ["ERROR", "FATAL", "disk full", "out of memory"] + +NODES = [ + { + "name": "shredder", + "host": "38.102.127.171", + "ssh_user": "doc", + "services": [], + "disks": ["/", "/mnt/raid5"], + "type": "remote", + "db": False, + "raid": True + }, + { + "name": "mastodon", + "host": "chatwithus.live", # Now points directly to your Mastodon server + "ssh_user": "root", + "services": ["nginx", "mastodon-web"], + "disks": ["/"], + "type": "remote", + "db": False, + "raid": False + }, + { + "name": "db1", + "host": "cluster.db1.genesishostingtechnologies.com", + "ssh_user": "doc", + "services": ["postgresql"], + "disks": ["/", "/var/lib/postgresql"], + "type": "remote", + "db": True, + "raid": False + }, + { + "name": "db2", + "host": "cluster.db2.genesishostingtechnologies.com", + "ssh_user": "doc", + "services": ["postgresql"], + "disks": ["/", "/var/lib/postgresql"], + "type": "remote", + "db": True, + "raid": False + }, +] + +# ==== Mastodon DM function ==== +def mastodon_dm(message): + url = f"{MASTODON_INSTANCE}/api/v1/statuses" + headers = {"Authorization": f"Bearer {MASTODON_TOKEN}"} + payload = { + "status": message, + "visibility": "direct", + "in_reply_to_account_id": MASTODON_USER_ID + } + resp = requests.post(url, headers=headers, data=payload) + if resp.status_code != 200: + print(f"Failed to send Mastodon DM: {resp.text}") + +# ==== SSH command runner ==== +def ssh_command(host, user, cmd): + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect(hostname=host, username=user, timeout=10) + stdin, stdout, stderr = ssh.exec_command(cmd) + out = stdout.read().decode().strip() + ssh.close() + return out + +# ==== Robust Remote disk check ==== +def check_remote_disk(host, user, path, node_name): + cmd = f"df --output=pcent {path} | tail -1 | tr -dc '0-9'" + out = ssh_command(host, user, cmd) + if not out: + return f"[{node_name}] ERROR: Disk {path} not found or could not check disk usage." + try: + percent = int(out) + except ValueError: + return f"[{node_name}] ERROR: Could not parse disk usage for {path}. Output was: '{out}'" + if percent > 90: + return f"[{node_name}] WARNING: Only {100-percent}% disk free on {path}." + # Inode check + cmd_inode = f"df --output=ipcent {path} | tail -1 | tr -dc '0-9'" + out_inode = ssh_command(host, user, cmd_inode) + if not out_inode: + return f"[{node_name}] ERROR: Disk {path} not found or could not check inode usage." + try: + percent_inode = int(out_inode) + except ValueError: + return f"[{node_name}] ERROR: Could not parse inode usage for {path}. Output was: '{out_inode}'" + if percent_inode > 90: + return f"[{node_name}] WARNING: Only {100-percent_inode}% inodes free on {path}." + return None + +# ==== SMART health check (for all disks) ==== +def check_remote_smart(host, user, node_name): + alerts = [] + # List block devices + cmd_lsblk = "lsblk -ndo NAME,TYPE | awk '$2==\"disk\" {print $1}'" + devs = ssh_command(host, user, cmd_lsblk) + if not devs: + alerts.append(f"[{node_name}] ERROR: Could not list block devices for SMART check.") + return alerts + for dev in devs.split(): + smart_cmd = f"sudo smartctl -H /dev/{dev}" + out = ssh_command(host, user, smart_cmd) + if "PASSED" in out: + continue # All good + elif "FAILED" in out or "Pre-fail" in out or "SMART support is: Unavailable" in out: + alerts.append(f"[{node_name}] CRITICAL: SMART health issue on /dev/{dev}!\n{out}") + elif "Unknown" in out or not out: + alerts.append(f"[{node_name}] ERROR: SMART status unknown on /dev/{dev}. Output: {out}") + # Optionally scan for other SMART errors + return alerts + +# ==== Remote service check ==== +def check_remote_service(host, user, service, node_name): + cmd = f"systemctl is-active {service}" + out = ssh_command(host, user, cmd) + if out.strip() != "active": + return f"[{node_name}] CRITICAL: Service {service} not running!" + return None + +# ==== Remote RAID md0 check (robust for all mdstat layouts) ==== +def check_remote_raid_md0(host, user, node_name): + try: + import re + ssh = paramiko.SSHClient() + ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh.connect(hostname=host, username=user, timeout=10) + stdin, stdout, stderr = ssh.exec_command("cat /proc/mdstat") + mdstat = stdout.read().decode() + + # Find the block for md0 and look for the [UU_] status + lines = mdstat.splitlines() + status = None + inside_md0 = False + for line in lines: + if line.startswith("md0"): + inside_md0 = True + elif inside_md0: + m = re.search(r"\[(U|_)+\]", line) + if m: + status = m.group(0) + break + # Stop searching if we hit a blank line or another array + if line.strip() == "" or ":" in line: + break + + ssh.close() + + if status is None: + return f"[{node_name}] CRITICAL: /dev/md0 RAID status string not found!" + if "_" in status: + return f"[{node_name}] WARNING: /dev/md0 RAID degraded! Status: {status}" + # All U's means all disks up + return None + + except Exception as e: + return f"[{node_name}] ERROR: Could not check RAID health remotely: {e}" + +# ==== Remote log scan ==== +def check_remote_logs(host, user, node_name): + alerts = [] + for log in LOG_FILES: + cmd = f"tail -500 {log}" + try: + out = ssh_command(host, user, cmd) + lines = out.split("\n") + for pattern in LOG_PATTERNS: + if any(pattern in line for line in lines): + alerts.append(f"[{node_name}] WARNING: Pattern '{pattern}' in {log}") + except Exception as e: + alerts.append(f"[{node_name}] ERROR: Could not read log {log}: {e}") + return alerts + +# ==== Remote PostgreSQL replication check ==== +def check_replication(host, node_name): + try: + import psycopg2 + conn = psycopg2.connect(host=host, dbname="postgres", user="postgres", connect_timeout=5) + cur = conn.cursor() + cur.execute("SELECT pg_is_in_recovery();") + is_replica = cur.fetchone()[0] + if is_replica: + cur.execute("SELECT EXTRACT(EPOCH FROM (now() - pg_last_xact_replay_timestamp()))::INT;") + lag = cur.fetchone()[0] + if lag is None: + return f"[{node_name}] CRITICAL: Standby not streaming! Replication down." + elif lag > 10: + return f"[{node_name}] WARNING: Replication lag is {lag} seconds." + cur.close() + conn.close() + except Exception as e: + return f"[{node_name}] ERROR: Could not check replication: {e}" + return None + +# ==== Main routine ==== +def main(): + problems = [] + + # Multi-node checks + for node in NODES: + # All checks via SSH + for disk in node["disks"]: + res = check_remote_disk(node["host"], node["ssh_user"], disk, node["name"]) + if res: problems.append(res) + # SMART check for all disks on this node + smart_alerts = check_remote_smart(node["host"], node["ssh_user"], node["name"]) + if smart_alerts: + problems.extend(smart_alerts) + for svc in node["services"]: + res = check_remote_service(node["host"], node["ssh_user"], svc, node["name"]) + if res: problems.append(res) + # Replication check + if node.get("db"): + res = check_replication(node["host"], node["name"]) + if res: problems.append(res) + # RAID check, only for nodes with "raid": True + if node.get("raid", False): + raid_health = check_remote_raid_md0(node["host"], node["ssh_user"], node["name"]) + if raid_health: + problems.append(raid_health) + # Log scan + logs = check_remote_logs(node["host"], node["ssh_user"], node["name"]) + if logs: + problems.extend(logs) + + # Send DM if anything wrong + if problems: + now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + msg = f"🚨 Genesis Radio Multi-Node Healthcheck {now} 🚨\n" + "\n".join(problems) + print(msg) + mastodon_dm(msg) + else: + print("OK: All checks passed.") + +if __name__ == "__main__": + main() diff --git a/miscellaneous/malips.sh b/miscellaneous/malips.sh index 0f540ff..48f5142 100755 --- a/miscellaneous/malips.sh +++ b/miscellaneous/malips.sh @@ -3,28 +3,30 @@ # Path to Snort's alert log (snort.alert.fast) SNORT_LOG="/var/log/snort/snort.alert.fast" -# Path to store blocked IPs (to avoid duplicates) -BLOCKED_IPS="/tmp/blocked_ips.txt" +# Database connection details +DB_HOST="38.102.127.166" +DB_USER="ipblocks_user" +DB_PASS="rusty2281" +DB_NAME="ipblocks" -# Function to block an IP using iptables +# Function to insert blocked IP into the PostgreSQL database block_ip() { local ip=$1 - # Check if IP is already blocked - if ! grep -q "$ip" "$BLOCKED_IPS"; then - # Add IP to iptables block list - sudo iptables -I INPUT -s "$ip" -j DROP + # Remove port if included in the IP + ip=${ip%%:*} - # Log the blocked IP in the blocked_ips file - echo "$ip" >> "$BLOCKED_IPS" - echo "Blocked IP: $ip" - fi + # Insert the blocked IP into the PostgreSQL database (into the blocked_ip_log table) + PGPASSWORD="$DB_PASS" psql -U "$DB_USER" -h "$DB_HOST" -d "$DB_NAME" -c "INSERT INTO blocked_ip_log (ip_address) VALUES ('$ip');" + + # Optionally print to confirm the insertion + echo "Blocked IP $ip inserted into the database log." } -# Ensure the blocked_ips.txt file exists and is writable -if [ ! -f "$BLOCKED_IPS" ]; then - sudo touch "$BLOCKED_IPS" - sudo chmod 666 "$BLOCKED_IPS" +# Ensure the log file exists and is readable +if [ ! -f "$SNORT_LOG" ]; then + echo "Snort log file not found!" + exit 1 fi # Monitor the snort.alert.fast file for new malicious IPs @@ -35,12 +37,12 @@ tail -F "$SNORT_LOG" | while read line; do # Extract source and destination IP addresses from Snort logs if echo "$line" | grep -q "ICMP PING NMAP"; then # Extract source IP (before "->") - ip=$(echo "$line" | awk -F' -> ' '{print $1}' | awk '{print $NF}') + ip=$(echo "$line" | awk -F' -> ' '{print $1}' | awk '{print $NF}' | cut -d':' -f1) echo "Found Source IP: $ip" # Debug: Show the IP being extracted block_ip "$ip" elif echo "$line" | grep -q "EXPLOIT"; then # Extract source IP (before "->") - ip=$(echo "$line" | awk -F' -> ' '{print $1}' | awk '{print $NF}') + ip=$(echo "$line" | awk -F' -> ' '{print $1}' | awk '{print $NF}' | cut -d':' -f1) echo "Found Source IP: $ip" # Debug: Show the IP being extracted block_ip "$ip" fi