SysOps: deploy-all — 2026-06-09 10:41 UTC
This commit is contained in:
sysops
@@ -0,0 +1,179 @@
|
||||
"""Daily maintenance checks for IT Ops tab."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from app.db import execute, fetch_all, fetch_one
|
||||
from app.connectors.proxmox import PROXMOX_HOST, VM_106_IP, _port_health, _run_ssh, check_docker_services
|
||||
|
||||
SEVERITY_ORDER = {"critical": 0, "warning": 1, "info": 2}
|
||||
|
||||
|
||||
def _iso_now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def _table_exists(name: str) -> bool:
|
||||
row = fetch_one(
|
||||
"""
|
||||
SELECT EXISTS(
|
||||
SELECT 1 FROM information_schema.tables
|
||||
WHERE table_schema = 'public' AND table_name = %s
|
||||
) AS ok
|
||||
""",
|
||||
(name,),
|
||||
)
|
||||
return bool(row and row.get("ok"))
|
||||
|
||||
|
||||
def list_notes(limit: int = 20, unresolved_only: bool = True) -> list[dict[str, Any]]:
|
||||
if not _table_exists("ops_maintenance_notes"):
|
||||
return []
|
||||
clauses = ["resolved = false"] if unresolved_only else []
|
||||
where = (" WHERE " + " AND ".join(clauses)) if clauses else ""
|
||||
rows = fetch_all(
|
||||
f"SELECT * FROM ops_maintenance_notes{where} ORDER BY created_at DESC LIMIT %s",
|
||||
(max(1, min(limit, 100)),),
|
||||
)
|
||||
out = []
|
||||
for r in rows:
|
||||
item = dict(r)
|
||||
if hasattr(item.get("created_at"), "isoformat"):
|
||||
item["created_at"] = item["created_at"].isoformat()
|
||||
out.append(item)
|
||||
return out
|
||||
|
||||
|
||||
def add_note(title: str, body: str = "", severity: str = "info", source: str = "sysops") -> dict[str, Any] | None:
|
||||
if not _table_exists("ops_maintenance_notes"):
|
||||
return None
|
||||
row = fetch_one(
|
||||
"""
|
||||
INSERT INTO ops_maintenance_notes (severity, title, body, source)
|
||||
VALUES (%s, %s, %s, %s)
|
||||
RETURNING *
|
||||
""",
|
||||
(severity, title[:255], body, source),
|
||||
)
|
||||
if not row:
|
||||
return None
|
||||
item = dict(row)
|
||||
if hasattr(item.get("created_at"), "isoformat"):
|
||||
item["created_at"] = item["created_at"].isoformat()
|
||||
return item
|
||||
|
||||
|
||||
def run_maintenance_scan() -> dict[str, Any]:
|
||||
findings: list[dict[str, Any]] = []
|
||||
|
||||
proxmox_up = _port_health(PROXMOX_HOST, 8006)
|
||||
if not proxmox_up:
|
||||
findings.append({
|
||||
"severity": "critical",
|
||||
"title": "Proxmox Web UI niet bereikbaar",
|
||||
"body": f"Poort 8006 op {PROXMOX_HOST} reageert niet. Controleer Dell R340 / Proxmox VE.",
|
||||
})
|
||||
|
||||
docker = check_docker_services()
|
||||
if not docker.get("ok"):
|
||||
findings.append({
|
||||
"severity": "warning",
|
||||
"title": "Docker status onbekend",
|
||||
"body": docker.get("error") or "Kon docker ps niet uitvoeren op VM106.",
|
||||
})
|
||||
else:
|
||||
containers = docker.get("containers") or []
|
||||
unhealthy = [
|
||||
c for c in containers
|
||||
if "unhealthy" in str(c.get("Status") or c.get("State") or "").lower()
|
||||
]
|
||||
if unhealthy:
|
||||
names = ", ".join(str(c.get("Names") or c.get("Name") or "?")[:40] for c in unhealthy[:5])
|
||||
findings.append({
|
||||
"severity": "warning",
|
||||
"title": f"{len(unhealthy)} container(s) unhealthy",
|
||||
"body": names,
|
||||
})
|
||||
|
||||
apt = _run_ssh(VM_106_IP, "apt list --upgradable 2>/dev/null | grep -v Listing | wc -l")
|
||||
if apt.get("ok"):
|
||||
try:
|
||||
count = int((apt.get("stdout") or "0").strip())
|
||||
except ValueError:
|
||||
count = 0
|
||||
if count > 0:
|
||||
findings.append({
|
||||
"severity": "info" if count < 10 else "warning",
|
||||
"title": f"{count} package updates beschikbaar op VM106",
|
||||
"body": "Voer maintenance uit wanneer gepland: apt update && apt upgrade",
|
||||
})
|
||||
|
||||
disk = _run_ssh(VM_106_IP, "df -h / | tail -1 | awk '{print $5}'")
|
||||
if disk.get("ok"):
|
||||
pct = (disk.get("stdout") or "").replace("%", "").strip()
|
||||
try:
|
||||
used = int(pct)
|
||||
if used >= 90:
|
||||
findings.append({
|
||||
"severity": "critical",
|
||||
"title": f"Schijf VM106 {used}% vol",
|
||||
"body": "Ruimte vrijmaken op root volume.",
|
||||
})
|
||||
elif used >= 80:
|
||||
findings.append({
|
||||
"severity": "warning",
|
||||
"title": f"Schijf VM106 {used}% vol",
|
||||
"body": "Plan opschoning binnenkort.",
|
||||
})
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
saved: list[dict[str, Any]] = []
|
||||
for f in findings:
|
||||
note = add_note(f["title"], f.get("body", ""), f.get("severity", "info"))
|
||||
if note:
|
||||
saved.append(note)
|
||||
|
||||
if not findings:
|
||||
note = add_note(
|
||||
"Geen openstaande maintenance — alles OK",
|
||||
f"Scan {_iso_now()}: Proxmox, Docker en schijf binnen normen.",
|
||||
"info",
|
||||
)
|
||||
if note:
|
||||
saved.append(note)
|
||||
|
||||
saved.sort(key=lambda n: SEVERITY_ORDER.get(str(n.get("severity")), 9))
|
||||
|
||||
try:
|
||||
from app.connectors.gitea_repo_sync import _record_activity
|
||||
from app.middleware import log_agent_event
|
||||
|
||||
summary = f"{len(findings)} bevinding(en) — scan {_iso_now()[:19]}"
|
||||
_record_activity(
|
||||
"maintenance_scan",
|
||||
f"Maintenance scan: {len(findings)} bevinding(en)",
|
||||
summary + "\n" + "\n".join(f"- {f['title']}" for f in findings[:8]),
|
||||
metadata={"findings_count": len(findings), "for_herman": True},
|
||||
)
|
||||
log_agent_event(
|
||||
agent_name="sysops",
|
||||
agent_type="sysops",
|
||||
event_type="maintenance_scan",
|
||||
title=f"Maintenance scan voltooid — {len(findings)} bevinding(en)",
|
||||
body=summary,
|
||||
metadata={"findings": [f.get("title") for f in findings[:10]], "for_herman": True},
|
||||
channel="sysops",
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"scanned_at": _iso_now(),
|
||||
"findings_count": len(findings),
|
||||
"notes": saved,
|
||||
}
|
||||
Reference in New Issue
Block a user