foodlinkk-command-center/tools-api/app/connectors/ops_maintenance.py

"""Daily maintenance checks for IT Ops tab."""
from __future__ import annotations

import json
import subprocess
from datetime import datetime, timezone
from typing import Any

from app.db import execute, fetch_all, fetch_one
from app.connectors.proxmox import PROXMOX_HOST, VM_106_IP, _port_health, _run_ssh, check_docker_services

SEVERITY_ORDER = {"critical": 0, "warning": 1, "info": 2}


def _iso_now() -> str:
    return datetime.now(timezone.utc).isoformat()


def _table_exists(name: str) -> bool:
    row = fetch_one(
        """
        SELECT EXISTS(
            SELECT 1 FROM information_schema.tables
            WHERE table_schema = 'public' AND table_name = %s
        ) AS ok
        """,
        (name,),
    )
    return bool(row and row.get("ok"))


def list_notes(limit: int = 20, unresolved_only: bool = True) -> list[dict[str, Any]]:
    if not _table_exists("ops_maintenance_notes"):
        return []
    clauses = ["resolved = false"] if unresolved_only else []
    where = (" WHERE " + " AND ".join(clauses)) if clauses else ""
    rows = fetch_all(
        f"SELECT * FROM ops_maintenance_notes{where} ORDER BY created_at DESC LIMIT %s",
        (max(1, min(limit, 100)),),
    )
    out = []
    for r in rows:
        item = dict(r)
        if hasattr(item.get("created_at"), "isoformat"):
            item["created_at"] = item["created_at"].isoformat()
        out.append(item)
    return out


def add_note(title: str, body: str = "", severity: str = "info", source: str = "sysops") -> dict[str, Any] | None:
    if not _table_exists("ops_maintenance_notes"):
        return None
    row = fetch_one(
        """
        INSERT INTO ops_maintenance_notes (severity, title, body, source)
        VALUES (%s, %s, %s, %s)
        RETURNING *
        """,
        (severity, title[:255], body, source),
    )
    if not row:
        return None
    item = dict(row)
    if hasattr(item.get("created_at"), "isoformat"):
        item["created_at"] = item["created_at"].isoformat()
    return item


def run_maintenance_scan() -> dict[str, Any]:
    findings: list[dict[str, Any]] = []

    proxmox_up = _port_health(PROXMOX_HOST, 8006)
    if not proxmox_up:
        findings.append({
            "severity": "critical",
            "title": "Proxmox Web UI niet bereikbaar",
            "body": f"Poort 8006 op {PROXMOX_HOST} reageert niet. Controleer Dell R340 / Proxmox VE.",
        })

    docker = check_docker_services()
    if not docker.get("ok"):
        findings.append({
            "severity": "warning",
            "title": "Docker status onbekend",
            "body": docker.get("error") or "Kon docker ps niet uitvoeren op VM106.",
        })
    else:
        containers = docker.get("containers") or []
        unhealthy = [
            c for c in containers
            if "unhealthy" in str(c.get("Status") or c.get("State") or "").lower()
        ]
        if unhealthy:
            names = ", ".join(str(c.get("Names") or c.get("Name") or "?")[:40] for c in unhealthy[:5])
            findings.append({
                "severity": "warning",
                "title": f"{len(unhealthy)} container(s) unhealthy",
                "body": names,
            })

    apt = _run_ssh(VM_106_IP, "apt list --upgradable 2>/dev/null | grep -v Listing | wc -l")
    if apt.get("ok"):
        try:
            count = int((apt.get("stdout") or "0").strip())
        except ValueError:
            count = 0
        if count > 0:
            findings.append({
                "severity": "info" if count < 10 else "warning",
                "title": f"{count} package updates beschikbaar op VM106",
                "body": "Voer maintenance uit wanneer gepland: apt update && apt upgrade",
            })

    disk = _run_ssh(VM_106_IP, "df -h / | tail -1 | awk '{print $5}'")
    if disk.get("ok"):
        pct = (disk.get("stdout") or "").replace("%", "").strip()
        try:
            used = int(pct)
            if used >= 90:
                findings.append({
                    "severity": "critical",
                    "title": f"Schijf VM106 {used}% vol",
                    "body": "Ruimte vrijmaken op root volume.",
                })
            elif used >= 80:
                findings.append({
                    "severity": "warning",
                    "title": f"Schijf VM106 {used}% vol",
                    "body": "Plan opschoning binnenkort.",
                })
        except ValueError:
            pass

    saved: list[dict[str, Any]] = []
    for f in findings:
        note = add_note(f["title"], f.get("body", ""), f.get("severity", "info"))
        if note:
            saved.append(note)

    if not findings:
        note = add_note(
            "Geen openstaande maintenance — alles OK",
            f"Scan {_iso_now()}: Proxmox, Docker en schijf binnen normen.",
            "info",
        )
        if note:
            saved.append(note)

    saved.sort(key=lambda n: SEVERITY_ORDER.get(str(n.get("severity")), 9))

    try:
        from app.connectors.gitea_repo_sync import _record_activity
        from app.middleware import log_agent_event

        summary = f"{len(findings)} bevinding(en) — scan {_iso_now()[:19]}"
        _record_activity(
            "maintenance_scan",
            f"Maintenance scan: {len(findings)} bevinding(en)",
            summary + "\n" + "\n".join(f"- {f['title']}" for f in findings[:8]),
            metadata={"findings_count": len(findings), "for_herman": True},
        )
        log_agent_event(
            agent_name="sysops",
            agent_type="sysops",
            event_type="maintenance_scan",
            title=f"Maintenance scan voltooid — {len(findings)} bevinding(en)",
            body=summary,
            metadata={"findings": [f.get("title") for f in findings[:10]], "for_herman": True},
            channel="sysops",
        )
    except Exception:
        pass

    return {
        "ok": True,
        "scanned_at": _iso_now(),
        "findings_count": len(findings),
        "notes": saved,
    }