foodlinkk-command-center/tools-api/app/connectors/proxmox.py

"""Proxmox infrastructure monitoring — Dell R340 · Proxmox VE · VM106 services."""
from __future__ import annotations

import json
import os
import shlex
import socket
import ssl
import subprocess
import time
from datetime import datetime, timezone
from typing import Any
from urllib.error import URLError
from urllib.request import Request, urlopen

from app.db import execute, fetch_all, fetch_one

PROXMOX_HOST = os.getenv("PROXMOX_HOST", "10.4.7.14")
PROXMOX_API_URL = f"https://{PROXMOX_HOST}:8006/api2/json"
SSH_USER = os.getenv("PROXMOX_SSH_USER", "aissa")
SSH_PASSWORD = os.getenv("PROXMOX_SSH_PASS", "Foodlinkk#2026")

VM_106_IP = os.getenv("VM106_IP", "10.4.7.18")
VM_106_ID = 106
VM_106_NAME = "dockervm"

HARDWARE_LABEL = "Dell PowerEdge R340"

SERVICE_CATALOG: list[dict[str, Any]] = [
    {"id": "svc-cockpit", "label": "Cockpit", "port": 8600, "url": f"http://{VM_106_IP}:8600", "role": "Command Center UI"},
    {"id": "svc-tools-api", "label": "Tools API", "port": 8700, "url": f"http://{VM_106_IP}:8700/docs", "role": "Connectors & API"},
    {"id": "svc-gitea", "label": "Gitea", "port": 3001, "url": f"http://{VM_106_IP}:3001", "role": "Git · config backups"},
    {"id": "svc-email-agent", "label": "Email Agent", "port": 8801, "url": f"http://{VM_106_IP}:8801", "role": "IMAP/SMTP sync"},
    {"id": "svc-grafana", "label": "Grafana", "port": 3002, "url": f"http://{VM_106_IP}:3002", "role": "Metrics dashboards"},
    {"id": "svc-prometheus", "label": "Prometheus", "port": 9090, "url": f"http://{VM_106_IP}:9090", "role": "Metrics scrape"},
    {"id": "svc-browser-agent", "label": "Browser Agent", "port": 7790, "url": f"http://{VM_106_IP}:7790", "role": "Web automation"},
]

LAYER_AGENTS: list[dict[str, Any]] = [
    {"id": "mon-sysops", "label": "SysOps", "agent_key": "sysops", "role": "Backup · Proxmox · Docker · maintenance"},
    {"id": "mon-research", "label": "Research", "agent_key": "research", "role": "Markt intel (approval gate)"},
]


def _iso_now() -> str:
    return datetime.now(timezone.utc).isoformat()


def _run_ssh(host: str, command: str, timeout: int = 12) -> dict[str, Any]:
    ssh_cmd = [
        "sshpass", "-p", SSH_PASSWORD, "ssh",
        "-o", "StrictHostKeyChecking=no",
        "-o", "UserKnownHostsFile=/dev/null",
        "-o", "ConnectTimeout=7",
        f"{SSH_USER}@{host}",
        command,
    ]
    try:
        proc = subprocess.run(  # noqa: S603
            ssh_cmd, capture_output=True, text=True, timeout=timeout, check=False,
        )
    except FileNotFoundError as exc:
        return {"ok": False, "error": f"ssh tooling missing: {exc}"}
    except subprocess.TimeoutExpired:
        return {"ok": False, "error": "ssh timeout"}
    return {
        "ok": proc.returncode == 0,
        "code": proc.returncode,
        "stdout": (proc.stdout or "").strip(),
        "stderr": (proc.stderr or "").strip(),
    }


def _http_get_json(url: str, headers: dict[str, str] | None = None, timeout: int = 8) -> dict[str, Any]:
    req = Request(url, headers=headers or {})
    ctx = ssl.create_default_context()
    ctx.check_hostname = False
    ctx.verify_mode = ssl.CERT_NONE
    with urlopen(req, timeout=timeout, context=ctx) as resp:  # noqa: S310
        return json.loads(resp.read().decode("utf-8"))


def _build_token_header(token_value: str) -> str:
    val = token_value.strip()
    return val if val.startswith("PVEAPIToken=") else f"PVEAPIToken={val}"


def _create_api_token_via_ssh() -> str | None:
    token_name = f"ops{int(time.time())}"
    cmd = (
        f"pveum user token add {shlex.quote(SSH_USER + '@pam')} {shlex.quote(token_name)} "
        "--privsep 0 --expire 0 --output-format json"
    )
    result = _run_ssh(PROXMOX_HOST, cmd, timeout=15)
    if not result.get("ok"):
        return None
    try:
        parsed = json.loads(result.get("stdout") or "{}")
    except json.JSONDecodeError:
        return None
    tokenid, secret = parsed.get("full-tokenid"), parsed.get("value")
    return f"PVEAPIToken={tokenid}={secret}" if tokenid and secret else None


def _fetch_proxmox_state() -> tuple[str, float, float, str, str, str | None]:
    token = os.getenv("PROXMOX_TOKEN")
    api_source, api_error = "none", None
    nodes: list[dict[str, Any]] = []

    if token:
        try:
            data = _http_get_json(f"{PROXMOX_API_URL}/nodes", headers={"Authorization": _build_token_header(token)})
            nodes = data.get("data") or []
            api_source = "api-token-env"
        except Exception as exc:  # noqa: BLE001
            api_error = str(exc)

    if not nodes:
        created = _create_api_token_via_ssh()
        if created:
            try:
                data = _http_get_json(f"{PROXMOX_API_URL}/nodes", headers={"Authorization": created})
                nodes = data.get("data") or []
                api_source = "api-token-ssh"
            except Exception as exc:  # noqa: BLE001
                api_error = str(exc)

    if not nodes:
        pvesh = _run_ssh(PROXMOX_HOST, "pvesh get /nodes --output-format json")
        if pvesh.get("ok"):
            try:
                nodes = json.loads(pvesh["stdout"] or "[]")
                api_source = "ssh-pvesh"
            except json.JSONDecodeError:
                api_error = pvesh.get("stderr") or "pvesh parse error"

    api_node = next((n for n in nodes if isinstance(n, dict) and (n.get("node") or "").strip()), None)
    cpu = float(api_node.get("cpu", 0)) if api_node else 0.0
    mem = float(api_node.get("mem", 0)) if api_node else 0.0
    status = str(api_node.get("status") or "unknown") if api_node else "unknown"

    vm_status = "unknown"
    qm = _run_ssh(PROXMOX_HOST, f"qm status {VM_106_ID} 2>/dev/null || qm list | grep '^{VM_106_ID} '")
    if qm.get("ok") and qm.get("stdout"):
        line = qm["stdout"].splitlines()[0].lower()
        if "running" in line:
            vm_status = "running"
        elif "stopped" in line:
            vm_status = "stopped"
        else:
            vm_status = "online" if _port_health(VM_106_IP, 22) else "offline"

    if status == "unknown" and _port_health(PROXMOX_HOST, 8006):
        status = "online"

    return status, cpu, mem, vm_status, api_source, api_error


def _port_health(host: str, port: int, timeout: float = 1.5) -> bool:
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.settimeout(timeout)
    try:
        return sock.connect_ex((host, port)) == 0
    finally:
        sock.close()


def check_docker_services() -> dict[str, Any]:
    result = _run_ssh(VM_106_IP, "docker ps --format json")
    if not result.get("ok"):
        result = _run_ssh(VM_106_IP, "docker ps --format '{{json .}}'")
    if not result.get("ok"):
        return {"ok": False, "error": result.get("stderr") or "docker check failed", "containers": []}
    containers: list[dict[str, Any]] = []
    for line in (result.get("stdout") or "").splitlines():
        line = line.strip()
        if not line:
            continue
        try:
            parsed = json.loads(line)
            containers.append(parsed if isinstance(parsed, dict) else {"raw": parsed})
        except json.JSONDecodeError:
            containers.append({"raw": line})
    return {"ok": True, "containers": containers}


def _online_services() -> list[dict[str, Any]]:
    docker_state = check_docker_services()
    docker_names = {
        str(c.get("Names") or c.get("Name") or "").lower(): c
        for c in docker_state.get("containers", [])
    }
    online: list[dict[str, Any]] = []
    for svc in SERVICE_CATALOG:
        up = _port_health(VM_106_IP, int(svc["port"]))
        if not up:
            continue
        hint = "running"
        for name, details in docker_names.items():
            slug = svc["id"].replace("svc-", "").replace("-", "")
            if slug in name.replace("-", ""):
                hint = str(details.get("State") or details.get("Status") or "running")
                break
        online.append({
            "id": svc["id"],
            "label": f"{svc['label']}:{svc['port']}",
            "type": "service",
            "status": "online",
            "host": VM_106_IP,
            "port": svc["port"],
            "url": svc["url"],
            "role": svc["role"],
            "hint": hint,
            "agent": "sysops",
            "children": [],
        })
    return online


def get_topology() -> dict[str, Any]:
    host_status, host_cpu, host_mem, vm_status, api_source, api_error = _fetch_proxmox_state()
    services = _online_services()
    proxmox_reachable = _port_health(PROXMOX_HOST, 8006)

    topology_nodes = [
        {
            "id": "hardware-r340",
            "label": HARDWARE_LABEL,
            "type": "hardware",
            "status": "online" if proxmox_reachable or _port_health(VM_106_IP, 8600) else "degraded",
            "role": "Bare metal host",
            "children": [
                {
                    "id": "proxmox-host",
                    "label": f"Proxmox VE · {PROXMOX_HOST}",
                    "type": "proxmox",
                    "status": host_status if proxmox_reachable else ("online" if _port_health(PROXMOX_HOST, 22) else "offline"),
                    "cpu": host_cpu,
                    "mem": host_mem,
                    "role": "Hypervisor · node pve",
                    "url": f"https://{PROXMOX_HOST}:8006",
                    "children": [
                        {
                            "id": "vm106-command",
                            "label": f"VM {VM_106_ID} · {VM_106_NAME} · {VM_106_IP}",
                            "type": "vm",
                            "status": vm_status if vm_status != "unknown" else ("online" if services else "offline"),
                            "vmid": VM_106_ID,
                            "role": "Foodlinkk Command Center",
                            "children": services,
                        }
                    ],
                }
            ],
        }
    ]

    return {
        "generated_at": _iso_now(),
        "nodes": topology_nodes,
        "layer_agents": LAYER_AGENTS,
        "layers": [
            {"id": "L0", "label": "Hardware", "y": 55},
            {"id": "L1", "label": "Hypervisor", "y": 175},
            {"id": "L2", "label": f"VM {VM_106_ID}", "y": 295},
            {"id": "L3", "label": "Online services", "y": 415},
            {"id": "L4", "label": "IT Agents", "y": 555},
        ],
        "meta": {
            "hardware": HARDWARE_LABEL,
            "proxmox_host": PROXMOX_HOST,
            "vm106_ip": VM_106_IP,
            "vm106_id": VM_106_ID,
            "services_online": len(services),
            "api_source": api_source,
            "api_error": api_error,
        },
    }


def get_status_summary() -> dict[str, Any]:
    topo = get_topology()
    flat: list[dict[str, Any]] = []

    def _collect(node: dict[str, Any]) -> None:
        flat.append(node)
        for child in node.get("children") or []:
            _collect(child)

    for root in topo.get("nodes") or []:
        _collect(root)

    total = len(flat)
    online = sum(1 for n in flat if str(n.get("status")).lower() in {"online", "running", "up"})
    degraded = sum(1 for n in flat if str(n.get("status")).lower() in {"unknown", "degraded"})
    offline = max(0, total - online - degraded)
    return {
        "generated_at": topo.get("generated_at"),
        "health": "healthy" if offline == 0 else ("degraded" if online > 0 else "down"),
        "counts": {"total": total, "online": online, "degraded": degraded, "offline": offline},
        "sources": topo.get("meta", {}),
        "topology": topo,
    }


def _table_exists(table_name: str) -> bool:
    row = fetch_one(
        """
        SELECT EXISTS(
            SELECT 1 FROM information_schema.tables
            WHERE table_schema = 'public' AND table_name = %s
        ) AS ok
        """,
        (table_name,),
    )
    return bool(row and row.get("ok"))


def poll_and_snapshot() -> dict[str, Any]:
    status = get_status_summary()
    if not _table_exists("infra_snapshots"):
        return {"ok": False, "saved": False, "reason": "infra_snapshots table not found", "status": status}

    cols = fetch_all(
        """
        SELECT column_name FROM information_schema.columns
        WHERE table_schema = 'public' AND table_name = 'infra_snapshots'
        ORDER BY ordinal_position
        """
    )
    colset = {c.get("column_name") for c in cols}
    payload = {
        "source": "proxmox",
        "topology": status.get("topology"),
        "summary": {k: v for k, v in status.items() if k != "topology"},
        "generated_at": status.get("generated_at"),
    }
    value_map: dict[str, Any] = {}
    if "source" in colset:
        value_map["source"] = "proxmox"
    if "provider" in colset:
        value_map["provider"] = "proxmox"
    if "snapshot" in colset:
        value_map["snapshot"] = json.dumps(payload)
    if "payload" in colset:
        value_map["payload"] = json.dumps(payload)
    if "topology" in colset:
        value_map["topology"] = json.dumps(status.get("topology"))
    if "summary" in colset:
        value_map["summary"] = json.dumps({k: v for k, v in status.items() if k != "topology"})
    if "created_at" in colset:
        value_map["created_at"] = datetime.now(timezone.utc)

    if not value_map:
        return {"ok": False, "saved": False, "reason": "infra_snapshots has no compatible columns", "status": status}

    columns = list(value_map.keys())
    sql = f"INSERT INTO infra_snapshots ({', '.join(columns)}) VALUES ({', '.join([ '%s' ] * len(columns))})"
    execute(sql, tuple(value_map[c] for c in columns))
    return {"ok": True, "saved": True, "columns": columns, "status": status}