foodlinkk-command-center/tools-api/app/connectors/proxmox.py

"""Proxmox infrastructure monitoring connector for Foodlinkk IT Ops."""
from __future__ import annotations

import json
import os
import shlex
import socket
import ssl
import subprocess
import time
from datetime import datetime, timezone
from typing import Any
from urllib.error import URLError
from urllib.request import Request, urlopen

from app.db import execute, fetch_all, fetch_one

PROXMOX_HOST = "10.4.7.14"
PROXMOX_API_URL = f"https://{PROXMOX_HOST}:8006/api2/json"
SSH_USER = "aissa"
SSH_PASSWORD = "Foodlinkk#2026"

VM_105_IP = "10.4.7.19"
VM_106_IP = "10.4.7.18"

SERVICE_LAYOUT: list[dict[str, Any]] = [
    {"id": "svc-cockpit", "label": "cockpit:8600", "host": VM_106_IP, "parent": "vm106-command", "port": 8600},
    {"id": "svc-tools-api", "label": "tools-api:8700", "host": VM_106_IP, "parent": "vm106-command", "port": 8700},
    {"id": "svc-email-agent", "label": "email-agent:8801", "host": VM_106_IP, "parent": "vm106-command", "port": 8801},
    {"id": "svc-gitea", "label": "gitea:3001", "host": VM_105_IP, "parent": "vm105-hermes", "port": 3001},
    {"id": "svc-ollama", "label": "ollama:11434", "host": VM_105_IP, "parent": "vm105-hermes", "port": 11434},
]


def _iso_now() -> str:
    return datetime.now(timezone.utc).isoformat()


def _run_ssh(host: str, command: str, timeout: int = 12) -> dict[str, Any]:
    ssh_cmd = [
        "sshpass",
        "-p",
        SSH_PASSWORD,
        "ssh",
        "-o",
        "StrictHostKeyChecking=no",
        "-o",
        "UserKnownHostsFile=/dev/null",
        "-o",
        "ConnectTimeout=7",
        f"{SSH_USER}@{host}",
        command,
    ]
    try:
        proc = subprocess.run(  # noqa: S603
            ssh_cmd,
            capture_output=True,
            text=True,
            timeout=timeout,
            check=False,
        )
    except FileNotFoundError as exc:
        return {"ok": False, "error": f"ssh tooling missing: {exc}"}
    except subprocess.TimeoutExpired:
        return {"ok": False, "error": "ssh timeout"}
    return {
        "ok": proc.returncode == 0,
        "code": proc.returncode,
        "stdout": (proc.stdout or "").strip(),
        "stderr": (proc.stderr or "").strip(),
    }


def _http_get_json(url: str, headers: dict[str, str] | None = None, timeout: int = 8) -> dict[str, Any]:
    req = Request(url, headers=headers or {})
    ctx = ssl.create_default_context()
    ctx.check_hostname = False
    ctx.verify_mode = ssl.CERT_NONE
    with urlopen(req, timeout=timeout, context=ctx) as resp:  # noqa: S310
        payload = resp.read().decode("utf-8")
    return json.loads(payload)


def _build_token_header(token_value: str) -> str:
    val = token_value.strip()
    if val.startswith("PVEAPIToken="):
        return val
    return f"PVEAPIToken={val}"


def _create_api_token_via_ssh() -> str | None:
    token_name = f"ops{int(time.time())}"
    cmd = (
        f"pveum user token add {shlex.quote(SSH_USER + '@pam')} {shlex.quote(token_name)} "
        "--privsep 0 --expire 0 --output-format json"
    )
    result = _run_ssh(PROXMOX_HOST, cmd, timeout=15)
    if not result.get("ok"):
        return None
    try:
        parsed = json.loads(result.get("stdout") or "{}")
    except json.JSONDecodeError:
        return None
    tokenid = parsed.get("full-tokenid")
    secret = parsed.get("value")
    if tokenid and secret:
        return f"PVEAPIToken={tokenid}={secret}"
    return None


def _fetch_nodes_via_api() -> tuple[list[dict[str, Any]], str, str | None]:
    token = os.getenv("PROXMOX_TOKEN")
    tried = []
    if token:
        tried.append("env-token")
        try:
            data = _http_get_json(
                f"{PROXMOX_API_URL}/nodes",
                headers={"Authorization": _build_token_header(token)},
            )
            return data.get("data") or [], "api-token-env", None
        except Exception as exc:  # noqa: BLE001
            tried.append(f"env-failed:{exc}")
    created = _create_api_token_via_ssh()
    if created:
        tried.append("ssh-created-token")
        try:
            data = _http_get_json(
                f"{PROXMOX_API_URL}/nodes",
                headers={"Authorization": created},
            )
            return data.get("data") or [], "api-token-ssh", None
        except Exception as exc:  # noqa: BLE001
            tried.append(f"ssh-token-failed:{exc}")
    return [], "none", ", ".join(tried) if tried else "no-token"


def _fetch_nodes_via_ssh() -> tuple[list[dict[str, Any]], str, str | None]:
    pvesh = _run_ssh(PROXMOX_HOST, "pvesh get /nodes --output-format json")
    if pvesh.get("ok"):
        try:
            return json.loads(pvesh["stdout"] or "[]"), "ssh-pvesh", None
        except json.JSONDecodeError:
            pass
    qm = _run_ssh(PROXMOX_HOST, "qm list")
    rows: list[dict[str, Any]] = []
    if qm.get("ok") and qm.get("stdout"):
        lines = (qm["stdout"] or "").splitlines()
        for line in lines[1:]:
            parts = line.split()
            if not parts:
                continue
            vmid = parts[0]
            rows.append(
                {
                    "node": "pve",
                    "type": "qemu",
                    "id": f"qemu/{vmid}",
                    "vmid": int(vmid) if vmid.isdigit() else vmid,
                    "status": parts[2] if len(parts) > 2 else "unknown",
                }
            )
        return rows, "ssh-qm-list", None
    err = pvesh.get("stderr") or qm.get("stderr") or "ssh lookup failed"
    return [], "none", err


def _port_health(host: str, port: int, timeout: float = 1.5) -> bool:
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    sock.settimeout(timeout)
    try:
        return sock.connect_ex((host, port)) == 0
    finally:
        sock.close()


def check_docker_services() -> dict[str, Any]:
    result = _run_ssh(VM_106_IP, "docker ps --format json")
    method = "docker-ps-json"
    if not result.get("ok"):
        result = _run_ssh(VM_106_IP, "docker ps --format '{{json .}}'")
        method = "docker-ps-template-json"
    if not result.get("ok"):
        return {"ok": False, "source": method, "error": result.get("stderr") or "docker check failed", "containers": []}

    containers: list[dict[str, Any]] = []
    for line in (result.get("stdout") or "").splitlines():
        line = line.strip()
        if not line:
            continue
        try:
            parsed = json.loads(line)
            containers.append(parsed if isinstance(parsed, dict) else {"raw": parsed})
        except json.JSONDecodeError:
            containers.append({"raw": line})
    return {"ok": True, "source": method, "containers": containers}


def get_topology() -> dict[str, Any]:
    api_nodes, api_source, api_error = _fetch_nodes_via_api()
    ssh_nodes: list[dict[str, Any]] = []
    ssh_source = "none"
    ssh_error: str | None = None
    if not api_nodes:
        ssh_nodes, ssh_source, ssh_error = _fetch_nodes_via_ssh()

    api_node = next((n for n in api_nodes if (n.get("node") or "").strip()), None) if api_nodes else None
    host_cpu = float(api_node.get("cpu", 0)) if api_node else 0.0
    host_mem = float(api_node.get("mem", 0)) if api_node else 0.0
    host_status = api_node.get("status") if api_node else "unknown"
    if host_status == "unknown" and ssh_nodes:
        host_status = "online"

    vm_states: dict[str, str] = {"105": "unknown", "106": "unknown"}
    source_rows = api_nodes or ssh_nodes
    for row in source_rows:
        vmid = str(row.get("vmid") or "").strip()
        if vmid in vm_states:
            vm_states[vmid] = str(row.get("status") or "unknown")

    docker_state = check_docker_services()
    docker_names = {
        str(c.get("Names") or c.get("Names.0") or c.get("Name") or "").lower(): c for c in docker_state.get("containers", [])
    }

    vm105_children: list[dict[str, Any]] = []
    vm106_children: list[dict[str, Any]] = []
    for svc in SERVICE_LAYOUT:
        up = _port_health(str(svc["host"]), int(svc["port"]))
        hinted = "unknown"
        for name, details in docker_names.items():
            if svc["label"].split(":")[0].replace("-", "") in name.replace("-", ""):
                hinted = str(details.get("State") or details.get("Status") or "running")
                break
        item = {
            "id": svc["id"],
            "label": svc["label"],
            "type": "service",
            "status": "online" if up else "offline",
            "cpu": None,
            "mem": None,
            "host": svc["host"],
            "hint": hinted,
            "children": [],
        }
        if svc["parent"] == "vm105-hermes":
            vm105_children.append(item)
        else:
            vm106_children.append(item)

    topology_nodes = [
        {
            "id": "proxmox-host",
            "label": f"proxmox-host ({PROXMOX_HOST})",
            "type": "proxmox",
            "status": host_status,
            "cpu": host_cpu,
            "mem": host_mem,
            "children": [
                {
                    "id": "vm105-hermes",
                    "label": f"vm105-hermes ({VM_105_IP})",
                    "type": "vm",
                    "status": vm_states["105"],
                    "cpu": None,
                    "mem": None,
                    "children": vm105_children,
                },
                {
                    "id": "vm106-command",
                    "label": f"vm106-command ({VM_106_IP})",
                    "type": "vm",
                    "status": vm_states["106"],
                    "cpu": None,
                    "mem": None,
                    "children": vm106_children,
                },
            ],
        }
    ]
    return {
        "generated_at": _iso_now(),
        "nodes": topology_nodes,
        "meta": {
            "proxmox_host": PROXMOX_HOST,
            "api_source": api_source,
            "api_error": api_error,
            "ssh_source": ssh_source,
            "ssh_error": ssh_error,
            "docker_source": docker_state.get("source"),
            "docker_ok": docker_state.get("ok", False),
        },
    }


def get_status_summary() -> dict[str, Any]:
    topo = get_topology()
    flat: list[dict[str, Any]] = []

    def _collect(node: dict[str, Any]) -> None:
        flat.append(node)
        for child in node.get("children") or []:
            _collect(child)

    for root in topo.get("nodes") or []:
        _collect(root)

    total = len(flat)
    online = sum(1 for n in flat if str(n.get("status")).lower() in {"online", "running", "up"})
    degraded = sum(1 for n in flat if str(n.get("status")).lower() in {"unknown", "degraded"})
    offline = max(0, total - online - degraded)
    return {
        "generated_at": topo.get("generated_at"),
        "health": "healthy" if offline == 0 else ("degraded" if online > 0 else "down"),
        "counts": {"total": total, "online": online, "degraded": degraded, "offline": offline},
        "sources": topo.get("meta", {}),
        "topology": topo,
    }


def _table_exists(table_name: str) -> bool:
    row = fetch_one(
        """
        SELECT EXISTS(
            SELECT 1
            FROM information_schema.tables
            WHERE table_schema = 'public'
              AND table_name = %s
        ) AS ok
        """,
        (table_name,),
    )
    return bool(row and row.get("ok"))


def poll_and_snapshot() -> dict[str, Any]:
    status = get_status_summary()
    if not _table_exists("infra_snapshots"):
        return {"ok": False, "saved": False, "reason": "infra_snapshots table not found", "status": status}

    cols = fetch_all(
        """
        SELECT column_name
        FROM information_schema.columns
        WHERE table_schema = 'public'
          AND table_name = 'infra_snapshots'
        ORDER BY ordinal_position
        """
    )
    colset = {c.get("column_name") for c in cols}
    payload = {
        "source": "proxmox",
        "topology": status.get("topology"),
        "summary": {k: v for k, v in status.items() if k != "topology"},
        "generated_at": status.get("generated_at"),
    }

    value_map: dict[str, Any] = {}
    if "source" in colset:
        value_map["source"] = "proxmox"
    if "provider" in colset:
        value_map["provider"] = "proxmox"
    if "snapshot" in colset:
        value_map["snapshot"] = json.dumps(payload)
    if "payload" in colset:
        value_map["payload"] = json.dumps(payload)
    if "topology" in colset:
        value_map["topology"] = json.dumps(status.get("topology"))
    if "summary" in colset:
        value_map["summary"] = json.dumps({k: v for k, v in status.items() if k != "topology"})
    if "created_at" in colset:
        value_map["created_at"] = datetime.now(timezone.utc)

    if not value_map:
        return {"ok": False, "saved": False, "reason": "infra_snapshots has no compatible columns", "status": status}

    columns = list(value_map.keys())
    placeholders = ", ".join(["%s"] * len(columns))
    sql = f"INSERT INTO infra_snapshots ({', '.join(columns)}) VALUES ({placeholders})"
    execute(sql, tuple(value_map[c] for c in columns))
    return {"ok": True, "saved": True, "columns": columns, "status": status}