"""Proxmox infrastructure monitoring connector for Foodlinkk IT Ops.""" from __future__ import annotations import json import os import shlex import socket import ssl import subprocess import time from datetime import datetime, timezone from typing import Any from urllib.error import URLError from urllib.request import Request, urlopen from app.db import execute, fetch_all, fetch_one PROXMOX_HOST = "10.4.7.14" PROXMOX_API_URL = f"https://{PROXMOX_HOST}:8006/api2/json" SSH_USER = "aissa" SSH_PASSWORD = "Foodlinkk#2026" VM_105_IP = "10.4.7.19" VM_106_IP = "10.4.7.18" SERVICE_LAYOUT: list[dict[str, Any]] = [ {"id": "svc-cockpit", "label": "cockpit:8600", "host": VM_106_IP, "parent": "vm106-command", "port": 8600}, {"id": "svc-tools-api", "label": "tools-api:8700", "host": VM_106_IP, "parent": "vm106-command", "port": 8700}, {"id": "svc-email-agent", "label": "email-agent:8801", "host": VM_106_IP, "parent": "vm106-command", "port": 8801}, {"id": "svc-gitea", "label": "gitea:3001", "host": VM_105_IP, "parent": "vm105-hermes", "port": 3001}, {"id": "svc-ollama", "label": "ollama:11434", "host": VM_105_IP, "parent": "vm105-hermes", "port": 11434}, ] def _iso_now() -> str: return datetime.now(timezone.utc).isoformat() def _run_ssh(host: str, command: str, timeout: int = 12) -> dict[str, Any]: ssh_cmd = [ "sshpass", "-p", SSH_PASSWORD, "ssh", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", "-o", "ConnectTimeout=7", f"{SSH_USER}@{host}", command, ] try: proc = subprocess.run( # noqa: S603 ssh_cmd, capture_output=True, text=True, timeout=timeout, check=False, ) except FileNotFoundError as exc: return {"ok": False, "error": f"ssh tooling missing: {exc}"} except subprocess.TimeoutExpired: return {"ok": False, "error": "ssh timeout"} return { "ok": proc.returncode == 0, "code": proc.returncode, "stdout": (proc.stdout or "").strip(), "stderr": (proc.stderr or "").strip(), } def _http_get_json(url: str, headers: dict[str, str] | None = None, timeout: int = 8) -> dict[str, Any]: req = Request(url, headers=headers or {}) ctx = ssl.create_default_context() ctx.check_hostname = False ctx.verify_mode = ssl.CERT_NONE with urlopen(req, timeout=timeout, context=ctx) as resp: # noqa: S310 payload = resp.read().decode("utf-8") return json.loads(payload) def _build_token_header(token_value: str) -> str: val = token_value.strip() if val.startswith("PVEAPIToken="): return val return f"PVEAPIToken={val}" def _create_api_token_via_ssh() -> str | None: token_name = f"ops{int(time.time())}" cmd = ( f"pveum user token add {shlex.quote(SSH_USER + '@pam')} {shlex.quote(token_name)} " "--privsep 0 --expire 0 --output-format json" ) result = _run_ssh(PROXMOX_HOST, cmd, timeout=15) if not result.get("ok"): return None try: parsed = json.loads(result.get("stdout") or "{}") except json.JSONDecodeError: return None tokenid = parsed.get("full-tokenid") secret = parsed.get("value") if tokenid and secret: return f"PVEAPIToken={tokenid}={secret}" return None def _fetch_nodes_via_api() -> tuple[list[dict[str, Any]], str, str | None]: token = os.getenv("PROXMOX_TOKEN") tried = [] if token: tried.append("env-token") try: data = _http_get_json( f"{PROXMOX_API_URL}/nodes", headers={"Authorization": _build_token_header(token)}, ) return data.get("data") or [], "api-token-env", None except Exception as exc: # noqa: BLE001 tried.append(f"env-failed:{exc}") created = _create_api_token_via_ssh() if created: tried.append("ssh-created-token") try: data = _http_get_json( f"{PROXMOX_API_URL}/nodes", headers={"Authorization": created}, ) return data.get("data") or [], "api-token-ssh", None except Exception as exc: # noqa: BLE001 tried.append(f"ssh-token-failed:{exc}") return [], "none", ", ".join(tried) if tried else "no-token" def _fetch_nodes_via_ssh() -> tuple[list[dict[str, Any]], str, str | None]: pvesh = _run_ssh(PROXMOX_HOST, "pvesh get /nodes --output-format json") if pvesh.get("ok"): try: return json.loads(pvesh["stdout"] or "[]"), "ssh-pvesh", None except json.JSONDecodeError: pass qm = _run_ssh(PROXMOX_HOST, "qm list") rows: list[dict[str, Any]] = [] if qm.get("ok") and qm.get("stdout"): lines = (qm["stdout"] or "").splitlines() for line in lines[1:]: parts = line.split() if not parts: continue vmid = parts[0] rows.append( { "node": "pve", "type": "qemu", "id": f"qemu/{vmid}", "vmid": int(vmid) if vmid.isdigit() else vmid, "status": parts[2] if len(parts) > 2 else "unknown", } ) return rows, "ssh-qm-list", None err = pvesh.get("stderr") or qm.get("stderr") or "ssh lookup failed" return [], "none", err def _port_health(host: str, port: int, timeout: float = 1.5) -> bool: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.settimeout(timeout) try: return sock.connect_ex((host, port)) == 0 finally: sock.close() def check_docker_services() -> dict[str, Any]: result = _run_ssh(VM_106_IP, "docker ps --format json") method = "docker-ps-json" if not result.get("ok"): result = _run_ssh(VM_106_IP, "docker ps --format '{{json .}}'") method = "docker-ps-template-json" if not result.get("ok"): return {"ok": False, "source": method, "error": result.get("stderr") or "docker check failed", "containers": []} containers: list[dict[str, Any]] = [] for line in (result.get("stdout") or "").splitlines(): line = line.strip() if not line: continue try: parsed = json.loads(line) containers.append(parsed if isinstance(parsed, dict) else {"raw": parsed}) except json.JSONDecodeError: containers.append({"raw": line}) return {"ok": True, "source": method, "containers": containers} def get_topology() -> dict[str, Any]: api_nodes, api_source, api_error = _fetch_nodes_via_api() ssh_nodes: list[dict[str, Any]] = [] ssh_source = "none" ssh_error: str | None = None if not api_nodes: ssh_nodes, ssh_source, ssh_error = _fetch_nodes_via_ssh() api_node = next((n for n in api_nodes if (n.get("node") or "").strip()), None) if api_nodes else None host_cpu = float(api_node.get("cpu", 0)) if api_node else 0.0 host_mem = float(api_node.get("mem", 0)) if api_node else 0.0 host_status = api_node.get("status") if api_node else "unknown" if host_status == "unknown" and ssh_nodes: host_status = "online" vm_states: dict[str, str] = {"105": "unknown", "106": "unknown"} source_rows = api_nodes or ssh_nodes for row in source_rows: vmid = str(row.get("vmid") or "").strip() if vmid in vm_states: vm_states[vmid] = str(row.get("status") or "unknown") docker_state = check_docker_services() docker_names = { str(c.get("Names") or c.get("Names.0") or c.get("Name") or "").lower(): c for c in docker_state.get("containers", []) } vm105_children: list[dict[str, Any]] = [] vm106_children: list[dict[str, Any]] = [] for svc in SERVICE_LAYOUT: up = _port_health(str(svc["host"]), int(svc["port"])) hinted = "unknown" for name, details in docker_names.items(): if svc["label"].split(":")[0].replace("-", "") in name.replace("-", ""): hinted = str(details.get("State") or details.get("Status") or "running") break item = { "id": svc["id"], "label": svc["label"], "type": "service", "status": "online" if up else "offline", "cpu": None, "mem": None, "host": svc["host"], "hint": hinted, "children": [], } if svc["parent"] == "vm105-hermes": vm105_children.append(item) else: vm106_children.append(item) topology_nodes = [ { "id": "proxmox-host", "label": f"proxmox-host ({PROXMOX_HOST})", "type": "proxmox", "status": host_status, "cpu": host_cpu, "mem": host_mem, "children": [ { "id": "vm105-hermes", "label": f"vm105-hermes ({VM_105_IP})", "type": "vm", "status": vm_states["105"], "cpu": None, "mem": None, "children": vm105_children, }, { "id": "vm106-command", "label": f"vm106-command ({VM_106_IP})", "type": "vm", "status": vm_states["106"], "cpu": None, "mem": None, "children": vm106_children, }, ], } ] return { "generated_at": _iso_now(), "nodes": topology_nodes, "meta": { "proxmox_host": PROXMOX_HOST, "api_source": api_source, "api_error": api_error, "ssh_source": ssh_source, "ssh_error": ssh_error, "docker_source": docker_state.get("source"), "docker_ok": docker_state.get("ok", False), }, } def get_status_summary() -> dict[str, Any]: topo = get_topology() flat: list[dict[str, Any]] = [] def _collect(node: dict[str, Any]) -> None: flat.append(node) for child in node.get("children") or []: _collect(child) for root in topo.get("nodes") or []: _collect(root) total = len(flat) online = sum(1 for n in flat if str(n.get("status")).lower() in {"online", "running", "up"}) degraded = sum(1 for n in flat if str(n.get("status")).lower() in {"unknown", "degraded"}) offline = max(0, total - online - degraded) return { "generated_at": topo.get("generated_at"), "health": "healthy" if offline == 0 else ("degraded" if online > 0 else "down"), "counts": {"total": total, "online": online, "degraded": degraded, "offline": offline}, "sources": topo.get("meta", {}), "topology": topo, } def _table_exists(table_name: str) -> bool: row = fetch_one( """ SELECT EXISTS( SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = %s ) AS ok """, (table_name,), ) return bool(row and row.get("ok")) def poll_and_snapshot() -> dict[str, Any]: status = get_status_summary() if not _table_exists("infra_snapshots"): return {"ok": False, "saved": False, "reason": "infra_snapshots table not found", "status": status} cols = fetch_all( """ SELECT column_name FROM information_schema.columns WHERE table_schema = 'public' AND table_name = 'infra_snapshots' ORDER BY ordinal_position """ ) colset = {c.get("column_name") for c in cols} payload = { "source": "proxmox", "topology": status.get("topology"), "summary": {k: v for k, v in status.items() if k != "topology"}, "generated_at": status.get("generated_at"), } value_map: dict[str, Any] = {} if "source" in colset: value_map["source"] = "proxmox" if "provider" in colset: value_map["provider"] = "proxmox" if "snapshot" in colset: value_map["snapshot"] = json.dumps(payload) if "payload" in colset: value_map["payload"] = json.dumps(payload) if "topology" in colset: value_map["topology"] = json.dumps(status.get("topology")) if "summary" in colset: value_map["summary"] = json.dumps({k: v for k, v in status.items() if k != "topology"}) if "created_at" in colset: value_map["created_at"] = datetime.now(timezone.utc) if not value_map: return {"ok": False, "saved": False, "reason": "infra_snapshots has no compatible columns", "status": status} columns = list(value_map.keys()) placeholders = ", ".join(["%s"] * len(columns)) sql = f"INSERT INTO infra_snapshots ({', '.join(columns)}) VALUES ({placeholders})" execute(sql, tuple(value_map[c] for c in columns)) return {"ok": True, "saved": True, "columns": columns, "status": status}