"""Proxmox infrastructure monitoring — Dell R340 · Proxmox VE · VM106 services.""" from __future__ import annotations import json import os import shlex import socket import ssl import subprocess import time from datetime import datetime, timezone from typing import Any from urllib.error import URLError from urllib.request import Request, urlopen from app.db import execute, fetch_all, fetch_one PROXMOX_HOST = os.getenv("PROXMOX_HOST", "10.4.7.14") PROXMOX_API_URL = f"https://{PROXMOX_HOST}:8006/api2/json" SSH_USER = os.getenv("PROXMOX_SSH_USER", "aissa") SSH_PASSWORD = os.getenv("PROXMOX_SSH_PASS", "Foodlinkk#2026") VM_106_IP = os.getenv("VM106_IP", "10.4.7.18") VM_106_ID = 106 VM_106_NAME = "dockervm" HARDWARE_LABEL = "Dell PowerEdge R340" SERVICE_CATALOG: list[dict[str, Any]] = [ {"id": "svc-cockpit", "label": "Cockpit", "port": 8600, "url": f"http://{VM_106_IP}:8600", "role": "Command Center UI"}, {"id": "svc-tools-api", "label": "Tools API", "port": 8700, "url": f"http://{VM_106_IP}:8700/docs", "role": "Connectors & API"}, {"id": "svc-gitea", "label": "Gitea", "port": 3001, "url": f"http://{VM_106_IP}:3001", "role": "Git · config backups"}, {"id": "svc-email-agent", "label": "Email Agent", "port": 8801, "url": f"http://{VM_106_IP}:8801", "role": "IMAP/SMTP sync"}, {"id": "svc-grafana", "label": "Grafana", "port": 3002, "url": f"http://{VM_106_IP}:3002", "role": "Metrics dashboards"}, {"id": "svc-prometheus", "label": "Prometheus", "port": 9090, "url": f"http://{VM_106_IP}:9090", "role": "Metrics scrape"}, {"id": "svc-browser-agent", "label": "Browser Agent", "port": 7790, "url": f"http://{VM_106_IP}:7790", "role": "Web automation"}, ] LAYER_AGENTS: list[dict[str, Any]] = [ {"id": "mon-sysops", "label": "SysOps", "agent_key": "sysops", "role": "Backup · Proxmox · Docker · maintenance"}, {"id": "mon-research", "label": "Research", "agent_key": "research", "role": "Markt intel (approval gate)"}, ] def _iso_now() -> str: return datetime.now(timezone.utc).isoformat() def _run_ssh(host: str, command: str, timeout: int = 12) -> dict[str, Any]: ssh_cmd = [ "sshpass", "-p", SSH_PASSWORD, "ssh", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", "-o", "ConnectTimeout=7", f"{SSH_USER}@{host}", command, ] try: proc = subprocess.run( # noqa: S603 ssh_cmd, capture_output=True, text=True, timeout=timeout, check=False, ) except FileNotFoundError as exc: return {"ok": False, "error": f"ssh tooling missing: {exc}"} except subprocess.TimeoutExpired: return {"ok": False, "error": "ssh timeout"} return { "ok": proc.returncode == 0, "code": proc.returncode, "stdout": (proc.stdout or "").strip(), "stderr": (proc.stderr or "").strip(), } def _http_get_json(url: str, headers: dict[str, str] | None = None, timeout: int = 8) -> dict[str, Any]: req = Request(url, headers=headers or {}) ctx = ssl.create_default_context() ctx.check_hostname = False ctx.verify_mode = ssl.CERT_NONE with urlopen(req, timeout=timeout, context=ctx) as resp: # noqa: S310 return json.loads(resp.read().decode("utf-8")) def _build_token_header(token_value: str) -> str: val = token_value.strip() return val if val.startswith("PVEAPIToken=") else f"PVEAPIToken={val}" def _create_api_token_via_ssh() -> str | None: token_name = f"ops{int(time.time())}" cmd = ( f"pveum user token add {shlex.quote(SSH_USER + '@pam')} {shlex.quote(token_name)} " "--privsep 0 --expire 0 --output-format json" ) result = _run_ssh(PROXMOX_HOST, cmd, timeout=15) if not result.get("ok"): return None try: parsed = json.loads(result.get("stdout") or "{}") except json.JSONDecodeError: return None tokenid, secret = parsed.get("full-tokenid"), parsed.get("value") return f"PVEAPIToken={tokenid}={secret}" if tokenid and secret else None def _fetch_proxmox_state() -> tuple[str, float, float, str, str, str | None]: token = os.getenv("PROXMOX_TOKEN") api_source, api_error = "none", None nodes: list[dict[str, Any]] = [] if token: try: data = _http_get_json(f"{PROXMOX_API_URL}/nodes", headers={"Authorization": _build_token_header(token)}) nodes = data.get("data") or [] api_source = "api-token-env" except Exception as exc: # noqa: BLE001 api_error = str(exc) if not nodes: created = _create_api_token_via_ssh() if created: try: data = _http_get_json(f"{PROXMOX_API_URL}/nodes", headers={"Authorization": created}) nodes = data.get("data") or [] api_source = "api-token-ssh" except Exception as exc: # noqa: BLE001 api_error = str(exc) if not nodes: pvesh = _run_ssh(PROXMOX_HOST, "pvesh get /nodes --output-format json") if pvesh.get("ok"): try: nodes = json.loads(pvesh["stdout"] or "[]") api_source = "ssh-pvesh" except json.JSONDecodeError: api_error = pvesh.get("stderr") or "pvesh parse error" api_node = next((n for n in nodes if isinstance(n, dict) and (n.get("node") or "").strip()), None) cpu = float(api_node.get("cpu", 0)) if api_node else 0.0 mem = float(api_node.get("mem", 0)) if api_node else 0.0 status = str(api_node.get("status") or "unknown") if api_node else "unknown" vm_status = "unknown" qm = _run_ssh(PROXMOX_HOST, f"qm status {VM_106_ID} 2>/dev/null || qm list | grep '^{VM_106_ID} '") if qm.get("ok") and qm.get("stdout"): line = qm["stdout"].splitlines()[0].lower() if "running" in line: vm_status = "running" elif "stopped" in line: vm_status = "stopped" else: vm_status = "online" if _port_health(VM_106_IP, 22) else "offline" if status == "unknown" and _port_health(PROXMOX_HOST, 8006): status = "online" return status, cpu, mem, vm_status, api_source, api_error def _port_health(host: str, port: int, timeout: float = 1.5) -> bool: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.settimeout(timeout) try: return sock.connect_ex((host, port)) == 0 finally: sock.close() def check_docker_services() -> dict[str, Any]: result = _run_ssh(VM_106_IP, "docker ps --format json") if not result.get("ok"): result = _run_ssh(VM_106_IP, "docker ps --format '{{json .}}'") if not result.get("ok"): return {"ok": False, "error": result.get("stderr") or "docker check failed", "containers": []} containers: list[dict[str, Any]] = [] for line in (result.get("stdout") or "").splitlines(): line = line.strip() if not line: continue try: parsed = json.loads(line) containers.append(parsed if isinstance(parsed, dict) else {"raw": parsed}) except json.JSONDecodeError: containers.append({"raw": line}) return {"ok": True, "containers": containers} def _online_services() -> list[dict[str, Any]]: docker_state = check_docker_services() docker_names = { str(c.get("Names") or c.get("Name") or "").lower(): c for c in docker_state.get("containers", []) } online: list[dict[str, Any]] = [] for svc in SERVICE_CATALOG: up = _port_health(VM_106_IP, int(svc["port"])) if not up: continue hint = "running" for name, details in docker_names.items(): slug = svc["id"].replace("svc-", "").replace("-", "") if slug in name.replace("-", ""): hint = str(details.get("State") or details.get("Status") or "running") break online.append({ "id": svc["id"], "label": f"{svc['label']}:{svc['port']}", "type": "service", "status": "online", "host": VM_106_IP, "port": svc["port"], "url": svc["url"], "role": svc["role"], "hint": hint, "agent": "sysops", "children": [], }) return online def get_topology() -> dict[str, Any]: host_status, host_cpu, host_mem, vm_status, api_source, api_error = _fetch_proxmox_state() services = _online_services() proxmox_reachable = _port_health(PROXMOX_HOST, 8006) topology_nodes = [ { "id": "hardware-r340", "label": HARDWARE_LABEL, "type": "hardware", "status": "online" if proxmox_reachable or _port_health(VM_106_IP, 8600) else "degraded", "role": "Bare metal host", "children": [ { "id": "proxmox-host", "label": f"Proxmox VE · {PROXMOX_HOST}", "type": "proxmox", "status": host_status if proxmox_reachable else ("online" if _port_health(PROXMOX_HOST, 22) else "offline"), "cpu": host_cpu, "mem": host_mem, "role": "Hypervisor · node pve", "url": f"https://{PROXMOX_HOST}:8006", "children": [ { "id": "vm106-command", "label": f"VM {VM_106_ID} · {VM_106_NAME} · {VM_106_IP}", "type": "vm", "status": vm_status if vm_status != "unknown" else ("online" if services else "offline"), "vmid": VM_106_ID, "role": "Foodlinkk Command Center", "children": services, } ], } ], } ] return { "generated_at": _iso_now(), "nodes": topology_nodes, "layer_agents": LAYER_AGENTS, "layers": [ {"id": "L0", "label": "Hardware", "y": 55}, {"id": "L1", "label": "Hypervisor", "y": 175}, {"id": "L2", "label": f"VM {VM_106_ID}", "y": 295}, {"id": "L3", "label": "Online services", "y": 415}, {"id": "L4", "label": "IT Agents", "y": 555}, ], "meta": { "hardware": HARDWARE_LABEL, "proxmox_host": PROXMOX_HOST, "vm106_ip": VM_106_IP, "vm106_id": VM_106_ID, "services_online": len(services), "api_source": api_source, "api_error": api_error, }, } def get_status_summary() -> dict[str, Any]: topo = get_topology() flat: list[dict[str, Any]] = [] def _collect(node: dict[str, Any]) -> None: flat.append(node) for child in node.get("children") or []: _collect(child) for root in topo.get("nodes") or []: _collect(root) total = len(flat) online = sum(1 for n in flat if str(n.get("status")).lower() in {"online", "running", "up"}) degraded = sum(1 for n in flat if str(n.get("status")).lower() in {"unknown", "degraded"}) offline = max(0, total - online - degraded) return { "generated_at": topo.get("generated_at"), "health": "healthy" if offline == 0 else ("degraded" if online > 0 else "down"), "counts": {"total": total, "online": online, "degraded": degraded, "offline": offline}, "sources": topo.get("meta", {}), "topology": topo, } def _table_exists(table_name: str) -> bool: row = fetch_one( """ SELECT EXISTS( SELECT 1 FROM information_schema.tables WHERE table_schema = 'public' AND table_name = %s ) AS ok """, (table_name,), ) return bool(row and row.get("ok")) def poll_and_snapshot() -> dict[str, Any]: status = get_status_summary() if not _table_exists("infra_snapshots"): return {"ok": False, "saved": False, "reason": "infra_snapshots table not found", "status": status} cols = fetch_all( """ SELECT column_name FROM information_schema.columns WHERE table_schema = 'public' AND table_name = 'infra_snapshots' ORDER BY ordinal_position """ ) colset = {c.get("column_name") for c in cols} payload = { "source": "proxmox", "topology": status.get("topology"), "summary": {k: v for k, v in status.items() if k != "topology"}, "generated_at": status.get("generated_at"), } value_map: dict[str, Any] = {} if "source" in colset: value_map["source"] = "proxmox" if "provider" in colset: value_map["provider"] = "proxmox" if "snapshot" in colset: value_map["snapshot"] = json.dumps(payload) if "payload" in colset: value_map["payload"] = json.dumps(payload) if "topology" in colset: value_map["topology"] = json.dumps(status.get("topology")) if "summary" in colset: value_map["summary"] = json.dumps({k: v for k, v in status.items() if k != "topology"}) if "created_at" in colset: value_map["created_at"] = datetime.now(timezone.utc) if not value_map: return {"ok": False, "saved": False, "reason": "infra_snapshots has no compatible columns", "status": status} columns = list(value_map.keys()) sql = f"INSERT INTO infra_snapshots ({', '.join(columns)}) VALUES ({', '.join([ '%s' ] * len(columns))})" execute(sql, tuple(value_map[c] for c in columns)) return {"ok": True, "saved": True, "columns": columns, "status": status}