food-cockpit/foodlinkk-command-center
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

SysOps: deploy-all — 2026-06-09 10:41 UTC

Browse Source
This commit is contained in:
sysops
2026-06-09 10:41:13 +00:00
parent 69fe67cc0e
commit 21ea3a2c81
82 changed files with 8906 additions and 981 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools-api/app/connectors/proxmox.py
+157 -178
View File
@@ -1,4 +1,4 @@
"""Proxmox infrastructure monitoring connector for Foodlinkk IT Ops."""
"""Proxmox infrastructure monitoring — Dell R340 · Proxmox VE · VM106 services."""
from __future__ import annotations
import json
@@ -15,20 +15,30 @@ from urllib.request import Request, urlopen
from app.db import execute, fetch_all, fetch_one
PROXMOX_HOST = "10.4.7.14"
PROXMOX_HOST = os.getenv("PROXMOX_HOST", "10.4.7.14")
PROXMOX_API_URL = f"https://{PROXMOX_HOST}:8006/api2/json"
SSH_USER = "aissa"
SSH_PASSWORD = "Foodlinkk#2026"
SSH_USER = os.getenv("PROXMOX_SSH_USER", "aissa")
SSH_PASSWORD = os.getenv("PROXMOX_SSH_PASS", "Foodlinkk#2026")
VM_105_IP = "10.4.7.19"
VM_106_IP = "10.4.7.18"
VM_106_IP = os.getenv("VM106_IP", "10.4.7.18")
VM_106_ID = 106
VM_106_NAME = "dockervm"
SERVICE_LAYOUT: list[dict[str, Any]] = [
{"id": "svc-cockpit", "label": "cockpit:8600", "host": VM_106_IP, "parent": "vm106-command", "port": 8600},
{"id": "svc-tools-api", "label": "tools-api:8700", "host": VM_106_IP, "parent": "vm106-command", "port": 8700},
{"id": "svc-email-agent", "label": "email-agent:8801", "host": VM_106_IP, "parent": "vm106-command", "port": 8801},
{"id": "svc-gitea", "label": "gitea:3001", "host": VM_105_IP, "parent": "vm105-hermes", "port": 3001},
{"id": "svc-ollama", "label": "ollama:11434", "host": VM_105_IP, "parent": "vm105-hermes", "port": 11434},
HARDWARE_LABEL = "Dell PowerEdge R340"
SERVICE_CATALOG: list[dict[str, Any]] = [
{"id": "svc-cockpit", "label": "Cockpit", "port": 8600, "url": f"http://{VM_106_IP}:8600", "role": "Command Center UI"},
{"id": "svc-tools-api", "label": "Tools API", "port": 8700, "url": f"http://{VM_106_IP}:8700/docs", "role": "Connectors & API"},
{"id": "svc-gitea", "label": "Gitea", "port": 3001, "url": f"http://{VM_106_IP}:3001", "role": "Git · config backups"},
{"id": "svc-email-agent", "label": "Email Agent", "port": 8801, "url": f"http://{VM_106_IP}:8801", "role": "IMAP/SMTP sync"},
{"id": "svc-grafana", "label": "Grafana", "port": 3002, "url": f"http://{VM_106_IP}:3002", "role": "Metrics dashboards"},
{"id": "svc-prometheus", "label": "Prometheus", "port": 9090, "url": f"http://{VM_106_IP}:9090", "role": "Metrics scrape"},
{"id": "svc-browser-agent", "label": "Browser Agent", "port": 7790, "url": f"http://{VM_106_IP}:7790", "role": "Web automation"},
]
LAYER_AGENTS: list[dict[str, Any]] = [
{"id": "mon-sysops", "label": "SysOps", "agent_key": "sysops", "role": "Backup · Proxmox · Docker · maintenance"},
{"id": "mon-research", "label": "Research", "agent_key": "research", "role": "Markt intel (approval gate)"},
]
@@ -38,26 +48,16 @@ def _iso_now() -> str:
def _run_ssh(host: str, command: str, timeout: int = 12) -> dict[str, Any]:
ssh_cmd = [
"sshpass",
"-p",
SSH_PASSWORD,
"ssh",
"-o",
"StrictHostKeyChecking=no",
"-o",
"UserKnownHostsFile=/dev/null",
"-o",
"ConnectTimeout=7",
"sshpass", "-p", SSH_PASSWORD, "ssh",
"-o", "StrictHostKeyChecking=no",
"-o", "UserKnownHostsFile=/dev/null",
"-o", "ConnectTimeout=7",
f"{SSH_USER}@{host}",
command,
]
try:
proc = subprocess.run( # noqa: S603
ssh_cmd,
capture_output=True,
text=True,
timeout=timeout,
check=False,
ssh_cmd, capture_output=True, text=True, timeout=timeout, check=False,
)
except FileNotFoundError as exc:
return {"ok": False, "error": f"ssh tooling missing: {exc}"}
@@ -77,15 +77,12 @@ def _http_get_json(url: str, headers: dict[str, str] | None = None, timeout: int
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
with urlopen(req, timeout=timeout, context=ctx) as resp: # noqa: S310
payload = resp.read().decode("utf-8")
return json.loads(payload)
return json.loads(resp.read().decode("utf-8"))
def _build_token_header(token_value: str) -> str:
val = token_value.strip()
if val.startswith("PVEAPIToken="):
return val
return f"PVEAPIToken={val}"
return val if val.startswith("PVEAPIToken=") else f"PVEAPIToken={val}"
def _create_api_token_via_ssh() -> str | None:
@@ -101,68 +98,62 @@ def _create_api_token_via_ssh() -> str | None:
parsed = json.loads(result.get("stdout") or "{}")
except json.JSONDecodeError:
return None
tokenid = parsed.get("full-tokenid")
secret = parsed.get("value")
if tokenid and secret:
return f"PVEAPIToken={tokenid}={secret}"
return None
tokenid, secret = parsed.get("full-tokenid"), parsed.get("value")
return f"PVEAPIToken={tokenid}={secret}" if tokenid and secret else None
def _fetch_nodes_via_api() -> tuple[list[dict[str, Any]], str, str | None]:
def _fetch_proxmox_state() -> tuple[str, float, float, str, str, str | None]:
token = os.getenv("PROXMOX_TOKEN")
tried = []
api_source, api_error = "none", None
nodes: list[dict[str, Any]] = []
if token:
tried.append("env-token")
try:
data = _http_get_json(
f"{PROXMOX_API_URL}/nodes",
headers={"Authorization": _build_token_header(token)},
)
return data.get("data") or [], "api-token-env", None
data = _http_get_json(f"{PROXMOX_API_URL}/nodes", headers={"Authorization": _build_token_header(token)})
nodes = data.get("data") or []
api_source = "api-token-env"
except Exception as exc: # noqa: BLE001
tried.append(f"env-failed:{exc}")
created = _create_api_token_via_ssh()
if created:
tried.append("ssh-created-token")
try:
data = _http_get_json(
f"{PROXMOX_API_URL}/nodes",
headers={"Authorization": created},
)
return data.get("data") or [], "api-token-ssh", None
except Exception as exc: # noqa: BLE001
tried.append(f"ssh-token-failed:{exc}")
return [], "none", ", ".join(tried) if tried else "no-token"
api_error = str(exc)
if not nodes:
created = _create_api_token_via_ssh()
if created:
try:
data = _http_get_json(f"{PROXMOX_API_URL}/nodes", headers={"Authorization": created})
nodes = data.get("data") or []
api_source = "api-token-ssh"
except Exception as exc: # noqa: BLE001
api_error = str(exc)
def _fetch_nodes_via_ssh() -> tuple[list[dict[str, Any]], str, str | None]:
pvesh = _run_ssh(PROXMOX_HOST, "pvesh get /nodes --output-format json")
if pvesh.get("ok"):
try:
return json.loads(pvesh["stdout"] or "[]"), "ssh-pvesh", None
except json.JSONDecodeError:
pass
qm = _run_ssh(PROXMOX_HOST, "qm list")
rows: list[dict[str, Any]] = []
if not nodes:
pvesh = _run_ssh(PROXMOX_HOST, "pvesh get /nodes --output-format json")
if pvesh.get("ok"):
try:
nodes = json.loads(pvesh["stdout"] or "[]")
api_source = "ssh-pvesh"
except json.JSONDecodeError:
api_error = pvesh.get("stderr") or "pvesh parse error"
api_node = next((n for n in nodes if isinstance(n, dict) and (n.get("node") or "").strip()), None)
cpu = float(api_node.get("cpu", 0)) if api_node else 0.0
mem = float(api_node.get("mem", 0)) if api_node else 0.0
status = str(api_node.get("status") or "unknown") if api_node else "unknown"
vm_status = "unknown"
qm = _run_ssh(PROXMOX_HOST, f"qm status {VM_106_ID} 2>/dev/null || qm list | grep '^{VM_106_ID} '")
if qm.get("ok") and qm.get("stdout"):
lines = (qm["stdout"] or "").splitlines()
for line in lines[1:]:
parts = line.split()
if not parts:
continue
vmid = parts[0]
rows.append(
{
"node": "pve",
"type": "qemu",
"id": f"qemu/{vmid}",
"vmid": int(vmid) if vmid.isdigit() else vmid,
"status": parts[2] if len(parts) > 2 else "unknown",
}
)
return rows, "ssh-qm-list", None
err = pvesh.get("stderr") or qm.get("stderr") or "ssh lookup failed"
return [], "none", err
line = qm["stdout"].splitlines()[0].lower()
if "running" in line:
vm_status = "running"
elif "stopped" in line:
vm_status = "stopped"
else:
vm_status = "online" if _port_health(VM_106_IP, 22) else "offline"
if status == "unknown" and _port_health(PROXMOX_HOST, 8006):
status = "online"
return status, cpu, mem, vm_status, api_source, api_error
def _port_health(host: str, port: int, timeout: float = 1.5) -> bool:
@@ -176,13 +167,10 @@ def _port_health(host: str, port: int, timeout: float = 1.5) -> bool:
def check_docker_services() -> dict[str, Any]:
result = _run_ssh(VM_106_IP, "docker ps --format json")
method = "docker-ps-json"
if not result.get("ok"):
result = _run_ssh(VM_106_IP, "docker ps --format '{{json .}}'")
method = "docker-ps-template-json"
if not result.get("ok"):
return {"ok": False, "source": method, "error": result.get("stderr") or "docker check failed", "containers": []}
return {"ok": False, "error": result.get("stderr") or "docker check failed", "containers": []}
containers: list[dict[str, Any]] = []
for line in (result.get("stdout") or "").splitlines():
line = line.strip()
@@ -193,102 +181,99 @@ def check_docker_services() -> dict[str, Any]:
containers.append(parsed if isinstance(parsed, dict) else {"raw": parsed})
except json.JSONDecodeError:
containers.append({"raw": line})
return {"ok": True, "source": method, "containers": containers}
return {"ok": True, "containers": containers}
def _online_services() -> list[dict[str, Any]]:
docker_state = check_docker_services()
docker_names = {
str(c.get("Names") or c.get("Name") or "").lower(): c
for c in docker_state.get("containers", [])
}
online: list[dict[str, Any]] = []
for svc in SERVICE_CATALOG:
up = _port_health(VM_106_IP, int(svc["port"]))
if not up:
continue
hint = "running"
for name, details in docker_names.items():
slug = svc["id"].replace("svc-", "").replace("-", "")
if slug in name.replace("-", ""):
hint = str(details.get("State") or details.get("Status") or "running")
break
online.append({
"id": svc["id"],
"label": f"{svc['label']}:{svc['port']}",
"type": "service",
"status": "online",
"host": VM_106_IP,
"port": svc["port"],
"url": svc["url"],
"role": svc["role"],
"hint": hint,
"agent": "sysops",
"children": [],
})
return online
def get_topology() -> dict[str, Any]:
api_nodes, api_source, api_error = _fetch_nodes_via_api()
ssh_nodes: list[dict[str, Any]] = []
ssh_source = "none"
ssh_error: str | None = None
if not api_nodes:
ssh_nodes, ssh_source, ssh_error = _fetch_nodes_via_ssh()
api_node = next((n for n in api_nodes if (n.get("node") or "").strip()), None) if api_nodes else None
host_cpu = float(api_node.get("cpu", 0)) if api_node else 0.0
host_mem = float(api_node.get("mem", 0)) if api_node else 0.0
host_status = api_node.get("status") if api_node else "unknown"
if host_status == "unknown" and ssh_nodes:
host_status = "online"
vm_states: dict[str, str] = {"105": "unknown", "106": "unknown"}
source_rows = api_nodes or ssh_nodes
for row in source_rows:
vmid = str(row.get("vmid") or "").strip()
if vmid in vm_states:
vm_states[vmid] = str(row.get("status") or "unknown")
docker_state = check_docker_services()
docker_names = {
str(c.get("Names") or c.get("Names.0") or c.get("Name") or "").lower(): c for c in docker_state.get("containers", [])
}
vm105_children: list[dict[str, Any]] = []
vm106_children: list[dict[str, Any]] = []
for svc in SERVICE_LAYOUT:
up = _port_health(str(svc["host"]), int(svc["port"]))
hinted = "unknown"
for name, details in docker_names.items():
if svc["label"].split(":")[0].replace("-", "") in name.replace("-", ""):
hinted = str(details.get("State") or details.get("Status") or "running")
break
item = {
"id": svc["id"],
"label": svc["label"],
"type": "service",
"status": "online" if up else "offline",
"cpu": None,
"mem": None,
"host": svc["host"],
"hint": hinted,
"children": [],
}
if svc["parent"] == "vm105-hermes":
vm105_children.append(item)
else:
vm106_children.append(item)
host_status, host_cpu, host_mem, vm_status, api_source, api_error = _fetch_proxmox_state()
services = _online_services()
proxmox_reachable = _port_health(PROXMOX_HOST, 8006)
topology_nodes = [
{
"id": "proxmox-host",
"label": f"proxmox-host ({PROXMOX_HOST})",
"type": "proxmox",
"status": host_status,
"cpu": host_cpu,
"mem": host_mem,
"id": "hardware-r340",
"label": HARDWARE_LABEL,
"type": "hardware",
"status": "online" if proxmox_reachable or _port_health(VM_106_IP, 8600) else "degraded",
"role": "Bare metal host",
"children": [
{
"id": "vm105-hermes",
"label": f"vm105-hermes ({VM_105_IP})",
"type": "vm",
"status": vm_states["105"],
"cpu": None,
"mem": None,
"children": vm105_children,
},
{
"id": "vm106-command",
"label": f"vm106-command ({VM_106_IP})",
"type": "vm",
"status": vm_states["106"],
"cpu": None,
"mem": None,
"children": vm106_children,
},
"id": "proxmox-host",
"label": f"Proxmox VE · {PROXMOX_HOST}",
"type": "proxmox",
"status": host_status if proxmox_reachable else ("online" if _port_health(PROXMOX_HOST, 22) else "offline"),
"cpu": host_cpu,
"mem": host_mem,
"role": "Hypervisor · node pve",
"url": f"https://{PROXMOX_HOST}:8006",
"children": [
{
"id": "vm106-command",
"label": f"VM {VM_106_ID} · {VM_106_NAME} · {VM_106_IP}",
"type": "vm",
"status": vm_status if vm_status != "unknown" else ("online" if services else "offline"),
"vmid": VM_106_ID,
"role": "Foodlinkk Command Center",
"children": services,
}
],
}
],
}
]
return {
"generated_at": _iso_now(),
"nodes": topology_nodes,
"layer_agents": LAYER_AGENTS,
"layers": [
{"id": "L0", "label": "Hardware", "y": 55},
{"id": "L1", "label": "Hypervisor", "y": 175},
{"id": "L2", "label": f"VM {VM_106_ID}", "y": 295},
{"id": "L3", "label": "Online services", "y": 415},
{"id": "L4", "label": "IT Agents", "y": 555},
],
"meta": {
"hardware": HARDWARE_LABEL,
"proxmox_host": PROXMOX_HOST,
"vm106_ip": VM_106_IP,
"vm106_id": VM_106_ID,
"services_online": len(services),
"api_source": api_source,
"api_error": api_error,
"ssh_source": ssh_source,
"ssh_error": ssh_error,
"docker_source": docker_state.get("source"),
"docker_ok": docker_state.get("ok", False),
},
}
@@ -322,10 +307,8 @@ def _table_exists(table_name: str) -> bool:
row = fetch_one(
"""
SELECT EXISTS(
SELECT 1
FROM information_schema.tables
WHERE table_schema = 'public'
AND table_name = %s
SELECT 1 FROM information_schema.tables
WHERE table_schema = 'public' AND table_name = %s
) AS ok
""",
(table_name,),
@@ -340,10 +323,8 @@ def poll_and_snapshot() -> dict[str, Any]:
cols = fetch_all(
"""
SELECT column_name
FROM information_schema.columns
WHERE table_schema = 'public'
AND table_name = 'infra_snapshots'
SELECT column_name FROM information_schema.columns
WHERE table_schema = 'public' AND table_name = 'infra_snapshots'
ORDER BY ordinal_position
"""
)
@@ -354,7 +335,6 @@ def poll_and_snapshot() -> dict[str, Any]:
"summary": {k: v for k, v in status.items() if k != "topology"},
"generated_at": status.get("generated_at"),
}
value_map: dict[str, Any] = {}
if "source" in colset:
value_map["source"] = "proxmox"
@@ -375,7 +355,6 @@ def poll_and_snapshot() -> dict[str, Any]:
return {"ok": False, "saved": False, "reason": "infra_snapshots has no compatible columns", "status": status}
columns = list(value_map.keys())
placeholders = ", ".join(["%s"] * len(columns))
sql = f"INSERT INTO infra_snapshots ({', '.join(columns)}) VALUES ({placeholders})"
sql = f"INSERT INTO infra_snapshots ({', '.join(columns)}) VALUES ({', '.join([ '%s' ] * len(columns))})"
execute(sql, tuple(value_map[c] for c in columns))
return {"ok": True, "saved": True, "columns": columns, "status": status}