Platform bundle: marketing publish, IT ops, packaging, agents mesh.
Volledige Foodlinkk Command Center uitbreiding met social automatisering, reclamefolder filters, Proxmox monitoring en documentatie.
This commit is contained in:
Aissa
@@ -0,0 +1,381 @@
|
||||
"""Proxmox infrastructure monitoring connector for Foodlinkk IT Ops."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import shlex
|
||||
import socket
|
||||
import ssl
|
||||
import subprocess
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
from urllib.error import URLError
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
from app.db import execute, fetch_all, fetch_one
|
||||
|
||||
PROXMOX_HOST = "10.4.7.14"
|
||||
PROXMOX_API_URL = f"https://{PROXMOX_HOST}:8006/api2/json"
|
||||
SSH_USER = "aissa"
|
||||
SSH_PASSWORD = "Foodlinkk#2026"
|
||||
|
||||
VM_105_IP = "10.4.7.19"
|
||||
VM_106_IP = "10.4.7.18"
|
||||
|
||||
SERVICE_LAYOUT: list[dict[str, Any]] = [
|
||||
{"id": "svc-cockpit", "label": "cockpit:8600", "host": VM_106_IP, "parent": "vm106-command", "port": 8600},
|
||||
{"id": "svc-tools-api", "label": "tools-api:8700", "host": VM_106_IP, "parent": "vm106-command", "port": 8700},
|
||||
{"id": "svc-email-agent", "label": "email-agent:8801", "host": VM_106_IP, "parent": "vm106-command", "port": 8801},
|
||||
{"id": "svc-gitea", "label": "gitea:3001", "host": VM_105_IP, "parent": "vm105-hermes", "port": 3001},
|
||||
{"id": "svc-ollama", "label": "ollama:11434", "host": VM_105_IP, "parent": "vm105-hermes", "port": 11434},
|
||||
]
|
||||
|
||||
|
||||
def _iso_now() -> str:
|
||||
return datetime.now(timezone.utc).isoformat()
|
||||
|
||||
|
||||
def _run_ssh(host: str, command: str, timeout: int = 12) -> dict[str, Any]:
|
||||
ssh_cmd = [
|
||||
"sshpass",
|
||||
"-p",
|
||||
SSH_PASSWORD,
|
||||
"ssh",
|
||||
"-o",
|
||||
"StrictHostKeyChecking=no",
|
||||
"-o",
|
||||
"UserKnownHostsFile=/dev/null",
|
||||
"-o",
|
||||
"ConnectTimeout=7",
|
||||
f"{SSH_USER}@{host}",
|
||||
command,
|
||||
]
|
||||
try:
|
||||
proc = subprocess.run( # noqa: S603
|
||||
ssh_cmd,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout,
|
||||
check=False,
|
||||
)
|
||||
except FileNotFoundError as exc:
|
||||
return {"ok": False, "error": f"ssh tooling missing: {exc}"}
|
||||
except subprocess.TimeoutExpired:
|
||||
return {"ok": False, "error": "ssh timeout"}
|
||||
return {
|
||||
"ok": proc.returncode == 0,
|
||||
"code": proc.returncode,
|
||||
"stdout": (proc.stdout or "").strip(),
|
||||
"stderr": (proc.stderr or "").strip(),
|
||||
}
|
||||
|
||||
|
||||
def _http_get_json(url: str, headers: dict[str, str] | None = None, timeout: int = 8) -> dict[str, Any]:
|
||||
req = Request(url, headers=headers or {})
|
||||
ctx = ssl.create_default_context()
|
||||
ctx.check_hostname = False
|
||||
ctx.verify_mode = ssl.CERT_NONE
|
||||
with urlopen(req, timeout=timeout, context=ctx) as resp: # noqa: S310
|
||||
payload = resp.read().decode("utf-8")
|
||||
return json.loads(payload)
|
||||
|
||||
|
||||
def _build_token_header(token_value: str) -> str:
|
||||
val = token_value.strip()
|
||||
if val.startswith("PVEAPIToken="):
|
||||
return val
|
||||
return f"PVEAPIToken={val}"
|
||||
|
||||
|
||||
def _create_api_token_via_ssh() -> str | None:
|
||||
token_name = f"ops{int(time.time())}"
|
||||
cmd = (
|
||||
f"pveum user token add {shlex.quote(SSH_USER + '@pam')} {shlex.quote(token_name)} "
|
||||
"--privsep 0 --expire 0 --output-format json"
|
||||
)
|
||||
result = _run_ssh(PROXMOX_HOST, cmd, timeout=15)
|
||||
if not result.get("ok"):
|
||||
return None
|
||||
try:
|
||||
parsed = json.loads(result.get("stdout") or "{}")
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
tokenid = parsed.get("full-tokenid")
|
||||
secret = parsed.get("value")
|
||||
if tokenid and secret:
|
||||
return f"PVEAPIToken={tokenid}={secret}"
|
||||
return None
|
||||
|
||||
|
||||
def _fetch_nodes_via_api() -> tuple[list[dict[str, Any]], str, str | None]:
|
||||
token = os.getenv("PROXMOX_TOKEN")
|
||||
tried = []
|
||||
if token:
|
||||
tried.append("env-token")
|
||||
try:
|
||||
data = _http_get_json(
|
||||
f"{PROXMOX_API_URL}/nodes",
|
||||
headers={"Authorization": _build_token_header(token)},
|
||||
)
|
||||
return data.get("data") or [], "api-token-env", None
|
||||
except Exception as exc: # noqa: BLE001
|
||||
tried.append(f"env-failed:{exc}")
|
||||
created = _create_api_token_via_ssh()
|
||||
if created:
|
||||
tried.append("ssh-created-token")
|
||||
try:
|
||||
data = _http_get_json(
|
||||
f"{PROXMOX_API_URL}/nodes",
|
||||
headers={"Authorization": created},
|
||||
)
|
||||
return data.get("data") or [], "api-token-ssh", None
|
||||
except Exception as exc: # noqa: BLE001
|
||||
tried.append(f"ssh-token-failed:{exc}")
|
||||
return [], "none", ", ".join(tried) if tried else "no-token"
|
||||
|
||||
|
||||
def _fetch_nodes_via_ssh() -> tuple[list[dict[str, Any]], str, str | None]:
|
||||
pvesh = _run_ssh(PROXMOX_HOST, "pvesh get /nodes --output-format json")
|
||||
if pvesh.get("ok"):
|
||||
try:
|
||||
return json.loads(pvesh["stdout"] or "[]"), "ssh-pvesh", None
|
||||
except json.JSONDecodeError:
|
||||
pass
|
||||
qm = _run_ssh(PROXMOX_HOST, "qm list")
|
||||
rows: list[dict[str, Any]] = []
|
||||
if qm.get("ok") and qm.get("stdout"):
|
||||
lines = (qm["stdout"] or "").splitlines()
|
||||
for line in lines[1:]:
|
||||
parts = line.split()
|
||||
if not parts:
|
||||
continue
|
||||
vmid = parts[0]
|
||||
rows.append(
|
||||
{
|
||||
"node": "pve",
|
||||
"type": "qemu",
|
||||
"id": f"qemu/{vmid}",
|
||||
"vmid": int(vmid) if vmid.isdigit() else vmid,
|
||||
"status": parts[2] if len(parts) > 2 else "unknown",
|
||||
}
|
||||
)
|
||||
return rows, "ssh-qm-list", None
|
||||
err = pvesh.get("stderr") or qm.get("stderr") or "ssh lookup failed"
|
||||
return [], "none", err
|
||||
|
||||
|
||||
def _port_health(host: str, port: int, timeout: float = 1.5) -> bool:
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
sock.settimeout(timeout)
|
||||
try:
|
||||
return sock.connect_ex((host, port)) == 0
|
||||
finally:
|
||||
sock.close()
|
||||
|
||||
|
||||
def check_docker_services() -> dict[str, Any]:
|
||||
result = _run_ssh(VM_106_IP, "docker ps --format json")
|
||||
method = "docker-ps-json"
|
||||
if not result.get("ok"):
|
||||
result = _run_ssh(VM_106_IP, "docker ps --format '{{json .}}'")
|
||||
method = "docker-ps-template-json"
|
||||
if not result.get("ok"):
|
||||
return {"ok": False, "source": method, "error": result.get("stderr") or "docker check failed", "containers": []}
|
||||
|
||||
containers: list[dict[str, Any]] = []
|
||||
for line in (result.get("stdout") or "").splitlines():
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
parsed = json.loads(line)
|
||||
containers.append(parsed if isinstance(parsed, dict) else {"raw": parsed})
|
||||
except json.JSONDecodeError:
|
||||
containers.append({"raw": line})
|
||||
return {"ok": True, "source": method, "containers": containers}
|
||||
|
||||
|
||||
def get_topology() -> dict[str, Any]:
|
||||
api_nodes, api_source, api_error = _fetch_nodes_via_api()
|
||||
ssh_nodes: list[dict[str, Any]] = []
|
||||
ssh_source = "none"
|
||||
ssh_error: str | None = None
|
||||
if not api_nodes:
|
||||
ssh_nodes, ssh_source, ssh_error = _fetch_nodes_via_ssh()
|
||||
|
||||
api_node = next((n for n in api_nodes if (n.get("node") or "").strip()), None) if api_nodes else None
|
||||
host_cpu = float(api_node.get("cpu", 0)) if api_node else 0.0
|
||||
host_mem = float(api_node.get("mem", 0)) if api_node else 0.0
|
||||
host_status = api_node.get("status") if api_node else "unknown"
|
||||
if host_status == "unknown" and ssh_nodes:
|
||||
host_status = "online"
|
||||
|
||||
vm_states: dict[str, str] = {"105": "unknown", "106": "unknown"}
|
||||
source_rows = api_nodes or ssh_nodes
|
||||
for row in source_rows:
|
||||
vmid = str(row.get("vmid") or "").strip()
|
||||
if vmid in vm_states:
|
||||
vm_states[vmid] = str(row.get("status") or "unknown")
|
||||
|
||||
docker_state = check_docker_services()
|
||||
docker_names = {
|
||||
str(c.get("Names") or c.get("Names.0") or c.get("Name") or "").lower(): c for c in docker_state.get("containers", [])
|
||||
}
|
||||
|
||||
vm105_children: list[dict[str, Any]] = []
|
||||
vm106_children: list[dict[str, Any]] = []
|
||||
for svc in SERVICE_LAYOUT:
|
||||
up = _port_health(str(svc["host"]), int(svc["port"]))
|
||||
hinted = "unknown"
|
||||
for name, details in docker_names.items():
|
||||
if svc["label"].split(":")[0].replace("-", "") in name.replace("-", ""):
|
||||
hinted = str(details.get("State") or details.get("Status") or "running")
|
||||
break
|
||||
item = {
|
||||
"id": svc["id"],
|
||||
"label": svc["label"],
|
||||
"type": "service",
|
||||
"status": "online" if up else "offline",
|
||||
"cpu": None,
|
||||
"mem": None,
|
||||
"host": svc["host"],
|
||||
"hint": hinted,
|
||||
"children": [],
|
||||
}
|
||||
if svc["parent"] == "vm105-hermes":
|
||||
vm105_children.append(item)
|
||||
else:
|
||||
vm106_children.append(item)
|
||||
|
||||
topology_nodes = [
|
||||
{
|
||||
"id": "proxmox-host",
|
||||
"label": f"proxmox-host ({PROXMOX_HOST})",
|
||||
"type": "proxmox",
|
||||
"status": host_status,
|
||||
"cpu": host_cpu,
|
||||
"mem": host_mem,
|
||||
"children": [
|
||||
{
|
||||
"id": "vm105-hermes",
|
||||
"label": f"vm105-hermes ({VM_105_IP})",
|
||||
"type": "vm",
|
||||
"status": vm_states["105"],
|
||||
"cpu": None,
|
||||
"mem": None,
|
||||
"children": vm105_children,
|
||||
},
|
||||
{
|
||||
"id": "vm106-command",
|
||||
"label": f"vm106-command ({VM_106_IP})",
|
||||
"type": "vm",
|
||||
"status": vm_states["106"],
|
||||
"cpu": None,
|
||||
"mem": None,
|
||||
"children": vm106_children,
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
return {
|
||||
"generated_at": _iso_now(),
|
||||
"nodes": topology_nodes,
|
||||
"meta": {
|
||||
"proxmox_host": PROXMOX_HOST,
|
||||
"api_source": api_source,
|
||||
"api_error": api_error,
|
||||
"ssh_source": ssh_source,
|
||||
"ssh_error": ssh_error,
|
||||
"docker_source": docker_state.get("source"),
|
||||
"docker_ok": docker_state.get("ok", False),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def get_status_summary() -> dict[str, Any]:
|
||||
topo = get_topology()
|
||||
flat: list[dict[str, Any]] = []
|
||||
|
||||
def _collect(node: dict[str, Any]) -> None:
|
||||
flat.append(node)
|
||||
for child in node.get("children") or []:
|
||||
_collect(child)
|
||||
|
||||
for root in topo.get("nodes") or []:
|
||||
_collect(root)
|
||||
|
||||
total = len(flat)
|
||||
online = sum(1 for n in flat if str(n.get("status")).lower() in {"online", "running", "up"})
|
||||
degraded = sum(1 for n in flat if str(n.get("status")).lower() in {"unknown", "degraded"})
|
||||
offline = max(0, total - online - degraded)
|
||||
return {
|
||||
"generated_at": topo.get("generated_at"),
|
||||
"health": "healthy" if offline == 0 else ("degraded" if online > 0 else "down"),
|
||||
"counts": {"total": total, "online": online, "degraded": degraded, "offline": offline},
|
||||
"sources": topo.get("meta", {}),
|
||||
"topology": topo,
|
||||
}
|
||||
|
||||
|
||||
def _table_exists(table_name: str) -> bool:
|
||||
row = fetch_one(
|
||||
"""
|
||||
SELECT EXISTS(
|
||||
SELECT 1
|
||||
FROM information_schema.tables
|
||||
WHERE table_schema = 'public'
|
||||
AND table_name = %s
|
||||
) AS ok
|
||||
""",
|
||||
(table_name,),
|
||||
)
|
||||
return bool(row and row.get("ok"))
|
||||
|
||||
|
||||
def poll_and_snapshot() -> dict[str, Any]:
|
||||
status = get_status_summary()
|
||||
if not _table_exists("infra_snapshots"):
|
||||
return {"ok": False, "saved": False, "reason": "infra_snapshots table not found", "status": status}
|
||||
|
||||
cols = fetch_all(
|
||||
"""
|
||||
SELECT column_name
|
||||
FROM information_schema.columns
|
||||
WHERE table_schema = 'public'
|
||||
AND table_name = 'infra_snapshots'
|
||||
ORDER BY ordinal_position
|
||||
"""
|
||||
)
|
||||
colset = {c.get("column_name") for c in cols}
|
||||
payload = {
|
||||
"source": "proxmox",
|
||||
"topology": status.get("topology"),
|
||||
"summary": {k: v for k, v in status.items() if k != "topology"},
|
||||
"generated_at": status.get("generated_at"),
|
||||
}
|
||||
|
||||
value_map: dict[str, Any] = {}
|
||||
if "source" in colset:
|
||||
value_map["source"] = "proxmox"
|
||||
if "provider" in colset:
|
||||
value_map["provider"] = "proxmox"
|
||||
if "snapshot" in colset:
|
||||
value_map["snapshot"] = json.dumps(payload)
|
||||
if "payload" in colset:
|
||||
value_map["payload"] = json.dumps(payload)
|
||||
if "topology" in colset:
|
||||
value_map["topology"] = json.dumps(status.get("topology"))
|
||||
if "summary" in colset:
|
||||
value_map["summary"] = json.dumps({k: v for k, v in status.items() if k != "topology"})
|
||||
if "created_at" in colset:
|
||||
value_map["created_at"] = datetime.now(timezone.utc)
|
||||
|
||||
if not value_map:
|
||||
return {"ok": False, "saved": False, "reason": "infra_snapshots has no compatible columns", "status": status}
|
||||
|
||||
columns = list(value_map.keys())
|
||||
placeholders = ", ".join(["%s"] * len(columns))
|
||||
sql = f"INSERT INTO infra_snapshots ({', '.join(columns)}) VALUES ({placeholders})"
|
||||
execute(sql, tuple(value_map[c] for c in columns))
|
||||
return {"ok": True, "saved": True, "columns": columns, "status": status}
|
||||
Reference in New Issue
Block a user