SysOps: deploy-all — 2026-06-09 10:41 UTC

This commit is contained in:
sysops
2026-06-09 10:41:13 +00:00
parent 69fe67cc0e
commit 21ea3a2c81
82 changed files with 8906 additions and 981 deletions
+144
View File
@@ -0,0 +1,144 @@
"""Daily configuration backup to Gitea — executed after CEO approval."""
from __future__ import annotations
import json
import os
import subprocess
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from app.db import execute, fetch_all, fetch_one
BACKUP_ROOT = Path(os.getenv("BACKUP_ROOT", "/data/backup-root"))
GITEA_REMOTE = os.getenv(
"GITEA_BACKUP_REMOTE",
"http://sysops:Foodlinkk%23SysOps2026@gitea:3001/aissa/foodlinkk-command-center.git",
)
GITEA_GIT_USER = os.getenv("GITEA_SYSOPS_USER", "sysops")
GITEA_GIT_EMAIL = os.getenv("GITEA_SYSOPS_EMAIL", "sysops@foodlinkk.local")
GITEA_GIT_NAME = os.getenv("GITEA_SYSOPS_NAME", "sysops")
GITEA_BRANCH = os.getenv("GITEA_BACKUP_BRANCH", "main")
BACKUP_PATHS = [
"docker-compose.yml",
".env.example",
"migrations",
"monitoring",
"cockpit/requirements.txt",
"tools-api/requirements.txt",
"deploy-all.sh",
"README.md",
]
def _iso_now() -> str:
return datetime.now(timezone.utc).isoformat()
def _run(cmd: list[str], cwd: Path | None = None, timeout: int = 120) -> dict[str, Any]:
try:
proc = subprocess.run( # noqa: S603
cmd,
cwd=str(cwd) if cwd else None,
capture_output=True,
text=True,
timeout=timeout,
check=False,
)
except subprocess.TimeoutExpired:
return {"ok": False, "error": "timeout", "cmd": cmd}
except FileNotFoundError as exc:
return {"ok": False, "error": str(exc), "cmd": cmd}
return {
"ok": proc.returncode == 0,
"code": proc.returncode,
"stdout": (proc.stdout or "").strip(),
"stderr": (proc.stderr or "").strip(),
"cmd": cmd,
}
def _table_exists(name: str) -> bool:
row = fetch_one(
"""
SELECT EXISTS(
SELECT 1 FROM information_schema.tables
WHERE table_schema = 'public' AND table_name = %s
) AS ok
""",
(name,),
)
return bool(row and row.get("ok"))
def _record_backup(
status: str,
message: str,
files_count: int = 0,
commit_ref: str | None = None,
approval_request_id: int | None = None,
payload: dict | None = None,
) -> dict[str, Any] | None:
if not _table_exists("config_backups"):
return None
row = fetch_one(
"""
INSERT INTO config_backups (approval_request_id, commit_ref, files_count, status, message, payload)
VALUES (%s, %s, %s, %s, %s, %s::jsonb)
RETURNING id, created_at
""",
(
approval_request_id,
commit_ref,
files_count,
status,
message,
json.dumps(payload or {}),
),
)
return dict(row) if row else None
def list_backups(limit: int = 20) -> list[dict[str, Any]]:
if not _table_exists("config_backups"):
return []
rows = fetch_all(
"SELECT * FROM config_backups ORDER BY created_at DESC LIMIT %s",
(max(1, min(limit, 100)),),
)
out = []
for r in rows:
item = dict(r)
if hasattr(item.get("created_at"), "isoformat"):
item["created_at"] = item["created_at"].isoformat()
out.append(item)
return out
def prepare_backup_manifest() -> dict[str, Any]:
root = BACKUP_ROOT
files: list[str] = []
if root.exists():
for rel in BACKUP_PATHS:
p = root / rel
if p.is_file():
files.append(rel)
elif p.is_dir():
for fp in p.rglob("*"):
if fp.is_file() and not any(x in str(fp) for x in [".git", "__pycache__", ".pyc"]):
files.append(str(fp.relative_to(root)))
return {
"root": str(root),
"root_exists": root.exists(),
"files": sorted(files)[:500],
"files_count": len(files),
"paths_configured": BACKUP_PATHS,
"generated_at": _iso_now(),
}
def run_gitea_backup(approval_request_id: int | None = None) -> dict[str, Any]:
from app.connectors.gitea_repo_sync import run_gitea_repo_sync
return run_gitea_repo_sync(reason="approved_backup", approval_request_id=approval_request_id)
+248
View File
@@ -0,0 +1,248 @@
"""Volledige codebase sync naar Gitea main — SysOps."""
from __future__ import annotations
import json
import os
import shutil
import subprocess
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from app.db import execute, fetch_all, fetch_one
from app.middleware import log_agent_event
REPO_ROOT = Path(os.getenv("BACKUP_ROOT", "/data/backup-root"))
GITEA_REMOTE = os.getenv(
"GITEA_BACKUP_REMOTE",
"http://sysops:Foodlinkk%23SysOps2026@gitea:3001/aissa/foodlinkk-command-center.git",
)
GITEA_GIT_USER = os.getenv("GITEA_SYSOPS_USER", "sysops")
GITEA_GIT_EMAIL = os.getenv("GITEA_SYSOPS_EMAIL", "sysops@foodlinkk.local")
GITEA_GIT_NAME = os.getenv("GITEA_SYSOPS_NAME", "sysops")
GITEA_BRANCH = os.getenv("GITEA_BACKUP_BRANCH", "main")
SKIP_NAMES = {
".git",
"__pycache__",
".env",
"node_modules",
".cursor",
".venv",
"venv",
"*.pyc",
}
def _iso_now() -> str:
return datetime.now(timezone.utc).isoformat()
def _run(cmd: list[str], cwd: Path | None = None, timeout: int = 180) -> dict[str, Any]:
try:
proc = subprocess.run( # noqa: S603
cmd,
cwd=str(cwd) if cwd else None,
capture_output=True,
text=True,
timeout=timeout,
check=False,
)
except subprocess.TimeoutExpired:
return {"ok": False, "error": "timeout", "cmd": cmd}
except FileNotFoundError as exc:
return {"ok": False, "error": str(exc), "cmd": cmd}
return {
"ok": proc.returncode == 0,
"code": proc.returncode,
"stdout": (proc.stdout or "").strip(),
"stderr": (proc.stderr or "").strip(),
"cmd": cmd,
}
def _table_exists(name: str) -> bool:
row = fetch_one(
"""
SELECT EXISTS(
SELECT 1 FROM information_schema.tables
WHERE table_schema = 'public' AND table_name = %s
) AS ok
""",
(name,),
)
return bool(row and row.get("ok"))
def _record_activity(
action_type: str,
title: str,
body: str = "",
commit_ref: str | None = None,
files_changed: int = 0,
status: str = "success",
metadata: dict | None = None,
) -> dict[str, Any] | None:
if not _table_exists("sysops_activity"):
return None
row = fetch_one(
"""
INSERT INTO sysops_activity (action_type, title, body, commit_ref, files_changed, status, metadata)
VALUES (%s, %s, %s, %s, %s, %s, %s::jsonb)
RETURNING id, created_at
""",
(action_type, title, body, commit_ref, files_changed, status, json.dumps(metadata or {})),
)
return dict(row) if row else None
def list_activity(limit: int = 30) -> list[dict[str, Any]]:
if not _table_exists("sysops_activity"):
return []
rows = fetch_all(
"SELECT * FROM sysops_activity ORDER BY created_at DESC LIMIT %s",
(max(1, min(limit, 100)),),
)
out = []
for r in rows:
item = dict(r)
if hasattr(item.get("created_at"), "isoformat"):
item["created_at"] = item["created_at"].isoformat()
out.append(item)
return out
def _should_skip(name: str) -> bool:
if name in SKIP_NAMES:
return True
if name.endswith(".pyc") or name.endswith(".pyo"):
return True
return False
def _copy_tree(src: Path, dest: Path) -> int:
"""Kopieer repo-inhoud naar git worktree; retour aantal bestanden."""
count = 0
if not src.exists():
return 0
for item in src.iterdir():
if _should_skip(item.name):
continue
target = dest / item.name
if item.is_file():
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(item, target)
count += 1
elif item.is_dir():
if target.exists():
shutil.rmtree(target, ignore_errors=True)
shutil.copytree(
item,
target,
ignore=shutil.ignore_patterns("__pycache__", "*.pyc", ".env", "node_modules", ".git"),
)
count += sum(1 for _ in target.rglob("*") if _.is_file())
return count
def run_gitea_repo_sync(reason: str = "deploy", approval_request_id: int | None = None) -> dict[str, Any]:
"""Push volledige codebase naar Gitea main branch."""
work = Path("/tmp/foodlinkk-gitea-sync")
if work.exists():
shutil.rmtree(work, ignore_errors=True)
work.mkdir(parents=True, exist_ok=True)
if not REPO_ROOT.exists():
msg = f"Repo root niet gevonden: {REPO_ROOT}"
_record_activity("gitea_sync", "Gitea sync mislukt", msg, status="failed")
return {"ok": False, "message": msg}
repo = work / "repo"
clone = _run(["git", "clone", "--depth", "1", "-b", GITEA_BRANCH, GITEA_REMOTE, str(repo)])
if not clone.get("ok"):
repo = work
_run(["git", "init", "-b", GITEA_BRANCH], cwd=repo)
_run(["git", "remote", "add", "origin", GITEA_REMOTE], cwd=repo)
_run(["git", "config", "user.email", GITEA_GIT_EMAIL], cwd=repo)
_run(["git", "config", "user.name", GITEA_GIT_NAME], cwd=repo)
copied = _copy_tree(REPO_ROOT, repo)
stamp = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
commit_msg = f"SysOps: {reason}{stamp}"
_run(["git", "add", "-A"], cwd=repo)
commit = _run(["git", "commit", "-m", commit_msg], cwd=repo)
nothing = "nothing to commit" in (commit.get("stdout", "") + commit.get("stderr", ""))
if not commit.get("ok") and not nothing:
msg = "Git commit mislukt: " + (commit.get("stderr") or "")
_record_activity("gitea_sync", "Gitea commit mislukt", msg, files_changed=copied, status="failed")
return {"ok": False, "message": msg, "copied": copied}
commit_ref = None
if not nothing:
push = _run(["git", "push", "-u", "origin", GITEA_BRANCH], cwd=repo, timeout=300)
if not push.get("ok"):
msg = "Git push mislukt: " + (push.get("stderr") or "")
_record_activity("gitea_sync", "Gitea push mislukt", msg, files_changed=copied, status="failed")
return {"ok": False, "message": msg, "copied": copied}
rev = _run(["git", "rev-parse", "--short", "HEAD"], cwd=repo)
commit_ref = rev.get("stdout") or None
summary = (
f"Volledige codebase gesynchroniseerd naar Gitea ({reason}). "
f"{copied} bestanden · commit {commit_ref or 'geen wijzigingen'}"
)
activity = _record_activity(
"gitea_sync",
f"Gitea push: {reason}",
summary,
commit_ref=commit_ref,
files_changed=copied,
status="success" if (commit_ref or nothing) else "skipped",
metadata={"reason": reason, "approval_request_id": approval_request_id, "nothing_to_commit": nothing},
)
try:
log_agent_event(
agent_name="sysops",
agent_type="sysops",
event_type="gitea_sync",
title=f"Gitea sync: {reason}" + (f" ({commit_ref})" if commit_ref else " — geen wijzigingen"),
body=summary,
metadata={
"commit_ref": commit_ref,
"files_changed": copied,
"reason": reason,
"for_herman": True,
},
channel="sysops",
)
except Exception:
pass
if _table_exists("config_backups"):
fetch_one(
"""
INSERT INTO config_backups (approval_request_id, commit_ref, files_count, status, message, payload)
VALUES (%s, %s, %s, %s, %s, %s::jsonb)
RETURNING id
""",
(
approval_request_id,
commit_ref,
copied,
"success" if commit_ref else "unchanged",
summary,
json.dumps({"reason": reason, "full_repo": True}),
),
)
return {
"ok": True,
"message": summary,
"commit_ref": commit_ref,
"files_count": copied,
"nothing_to_commit": nothing,
"activity_id": (activity or {}).get("id"),
}
+179
View File
@@ -0,0 +1,179 @@
"""Daily maintenance checks for IT Ops tab."""
from __future__ import annotations
import json
import subprocess
from datetime import datetime, timezone
from typing import Any
from app.db import execute, fetch_all, fetch_one
from app.connectors.proxmox import PROXMOX_HOST, VM_106_IP, _port_health, _run_ssh, check_docker_services
SEVERITY_ORDER = {"critical": 0, "warning": 1, "info": 2}
def _iso_now() -> str:
return datetime.now(timezone.utc).isoformat()
def _table_exists(name: str) -> bool:
row = fetch_one(
"""
SELECT EXISTS(
SELECT 1 FROM information_schema.tables
WHERE table_schema = 'public' AND table_name = %s
) AS ok
""",
(name,),
)
return bool(row and row.get("ok"))
def list_notes(limit: int = 20, unresolved_only: bool = True) -> list[dict[str, Any]]:
if not _table_exists("ops_maintenance_notes"):
return []
clauses = ["resolved = false"] if unresolved_only else []
where = (" WHERE " + " AND ".join(clauses)) if clauses else ""
rows = fetch_all(
f"SELECT * FROM ops_maintenance_notes{where} ORDER BY created_at DESC LIMIT %s",
(max(1, min(limit, 100)),),
)
out = []
for r in rows:
item = dict(r)
if hasattr(item.get("created_at"), "isoformat"):
item["created_at"] = item["created_at"].isoformat()
out.append(item)
return out
def add_note(title: str, body: str = "", severity: str = "info", source: str = "sysops") -> dict[str, Any] | None:
if not _table_exists("ops_maintenance_notes"):
return None
row = fetch_one(
"""
INSERT INTO ops_maintenance_notes (severity, title, body, source)
VALUES (%s, %s, %s, %s)
RETURNING *
""",
(severity, title[:255], body, source),
)
if not row:
return None
item = dict(row)
if hasattr(item.get("created_at"), "isoformat"):
item["created_at"] = item["created_at"].isoformat()
return item
def run_maintenance_scan() -> dict[str, Any]:
findings: list[dict[str, Any]] = []
proxmox_up = _port_health(PROXMOX_HOST, 8006)
if not proxmox_up:
findings.append({
"severity": "critical",
"title": "Proxmox Web UI niet bereikbaar",
"body": f"Poort 8006 op {PROXMOX_HOST} reageert niet. Controleer Dell R340 / Proxmox VE.",
})
docker = check_docker_services()
if not docker.get("ok"):
findings.append({
"severity": "warning",
"title": "Docker status onbekend",
"body": docker.get("error") or "Kon docker ps niet uitvoeren op VM106.",
})
else:
containers = docker.get("containers") or []
unhealthy = [
c for c in containers
if "unhealthy" in str(c.get("Status") or c.get("State") or "").lower()
]
if unhealthy:
names = ", ".join(str(c.get("Names") or c.get("Name") or "?")[:40] for c in unhealthy[:5])
findings.append({
"severity": "warning",
"title": f"{len(unhealthy)} container(s) unhealthy",
"body": names,
})
apt = _run_ssh(VM_106_IP, "apt list --upgradable 2>/dev/null | grep -v Listing | wc -l")
if apt.get("ok"):
try:
count = int((apt.get("stdout") or "0").strip())
except ValueError:
count = 0
if count > 0:
findings.append({
"severity": "info" if count < 10 else "warning",
"title": f"{count} package updates beschikbaar op VM106",
"body": "Voer maintenance uit wanneer gepland: apt update && apt upgrade",
})
disk = _run_ssh(VM_106_IP, "df -h / | tail -1 | awk '{print $5}'")
if disk.get("ok"):
pct = (disk.get("stdout") or "").replace("%", "").strip()
try:
used = int(pct)
if used >= 90:
findings.append({
"severity": "critical",
"title": f"Schijf VM106 {used}% vol",
"body": "Ruimte vrijmaken op root volume.",
})
elif used >= 80:
findings.append({
"severity": "warning",
"title": f"Schijf VM106 {used}% vol",
"body": "Plan opschoning binnenkort.",
})
except ValueError:
pass
saved: list[dict[str, Any]] = []
for f in findings:
note = add_note(f["title"], f.get("body", ""), f.get("severity", "info"))
if note:
saved.append(note)
if not findings:
note = add_note(
"Geen openstaande maintenance — alles OK",
f"Scan {_iso_now()}: Proxmox, Docker en schijf binnen normen.",
"info",
)
if note:
saved.append(note)
saved.sort(key=lambda n: SEVERITY_ORDER.get(str(n.get("severity")), 9))
try:
from app.connectors.gitea_repo_sync import _record_activity
from app.middleware import log_agent_event
summary = f"{len(findings)} bevinding(en) — scan {_iso_now()[:19]}"
_record_activity(
"maintenance_scan",
f"Maintenance scan: {len(findings)} bevinding(en)",
summary + "\n" + "\n".join(f"- {f['title']}" for f in findings[:8]),
metadata={"findings_count": len(findings), "for_herman": True},
)
log_agent_event(
agent_name="sysops",
agent_type="sysops",
event_type="maintenance_scan",
title=f"Maintenance scan voltooid — {len(findings)} bevinding(en)",
body=summary,
metadata={"findings": [f.get("title") for f in findings[:10]], "for_herman": True},
channel="sysops",
)
except Exception:
pass
return {
"ok": True,
"scanned_at": _iso_now(),
"findings_count": len(findings),
"notes": saved,
}
@@ -0,0 +1,83 @@
"""Register agent outputs as project assets (shared DB helpers)."""
from __future__ import annotations
import json
from typing import Any, Optional
from app.db import execute, fetch_one
def get_or_create_agent_project(agent_key: str, client_id: int | None = None) -> int:
key = (agent_key or "agent").strip().lower()
row = fetch_one(
"""
SELECT id FROM cockpit_projects
WHERE metadata->>'auto_agent' = %s AND status = 'active'
ORDER BY updated_at DESC LIMIT 1
""",
(key,),
)
if row:
return int(row["id"])
title = f"Agent · {key}"
created = fetch_one(
"""
INSERT INTO cockpit_projects (name, client_id, description, created_by, metadata, updated_at)
VALUES (%s, %s, %s, %s, %s::jsonb, NOW())
RETURNING id
""",
(
title,
client_id,
f"Automatisch project voor {key} output",
key,
json.dumps({"auto_agent": key}),
),
)
return int(created["id"])
def register_agent_output(
asset_type: str,
title: str,
ref_id: str | None = None,
payload: dict | None = None,
project_id: int | None = None,
source_agent: str | None = None,
created_by: str | None = None,
file_path: str | None = None,
) -> dict[str, Any] | None:
agent = (source_agent or created_by or "agent").strip().lower()
pid = project_id or get_or_create_agent_project(agent)
if ref_id:
existing = fetch_one(
"""
SELECT id FROM project_assets
WHERE project_id = %s AND asset_type = %s AND ref_id = %s
LIMIT 1
""",
(pid, asset_type, ref_id),
)
if existing:
return {"id": existing["id"], "project_id": pid, "dedup": True}
row = fetch_one(
"""
INSERT INTO project_assets (project_id, asset_type, ref_id, title, file_path, payload, created_by, source_agent)
VALUES (%s, %s, %s, %s, %s, %s::jsonb, %s, %s)
RETURNING id, project_id, asset_type, ref_id, title
""",
(
pid,
asset_type,
ref_id,
title[:255],
file_path,
json.dumps(payload or {}),
created_by or agent,
agent,
),
)
execute("UPDATE cockpit_projects SET updated_at = NOW() WHERE id = %s", (pid,))
return dict(row) if row else None
+157 -178
View File
@@ -1,4 +1,4 @@
"""Proxmox infrastructure monitoring connector for Foodlinkk IT Ops."""
"""Proxmox infrastructure monitoring — Dell R340 · Proxmox VE · VM106 services."""
from __future__ import annotations
import json
@@ -15,20 +15,30 @@ from urllib.request import Request, urlopen
from app.db import execute, fetch_all, fetch_one
PROXMOX_HOST = "10.4.7.14"
PROXMOX_HOST = os.getenv("PROXMOX_HOST", "10.4.7.14")
PROXMOX_API_URL = f"https://{PROXMOX_HOST}:8006/api2/json"
SSH_USER = "aissa"
SSH_PASSWORD = "Foodlinkk#2026"
SSH_USER = os.getenv("PROXMOX_SSH_USER", "aissa")
SSH_PASSWORD = os.getenv("PROXMOX_SSH_PASS", "Foodlinkk#2026")
VM_105_IP = "10.4.7.19"
VM_106_IP = "10.4.7.18"
VM_106_IP = os.getenv("VM106_IP", "10.4.7.18")
VM_106_ID = 106
VM_106_NAME = "dockervm"
SERVICE_LAYOUT: list[dict[str, Any]] = [
{"id": "svc-cockpit", "label": "cockpit:8600", "host": VM_106_IP, "parent": "vm106-command", "port": 8600},
{"id": "svc-tools-api", "label": "tools-api:8700", "host": VM_106_IP, "parent": "vm106-command", "port": 8700},
{"id": "svc-email-agent", "label": "email-agent:8801", "host": VM_106_IP, "parent": "vm106-command", "port": 8801},
{"id": "svc-gitea", "label": "gitea:3001", "host": VM_105_IP, "parent": "vm105-hermes", "port": 3001},
{"id": "svc-ollama", "label": "ollama:11434", "host": VM_105_IP, "parent": "vm105-hermes", "port": 11434},
HARDWARE_LABEL = "Dell PowerEdge R340"
SERVICE_CATALOG: list[dict[str, Any]] = [
{"id": "svc-cockpit", "label": "Cockpit", "port": 8600, "url": f"http://{VM_106_IP}:8600", "role": "Command Center UI"},
{"id": "svc-tools-api", "label": "Tools API", "port": 8700, "url": f"http://{VM_106_IP}:8700/docs", "role": "Connectors & API"},
{"id": "svc-gitea", "label": "Gitea", "port": 3001, "url": f"http://{VM_106_IP}:3001", "role": "Git · config backups"},
{"id": "svc-email-agent", "label": "Email Agent", "port": 8801, "url": f"http://{VM_106_IP}:8801", "role": "IMAP/SMTP sync"},
{"id": "svc-grafana", "label": "Grafana", "port": 3002, "url": f"http://{VM_106_IP}:3002", "role": "Metrics dashboards"},
{"id": "svc-prometheus", "label": "Prometheus", "port": 9090, "url": f"http://{VM_106_IP}:9090", "role": "Metrics scrape"},
{"id": "svc-browser-agent", "label": "Browser Agent", "port": 7790, "url": f"http://{VM_106_IP}:7790", "role": "Web automation"},
]
LAYER_AGENTS: list[dict[str, Any]] = [
{"id": "mon-sysops", "label": "SysOps", "agent_key": "sysops", "role": "Backup · Proxmox · Docker · maintenance"},
{"id": "mon-research", "label": "Research", "agent_key": "research", "role": "Markt intel (approval gate)"},
]
@@ -38,26 +48,16 @@ def _iso_now() -> str:
def _run_ssh(host: str, command: str, timeout: int = 12) -> dict[str, Any]:
ssh_cmd = [
"sshpass",
"-p",
SSH_PASSWORD,
"ssh",
"-o",
"StrictHostKeyChecking=no",
"-o",
"UserKnownHostsFile=/dev/null",
"-o",
"ConnectTimeout=7",
"sshpass", "-p", SSH_PASSWORD, "ssh",
"-o", "StrictHostKeyChecking=no",
"-o", "UserKnownHostsFile=/dev/null",
"-o", "ConnectTimeout=7",
f"{SSH_USER}@{host}",
command,
]
try:
proc = subprocess.run( # noqa: S603
ssh_cmd,
capture_output=True,
text=True,
timeout=timeout,
check=False,
ssh_cmd, capture_output=True, text=True, timeout=timeout, check=False,
)
except FileNotFoundError as exc:
return {"ok": False, "error": f"ssh tooling missing: {exc}"}
@@ -77,15 +77,12 @@ def _http_get_json(url: str, headers: dict[str, str] | None = None, timeout: int
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
with urlopen(req, timeout=timeout, context=ctx) as resp: # noqa: S310
payload = resp.read().decode("utf-8")
return json.loads(payload)
return json.loads(resp.read().decode("utf-8"))
def _build_token_header(token_value: str) -> str:
val = token_value.strip()
if val.startswith("PVEAPIToken="):
return val
return f"PVEAPIToken={val}"
return val if val.startswith("PVEAPIToken=") else f"PVEAPIToken={val}"
def _create_api_token_via_ssh() -> str | None:
@@ -101,68 +98,62 @@ def _create_api_token_via_ssh() -> str | None:
parsed = json.loads(result.get("stdout") or "{}")
except json.JSONDecodeError:
return None
tokenid = parsed.get("full-tokenid")
secret = parsed.get("value")
if tokenid and secret:
return f"PVEAPIToken={tokenid}={secret}"
return None
tokenid, secret = parsed.get("full-tokenid"), parsed.get("value")
return f"PVEAPIToken={tokenid}={secret}" if tokenid and secret else None
def _fetch_nodes_via_api() -> tuple[list[dict[str, Any]], str, str | None]:
def _fetch_proxmox_state() -> tuple[str, float, float, str, str, str | None]:
token = os.getenv("PROXMOX_TOKEN")
tried = []
api_source, api_error = "none", None
nodes: list[dict[str, Any]] = []
if token:
tried.append("env-token")
try:
data = _http_get_json(
f"{PROXMOX_API_URL}/nodes",
headers={"Authorization": _build_token_header(token)},
)
return data.get("data") or [], "api-token-env", None
data = _http_get_json(f"{PROXMOX_API_URL}/nodes", headers={"Authorization": _build_token_header(token)})
nodes = data.get("data") or []
api_source = "api-token-env"
except Exception as exc: # noqa: BLE001
tried.append(f"env-failed:{exc}")
created = _create_api_token_via_ssh()
if created:
tried.append("ssh-created-token")
try:
data = _http_get_json(
f"{PROXMOX_API_URL}/nodes",
headers={"Authorization": created},
)
return data.get("data") or [], "api-token-ssh", None
except Exception as exc: # noqa: BLE001
tried.append(f"ssh-token-failed:{exc}")
return [], "none", ", ".join(tried) if tried else "no-token"
api_error = str(exc)
if not nodes:
created = _create_api_token_via_ssh()
if created:
try:
data = _http_get_json(f"{PROXMOX_API_URL}/nodes", headers={"Authorization": created})
nodes = data.get("data") or []
api_source = "api-token-ssh"
except Exception as exc: # noqa: BLE001
api_error = str(exc)
def _fetch_nodes_via_ssh() -> tuple[list[dict[str, Any]], str, str | None]:
pvesh = _run_ssh(PROXMOX_HOST, "pvesh get /nodes --output-format json")
if pvesh.get("ok"):
try:
return json.loads(pvesh["stdout"] or "[]"), "ssh-pvesh", None
except json.JSONDecodeError:
pass
qm = _run_ssh(PROXMOX_HOST, "qm list")
rows: list[dict[str, Any]] = []
if not nodes:
pvesh = _run_ssh(PROXMOX_HOST, "pvesh get /nodes --output-format json")
if pvesh.get("ok"):
try:
nodes = json.loads(pvesh["stdout"] or "[]")
api_source = "ssh-pvesh"
except json.JSONDecodeError:
api_error = pvesh.get("stderr") or "pvesh parse error"
api_node = next((n for n in nodes if isinstance(n, dict) and (n.get("node") or "").strip()), None)
cpu = float(api_node.get("cpu", 0)) if api_node else 0.0
mem = float(api_node.get("mem", 0)) if api_node else 0.0
status = str(api_node.get("status") or "unknown") if api_node else "unknown"
vm_status = "unknown"
qm = _run_ssh(PROXMOX_HOST, f"qm status {VM_106_ID} 2>/dev/null || qm list | grep '^{VM_106_ID} '")
if qm.get("ok") and qm.get("stdout"):
lines = (qm["stdout"] or "").splitlines()
for line in lines[1:]:
parts = line.split()
if not parts:
continue
vmid = parts[0]
rows.append(
{
"node": "pve",
"type": "qemu",
"id": f"qemu/{vmid}",
"vmid": int(vmid) if vmid.isdigit() else vmid,
"status": parts[2] if len(parts) > 2 else "unknown",
}
)
return rows, "ssh-qm-list", None
err = pvesh.get("stderr") or qm.get("stderr") or "ssh lookup failed"
return [], "none", err
line = qm["stdout"].splitlines()[0].lower()
if "running" in line:
vm_status = "running"
elif "stopped" in line:
vm_status = "stopped"
else:
vm_status = "online" if _port_health(VM_106_IP, 22) else "offline"
if status == "unknown" and _port_health(PROXMOX_HOST, 8006):
status = "online"
return status, cpu, mem, vm_status, api_source, api_error
def _port_health(host: str, port: int, timeout: float = 1.5) -> bool:
@@ -176,13 +167,10 @@ def _port_health(host: str, port: int, timeout: float = 1.5) -> bool:
def check_docker_services() -> dict[str, Any]:
result = _run_ssh(VM_106_IP, "docker ps --format json")
method = "docker-ps-json"
if not result.get("ok"):
result = _run_ssh(VM_106_IP, "docker ps --format '{{json .}}'")
method = "docker-ps-template-json"
if not result.get("ok"):
return {"ok": False, "source": method, "error": result.get("stderr") or "docker check failed", "containers": []}
return {"ok": False, "error": result.get("stderr") or "docker check failed", "containers": []}
containers: list[dict[str, Any]] = []
for line in (result.get("stdout") or "").splitlines():
line = line.strip()
@@ -193,102 +181,99 @@ def check_docker_services() -> dict[str, Any]:
containers.append(parsed if isinstance(parsed, dict) else {"raw": parsed})
except json.JSONDecodeError:
containers.append({"raw": line})
return {"ok": True, "source": method, "containers": containers}
return {"ok": True, "containers": containers}
def _online_services() -> list[dict[str, Any]]:
docker_state = check_docker_services()
docker_names = {
str(c.get("Names") or c.get("Name") or "").lower(): c
for c in docker_state.get("containers", [])
}
online: list[dict[str, Any]] = []
for svc in SERVICE_CATALOG:
up = _port_health(VM_106_IP, int(svc["port"]))
if not up:
continue
hint = "running"
for name, details in docker_names.items():
slug = svc["id"].replace("svc-", "").replace("-", "")
if slug in name.replace("-", ""):
hint = str(details.get("State") or details.get("Status") or "running")
break
online.append({
"id": svc["id"],
"label": f"{svc['label']}:{svc['port']}",
"type": "service",
"status": "online",
"host": VM_106_IP,
"port": svc["port"],
"url": svc["url"],
"role": svc["role"],
"hint": hint,
"agent": "sysops",
"children": [],
})
return online
def get_topology() -> dict[str, Any]:
api_nodes, api_source, api_error = _fetch_nodes_via_api()
ssh_nodes: list[dict[str, Any]] = []
ssh_source = "none"
ssh_error: str | None = None
if not api_nodes:
ssh_nodes, ssh_source, ssh_error = _fetch_nodes_via_ssh()
api_node = next((n for n in api_nodes if (n.get("node") or "").strip()), None) if api_nodes else None
host_cpu = float(api_node.get("cpu", 0)) if api_node else 0.0
host_mem = float(api_node.get("mem", 0)) if api_node else 0.0
host_status = api_node.get("status") if api_node else "unknown"
if host_status == "unknown" and ssh_nodes:
host_status = "online"
vm_states: dict[str, str] = {"105": "unknown", "106": "unknown"}
source_rows = api_nodes or ssh_nodes
for row in source_rows:
vmid = str(row.get("vmid") or "").strip()
if vmid in vm_states:
vm_states[vmid] = str(row.get("status") or "unknown")
docker_state = check_docker_services()
docker_names = {
str(c.get("Names") or c.get("Names.0") or c.get("Name") or "").lower(): c for c in docker_state.get("containers", [])
}
vm105_children: list[dict[str, Any]] = []
vm106_children: list[dict[str, Any]] = []
for svc in SERVICE_LAYOUT:
up = _port_health(str(svc["host"]), int(svc["port"]))
hinted = "unknown"
for name, details in docker_names.items():
if svc["label"].split(":")[0].replace("-", "") in name.replace("-", ""):
hinted = str(details.get("State") or details.get("Status") or "running")
break
item = {
"id": svc["id"],
"label": svc["label"],
"type": "service",
"status": "online" if up else "offline",
"cpu": None,
"mem": None,
"host": svc["host"],
"hint": hinted,
"children": [],
}
if svc["parent"] == "vm105-hermes":
vm105_children.append(item)
else:
vm106_children.append(item)
host_status, host_cpu, host_mem, vm_status, api_source, api_error = _fetch_proxmox_state()
services = _online_services()
proxmox_reachable = _port_health(PROXMOX_HOST, 8006)
topology_nodes = [
{
"id": "proxmox-host",
"label": f"proxmox-host ({PROXMOX_HOST})",
"type": "proxmox",
"status": host_status,
"cpu": host_cpu,
"mem": host_mem,
"id": "hardware-r340",
"label": HARDWARE_LABEL,
"type": "hardware",
"status": "online" if proxmox_reachable or _port_health(VM_106_IP, 8600) else "degraded",
"role": "Bare metal host",
"children": [
{
"id": "vm105-hermes",
"label": f"vm105-hermes ({VM_105_IP})",
"type": "vm",
"status": vm_states["105"],
"cpu": None,
"mem": None,
"children": vm105_children,
},
{
"id": "vm106-command",
"label": f"vm106-command ({VM_106_IP})",
"type": "vm",
"status": vm_states["106"],
"cpu": None,
"mem": None,
"children": vm106_children,
},
"id": "proxmox-host",
"label": f"Proxmox VE · {PROXMOX_HOST}",
"type": "proxmox",
"status": host_status if proxmox_reachable else ("online" if _port_health(PROXMOX_HOST, 22) else "offline"),
"cpu": host_cpu,
"mem": host_mem,
"role": "Hypervisor · node pve",
"url": f"https://{PROXMOX_HOST}:8006",
"children": [
{
"id": "vm106-command",
"label": f"VM {VM_106_ID} · {VM_106_NAME} · {VM_106_IP}",
"type": "vm",
"status": vm_status if vm_status != "unknown" else ("online" if services else "offline"),
"vmid": VM_106_ID,
"role": "Foodlinkk Command Center",
"children": services,
}
],
}
],
}
]
return {
"generated_at": _iso_now(),
"nodes": topology_nodes,
"layer_agents": LAYER_AGENTS,
"layers": [
{"id": "L0", "label": "Hardware", "y": 55},
{"id": "L1", "label": "Hypervisor", "y": 175},
{"id": "L2", "label": f"VM {VM_106_ID}", "y": 295},
{"id": "L3", "label": "Online services", "y": 415},
{"id": "L4", "label": "IT Agents", "y": 555},
],
"meta": {
"hardware": HARDWARE_LABEL,
"proxmox_host": PROXMOX_HOST,
"vm106_ip": VM_106_IP,
"vm106_id": VM_106_ID,
"services_online": len(services),
"api_source": api_source,
"api_error": api_error,
"ssh_source": ssh_source,
"ssh_error": ssh_error,
"docker_source": docker_state.get("source"),
"docker_ok": docker_state.get("ok", False),
},
}
@@ -322,10 +307,8 @@ def _table_exists(table_name: str) -> bool:
row = fetch_one(
"""
SELECT EXISTS(
SELECT 1
FROM information_schema.tables
WHERE table_schema = 'public'
AND table_name = %s
SELECT 1 FROM information_schema.tables
WHERE table_schema = 'public' AND table_name = %s
) AS ok
""",
(table_name,),
@@ -340,10 +323,8 @@ def poll_and_snapshot() -> dict[str, Any]:
cols = fetch_all(
"""
SELECT column_name
FROM information_schema.columns
WHERE table_schema = 'public'
AND table_name = 'infra_snapshots'
SELECT column_name FROM information_schema.columns
WHERE table_schema = 'public' AND table_name = 'infra_snapshots'
ORDER BY ordinal_position
"""
)
@@ -354,7 +335,6 @@ def poll_and_snapshot() -> dict[str, Any]:
"summary": {k: v for k, v in status.items() if k != "topology"},
"generated_at": status.get("generated_at"),
}
value_map: dict[str, Any] = {}
if "source" in colset:
value_map["source"] = "proxmox"
@@ -375,7 +355,6 @@ def poll_and_snapshot() -> dict[str, Any]:
return {"ok": False, "saved": False, "reason": "infra_snapshots has no compatible columns", "status": status}
columns = list(value_map.keys())
placeholders = ", ".join(["%s"] * len(columns))
sql = f"INSERT INTO infra_snapshots ({', '.join(columns)}) VALUES ({placeholders})"
sql = f"INSERT INTO infra_snapshots ({', '.join(columns)}) VALUES ({', '.join([ '%s' ] * len(columns))})"
execute(sql, tuple(value_map[c] for c in columns))
return {"ok": True, "saved": True, "columns": columns, "status": status}