atc-agents/api/main.py

"""ATC Command Center API — FastAPI backend."""

from __future__ import annotations

import asyncio
import json
import os
import time
import uuid
from contextlib import asynccontextmanager
from datetime import datetime, timezone
from typing import Any

import httpx
import redis.asyncio as aioredis
from fastapi import FastAPI, File, UploadFile, WebSocket, WebSocketDisconnect
from fastapi.middleware.cors import CORSMiddleware
from agent_terminal import (
    get_all_terminals,
    get_terminal_lines,
    init_terminals,
    make_logger,
    set_terminal_publisher,
    terminal_log,
)
from lab_context import collect_full_lab_context, format_context_for_agent
from presentation import build_presentation_payload, render_presentation_html
from presentation_upload import get_deck, list_decks, save_upload
from presentation_static import get_static_deck, list_static_decks
from storage_s3 import router as storage_s3_router
from node_registry import NODE_IDS, NODE_AGENT, NODE_REGISTRY, is_node_id
from node_ops import build_node_detail, probe_node, run_node_probe_task
from approval_service import (
    APPROVAL_ACTION_TYPES,
    approval_stats,
    create_approval_request,
    decide_approval_request,
    detect_agent_proposed_action,
    detect_approval_intent,
    list_approvals,
)
from db import SessionLocal, db_health, init_database
from supervisor import mirror_terminal_line, mirror_to_supervisors

from workload import build_workload_payload

_workload_cache: dict[str, Any] = {"ts": 0.0, "data": None}
_presentation_cache: dict[str, Any] = {"ts": 0.0, "data": None}
WORKLOAD_CACHE_TTL = 30.0
PRESENTATION_CACHE_TTL = 45.0

from pydantic import BaseModel, Field
from sqlalchemy import Column, DateTime, String, Text, select
from sqlalchemy.orm import DeclarativeBase

REDIS_URL = os.getenv("REDIS_URL", "redis://redis:6379/0")
DOCKHAND_URL = os.getenv("DOCKHAND_URL", "http://10.0.21.45:8082")
GPU_URL = os.getenv("GPU_URL", "http://10.0.20.106:9000")
GPU_UI_URL = os.getenv("GPU_UI_URL", GPU_URL)
LLM_URL = os.getenv("LLM_URL", "http://10.0.20.106:8001/v1")
LLM_MODEL = os.getenv("LLM_MODEL", "gpt-4o")
LLM_API_KEY = os.getenv("LLM_API_KEY", "sk-local")
LLM_TIMEOUT = float(os.getenv("LLM_TIMEOUT", "120"))

AGENTS = [
    {
        "id": "etl-guardian",
        "name": "ETL Guardian",
        "color": "#00f0ff",
        "zone": "etl",
        "role": "Airflow, Kafka, Debezium, S3 pipeline",
        "icon": "⚡",
        "motto": "Pipelines never sleep",
        "capabilities": ["Airflow", "Kafka", "Debezium", "S3", "Connectors"],
        "suggested_prompts": [
            "How is Debezium doing?",
            "Are all Airflow DAGs healthy?",
            "Kafka connector status?",
        ],
    },
    {
        "id": "lakehouse-ops",
        "name": "Lakehouse Ops",
        "color": "#ff00aa",
        "zone": "lakehouse",
        "role": "Spark, Trino, Iceberg",
        "icon": "🏔️",
        "motto": "Query the lake, trust the table",
        "capabilities": ["Spark", "Trino", "Iceberg", "Delta", "SQL"],
        "suggested_prompts": [
            "Lakehouse stack status?",
            "Is Trino reachable?",
            "How many lakehouse containers are running?",
        ],
    },
    {
        "id": "data-custodian",
        "name": "Data Custodian",
        "color": "#ffaa00",
        "zone": "db",
        "role": "PostgreSQL, MySQL, Mongo, Cassandra, Neo4j",
        "icon": "🛡️",
        "motto": "Guardian of every row",
        "capabilities": ["PostgreSQL", "MySQL", "MongoDB", "Cassandra", "Neo4j"],
        "suggested_prompts": [
            "Hoeveel data zit er in de databases?",
            "Wat staat er in PostgreSQL sales_orders?",
            "MongoDB supplychain overzicht",
        ],
    },
    {
        "id": "hadoop-ranger",
        "name": "Hadoop Ranger",
        "color": "#39ff14",
        "zone": "hadoop",
        "role": "HDFS, YARN cluster",
        "icon": "🌲",
        "motto": "Patrol the data forest",
        "capabilities": ["HDFS", "YARN", "NameNode", "DataNodes"],
        "suggested_prompts": [
            "Is HDFS NameNode up?",
            "Hadoop cluster status?",
            "YARN nodes healthy?",
        ],
    },
    {
        "id": "infra-sentinel",
        "name": "Infra Sentinel",
        "color": "#9b72cf",
        "zone": "docker",
        "role": "Docker, Proxmox, GPU, monitoring",
        "icon": "👁️",
        "motto": "See everything, miss nothing",
        "capabilities": ["Docker", "Proxmox", "GPU", "vLLM", "Monitoring"],
        "suggested_prompts": [
            "GPU status?",
            "Which LLM model is running?",
            "Docker container overview",
        ],
    },
    {
        "id": "mo-commander",
        "name": "Mo · Command",
        "color": "#4c9aed",
        "zone": "command",
        "role": "Supervisor — full event intel, ingress, approvals",
        "icon": "🎯",
        "motto": "Nothing happens without Mo knowing",
        "supervisor": True,
        "person": "mo",
        "capabilities": ["Events", "Ingress", "Approvals", "Agent dispatch", "Network IN"],
        "suggested_prompts": [
            "What happened today?",
            "What events came in?",
            "Pipeline status overview",
        ],
    },
    {
        "id": "bart-commander",
        "name": "Bart · Ops",
        "color": "#3fb950",
        "zone": "command",
        "role": "Supervisor — egress, MCP comms, network OUT",
        "icon": "📡",
        "motto": "All traffic flows through Bart",
        "supervisor": True,
        "person": "bart",
        "capabilities": ["Egress", "MCP routing", "Network OUT", "GPU inference", "S3 writes"],
        "suggested_prompts": [
            "What is leaving the cluster?",
            "MCP agent communication status?",
            "Network egress overview",
        ],
    },
    {
        "id": "network-watcher",
        "name": "Network Watcher",
        "color": "#58a6ff",
        "zone": "network",
        "role": "VLAN 20/21 traffic, data in & out paths",
        "icon": "🌐",
        "motto": "Every packet tells a story",
        "capabilities": ["VLAN 20", "VLAN 21", "Ingress", "Egress", "Firewall paths"],
        "suggested_prompts": [
            "Data ingress status?",
            "What leaves the cluster?",
            "Network path to S3?",
        ],
    },
    {
        "id": "mcp-coordinator",
        "name": "MCP Coordinator",
        "color": "#f778ba",
        "zone": "mcp",
        "role": "MCP hub — routes all agent tool calls & comms",
        "icon": "🔀",
        "motto": "Route once, deliver everywhere",
        "capabilities": ["MCP servers", "Tool routing", "Agent relay", "WebSocket bus"],
        "suggested_prompts": [
            "Which MCP agents are active?",
            "MCP hub route status?",
            "Agent communication overview",
        ],
    },
]

ZONES = [
    {"id": "docker", "label": "DOCKER RACK", "x": 8, "color": "#b366ff"},
    {"id": "db", "label": "DB VAULT", "x": 28, "color": "#ffaa00"},
    {"id": "lakehouse", "label": "LAKEHOUSE HUB", "x": 50, "color": "#ff00aa"},
    {"id": "hadoop", "label": "HADOOP CLUSTER", "x": 72, "color": "#39ff14"},
    {"id": "etl", "label": "ETL PIPE", "x": 92, "color": "#00f0ff"},
]

INTENT_KEYWORDS: dict[str, list[str]] = {
    "data-custodian": ["database", "postgres", "postgresql", "mysql", "mongo", "mongodb", "cassandra", "neo4j", "sql", "db "],
    "lakehouse-ops": ["trino", "spark", "lakehouse", "iceberg", "query", "table"],
    "hadoop-ranger": [
        "hadoop", "hdfs", "yarn", "datanode", "namenode", "replicatie", "replication",
        "rf factor", "opslag", "bestanden", "blocks", "cluster opslag", "data op",
    ],
    "infra-sentinel": ["docker", "container", "vm", "proxmox", "infra", "grafana", "gpu", "vllm", "llm", "nvidia", "inference", "model"],
    "etl-guardian": ["airflow", "dag", "debezium", "kafka", "connector", "etl", "pipeline", "s3"],
    "network-watcher": ["network", "vlan", "ingress", "egress", "traffic", "packet", "firewall", "route"],
    "mcp-coordinator": ["mcp", "tool", "router", "relay", "websocket", "hub"],
    "mo-commander": ["mo", "supervisor", "events", "overzicht", "alles", "gebeurd"],
    "bart-commander": ["bart", "egress", "uitgaand", "communicatie", "mcp comm"],
}


class Base(DeclarativeBase):
    pass


class FeedEntry(Base):
    __tablename__ = "feed"

    id = Column(String, primary_key=True)
    ts = Column(DateTime, default=lambda: datetime.now(timezone.utc))
    agent_id = Column(String)
    level = Column(String, default="info")
    message = Column(Text)


class Approval(Base):
    __tablename__ = "approvals"

    id = Column(String, primary_key=True)
    ts = Column(DateTime, default=lambda: datetime.now(timezone.utc))
    agent_id = Column(String)
    action = Column(Text)
    reason = Column(Text)
    status = Column(String, default="pending")
    action_type = Column(String, default="generic.mutate")
    target = Column(Text, default="")
    payload = Column(Text, default="{}")
    decided_by = Column(String, nullable=True)
    decide_note = Column(Text, nullable=True)
    decided_at = Column(DateTime, nullable=True)
    priority = Column(String, default="normal")


_db_info = init_database(Base)

redis_client: aioredis.Redis | None = None
ws_clients: set[WebSocket] = set()


class PromptRequest(BaseModel):
    message: str = Field(min_length=1, max_length=2000)
    agent_id: str | None = None


class NodeAskRequest(BaseModel):
    message: str = Field(min_length=1, max_length=2000)


class ApprovalCreateRequest(BaseModel):
    agent_id: str = Field(min_length=1, max_length=64)
    action: str = Field(min_length=1, max_length=2000)
    reason: str = Field(min_length=1, max_length=2000)
    action_type: str = "generic.mutate"
    target: str = ""
    payload: dict[str, Any] | None = None
    priority: str = "normal"


class ApprovalDecision(BaseModel):
    approved: bool
    decided_by: str = "mo-commander"
    note: str = ""


def route_agent(message: str) -> str:
    lower = message.lower()
    # Storage/data questions default to Hadoop unless clearly about databases
    if any(w in lower for w in ("data", "opslag", "gb", "replicatie", "replication", "hdfs", "hadoop")):
        if not any(w in lower for w in ("postgres", "mysql", "mongo", "database", "sql", "neo4j", "cassandra")):
            return "hadoop-ranger"
    scores = {aid: sum(1 for kw in kws if kw in lower) for aid, kws in INTENT_KEYWORDS.items()}
    best = max(scores, key=scores.get)
    if scores[best] == 0:
        return "infra-sentinel"
    return best


async def gather_agent_context(
    agent_id: str,
    status: dict[str, Any],
    log: Any | None = None,
) -> str:
    """Full lab snapshot for vLLM — all domains, agent's primary domain highlighted."""
    snapshot = await collect_full_lab_context(gpu_data=status.get("gpu"), log=log)
    snapshot["domains_summary"] = status.get("domains", {})
    ctx = format_context_for_agent(agent_id, snapshot)
    agent_lines = ["", "=== AGENTS & SUPERVISORS ==="]
    for a in AGENTS:
        sup = " [supervisor]" if a.get("supervisor") else ""
        agent_lines.append(f"  - {a['name']} ({a['id']}){sup}: {a['role']}")
    ctx = ctx + "\n".join(agent_lines)
    if log:
        await log("info", "fetch", f"▸ Context assembled: {len(ctx)} chars for LLM")
    return ctx


async def ask_llm(
    agent_id: str,
    message: str,
    context: str,
    log: Any | None = None,
) -> str | None:
    agent = next(a for a in AGENTS if a["id"] == agent_id)
    system = f"""Je bent {agent['name']}, een autonomous ops agent in het Dell ATC data lab.
Specialisatie: {agent['role']}.
Motto: {agent.get('motto', '')}

Je antwoordt namens je domein maar hebt zicht op de HELE lab stack: Docker, databases, lakehouse (Trino/Spark/Kafka Connect), ETL (Airflow/Kafka), Hadoop HDFS, en GPU/vLLM.

Regels:
- Antwoord in dezelfde taal als de gebruiker (Nederlands of Engels).
- Je hebt volledige zicht op de HELE cluster: alle VMs, zones, connectors, GPU, Hadoop, ObjectScale en Command Center.
- Gebruik ALLEEN de live data hieronder — verzin geen hosts, poorten, cijfers of connector namen.
- Gebruik exact de container/connector namen uit de data (bijv. mysql-hr-connector, niet "Debezium").
- Als iets DOWN of 0 GB is, zeg dat eerlijk.
- Kort en behulpzaam (max ~10 zinnen); bullet lists mogen als het overzicht helpt.

--- LIVE LAB DATA (primary domain eerst, daarna volledige stack) ---
{context}
"""
    if log:
        await log("info", "llm", f"▸ Querying vLLM model={LLM_MODEL}")
        await log("cmd", "llm", f"$ POST {LLM_URL.rstrip('/')}/chat/completions")
        await log("info", "llm", f"  user: {message[:160]}{'…' if len(message) > 160 else ''}")
    try:
        async with httpx.AsyncClient(timeout=LLM_TIMEOUT) as client:
            t0 = time.monotonic()
            r = await client.post(
                f"{LLM_URL.rstrip('/')}/chat/completions",
                headers={
                    "Authorization": f"Bearer {LLM_API_KEY}",
                    "Content-Type": "application/json",
                },
                json={
                    "model": LLM_MODEL,
                    "messages": [
                        {"role": "system", "content": system},
                        {"role": "user", "content": message},
                    ],
                    "max_tokens": 800,
                    "temperature": 0.25,
                },
            )
            r.raise_for_status()
            content = r.json()["choices"][0]["message"]["content"].strip()
            ms = int((time.monotonic() - t0) * 1000)
            if content and content.strip("!"):
                if log:
                    await log("ok", "llm", f"← vLLM response {len(content)} chars ({ms}ms)")
                    preview = content.replace("\n", " ")[:180]
                    await log("info", "llm", f"  » {preview}{'…' if len(content) > 180 else ''}")
                return content
            if log:
                await log("warn", "llm", f"← Empty or invalid LLM output ({ms}ms)")
    except Exception as exc:
        if log:
            await log("err", "llm", f"✗ vLLM error: {exc}")
    return None


def fallback_answer(agent_id: str, context: str) -> str:
    agent_name = next(a["name"] for a in AGENTS if a["id"] == agent_id)
    return f"**{agent_name}** (offline LLM — ruwe data):\n\n{context}"


async def publish_event(event: dict[str, Any]) -> None:
    payload = json.dumps(event, default=str)
    if redis_client:
        await redis_client.publish("ops", payload)
    dead = []
    for ws in ws_clients:
        try:
            await ws.send_text(payload)
        except Exception:
            dead.append(ws)
    for ws in dead:
        ws_clients.discard(ws)

    et = event.get("type")
    if et == "feed":
        entry = event.get("entry") or {}
        await mirror_to_supervisors(
            entry.get("agent_id", "?"),
            entry.get("message", ""),
            level=entry.get("level", "info"),
        )
    elif et == "terminal":
        await mirror_terminal_line(event.get("line") or {})
    elif et in ("agent_dispatch", "agent_fetch", "agent_return"):
        aid = event.get("agent_id", "?")
        zone = event.get("zone", "")
        await mirror_to_supervisors(aid, f"{et} → zone {zone}", level="info", phase="dispatch")


def add_feed(agent_id: str, message: str, level: str = "info") -> dict:
    entry_id = str(uuid.uuid4())[:8]
    with SessionLocal() as db:
        row = FeedEntry(id=entry_id, agent_id=agent_id, message=message, level=level)
        db.add(row)
        db.commit()
    return {
        "id": entry_id,
        "ts": datetime.now(timezone.utc).isoformat(),
        "agent_id": agent_id,
        "message": message,
        "level": level,
    }


async def dockhand_env_containers(env_id: int) -> list[dict]:
    try:
        async with httpx.AsyncClient(timeout=8.0) as client:
            r = await client.get(f"{DOCKHAND_URL}/api/containers", params={"env": env_id})
            r.raise_for_status()
            return r.json()
    except Exception:
        return []


async def probe_url(url: str) -> bool:
    try:
        async with httpx.AsyncClient(timeout=4.0, verify=False) as client:
            r = await client.get(url)
            return r.status_code < 500
    except Exception:
        return False


async def collect_gpu() -> dict[str, Any]:
    host = GPU_URL.replace("http://", "").replace("https://", "").split("/")[0]
    base = {"ok": False, "host": host, "ui_url": GPU_UI_URL}

    try:
        async with httpx.AsyncClient(timeout=6.0) as client:
            metrics_r, model_r, integration_r = await asyncio.gather(
                client.get(f"{GPU_URL}/api/gpu/metrics"),
                client.get(f"{GPU_URL}/api/active-model"),
                client.get(f"{GPU_URL}/api/integration"),
                return_exceptions=True,
            )

            gpus: list[dict[str, Any]] = []
            if isinstance(metrics_r, httpx.Response) and metrics_r.status_code == 200:
                current = metrics_r.json().get("current", {})
                gpus = [
                    {
                        "index": g["index"],
                        "name": g["name"],
                        "util_gpu": g.get("util_gpu", 0),
                        "memory_used_mib": g.get("memory_used_mib", 0),
                        "memory_total_mib": g.get("memory_total_mib", 0),
                        "temperature_c": g.get("temperature_c", 0),
                        "power_w": g.get("power_w", 0),
                    }
                    for g in current.get("gpus", [])
                ]

            active_model = None
            inference_active = False
            vllm_url = None

            if isinstance(model_r, httpx.Response) and model_r.status_code == 200:
                model_data = model_r.json()
                active_model = model_data.get("name")
                inference_active = bool(model_data.get("inference_active"))
                vllm_url = model_data.get("base_url")

            if isinstance(integration_r, httpx.Response) and integration_r.status_code == 200:
                integ = integration_r.json()
                if not active_model:
                    active_model = integ.get("active_name")
                if not inference_active:
                    inference_active = bool(integ.get("inference_active"))
                if not vllm_url:
                    vllm_url = integ.get("recommended_base_url")

            return {
                **base,
                "ok": len(gpus) > 0 or inference_active,
                "inference_active": inference_active,
                "active_model": active_model,
                "vllm_url": vllm_url,
                "gpu_count": len(gpus),
                "gpus": gpus,
            }
    except Exception as exc:
        return {**base, "error": str(exc)}


async def collect_status() -> dict[str, Any]:
    db_containers = await dockhand_env_containers(5)
    db_running = sum(1 for c in db_containers if c.get("state") == "running")
    db_total = len(db_containers) or 6

    lake_containers = await dockhand_env_containers(9)
    lake_running = sum(1 for c in lake_containers if c.get("state") == "running")
    lake_total = len(lake_containers) or 6

    docker_containers = await dockhand_env_containers(1)
    docker_running = sum(1 for c in docker_containers if c.get("state") == "running")

    hdfs_ok = await probe_url("http://10.0.21.61:9870")
    kafka_ok = await probe_url("http://10.0.21.36:9000")
    airflow_ok = await probe_url("http://10.0.21.55:8080")

    def level(running: int, total: int) -> str:
        if total == 0:
            return "unknown"
        ratio = running / total
        if ratio >= 0.9:
            return "ok"
        if ratio >= 0.5:
            return "warn"
        return "down"

    gpu = await collect_gpu()
    gpu_level = "ok" if gpu.get("ok") and gpu.get("inference_active") else ("warn" if gpu.get("ok") else "down")
    gpu_label = gpu.get("active_model") or (f"{gpu.get('gpu_count', 0)} GPUs" if gpu.get("ok") else "offline")

    return {
        "ts": datetime.now(timezone.utc).isoformat(),
        "domains": {
            "docker": {"level": "ok" if docker_running >= 5 else "warn", "label": f"{docker_running} containers", "running": docker_running},
            "databases": {"level": level(db_running, db_total), "label": f"{db_running}/{db_total} up", "running": db_running, "total": db_total},
            "lakehouse": {"level": level(lake_running, lake_total), "label": f"{lake_running}/{lake_total} up", "running": lake_running, "total": lake_total},
            "hadoop": {"level": "ok" if hdfs_ok else "warn", "label": "NN up" if hdfs_ok else "NN check"},
            "etl": {"level": "ok" if kafka_ok and airflow_ok else "warn", "label": "Kafka+Airflow"},
            "gpu": {"level": gpu_level, "label": gpu_label},
        },
        "gpu": gpu,
        "kafka_ok": kafka_ok,
        "airflow_ok": airflow_ok,
        "hdfs_ok": hdfs_ok,
    }




async def _run_agent_task_safe(agent_id: str, message: str, prompt_id: str) -> None:
  try:
    await run_agent_task(agent_id, message, prompt_id)
  except Exception as exc:
    agent_name = next((a["name"] for a in AGENTS if a["id"] == agent_id), agent_id)
    err = f"Sorry — {agent_name} could not complete your request: {exc}"
    await terminal_log(agent_id, f"[{prompt_id}] ✗ Error: {exc}", level="err", phase="error", prompt_id=prompt_id)
    feed = add_feed(agent_id, f"{agent_name} failed: {str(exc)[:80]}", "err")
    await publish_event({"type": "feed", "entry": feed})
    await publish_event({
      "type": "prompt_result",
      "prompt_id": prompt_id,
      "agent_id": agent_id,
      "answer": err,
    })


async def run_agent_task(agent_id: str, message: str, prompt_id: str) -> str:
    zone = next(a["zone"] for a in AGENTS if a["id"] == agent_id)
    agent_name = next(a["name"] for a in AGENTS if a["id"] == agent_id)
    log = make_logger(agent_id, prompt_id)
    approval_created = False

    intent = detect_approval_intent(message)
    if intent:
        with SessionLocal() as db:
            await create_approval_request(
                db=db,
                ApprovalModel=Approval,
                agent_id=agent_id,
                action=intent["action"],
                reason=intent["reason"],
                action_type=intent["action_type"],
                terminal_log=terminal_log,
                mirror_supervisors=mirror_to_supervisors,
                publish=publish_event,
                add_feed=add_feed,
            )
        approval_created = True
        await terminal_log(
            agent_id,
            f"[{prompt_id}] Mutating request detected — approval queued for Mo & Bart",
            level="warn",
            phase="approval",
            prompt_id=prompt_id,
        )

    await terminal_log(
        agent_id,
        f"[{prompt_id}] ▶ Mission accepted: {message}",
        level="info",
        phase="dispatch",
        prompt_id=prompt_id,
    )
    await publish_event({"type": "agent_dispatch", "agent_id": agent_id, "zone": zone, "prompt_id": prompt_id})
    await asyncio.sleep(0.4)
    await terminal_log(agent_id, f"[{prompt_id}] Walking to zone: {zone}", level="info", phase="dispatch", prompt_id=prompt_id)
    await publish_event({"type": "agent_fetch", "agent_id": agent_id, "zone": zone, "prompt_id": prompt_id})

    await log("info", "fetch", f"[{prompt_id}] Collecting live lab metrics…")
    status = await collect_status()
    context = await gather_agent_context(agent_id, status, log=log)

    answer = await ask_llm(agent_id, message, context, log=log)
    if not answer:
        await log("warn", "llm", "LLM fallback — returning raw context")
        answer = fallback_answer(agent_id, context)

    if not approval_created:
        proposed = detect_agent_proposed_action(answer, message)
        if proposed:
            with SessionLocal() as db:
                await create_approval_request(
                    db=db,
                    ApprovalModel=Approval,
                    agent_id=agent_id,
                    action=proposed["action"],
                    reason=proposed["reason"],
                    action_type=proposed["action_type"],
                    target=proposed.get("target", ""),
                    terminal_log=terminal_log,
                    mirror_supervisors=mirror_to_supervisors,
                    publish=publish_event,
                    add_feed=add_feed,
                )
            approval_created = True
            answer = (
                f"{answer}\n\n⏸ **Approval required** — this action is in the Approval Inbox. "
                f"Mo & Bart have been notified and must approve before we execute."
            )

    await asyncio.sleep(0.3)
    await terminal_log(agent_id, f"[{prompt_id}] ✓ Mission complete", level="ok", phase="done", prompt_id=prompt_id)
    await publish_event({"type": "agent_return", "agent_id": agent_id, "zone": zone, "prompt_id": prompt_id})
    feed = add_feed(agent_id, f"{agent_name} completed a response (see Comms)", "info")
    await publish_event({"type": "feed", "entry": feed})
    await publish_event({"type": "prompt_result", "prompt_id": prompt_id, "agent_id": agent_id, "answer": answer})
    return answer


async def heartbeat_loop() -> None:
    while True:
        try:
            status = await collect_status()
            workload = await collect_workload()
            await publish_event({"type": "status", "data": status})
            await publish_event({"type": "workload", "data": workload})
            for domain, info in status["domains"].items():
                if info["level"] == "down":
                    agent = "data-custodian" if domain == "databases" else "infra-sentinel"
                    feed = add_feed(agent, f"Alert: {domain} is DOWN ({info['label']})", "warn")
                    await publish_event({"type": "feed", "entry": feed})
        except Exception as exc:
            await publish_event({"type": "error", "message": str(exc)})
        await asyncio.sleep(60)


@asynccontextmanager
async def lifespan(app: FastAPI):
    global redis_client
    redis_client = aioredis.from_url(REDIS_URL, decode_responses=True)
    set_terminal_publisher(publish_event)
    init_terminals([a["id"] for a in AGENTS] + NODE_IDS)
    for a in AGENTS:
        await terminal_log(a["id"], f"{a['name']} terminal online — awaiting missions", level="info", phase="boot")
    for nid in NODE_IDS:
        if nid not in NODE_REGISTRY:
            continue
        meta = NODE_REGISTRY[nid]
        await terminal_log(nid, f"{meta['label']} shell ready — click node to connect", level="info", phase="boot")
    task = asyncio.create_task(heartbeat_loop())
    add_feed("infra-sentinel", "ATC Command Center API online", "info")
    yield
    task.cancel()
    if redis_client:
        await redis_client.close()


app = FastAPI(title="ATC Command Center API", lifespan=lifespan)
app.include_router(storage_s3_router)
app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)


@app.get("/api/health")
async def health():
    llm_ok = False
    try:
        async with httpx.AsyncClient(timeout=4.0) as client:
            r = await client.get(f"{LLM_URL.rstrip('/')}/models", headers={"Authorization": f"Bearer {LLM_API_KEY}"})
            llm_ok = r.status_code == 200
    except Exception:
        pass
    return {
        "ok": True,
        "ts": datetime.now(timezone.utc).isoformat(),
        "llm_url": LLM_URL,
        "llm_ok": llm_ok,
        "llm_model": LLM_MODEL,
        "database": db_health(),
        "db_init": _db_info,
    }


async def collect_workload(*, fast: bool = True, use_cache: bool = True) -> dict[str, Any]:
    import time as _time
    now = _time.time()
    if use_cache and _workload_cache.get("data") and now - float(_workload_cache.get("ts") or 0) < WORKLOAD_CACHE_TTL:
        return _workload_cache["data"]
    gpu = await collect_gpu()
    snap = await collect_full_lab_context(gpu_data=gpu, include_inventory=not fast)
    payload = build_workload_payload(snap)
    _workload_cache["ts"] = now
    _workload_cache["data"] = payload
    return payload




async def get_presentation_data(*, use_cache: bool = True) -> dict[str, Any]:
    import time as _time
    now = _time.time()
    if use_cache and _presentation_cache.get("data") and now - float(_presentation_cache.get("ts") or 0) < PRESENTATION_CACHE_TTL:
        return _presentation_cache["data"]
    gpu = await collect_gpu()
    snap = await collect_full_lab_context(gpu_data=gpu, include_inventory=False)
    data = build_presentation_payload(snap)
    data["source"] = "live"
    _presentation_cache["ts"] = now
    _presentation_cache["data"] = data
    return data


@app.get("/api/presentation")
async def get_presentation():
    return await get_presentation_data()


@app.get("/api/presentation/html")
async def get_presentation_html():
    from fastapi.responses import HTMLResponse
    payload = await get_presentation_data()
    return HTMLResponse(render_presentation_html(payload))


@app.get("/api/presentation/decks")
async def get_presentation_decks():
    return {"live": True, "builtin": list_static_decks(), "uploaded": list_decks()}


@app.get("/api/presentation/decks/{deck_id}")
async def get_presentation_deck(deck_id: str):
    if deck_id == "live":
        return await get_presentation_data()
    deck = get_static_deck(deck_id) or get_deck(deck_id)
    if not deck:
        return {"error": "deck not found"}
    return deck


@app.get("/api/presentation/decks/{deck_id}/html")
async def get_presentation_deck_html(deck_id: str):
    from fastapi.responses import HTMLResponse
    if deck_id == "live":
        payload = await get_presentation_data()
    else:
        payload = get_static_deck(deck_id) or get_deck(deck_id)
        if not payload:
            return HTMLResponse("<h1>Deck not found</h1>", status_code=404)
    return HTMLResponse(render_presentation_html(payload))


@app.post("/api/presentation/upload")
async def upload_presentation(file: UploadFile = File(...)):
    content = await file.read()
    if len(content) > 50 * 1024 * 1024:
        return {"error": "file too large (max 50MB)"}
    deck = await save_upload(file.filename or "upload.pptx", content)
    return {"ok": True, "deck": deck}

@app.get("/api/workload")
async def get_workload(fast: bool = True):
    return await collect_workload(fast=fast, use_cache=True)


@app.get("/api/status")
async def get_status():
    return await collect_status()


@app.get("/api/gpu")
async def get_gpu():
    return await collect_gpu()


def agent_stats() -> dict[str, dict[str, Any]]:
    stats: dict[str, dict[str, Any]] = {a["id"]: {"tasks": 0, "last_active": None, "alerts": 0} for a in AGENTS}
    with SessionLocal() as db:
        rows = db.execute(select(FeedEntry).order_by(FeedEntry.ts.desc()).limit(200)).scalars().all()
    for r in rows:
        aid = r.agent_id
        if aid not in stats:
            continue
        stats[aid]["tasks"] += 1
        if r.level == "warn":
            stats[aid]["alerts"] += 1
        if stats[aid]["last_active"] is None and r.ts:
            stats[aid]["last_active"] = r.ts.isoformat()
    return stats


@app.get("/api/agents")
async def get_agents():
    stats = agent_stats()
    enriched = [{**a, "stats": stats.get(a["id"], {})} for a in AGENTS]
    return {"agents": enriched, "zones": ZONES}


@app.get("/api/terminals")
async def get_terminals(limit: int = 200):
    return {"terminals": get_all_terminals(limit)}


@app.get("/api/terminals/{subject_id}")
async def get_subject_terminal(subject_id: str, limit: int = 200):
    valid_agents = {a["id"] for a in AGENTS}
    if subject_id not in valid_agents and not is_node_id(subject_id):
        return {"error": "unknown subject"}
    return {"agent_id": subject_id, "lines": get_terminal_lines(subject_id, limit)}


@app.get("/api/nodes")
async def list_nodes():
    workload = await collect_workload()
    nodes = workload.get("topology", {}).get("nodes", [])
    return {"nodes": [{"id": n["id"], "label": n["label"], "ip": n["ip"], "level": n["level"]} for n in nodes]}


@app.get("/api/nodes/{node_id}")
async def get_node(node_id: str):
    if not is_node_id(node_id):
        return {"error": "unknown node"}
    workload = await collect_workload()
    wn = next((n for n in workload.get("topology", {}).get("nodes", []) if n["id"] == node_id), None)
    gpu = await collect_gpu()
    snap = await collect_full_lab_context(gpu_data=gpu)
    return build_node_detail(node_id, snap, wn)


@app.post("/api/nodes/{node_id}/probe")
async def post_node_probe(node_id: str):
    if not is_node_id(node_id):
        return {"error": "unknown node"}
    asyncio.create_task(run_node_probe_task(node_id))
    return {"ok": True, "node_id": node_id, "status": "probing"}


async def run_node_ask_task(node_id: str, message: str) -> None:
    agent_id = NODE_AGENT.get(node_id, "infra-sentinel")
    meta = NODE_REGISTRY[node_id]
    await terminal_log(node_id, f"▶ Query: {message}", level="info", phase="ask")
    await terminal_log(node_id, f"→ Routing to agent {agent_id}", level="info", phase="ask")
    log = make_logger(node_id)
    status = await collect_status()
    context = await gather_agent_context(agent_id, status, log=log)
    node_ctx = f"\n\n=== FOCUSED NODE: {meta['label']} ({meta['ip']}) ===\n{meta.get('description', '')}\n"
    answer = await ask_llm(agent_id, message, context + node_ctx, log=log)
    if not answer:
        answer = fallback_answer(agent_id, context)
    await terminal_log(node_id, f"◆ {answer}", level="llm", phase="answer")
    await publish_event({"type": "node_ask_result", "node_id": node_id, "agent_id": agent_id, "answer": answer})


@app.post("/api/nodes/{node_id}/ask")
async def post_node_ask(node_id: str, body: NodeAskRequest):
    if not is_node_id(node_id):
        return {"error": "unknown node"}
    asyncio.create_task(run_node_ask_task(node_id, body.message))
    return {"ok": True, "node_id": node_id, "agent_id": NODE_AGENT.get(node_id), "status": "processing"}


@app.get("/api/feed")
async def get_feed(limit: int = 50):
    with SessionLocal() as db:
        rows = db.execute(select(FeedEntry).order_by(FeedEntry.ts.desc()).limit(limit)).scalars().all()
    return {
        "entries": [
            {
                "id": r.id,
                "ts": r.ts.isoformat() if r.ts else None,
                "agent_id": r.agent_id,
                "message": r.message,
                "level": r.level,
            }
            for r in rows
        ]
    }


@app.get("/api/approvals")
async def get_approvals(status: str = "pending", limit: int = 100):
    with SessionLocal() as db:
        items = list_approvals(db, Approval, status=status, limit=limit)
        stats = approval_stats(db, Approval)
    return {"approvals": items, "stats": stats, "action_types": APPROVAL_ACTION_TYPES}


@app.get("/api/approvals/stats")
async def get_approval_stats():
    with SessionLocal() as db:
        return approval_stats(db, Approval)


@app.post("/api/approvals")
async def post_approval(body: ApprovalCreateRequest):
    valid_ids = {a["id"] for a in AGENTS}
    if body.agent_id not in valid_ids:
        return {"error": "unknown agent_id"}
    if body.action_type not in APPROVAL_ACTION_TYPES:
        body.action_type = "generic.mutate"
    with SessionLocal() as db:
        item = await create_approval_request(
            db=db,
            ApprovalModel=Approval,
            agent_id=body.agent_id,
            action=body.action,
            reason=body.reason,
            action_type=body.action_type,
            target=body.target,
            payload=body.payload,
            priority=body.priority,
            terminal_log=terminal_log,
            mirror_supervisors=mirror_to_supervisors,
            publish=publish_event,
            add_feed=add_feed,
        )
    return {"ok": True, "approval": item}


@app.post("/api/approvals/{approval_id}/decide")
async def decide_approval(approval_id: str, body: ApprovalDecision):
    valid_supervisors = {"mo-commander", "bart-commander"}
    decided_by = body.decided_by if body.decided_by in valid_supervisors else "mo-commander"
    with SessionLocal() as db:
        item = await decide_approval_request(
            db=db,
            ApprovalModel=Approval,
            approval_id=approval_id,
            approved=body.approved,
            decided_by=decided_by,
            note=body.note,
            terminal_log=terminal_log,
            publish=publish_event,
            add_feed=add_feed,
        )
    if not item:
        return {"error": "not found"}
    return {"ok": True, "approval": item}


@app.post("/api/prompt")
async def post_prompt(body: PromptRequest):
    prompt_id = str(uuid.uuid4())[:8]
    valid_ids = {a["id"] for a in AGENTS}
    agent_id = body.agent_id if body.agent_id in valid_ids else route_agent(body.message)
    add_feed(agent_id, f"Prompt received: {body.message}", "info")
    asyncio.create_task(_run_agent_task_safe(agent_id, body.message, prompt_id))
    return {"prompt_id": prompt_id, "agent_id": agent_id, "status": "dispatched"}


@app.websocket("/api/ws/ops")
async def ws_ops(websocket: WebSocket):
    await websocket.accept()
    ws_clients.add(websocket)
    try:
        status = await collect_status()
        workload = await collect_workload()
        await websocket.send_text(json.dumps({"type": "status", "data": status}, default=str))
        await websocket.send_text(json.dumps({"type": "workload", "data": workload}, default=str))
        await websocket.send_text(json.dumps({
            "type": "terminal_history",
            "terminals": get_all_terminals(150),
        }, default=str))
        while True:
            await websocket.receive_text()
    except WebSocketDisconnect:
        pass
    finally:
        ws_clients.discard(websocket)