atc-agents/api/main.py

"""ATC Command Center API — FastAPI backend."""

from __future__ import annotations

import asyncio
import json
import os
import time
import uuid
from contextlib import asynccontextmanager
from datetime import datetime, timezone
from typing import Any

import httpx
import redis.asyncio as aioredis
from fastapi import Body, FastAPI, File, UploadFile, WebSocket, WebSocketDisconnect
from fastapi.middleware.cors import CORSMiddleware
import auth as cockpit_auth
from agent_terminal import (
    get_all_terminals,
    get_terminal_lines,
    init_terminals,
    make_logger,
    set_terminal_publisher,
    terminal_log,
)
from lab_context import collect_full_lab_context, format_context_for_agent
from presentation import build_presentation_payload, render_presentation_html
from presentation_upload import (
    clear_live_override,
    create_deck,
    delete_deck,
    get_asset_path,
    get_deck,
    get_live_override,
    list_decks,
    save_deck,
    save_image,
    save_live_override,
    save_upload,
)
from presentation_static import get_static_deck, list_static_decks
from storage_s3 import router as storage_s3_router
from hdfs_api import router as hdfs_router
from pipeline_ops import router as pipeline_router
from hadoop_analytics import router as hadoop_router
from elasticsearch_api import router as elasticsearch_router
from sql_console import router as sql_router
from agent_ops import router as agent_ops_router, agent_dml_loop, etl_agent_loop, custodian_offload_loop
from agent_activity import agent_activity_loop
from streaming_ops import connector_autoheal_loop
from cdc_consumer import router as cdc_router, cdc_consumer_loop
from movements import router as movements_router
from movements import MOVEMENT_BY_ID, trigger_and_watch
from dataflow import router as dataflow_router
from streaming_ops import router as streaming_router
from spark_workbench import router as spark_workbench_router
from pii_catalog import router as pii_router
from trino_federated import router as federated_router
from etl_offload import router as etl_offload_router
from lineage import router as lineage_router
from dq_monitor import router as dq_router
from observability import router as observability_router
from catalog_governance import router as governance_router
from ssh_terminal import ssh_session
from node_registry import NODE_IDS, NODE_AGENT, NODE_REGISTRY, is_node_id
from node_ops import build_node_detail, probe_node, run_node_probe_task
from approval_service import (
    APPROVAL_ACTION_TYPES,
    approval_stats,
    create_approval_request,
    decide_approval_request,
    detect_agent_proposed_action,
    detect_approval_intent,
    list_approvals,
)
from db import SessionLocal, db_health, init_database
from supervisor import mirror_terminal_line, mirror_to_supervisors

from workload import build_workload_payload
from gpu_config import (
    get_gpu_config,
    get_gpu_config_payload,
    get_gpu_urls,
    reset_gpu_config,
    save_gpu_config,
    test_gpu_target,
)

_workload_cache: dict[str, Any] = {"ts": 0.0, "data": None}
_presentation_cache: dict[str, Any] = {"ts": 0.0, "data": None}
WORKLOAD_CACHE_TTL = 30.0
PRESENTATION_CACHE_TTL = 45.0

from pydantic import BaseModel, Field
from sqlalchemy import Column, DateTime, String, Text, select
from sqlalchemy.orm import DeclarativeBase

REDIS_URL = os.getenv("REDIS_URL", "redis://redis:6379/0")
DOCKHAND_URL = os.getenv("DOCKHAND_URL", "http://10.0.21.45:8082")
DOCKHAND_API_TOKEN = os.getenv("DOCKHAND_API_TOKEN", "")


def _dockhand_headers() -> dict[str, str]:
    if DOCKHAND_API_TOKEN:
        return {"Authorization": f"Bearer {DOCKHAND_API_TOKEN}"}
    return {}

GPU_URL = os.getenv("GPU_URL", "http://10.0.10.106:9000")
GPU_UI_URL = os.getenv("GPU_UI_URL", GPU_URL)
LLM_URL = os.getenv("LLM_URL", "http://10.0.10.106:8001/v1")
LLM_MODEL = os.getenv("LLM_MODEL", "gpt-4o")
LLM_API_KEY = os.getenv("LLM_API_KEY", "sk-local")
LLM_TIMEOUT = float(os.getenv("LLM_TIMEOUT", "120"))
# Llama-3-70B GPTQ on V100 is capped at 4096; keep a hard safety budget.
LLM_MAX_MODEL_LEN = int(os.getenv("LLM_MAX_MODEL_LEN", "4096"))
# Conservative estimate: Llama tokenizers often use ~2.2–2.8 chars/token on English+lab text.
LLM_CHARS_PER_TOKEN = float(os.getenv("LLM_CHARS_PER_TOKEN", "2.4"))
LLM_CONTEXT_MARGIN = int(os.getenv("LLM_CONTEXT_MARGIN", "160"))
LLM_MAX_OUTPUT = int(os.getenv("LLM_MAX_OUTPUT", "256"))
LLM_MAX_CONTEXT_CHARS = int(os.getenv("LLM_MAX_CONTEXT_CHARS", "5500"))


def _estimate_tokens(text: str) -> int:
    # Slightly inflate so we never underestimate vs vLLM's tokenizer.
    return max(1, int(len(text) / LLM_CHARS_PER_TOKEN) + 32)


def _compact_lab_context(context: str) -> str:
    """Keep primary section + short per-domain summaries; drop verbose inventory lines."""
    lines = context.splitlines()
    out: list[str] = []
    in_full = False
    detail = 0
    max_detail = 4
    for line in lines:
        if line.startswith("=== FULL LAB"):
            in_full = True
            out.append(line)
            continue
        if line.startswith("=== PRIMARY"):
            in_full = False
            detail = 0
            out.append(line)
            continue
        if line.startswith("--- "):
            detail = 0
            out.append(line)
            continue
        # Drop long platform-capabilities essays if present — keep a one-liner marker
        if line.startswith("=== PLATFORM CAPABILITIES"):
            out.append(line)
            out.append("  (see Command Center UI for full feature list)")
            continue
        if out and out[-1].startswith("  (see Command Center"):
            if line.startswith("===") or line.startswith("--- ") or line.startswith("=== AGENTS") or line.startswith("=== DATA MASKING") or line.startswith("=== PII MASKING"):
                pass
            else:
                continue
        # Always keep masking / PII evidence sections in full (demo-critical)
        if line.startswith("=== DATA MASKING") or line.startswith("=== PII MASKING"):
            # flush remaining lines of this section without detail limits by marking
            out.append(line)
            continue
        # Limit bullet detail in general lab dump, but keep masking evidence intact
        keep_full = any(s in "\n".join(out[-5:]) for s in ("=== DATA MASKING", "=== PII MASKING"))
        if (line.startswith("  - ") or line.startswith("    - ")) and not keep_full:
            detail += 1
            if detail > max_detail:
                if detail == max_detail + 1:
                    out.append("    …")
                continue
        out.append(line)
    return "\n".join(out)


def _truncate_for_llm(context: str, max_chars: int) -> str:
    context = _compact_lab_context(context)
    if len(context) <= max_chars:
        return context
    # Prefer keeping PRIMARY section; cut FULL LAB first
    primary_end = context.find("=== FULL LAB")
    if primary_end > 200:
        head = context[:primary_end].rstrip()
        tail_budget = max(400, max_chars - len(head) - 80)
        tail = context[primary_end: primary_end + tail_budget]
        trimmed = head + "\n" + tail
    else:
        trimmed = context[:max_chars]
    if len(trimmed) > max_chars:
        trimmed = trimmed[: max_chars - 60].rsplit("\n", 1)[0]
    if len(context) > len(trimmed):
        trimmed += f"\n\n[… truncated for {LLM_MAX_MODEL_LEN}-token model window …]"
    return trimmed


def _fit_llm_payload(system_rules: str, context: str, user_message: str) -> tuple[str, int]:
    """Fit prompt+completion into the served model length with a safety margin."""
    user_tok = _estimate_tokens(user_message)
    rules_tok = _estimate_tokens(system_rules)
    budget = LLM_MAX_MODEL_LEN - LLM_CONTEXT_MARGIN
    max_out = min(LLM_MAX_OUTPUT, 256)

    # Absolute char cap first (independent of estimate errors)
    context = _truncate_for_llm(context, LLM_MAX_CONTEXT_CHARS)

    for _ in range(6):
        ctx_budget_tok = budget - user_tok - rules_tok - max_out
        if ctx_budget_tok < 200:
            max_out = max(64, max_out // 2)
            continue
        ctx_max_chars = max(600, int(ctx_budget_tok * LLM_CHARS_PER_TOKEN * 0.85))
        fitted = _truncate_for_llm(context, ctx_max_chars)
        total = rules_tok + _estimate_tokens(fitted) + user_tok + max_out
        if total <= budget:
            return fitted, max_out
        # Still too big — shrink context harder, then output
        context = fitted
        LLM_MAX = max(800, int(len(fitted) * 0.7))
        context = _truncate_for_llm(context, LLM_MAX)
        max_out = max(64, max_out - 32)

    return _truncate_for_llm(context, 800), 64


AGENTS = [
    {
        "id": "etl-guardian",
        "name": "ETL Guardian",
        "color": "#00f0ff",
        "zone": "etl",
        "role": "Airflow, Kafka, Debezium, S3 pipeline",
        "icon": "⚡",
        "motto": "Pipelines never sleep",
        "capabilities": ["Airflow", "Kafka", "Debezium", "S3", "Connectors"],
        "suggested_prompts": [
            "How is Debezium doing?",
            "Are all Airflow DAGs healthy?",
            "Kafka connector status?",
        ],
    },
    {
        "id": "lakehouse-ops",
        "name": "Lakehouse Ops",
        "color": "#ff00aa",
        "zone": "lakehouse",
        "role": "Spark, Trino, Iceberg",
        "icon": "🏔️",
        "motto": "Query the lake, trust the table",
        "capabilities": ["Spark", "Trino", "Iceberg", "Delta", "SQL"],
        "suggested_prompts": [
            "Lakehouse stack status?",
            "Is Trino reachable?",
            "How many lakehouse containers are running?",
        ],
    },
    {
        "id": "data-custodian",
        "name": "Data Custodian",
        "color": "#ffaa00",
        "zone": "db",
        "role": "PostgreSQL, MySQL, Mongo, Cassandra, Neo4j",
        "icon": "🛡️",
        "motto": "Guardian of every row",
        "capabilities": ["PostgreSQL", "MySQL", "MongoDB", "Cassandra", "Neo4j"],
        "suggested_prompts": [
            "How much data is in the databases?",
            "What is in PostgreSQL sales_orders?",
            "MongoDB supplychain overview",
        ],
    },
    {
        "id": "hadoop-ranger",
        "name": "Hadoop Ranger",
        "color": "#39ff14",
        "zone": "hadoop",
        "role": "HDFS, YARN cluster",
        "icon": "🌲",
        "motto": "Patrol the data forest",
        "capabilities": ["HDFS", "YARN", "NameNode", "DataNodes"],
        "suggested_prompts": [
            "Is HDFS NameNode up?",
            "Hadoop cluster status?",
            "YARN nodes healthy?",
        ],
    },
    {
        "id": "infra-sentinel",
        "name": "Infra Sentinel",
        "color": "#9b72cf",
        "zone": "docker",
        "role": "Docker, Proxmox, GPU, monitoring",
        "icon": "👁️",
        "motto": "See everything, miss nothing",
        "capabilities": ["Docker", "Proxmox", "GPU", "vLLM", "Monitoring"],
        "suggested_prompts": [
            "GPU status?",
            "Which LLM model is running?",
            "Docker container overview",
        ],
    },
    {
        "id": "mo-commander",
        "name": "Mo · Command",
        "color": "#4c9aed",
        "zone": "command",
        "role": "Supervisor — full event intel, ingress, approvals",
        "icon": "🎯",
        "motto": "Nothing happens without Mo knowing",
        "supervisor": True,
        "person": "mo",
        "capabilities": ["Events", "Ingress", "Approvals", "Agent dispatch", "Network IN"],
        "suggested_prompts": [
            "What happened today?",
            "What events came in?",
            "Pipeline status overview",
        ],
    },
    {
        "id": "bart-commander",
        "name": "Bart · Ops",
        "color": "#3fb950",
        "zone": "command",
        "role": "Supervisor — egress, MCP comms, network OUT",
        "icon": "📡",
        "motto": "All traffic flows through Bart",
        "supervisor": True,
        "person": "bart",
        "capabilities": ["Egress", "MCP routing", "Network OUT", "GPU inference", "S3 writes"],
        "suggested_prompts": [
            "What is leaving the cluster?",
            "MCP agent communication status?",
            "Network egress overview",
        ],
    },
    {
        "id": "network-watcher",
        "name": "Network Watcher",
        "color": "#58a6ff",
        "zone": "network",
        "role": "VLAN 20/21 traffic, data in & out paths",
        "icon": "🌐",
        "motto": "Every packet tells a story",
        "capabilities": ["VLAN 20", "VLAN 21", "Ingress", "Egress", "Firewall paths"],
        "suggested_prompts": [
            "Data ingress status?",
            "What leaves the cluster?",
            "Network path to S3?",
        ],
    },
    {
        "id": "mcp-coordinator",
        "name": "MCP Coordinator",
        "color": "#f778ba",
        "zone": "mcp",
        "role": "MCP hub — routes all agent tool calls & comms",
        "icon": "🔀",
        "motto": "Route once, deliver everywhere",
        "capabilities": ["MCP servers", "Tool routing", "Agent relay", "WebSocket bus"],
        "suggested_prompts": [
            "Which MCP agents are active?",
            "MCP hub route status?",
            "Agent communication overview",
        ],
    },
]

ZONES = [
    {"id": "docker", "label": "DOCKER RACK", "x": 8, "color": "#b366ff"},
    {"id": "db", "label": "DB VAULT", "x": 28, "color": "#ffaa00"},
    {"id": "lakehouse", "label": "LAKEHOUSE HUB", "x": 50, "color": "#ff00aa"},
    {"id": "hadoop", "label": "HADOOP CLUSTER", "x": 72, "color": "#39ff14"},
    {"id": "etl", "label": "ETL PIPE", "x": 92, "color": "#00f0ff"},
]

INTENT_KEYWORDS: dict[str, list[str]] = {
    "data-custodian": [
        "database", "postgres", "postgresql", "mysql", "mongo", "mongodb", "cassandra", "neo4j", "sql", "db ",
        "pii", "mask", "masked", "masking", "email", "e-mail", "phone", "iban", "address", "customer",
        "employee", "gdpr", "privacy", "sensitive", "personal", "name", "ssn", "national_id",
    ],
    "lakehouse-ops": ["trino", "spark", "lakehouse", "iceberg", "query", "table"],
    "hadoop-ranger": [
        "hadoop", "hdfs", "yarn", "datanode", "namenode", "replicatie", "replication",
        "rf factor", "opslag", "bestanden", "blocks", "cluster opslag", "data op",
    ],
    "infra-sentinel": ["docker", "container", "vm", "proxmox", "infra", "grafana", "gpu", "vllm", "llm", "nvidia", "inference", "model"],
    "etl-guardian": ["airflow", "dag", "debezium", "kafka", "connector", "etl", "pipeline", "s3"],
    "network-watcher": ["network", "vlan", "ingress", "egress", "traffic", "packet", "firewall", "route"],
    "mcp-coordinator": ["mcp", "tool", "router", "relay", "websocket", "hub"],
    "mo-commander": ["mo", "supervisor", "events", "overzicht", "alles", "gebeurd"],
    "bart-commander": ["bart", "egress", "uitgaand", "communicatie", "mcp comm"],
}


class Base(DeclarativeBase):
    pass


class FeedEntry(Base):
    __tablename__ = "feed"

    id = Column(String, primary_key=True)
    ts = Column(DateTime, default=lambda: datetime.now(timezone.utc))
    agent_id = Column(String)
    level = Column(String, default="info")
    message = Column(Text)


class Approval(Base):
    __tablename__ = "approvals"

    id = Column(String, primary_key=True)
    ts = Column(DateTime, default=lambda: datetime.now(timezone.utc))
    agent_id = Column(String)
    action = Column(Text)
    reason = Column(Text)
    status = Column(String, default="pending")
    action_type = Column(String, default="generic.mutate")
    target = Column(Text, default="")
    payload = Column(Text, default="{}")
    decided_by = Column(String, nullable=True)
    decide_note = Column(Text, nullable=True)
    decided_at = Column(DateTime, nullable=True)
    priority = Column(String, default="normal")


_db_info = init_database(Base)

redis_client: aioredis.Redis | None = None
ws_clients: set[WebSocket] = set()


class PromptRequest(BaseModel):
    message: str = Field(min_length=1, max_length=2000)
    agent_id: str | None = None


class NodeAskRequest(BaseModel):
    message: str = Field(min_length=1, max_length=2000)


class ApprovalCreateRequest(BaseModel):
    agent_id: str = Field(min_length=1, max_length=64)
    action: str = Field(min_length=1, max_length=2000)
    reason: str = Field(min_length=1, max_length=2000)
    action_type: str = "generic.mutate"
    target: str = ""
    payload: dict[str, Any] | None = None
    priority: str = "normal"


class ApprovalDecision(BaseModel):
    approved: bool
    decided_by: str = "mo-commander"
    note: str = ""


def route_agent(message: str) -> str:
    lower = message.lower()
    # PII / privacy questions always go to Data Custodian (masking demo path)
    pii_words = (
        "pii", "mask", "masked", "masking", "email", "e-mail", "mail adres", "mail address",
        "phone", "telefoon", "iban", "address", "adres", "customer name", "employee",
        "gdpr", "privacy", "sensitive", "personal", "national_id", "ssn", "gevoelig",
    )
    if any(w in lower for w in pii_words):
        return "data-custodian"
    # Storage/data questions default to Hadoop unless clearly about databases
    if any(w in lower for w in ("opslag", "gb", "replicatie", "replication", "hdfs", "hadoop", "datanode", "namenode")):
        if not any(w in lower for w in ("postgres", "mysql", "mongo", "database", "sql", "neo4j", "cassandra", "pii", "email")):
            return "hadoop-ranger"
    if any(w in lower for w in ("gpu", "vllm", "llm", "nvidia", "inference", "vram", "model")):
        return "infra-sentinel"
    if any(w in lower for w in ("kafka", "airflow", "debezium", "connector", "etl", "pipeline", "dag")):
        return "etl-guardian"
    if any(w in lower for w in ("trino", "spark", "iceberg", "lakehouse")):
        return "lakehouse-ops"
    scores = {aid: sum(1 for kw in kws if kw in lower) for aid, kws in INTENT_KEYWORDS.items()}
    best = max(scores, key=scores.get)
    if scores[best] == 0:
        return "infra-sentinel"
    return best



def _is_pii_question(message: str) -> bool:
    lower = message.lower()
    return any(w in lower for w in (
        "pii", "mask", "masked", "masking", "unmask", "visible", "email", "e-mail", "phone",
        "iban", "address", "adres", "customer", "employee", "privacy", "gdpr", "sensitive",
        "personal", "gevoelig", "name", "telefoon", "mail", "data flow", "national_id",
        "ssn", "bsn", "geboorte", "birth",
    ))


async def _pii_evidence_block(message: str, log: Any | None = None) -> str:
    """Live policy + samples synced with Data Flow masking toggles."""
    try:
        from pii_catalog import build_policy_evidence
        return build_policy_evidence()
    except Exception as exc:
        return f"=== PII MASKING EVIDENCE ===\n(unavailable: {exc})"


async def gather_agent_context(
    agent_id: str,
    status: dict[str, Any],
    log: Any | None = None,
    message: str | None = None,
) -> str:
    """Full lab snapshot for vLLM — all domains, agent's primary domain highlighted."""
    snapshot = await collect_full_lab_context(gpu_data=status.get("gpu"), log=log, include_inventory=False)
    snapshot["domains_summary"] = status.get("domains", {})
    ctx = format_context_for_agent(agent_id, snapshot)
    agent_lines = ["", "=== AGENTS & SUPERVISORS ==="]
    for a in AGENTS:
        sup = " [supervisor]" if a.get("supervisor") else ""
        agent_lines.append(f"  - {a['name']} ({a['id']}){sup}: {a['role']}")
    ctx = ctx + "\n".join(agent_lines)
    try:
        from platform_context import build_masking_section
        # Fresh policy so chat mirrors Data Flow toggles (skip huge business catalog).
        ctx = ctx + "\n\n" + build_masking_section(fresh=True)
    except Exception:
        try:
            from platform_context import build_llm_addendum
            ctx = ctx + "\n\n" + build_llm_addendum()
        except Exception:
            pass
    if message and _is_pii_question(message):
        try:
            evidence = await _pii_evidence_block(message, log=log)
            ctx = ctx + "\n\n" + evidence
            if log:
                await log("ok", "fetch", "▸ PII masking evidence attached (synced with Data Flow)")
        except Exception as exc:
            if log:
                await log("warn", "fetch", f"▸ PII evidence skipped: {exc}")
    if log:
        await log("info", "fetch", f"▸ Context assembled: {len(ctx)} chars for LLM")
    return ctx


async def ask_llm(
    agent_id: str,
    message: str,
    context: str,
    log: Any | None = None,
) -> str | None:
    agent = next(a for a in AGENTS if a["id"] == agent_id)
    # Deterministic PII path: always mirror Data Flow masked vs visible toggles.
    if _is_pii_question(message):
        try:
            from pii_catalog import format_pii_chat_answer
            answer = format_pii_chat_answer(message)
            if log:
                await log("ok", "pii", "▸ Returning Data Flow–synced masking answer (masked + visible)")
            return answer
        except Exception as exc:
            if log:
                await log("warn", "pii", f"▸ PII answer builder failed: {exc}")
    rules = f"""You are {agent['name']} ({agent['role']}) in the Dell ATC data lab.
Answer in English, briefly (max ~8 sentences). Use ONLY the live data below — never invent hosts/ports/numbers.
If data is missing or DOWN, say so.

PII / masking rules (critical — synced with Data Flow tab):
- MASKED columns: NEVER reveal raw values; quote the token 🔒 MASKED when present.
- VISIBLE columns (operator opted out in Data Flow): you MAY report the real sample values and say they are visible by policy.
- Never invent emails, phones, names, IBANs, or addresses that are not in the live samples.
- If asked what is masked vs visible, list columns from the DATA MASKING POLICY / PII EVIDENCE sections.

--- LIVE LAB DATA ---"""
    fitted_ctx, max_tokens = _fit_llm_payload(rules, context, message)
    system = rules + "\n" + fitted_ctx
    urls = get_gpu_urls()
    llm_url = urls["llm_url"]
    if log:
        est = _estimate_tokens(system) + _estimate_tokens(message)
        await log("info", "llm", f"▸ Querying vLLM model={LLM_MODEL} @ {urls['host']} (~{est}+{max_tokens} tok)")
        if len(context) > len(fitted_ctx):
            await log("warn", "llm", f"  context trimmed {len(context)} → {len(fitted_ctx)} chars")
        await log("cmd", "llm", f"$ POST {llm_url.rstrip('/')}/chat/completions")
        await log("info", "llm", f"  user: {message[:160]}{'…' if len(message) > 160 else ''}")
    try:
        async with httpx.AsyncClient(timeout=LLM_TIMEOUT) as client:
            t0 = time.monotonic()
            r = await client.post(
                f"{llm_url.rstrip('/')}/chat/completions",
                headers={
                    "Authorization": f"Bearer {LLM_API_KEY}",
                    "Content-Type": "application/json",
                },
                json={
                    "model": LLM_MODEL,
                    "messages": [
                        {"role": "system", "content": system},
                        {"role": "user", "content": message},
                    ],
                    "max_tokens": max_tokens,
                    "temperature": 0.25,
                },
            )
            r.raise_for_status()
            content = r.json()["choices"][0]["message"]["content"].strip()
            ms = int((time.monotonic() - t0) * 1000)
            if content and content.strip("!"):
                if log:
                    await log("ok", "llm", f"← vLLM response {len(content)} chars ({ms}ms)")
                    preview = content.replace("\n", " ")[:180]
                    await log("info", "llm", f"  » {preview}{'…' if len(content) > 180 else ''}")
                return content
            if log:
                await log("warn", "llm", f"← Empty or invalid LLM output ({ms}ms)")
    except httpx.HTTPStatusError as exc:
        detail = exc.response.text[:200] if exc.response is not None else str(exc)
        if log:
            await log("err", "llm", f"✗ vLLM HTTP {exc.response.status_code}: {detail}")
        # One hard retry with a minimal context if we blew the window.
        if exc.response is not None and exc.response.status_code == 400 and "maximum context length" in detail:
            tiny = _truncate_for_llm(context, 1200)
            system2 = rules + "\n" + tiny
            max2 = 128
            if log:
                await log("warn", "llm", f"  retry with tiny context ({len(tiny)} chars, max_tokens={max2})")
            try:
                async with httpx.AsyncClient(timeout=LLM_TIMEOUT) as client:
                    r2 = await client.post(
                        f"{llm_url.rstrip('/')}/chat/completions",
                        headers={
                            "Authorization": f"Bearer {LLM_API_KEY}",
                            "Content-Type": "application/json",
                        },
                        json={
                            "model": LLM_MODEL,
                            "messages": [
                                {"role": "system", "content": system2},
                                {"role": "user", "content": message},
                            ],
                            "max_tokens": max2,
                            "temperature": 0.25,
                        },
                    )
                    r2.raise_for_status()
                    content2 = r2.json()["choices"][0]["message"]["content"].strip()
                    if content2:
                        if log:
                            await log("ok", "llm", f"← vLLM retry OK {len(content2)} chars")
                        return content2
            except Exception as exc2:
                if log:
                    await log("err", "llm", f"✗ vLLM retry failed: {exc2}")
    except Exception as exc:
        if log:
            await log("err", "llm", f"✗ vLLM error: {exc}")
    return None


def fallback_answer(agent_id: str, context: str, user_message: str = "") -> str:
    agent_name = next(a["name"] for a in AGENTS if a["id"] == agent_id)
    if user_message and _is_pii_question(user_message):
        try:
            from pii_catalog import format_pii_chat_answer
            return f"**{agent_name}**\n\n" + format_pii_chat_answer(user_message)
        except Exception:
            marker = "=== PII MASKING EVIDENCE"
            if marker in context:
                return (
                    f"**{agent_name}** — masking policy (synced with Data Flow):\n\n"
                    + context[context.index(marker):].strip()
                )
    preview_lines: list[str] = []
    for line in context.splitlines():
        if line.startswith(("=== PRIMARY", "Health summary", "ATC Lab", "--- ")):
            preview_lines.append(line)
        if len(preview_lines) >= 14:
            break
    hint = "\n".join(preview_lines) if preview_lines else "Lab snapshot collected; LLM unavailable."
    q = f"\n\nYour question: _{user_message[:200]}_" if user_message else ""
    return (
        f"**{agent_name}** — I could not get a reply from the GPU LLM "
        f"(context window or vLLM error).{q}\n\n"
        "Try a short, specific question "
        "(e.g. *How many GPUs are online?* or *Is Kafka healthy?*).\n\n"
        f"Quick snapshot:\n{hint}"
    )


async def publish_event(event: dict[str, Any]) -> None:
    payload = json.dumps(event, default=str)
    if redis_client:
        await redis_client.publish("ops", payload)
    dead = []
    for ws in ws_clients:
        try:
            await ws.send_text(payload)
        except Exception:
            dead.append(ws)
    for ws in dead:
        ws_clients.discard(ws)

    et = event.get("type")
    if et == "feed":
        entry = event.get("entry") or {}
        await mirror_to_supervisors(
            entry.get("agent_id", "?"),
            entry.get("message", ""),
            level=entry.get("level", "info"),
        )
    elif et == "terminal":
        await mirror_terminal_line(event.get("line") or {})
    elif et in ("agent_dispatch", "agent_fetch", "agent_return"):
        aid = event.get("agent_id", "?")
        zone = event.get("zone", "")
        await mirror_to_supervisors(aid, f"{et} → zone {zone}", level="info", phase="dispatch")


def add_feed(agent_id: str, message: str, level: str = "info") -> dict:
    entry_id = str(uuid.uuid4())[:8]
    with SessionLocal() as db:
        row = FeedEntry(id=entry_id, agent_id=agent_id, message=message, level=level)
        db.add(row)
        db.commit()
    return {
        "id": entry_id,
        "ts": datetime.now(timezone.utc).isoformat(),
        "agent_id": agent_id,
        "message": message,
        "level": level,
    }


async def dockhand_env_containers(env_id: int) -> list[dict]:
    try:
        async with httpx.AsyncClient(timeout=8.0) as client:
            r = await client.get(
                f"{DOCKHAND_URL}/api/containers",
                params={"env": env_id},
                headers=_dockhand_headers(),
            )
            r.raise_for_status()
            return r.json()
    except Exception:
        return []


async def probe_url(url: str) -> bool:
    try:
        async with httpx.AsyncClient(timeout=4.0, verify=False) as client:
            r = await client.get(url)
            return r.status_code < 500
    except Exception:
        return False


async def collect_gpu() -> dict[str, Any]:
    urls = get_gpu_urls()
    cfg = get_gpu_config()
    gpu_url = urls["gpu_url"]
    host = urls["host"]
    base = {
        "ok": False,
        "host": host,
        "ip": host,
        "ui_url": urls["gpu_ui_url"],
        "config_source": cfg.get("source", "env"),
        "preset_id": cfg.get("preset_id"),
        "config_label": cfg.get("label"),
    }

    try:
        async with httpx.AsyncClient(timeout=6.0) as client:
            metrics_r, model_r, integration_r = await asyncio.gather(
                client.get(f"{gpu_url}/api/gpu/metrics"),
                client.get(f"{gpu_url}/api/active-model"),
                client.get(f"{gpu_url}/api/integration"),
                return_exceptions=True,
            )

            gpus: list[dict[str, Any]] = []
            if isinstance(metrics_r, httpx.Response) and metrics_r.status_code == 200:
                current = metrics_r.json().get("current", {})
                gpus = [
                    {
                        "index": g["index"],
                        "name": g["name"],
                        "util_gpu": g.get("util_gpu", 0),
                        "memory_used_mib": g.get("memory_used_mib", 0),
                        "memory_total_mib": g.get("memory_total_mib", 0),
                        "temperature_c": g.get("temperature_c", 0),
                        "power_w": g.get("power_w", 0),
                    }
                    for g in current.get("gpus", [])
                ]

            active_model = None
            inference_active = False
            vllm_url = None

            if isinstance(model_r, httpx.Response) and model_r.status_code == 200:
                model_data = model_r.json()
                active_model = model_data.get("name")
                inference_active = bool(model_data.get("inference_active"))
                vllm_url = model_data.get("base_url")

            if isinstance(integration_r, httpx.Response) and integration_r.status_code == 200:
                integ = integration_r.json()
                if not active_model:
                    active_model = integ.get("active_name")
                if not inference_active:
                    inference_active = bool(integ.get("inference_active"))
                if not vllm_url:
                    vllm_url = integ.get("recommended_base_url")

            return {
                **base,
                "ok": len(gpus) > 0 or inference_active,
                "inference_active": inference_active,
                "active_model": active_model,
                "vllm_url": vllm_url,
                "gpu_count": len(gpus),
                "gpus": gpus,
            }
    except Exception as exc:
        return {**base, "error": str(exc)}


async def collect_status() -> dict[str, Any]:
    db_containers = await dockhand_env_containers(5)
    db_running = sum(1 for c in db_containers if c.get("state") == "running")
    db_total = len(db_containers) or 6

    lake_containers = await dockhand_env_containers(9)
    lake_running = sum(1 for c in lake_containers if c.get("state") == "running")
    lake_total = len(lake_containers) or 6

    docker_containers = await dockhand_env_containers(1)
    docker_running = sum(1 for c in docker_containers if c.get("state") == "running")

    hdfs_ok = await probe_url("http://10.0.21.61:9870")
    kafka_ok = await probe_url("http://10.0.21.36:9000")
    airflow_ok = await probe_url("http://10.0.21.55:8080")

    def level(running: int, total: int) -> str:
        if total == 0:
            return "unknown"
        ratio = running / total
        if ratio >= 0.9:
            return "ok"
        if ratio >= 0.5:
            return "warn"
        return "down"

    gpu = await collect_gpu()
    gpu_level = "ok" if gpu.get("ok") and gpu.get("inference_active") else ("warn" if gpu.get("ok") else "down")
    gpu_label = gpu.get("active_model") or (f"{gpu.get('gpu_count', 0)} GPUs" if gpu.get("ok") else "offline")

    return {
        "ts": datetime.now(timezone.utc).isoformat(),
        "domains": {
            "docker": {"level": "ok" if docker_running >= 5 else "warn", "label": f"{docker_running} containers", "running": docker_running},
            "databases": {"level": level(db_running, db_total), "label": f"{db_running}/{db_total} up", "running": db_running, "total": db_total},
            "lakehouse": {"level": level(lake_running, lake_total), "label": f"{lake_running}/{lake_total} up", "running": lake_running, "total": lake_total},
            "hadoop": {"level": "ok" if hdfs_ok else "warn", "label": "NN up" if hdfs_ok else "NN check"},
            "etl": {"level": "ok" if kafka_ok and airflow_ok else "warn", "label": "Kafka+Airflow"},
            "gpu": {"level": gpu_level, "label": gpu_label},
        },
        "gpu": gpu,
        "kafka_ok": kafka_ok,
        "airflow_ok": airflow_ok,
        "hdfs_ok": hdfs_ok,
    }




async def _run_agent_task_safe(agent_id: str, message: str, prompt_id: str) -> None:
  try:
    await run_agent_task(agent_id, message, prompt_id)
  except Exception as exc:
    agent_name = next((a["name"] for a in AGENTS if a["id"] == agent_id), agent_id)
    err = f"Sorry — {agent_name} could not complete your request: {exc}"
    await terminal_log(agent_id, f"[{prompt_id}] ✗ Error: {exc}", level="err", phase="error", prompt_id=prompt_id)
    feed = add_feed(agent_id, f"{agent_name} failed: {str(exc)[:80]}", "err")
    await publish_event({"type": "feed", "entry": feed})
    await publish_event({
      "type": "prompt_result",
      "prompt_id": prompt_id,
      "agent_id": agent_id,
      "answer": err,
    })


async def run_agent_task(agent_id: str, message: str, prompt_id: str) -> str:
    zone = next(a["zone"] for a in AGENTS if a["id"] == agent_id)
    agent_name = next(a["name"] for a in AGENTS if a["id"] == agent_id)
    log = make_logger(agent_id, prompt_id)
    approval_created = False

    intent = detect_approval_intent(message)
    if intent:
        with SessionLocal() as db:
            await create_approval_request(
                db=db,
                ApprovalModel=Approval,
                agent_id=agent_id,
                action=intent["action"],
                reason=intent["reason"],
                action_type=intent["action_type"],
                terminal_log=terminal_log,
                mirror_supervisors=mirror_to_supervisors,
                publish=publish_event,
                add_feed=add_feed,
            )
        approval_created = True
        await terminal_log(
            agent_id,
            f"[{prompt_id}] Mutating request detected — approval queued for Mo & Bart",
            level="warn",
            phase="approval",
            prompt_id=prompt_id,
        )

    await terminal_log(
        agent_id,
        f"[{prompt_id}] ▶ Mission accepted: {message}",
        level="info",
        phase="dispatch",
        prompt_id=prompt_id,
    )
    await publish_event({"type": "agent_dispatch", "agent_id": agent_id, "zone": zone, "prompt_id": prompt_id})
    await asyncio.sleep(0.4)
    await terminal_log(agent_id, f"[{prompt_id}] Walking to zone: {zone}", level="info", phase="dispatch", prompt_id=prompt_id)
    await publish_event({"type": "agent_fetch", "agent_id": agent_id, "zone": zone, "prompt_id": prompt_id})

    await log("info", "fetch", f"[{prompt_id}] Collecting live lab metrics…")
    status = await collect_status()
    context = await gather_agent_context(agent_id, status, log=log, message=message)

    answer = await ask_llm(agent_id, message, context, log=log)
    if not answer:
        await log("warn", "llm", "LLM fallback — returning raw context")
        answer = fallback_answer(agent_id, context, message)

    if not approval_created:
        proposed = detect_agent_proposed_action(answer, message)
        if proposed:
            with SessionLocal() as db:
                await create_approval_request(
                    db=db,
                    ApprovalModel=Approval,
                    agent_id=agent_id,
                    action=proposed["action"],
                    reason=proposed["reason"],
                    action_type=proposed["action_type"],
                    target=proposed.get("target", ""),
                    terminal_log=terminal_log,
                    mirror_supervisors=mirror_to_supervisors,
                    publish=publish_event,
                    add_feed=add_feed,
                )
            approval_created = True
            answer = (
                f"{answer}\n\n⏸ **Approval required** — this action is in the Approval Inbox. "
                f"Mo & Bart have been notified and must approve before we execute."
            )

    await asyncio.sleep(0.3)
    await terminal_log(agent_id, f"[{prompt_id}] ✓ Mission complete", level="ok", phase="done", prompt_id=prompt_id)
    await publish_event({"type": "agent_return", "agent_id": agent_id, "zone": zone, "prompt_id": prompt_id})
    feed = add_feed(agent_id, f"{agent_name} completed a response (see Comms)", "info")
    await publish_event({"type": "feed", "entry": feed})
    await publish_event({"type": "prompt_result", "prompt_id": prompt_id, "agent_id": agent_id, "answer": answer})
    return answer


async def heartbeat_loop() -> None:
    while True:
        try:
            status = await collect_status()
            workload = await collect_workload()
            await publish_event({"type": "status", "data": status})
            await publish_event({"type": "workload", "data": workload})
            for domain, info in status["domains"].items():
                if info["level"] == "down":
                    agent = "data-custodian" if domain == "databases" else "infra-sentinel"
                    feed = add_feed(agent, f"Alert: {domain} is DOWN ({info['label']})", "warn")
                    await publish_event({"type": "feed", "entry": feed})
        except Exception as exc:
            await publish_event({"type": "error", "message": str(exc)})
        await asyncio.sleep(60)


@asynccontextmanager
async def lifespan(app: FastAPI):
    global redis_client
    redis_client = aioredis.from_url(REDIS_URL, decode_responses=True)
    set_terminal_publisher(publish_event)
    init_terminals([a["id"] for a in AGENTS] + NODE_IDS)
    for a in AGENTS:
        await terminal_log(a["id"], f"{a['name']} terminal online — awaiting missions", level="info", phase="boot")
    for nid in NODE_IDS:
        if nid not in NODE_REGISTRY:
            continue
        meta = NODE_REGISTRY[nid]
        await terminal_log(nid, f"{meta['label']} shell ready — click node to connect", level="info", phase="boot")
    task = asyncio.create_task(heartbeat_loop())
    dml_task = asyncio.create_task(agent_dml_loop())
    cdc_task = asyncio.create_task(cdc_consumer_loop())
    etl_task = asyncio.create_task(etl_agent_loop())
    cust_task = asyncio.create_task(custodian_offload_loop())
    act_task = asyncio.create_task(agent_activity_loop())
    heal_task = asyncio.create_task(connector_autoheal_loop())
    add_feed("infra-sentinel", "ATC Command Center API online", "info")
    yield
    task.cancel()
    dml_task.cancel()
    cdc_task.cancel()
    etl_task.cancel()
    cust_task.cancel()
    act_task.cancel()
    heal_task.cancel()
    if redis_client:
        await redis_client.close()


app = FastAPI(title="ATC Command Center API", lifespan=lifespan)
app.include_router(storage_s3_router)
app.include_router(hdfs_router)
app.include_router(pipeline_router)
app.include_router(hadoop_router)
app.include_router(elasticsearch_router)
app.include_router(sql_router)
app.include_router(agent_ops_router)
app.include_router(cdc_router)
app.include_router(movements_router)
app.include_router(dataflow_router)
app.include_router(streaming_router)
app.include_router(spark_workbench_router)
app.include_router(pii_router)
app.include_router(federated_router)
app.include_router(etl_offload_router)
app.include_router(lineage_router)
app.include_router(dq_router)
app.include_router(observability_router)
app.include_router(governance_router)
app.add_middleware(
    CORSMiddleware,
    allow_origins=["*"],
    allow_credentials=True,
    allow_methods=["*"],
    allow_headers=["*"],
)

# Authentik OIDC session + API guard
cockpit_auth.init_auth_middleware(app)
cockpit_auth.setup_auth(app)


@app.get("/api/health")
async def health():
    llm_ok = False
    try:
        async with httpx.AsyncClient(timeout=4.0) as client:
            r = await client.get(f"{LLM_URL.rstrip('/')}/models", headers={"Authorization": f"Bearer {LLM_API_KEY}"})
            llm_ok = r.status_code == 200
    except Exception:
        pass
    return {
        "ok": True,
        "ts": datetime.now(timezone.utc).isoformat(),
        "llm_url": LLM_URL,
        "llm_ok": llm_ok,
        "llm_model": LLM_MODEL,
        "database": db_health(),
        "db_init": _db_info,
    }


async def collect_workload(*, fast: bool = True, use_cache: bool = True) -> dict[str, Any]:
    import time as _time
    now = _time.time()
    if use_cache and _workload_cache.get("data") and now - float(_workload_cache.get("ts") or 0) < WORKLOAD_CACHE_TTL:
        return _workload_cache["data"]
    gpu = await collect_gpu()
    snap = await collect_full_lab_context(gpu_data=gpu, include_inventory=not fast)
    payload = build_workload_payload(snap)
    _workload_cache["ts"] = now
    _workload_cache["data"] = payload
    return payload




async def get_presentation_data(*, use_cache: bool = True) -> dict[str, Any]:
    import time as _time
    now = _time.time()
    if use_cache and _presentation_cache.get("data") and now - float(_presentation_cache.get("ts") or 0) < PRESENTATION_CACHE_TTL:
        return _presentation_cache["data"]
    gpu = await collect_gpu()
    snap = await collect_full_lab_context(gpu_data=gpu, include_inventory=False)
    data = build_presentation_payload(snap)
    override = get_live_override()
    if override and override.get("slides"):
        data["title"] = override.get("title") or data.get("title")
        data["subtitle"] = override.get("subtitle") or data.get("subtitle", "")
        data["slides"] = override["slides"]
        data["slide_count"] = len(override["slides"])
        data["edited"] = True
        data["override_ts"] = override.get("ts")
    else:
        data["edited"] = False
    data["source"] = "live"
    data["id"] = "live"
    _presentation_cache["ts"] = now
    _presentation_cache["data"] = data
    return data


@app.get("/api/presentation")
async def get_presentation():
    return await get_presentation_data()


@app.get("/api/presentation/html")
async def get_presentation_html():
    from fastapi.responses import HTMLResponse
    payload = await get_presentation_data()
    return HTMLResponse(render_presentation_html(payload))


@app.get("/api/presentation/decks")
async def get_presentation_decks():
    return {"live": True, "builtin": list_static_decks(), "uploaded": list_decks()}


@app.get("/api/presentation/decks/{deck_id}")
async def get_presentation_deck(deck_id: str):
    if deck_id == "live":
        return await get_presentation_data()
    deck = get_static_deck(deck_id) or get_deck(deck_id)
    if not deck:
        return {"error": "deck not found"}
    return deck


@app.get("/api/presentation/decks/{deck_id}/html")
async def get_presentation_deck_html(deck_id: str):
    from fastapi.responses import HTMLResponse
    if deck_id == "live":
        payload = await get_presentation_data()
    else:
        payload = get_static_deck(deck_id) or get_deck(deck_id)
        if not payload:
            return HTMLResponse("<h1>Deck not found</h1>", status_code=404)
    return HTMLResponse(render_presentation_html(payload))


@app.post("/api/presentation/upload")
async def upload_presentation(file: UploadFile = File(...)):
    content = await file.read()
    if len(content) > 50 * 1024 * 1024:
        return {"error": "file too large (max 50MB)"}
    deck = await save_upload(file.filename or "upload.pptx", content)
    return {"ok": True, "deck": deck}


@app.post("/api/presentation/decks/new")
async def create_presentation_deck(body: dict[str, Any] | None = Body(default=None)):
    body = body or {}
    title = str(body.get("title") or "Untitled deck")
    seed_id = body.get("from")
    slides = None
    if seed_id:
        src = get_static_deck(seed_id) or get_deck(seed_id)
        if seed_id == "live":
            src = await get_presentation_data()
        if src and src.get("slides"):
            slides = [
                {
                    "id": s.get("id") or f"slide-{i}",
                    "title": s.get("title") or f"Slide {i}",
                    "subtitle": s.get("subtitle") or "",
                    "bullets": list(s.get("bullets") or []),
                    "image": s.get("image") or "",
                    "kind": s.get("kind") or "narrative",
                    **({"animation": s["animation"]} if s.get("animation") else {}),
                }
                for i, s in enumerate(src["slides"], start=1)
            ]
            if not title or title == "Untitled deck":
                title = f"{src.get('title', 'Deck')} (copy)"
    deck = create_deck(title, slides)
    return {"ok": True, "deck": deck}


@app.put("/api/presentation/decks/{deck_id}")
async def update_presentation_deck(deck_id: str, body: dict[str, Any] = Body(...)):
    if deck_id == "live":
        save_live_override(body)
        _presentation_cache["ts"] = 0
        _presentation_cache["data"] = None
        payload = await get_presentation_data(use_cache=False)
        return {"ok": True, "deck": payload}
    deck = save_deck(deck_id, body)
    if not deck:
        return {"error": "deck not found or not editable"}
    return {"ok": True, "deck": deck}


@app.post("/api/presentation/live/reset")
async def reset_live_presentation():
    clear_live_override()
    _presentation_cache["ts"] = 0
    _presentation_cache["data"] = None
    payload = await get_presentation_data(use_cache=False)
    return {"ok": True, "deck": payload}


@app.delete("/api/presentation/decks/{deck_id}")
async def remove_presentation_deck(deck_id: str):
    return {"ok": delete_deck(deck_id)}


@app.post("/api/presentation/decks/{deck_id}/image")
async def upload_presentation_image(deck_id: str, file: UploadFile = File(...)):
    content = await file.read()
    if len(content) > 12 * 1024 * 1024:
        return {"error": "image too large (max 12MB)"}
    result = save_image(deck_id, file.filename or "image.png", content)
    if not result:
        return {"error": "deck not found"}
    return {"ok": True, **result}


@app.get("/api/presentation/decks/{deck_id}/assets/{name}")
async def get_presentation_asset(deck_id: str, name: str):
    from fastapi.responses import FileResponse, Response
    path = get_asset_path(deck_id, name)
    if not path:
        return Response(status_code=404)
    return FileResponse(str(path))

@app.get("/api/workload")
async def get_workload(fast: bool = True):
    return await collect_workload(fast=fast, use_cache=True)


@app.get("/api/status")
async def get_status():
    return await collect_status()


@app.get("/api/gpu")
async def get_gpu():
    return await collect_gpu()

@app.get("/api/gpu/config")
async def get_gpu_config_endpoint():
    return get_gpu_config_payload()


@app.post("/api/gpu/config")
async def post_gpu_config(body: dict[str, Any]):
    try:
        saved = save_gpu_config(
            preset_id=body.get("preset_id"),
            host=body.get("host"),
            gpu_ui_port=body.get("gpu_ui_port"),
            llm_port=body.get("llm_port"),
        )
        return {"ok": True, "active": saved, "presets": get_gpu_config_payload()["presets"]}
    except ValueError as exc:
        return JSONResponse({"ok": False, "detail": str(exc)}, status_code=400)


@app.post("/api/gpu/config/test")
async def post_gpu_config_test(body: dict[str, Any]):
    return await test_gpu_target(
        preset_id=body.get("preset_id"),
        host=body.get("host"),
        gpu_ui_port=body.get("gpu_ui_port"),
        llm_port=body.get("llm_port"),
    )


@app.delete("/api/gpu/config")
async def delete_gpu_config():
    active = reset_gpu_config()
    return {"ok": True, "active": active}



def agent_stats() -> dict[str, dict[str, Any]]:
    stats: dict[str, dict[str, Any]] = {a["id"]: {"tasks": 0, "last_active": None, "alerts": 0} for a in AGENTS}
    with SessionLocal() as db:
        rows = db.execute(select(FeedEntry).order_by(FeedEntry.ts.desc()).limit(200)).scalars().all()
    for r in rows:
        aid = r.agent_id
        if aid not in stats:
            continue
        stats[aid]["tasks"] += 1
        if r.level == "warn":
            stats[aid]["alerts"] += 1
        if stats[aid]["last_active"] is None and r.ts:
            stats[aid]["last_active"] = r.ts.isoformat()
    return stats


@app.get("/api/agents")
async def get_agents():
    stats = agent_stats()
    enriched = [{**a, "stats": stats.get(a["id"], {})} for a in AGENTS]
    return {"agents": enriched, "zones": ZONES}


@app.get("/api/terminals")
async def get_terminals(limit: int = 200):
    return {"terminals": get_all_terminals(limit)}


@app.get("/api/terminals/{subject_id}")
async def get_subject_terminal(subject_id: str, limit: int = 200):
    valid_agents = {a["id"] for a in AGENTS}
    if subject_id not in valid_agents and not is_node_id(subject_id):
        return {"error": "unknown subject"}
    return {"agent_id": subject_id, "lines": get_terminal_lines(subject_id, limit)}


@app.get("/api/nodes")
async def list_nodes():
    workload = await collect_workload()
    nodes = workload.get("topology", {}).get("nodes", [])
    return {"nodes": [{"id": n["id"], "label": n["label"], "ip": n["ip"], "level": n["level"]} for n in nodes]}


@app.get("/api/nodes/{node_id}")
async def get_node(node_id: str):
    if not is_node_id(node_id):
        return {"error": "unknown node"}
    workload = await collect_workload()
    wn = next((n for n in workload.get("topology", {}).get("nodes", []) if n["id"] == node_id), None)
    gpu = await collect_gpu()
    snap = await collect_full_lab_context(gpu_data=gpu)
    return build_node_detail(node_id, snap, wn)


@app.post("/api/nodes/{node_id}/probe")
async def post_node_probe(node_id: str):
    if not is_node_id(node_id):
        return {"error": "unknown node"}
    asyncio.create_task(run_node_probe_task(node_id))
    return {"ok": True, "node_id": node_id, "status": "probing"}


async def run_node_ask_task(node_id: str, message: str) -> None:
    agent_id = NODE_AGENT.get(node_id, "infra-sentinel")
    meta = NODE_REGISTRY[node_id]
    await terminal_log(node_id, f"▶ Query: {message}", level="info", phase="ask")
    await terminal_log(node_id, f"→ Routing to agent {agent_id}", level="info", phase="ask")
    log = make_logger(node_id)
    status = await collect_status()
    context = await gather_agent_context(agent_id, status, log=log, message=message)
    node_ctx = f"\n\n=== FOCUSED NODE: {meta['label']} ({meta['ip']}) ===\n{meta.get('description', '')}\n"
    answer = await ask_llm(agent_id, message, context + node_ctx, log=log)
    if not answer:
        answer = fallback_answer(agent_id, context, message)
    await terminal_log(node_id, f"◆ {answer}", level="llm", phase="answer")
    await publish_event({"type": "node_ask_result", "node_id": node_id, "agent_id": agent_id, "answer": answer})


@app.post("/api/nodes/{node_id}/ask")
async def post_node_ask(node_id: str, body: NodeAskRequest):
    if not is_node_id(node_id):
        return {"error": "unknown node"}
    asyncio.create_task(run_node_ask_task(node_id, body.message))
    return {"ok": True, "node_id": node_id, "agent_id": NODE_AGENT.get(node_id), "status": "processing"}


@app.get("/api/feed")
async def get_feed(limit: int = 50):
    with SessionLocal() as db:
        rows = db.execute(select(FeedEntry).order_by(FeedEntry.ts.desc()).limit(limit)).scalars().all()
    return {
        "entries": [
            {
                "id": r.id,
                "ts": r.ts.isoformat() if r.ts else None,
                "agent_id": r.agent_id,
                "message": r.message,
                "level": r.level,
            }
            for r in rows
        ]
    }


@app.get("/api/approvals")
async def get_approvals(status: str = "pending", limit: int = 100):
    with SessionLocal() as db:
        items = list_approvals(db, Approval, status=status, limit=limit)
        stats = approval_stats(db, Approval)
    return {"approvals": items, "stats": stats, "action_types": APPROVAL_ACTION_TYPES}


@app.get("/api/approvals/stats")
async def get_approval_stats():
    with SessionLocal() as db:
        return approval_stats(db, Approval)


@app.post("/api/approvals")
async def post_approval(body: ApprovalCreateRequest):
    valid_ids = {a["id"] for a in AGENTS}
    if body.agent_id not in valid_ids:
        return {"error": "unknown agent_id"}
    if body.action_type not in APPROVAL_ACTION_TYPES:
        body.action_type = "generic.mutate"
    with SessionLocal() as db:
        item = await create_approval_request(
            db=db,
            ApprovalModel=Approval,
            agent_id=body.agent_id,
            action=body.action,
            reason=body.reason,
            action_type=body.action_type,
            target=body.target,
            payload=body.payload,
            priority=body.priority,
            terminal_log=terminal_log,
            mirror_supervisors=mirror_to_supervisors,
            publish=publish_event,
            add_feed=add_feed,
        )
    return {"ok": True, "approval": item}


@app.post("/api/approvals/{approval_id}/decide")
async def decide_approval(approval_id: str, body: ApprovalDecision):
    valid_supervisors = {"mo-commander", "bart-commander"}
    decided_by = body.decided_by if body.decided_by in valid_supervisors else "mo-commander"
    with SessionLocal() as db:
        item = await decide_approval_request(
            db=db,
            ApprovalModel=Approval,
            approval_id=approval_id,
            approved=body.approved,
            decided_by=decided_by,
            note=body.note,
            terminal_log=terminal_log,
            publish=publish_event,
            add_feed=add_feed,
        )
    if not item:
        return {"error": "not found"}

    # Approval-gated executor: run the underlying action once a supervisor approves it.
    if item.get("status") == "approved":
        payload = item.get("payload") or {}
        if isinstance(payload, dict) and payload.get("executor") == "movement":
            mid = payload.get("movement_id")
            if mid in MOVEMENT_BY_ID:
                add_feed("etl-guardian", f"Approval {approval_id} granted — executing movement '{mid}'", "info")
                asyncio.create_task(trigger_and_watch(mid, payload.get("conf")))

    return {"ok": True, "approval": item}


@app.post("/api/prompt")
async def post_prompt(body: PromptRequest):
    prompt_id = str(uuid.uuid4())[:8]
    valid_ids = {a["id"] for a in AGENTS}
    agent_id = body.agent_id if body.agent_id in valid_ids else route_agent(body.message)
    add_feed(agent_id, f"Prompt received: {body.message}", "info")
    asyncio.create_task(_run_agent_task_safe(agent_id, body.message, prompt_id))
    return {"prompt_id": prompt_id, "agent_id": agent_id, "status": "dispatched"}


@app.websocket("/api/ws/ssh")
async def ws_ssh(websocket: WebSocket):
    await ssh_session(websocket)


@app.websocket("/api/ws/ops")
async def ws_ops(websocket: WebSocket):
    await websocket.accept()
    ws_clients.add(websocket)
    try:
        status = await collect_status()
        workload = await collect_workload()
        await websocket.send_text(json.dumps({"type": "status", "data": status}, default=str))
        await websocket.send_text(json.dumps({"type": "workload", "data": workload}, default=str))
        await websocket.send_text(json.dumps({
            "type": "terminal_history",
            "terminals": get_all_terminals(150),
        }, default=str))
        while True:
            await websocket.receive_text()
    except WebSocketDisconnect:
        pass
    finally:
        ws_clients.discard(websocket)