"""ATC Command Center API β€” FastAPI backend.""" from __future__ import annotations import asyncio import json import os import time import uuid from contextlib import asynccontextmanager from datetime import datetime, timezone from typing import Any import httpx import redis.asyncio as aioredis from fastapi import Body, FastAPI, File, UploadFile, WebSocket, WebSocketDisconnect from fastapi.middleware.cors import CORSMiddleware from agent_terminal import ( get_all_terminals, get_terminal_lines, init_terminals, make_logger, set_terminal_publisher, terminal_log, ) from lab_context import collect_full_lab_context, format_context_for_agent from presentation import build_presentation_payload, render_presentation_html from presentation_upload import ( clear_live_override, create_deck, delete_deck, get_asset_path, get_deck, get_live_override, list_decks, save_deck, save_image, save_live_override, save_upload, ) from presentation_static import get_static_deck, list_static_decks from storage_s3 import router as storage_s3_router from hdfs_api import router as hdfs_router from pipeline_ops import router as pipeline_router from hadoop_analytics import router as hadoop_router from elasticsearch_api import router as elasticsearch_router from sql_console import router as sql_router from agent_ops import router as agent_ops_router, agent_dml_loop, etl_agent_loop, custodian_offload_loop from cdc_consumer import router as cdc_router, cdc_consumer_loop from movements import router as movements_router from movements import MOVEMENT_BY_ID, trigger_and_watch from dataflow import router as dataflow_router from streaming_ops import router as streaming_router from spark_workbench import router as spark_workbench_router from pii_catalog import router as pii_router from ssh_terminal import ssh_session from node_registry import NODE_IDS, NODE_AGENT, NODE_REGISTRY, is_node_id from node_ops import build_node_detail, probe_node, run_node_probe_task from approval_service import ( APPROVAL_ACTION_TYPES, approval_stats, create_approval_request, decide_approval_request, detect_agent_proposed_action, detect_approval_intent, list_approvals, ) from db import SessionLocal, db_health, init_database from supervisor import mirror_terminal_line, mirror_to_supervisors from workload import build_workload_payload _workload_cache: dict[str, Any] = {"ts": 0.0, "data": None} _presentation_cache: dict[str, Any] = {"ts": 0.0, "data": None} WORKLOAD_CACHE_TTL = 30.0 PRESENTATION_CACHE_TTL = 45.0 from pydantic import BaseModel, Field from sqlalchemy import Column, DateTime, String, Text, select from sqlalchemy.orm import DeclarativeBase REDIS_URL = os.getenv("REDIS_URL", "redis://redis:6379/0") DOCKHAND_URL = os.getenv("DOCKHAND_URL", "http://10.0.21.45:8082") GPU_URL = os.getenv("GPU_URL", "http://10.0.20.106:9000") GPU_UI_URL = os.getenv("GPU_UI_URL", GPU_URL) LLM_URL = os.getenv("LLM_URL", "http://10.0.20.106:8001/v1") LLM_MODEL = os.getenv("LLM_MODEL", "gpt-4o") LLM_API_KEY = os.getenv("LLM_API_KEY", "sk-local") LLM_TIMEOUT = float(os.getenv("LLM_TIMEOUT", "120")) AGENTS = [ { "id": "etl-guardian", "name": "ETL Guardian", "color": "#00f0ff", "zone": "etl", "role": "Airflow, Kafka, Debezium, S3 pipeline", "icon": "⚑", "motto": "Pipelines never sleep", "capabilities": ["Airflow", "Kafka", "Debezium", "S3", "Connectors"], "suggested_prompts": [ "How is Debezium doing?", "Are all Airflow DAGs healthy?", "Kafka connector status?", ], }, { "id": "lakehouse-ops", "name": "Lakehouse Ops", "color": "#ff00aa", "zone": "lakehouse", "role": "Spark, Trino, Iceberg", "icon": "πŸ”οΈ", "motto": "Query the lake, trust the table", "capabilities": ["Spark", "Trino", "Iceberg", "Delta", "SQL"], "suggested_prompts": [ "Lakehouse stack status?", "Is Trino reachable?", "How many lakehouse containers are running?", ], }, { "id": "data-custodian", "name": "Data Custodian", "color": "#ffaa00", "zone": "db", "role": "PostgreSQL, MySQL, Mongo, Cassandra, Neo4j", "icon": "πŸ›‘οΈ", "motto": "Guardian of every row", "capabilities": ["PostgreSQL", "MySQL", "MongoDB", "Cassandra", "Neo4j"], "suggested_prompts": [ "How much data is in the databases?", "What is in PostgreSQL sales_orders?", "MongoDB supplychain overview", ], }, { "id": "hadoop-ranger", "name": "Hadoop Ranger", "color": "#39ff14", "zone": "hadoop", "role": "HDFS, YARN cluster", "icon": "🌲", "motto": "Patrol the data forest", "capabilities": ["HDFS", "YARN", "NameNode", "DataNodes"], "suggested_prompts": [ "Is HDFS NameNode up?", "Hadoop cluster status?", "YARN nodes healthy?", ], }, { "id": "infra-sentinel", "name": "Infra Sentinel", "color": "#9b72cf", "zone": "docker", "role": "Docker, Proxmox, GPU, monitoring", "icon": "πŸ‘οΈ", "motto": "See everything, miss nothing", "capabilities": ["Docker", "Proxmox", "GPU", "vLLM", "Monitoring"], "suggested_prompts": [ "GPU status?", "Which LLM model is running?", "Docker container overview", ], }, { "id": "mo-commander", "name": "Mo Β· Command", "color": "#4c9aed", "zone": "command", "role": "Supervisor β€” full event intel, ingress, approvals", "icon": "🎯", "motto": "Nothing happens without Mo knowing", "supervisor": True, "person": "mo", "capabilities": ["Events", "Ingress", "Approvals", "Agent dispatch", "Network IN"], "suggested_prompts": [ "What happened today?", "What events came in?", "Pipeline status overview", ], }, { "id": "bart-commander", "name": "Bart Β· Ops", "color": "#3fb950", "zone": "command", "role": "Supervisor β€” egress, MCP comms, network OUT", "icon": "πŸ“‘", "motto": "All traffic flows through Bart", "supervisor": True, "person": "bart", "capabilities": ["Egress", "MCP routing", "Network OUT", "GPU inference", "S3 writes"], "suggested_prompts": [ "What is leaving the cluster?", "MCP agent communication status?", "Network egress overview", ], }, { "id": "network-watcher", "name": "Network Watcher", "color": "#58a6ff", "zone": "network", "role": "VLAN 20/21 traffic, data in & out paths", "icon": "🌐", "motto": "Every packet tells a story", "capabilities": ["VLAN 20", "VLAN 21", "Ingress", "Egress", "Firewall paths"], "suggested_prompts": [ "Data ingress status?", "What leaves the cluster?", "Network path to S3?", ], }, { "id": "mcp-coordinator", "name": "MCP Coordinator", "color": "#f778ba", "zone": "mcp", "role": "MCP hub β€” routes all agent tool calls & comms", "icon": "πŸ”€", "motto": "Route once, deliver everywhere", "capabilities": ["MCP servers", "Tool routing", "Agent relay", "WebSocket bus"], "suggested_prompts": [ "Which MCP agents are active?", "MCP hub route status?", "Agent communication overview", ], }, ] ZONES = [ {"id": "docker", "label": "DOCKER RACK", "x": 8, "color": "#b366ff"}, {"id": "db", "label": "DB VAULT", "x": 28, "color": "#ffaa00"}, {"id": "lakehouse", "label": "LAKEHOUSE HUB", "x": 50, "color": "#ff00aa"}, {"id": "hadoop", "label": "HADOOP CLUSTER", "x": 72, "color": "#39ff14"}, {"id": "etl", "label": "ETL PIPE", "x": 92, "color": "#00f0ff"}, ] INTENT_KEYWORDS: dict[str, list[str]] = { "data-custodian": ["database", "postgres", "postgresql", "mysql", "mongo", "mongodb", "cassandra", "neo4j", "sql", "db "], "lakehouse-ops": ["trino", "spark", "lakehouse", "iceberg", "query", "table"], "hadoop-ranger": [ "hadoop", "hdfs", "yarn", "datanode", "namenode", "replicatie", "replication", "rf factor", "opslag", "bestanden", "blocks", "cluster opslag", "data op", ], "infra-sentinel": ["docker", "container", "vm", "proxmox", "infra", "grafana", "gpu", "vllm", "llm", "nvidia", "inference", "model"], "etl-guardian": ["airflow", "dag", "debezium", "kafka", "connector", "etl", "pipeline", "s3"], "network-watcher": ["network", "vlan", "ingress", "egress", "traffic", "packet", "firewall", "route"], "mcp-coordinator": ["mcp", "tool", "router", "relay", "websocket", "hub"], "mo-commander": ["mo", "supervisor", "events", "overzicht", "alles", "gebeurd"], "bart-commander": ["bart", "egress", "uitgaand", "communicatie", "mcp comm"], } class Base(DeclarativeBase): pass class FeedEntry(Base): __tablename__ = "feed" id = Column(String, primary_key=True) ts = Column(DateTime, default=lambda: datetime.now(timezone.utc)) agent_id = Column(String) level = Column(String, default="info") message = Column(Text) class Approval(Base): __tablename__ = "approvals" id = Column(String, primary_key=True) ts = Column(DateTime, default=lambda: datetime.now(timezone.utc)) agent_id = Column(String) action = Column(Text) reason = Column(Text) status = Column(String, default="pending") action_type = Column(String, default="generic.mutate") target = Column(Text, default="") payload = Column(Text, default="{}") decided_by = Column(String, nullable=True) decide_note = Column(Text, nullable=True) decided_at = Column(DateTime, nullable=True) priority = Column(String, default="normal") _db_info = init_database(Base) redis_client: aioredis.Redis | None = None ws_clients: set[WebSocket] = set() class PromptRequest(BaseModel): message: str = Field(min_length=1, max_length=2000) agent_id: str | None = None class NodeAskRequest(BaseModel): message: str = Field(min_length=1, max_length=2000) class ApprovalCreateRequest(BaseModel): agent_id: str = Field(min_length=1, max_length=64) action: str = Field(min_length=1, max_length=2000) reason: str = Field(min_length=1, max_length=2000) action_type: str = "generic.mutate" target: str = "" payload: dict[str, Any] | None = None priority: str = "normal" class ApprovalDecision(BaseModel): approved: bool decided_by: str = "mo-commander" note: str = "" def route_agent(message: str) -> str: lower = message.lower() # Storage/data questions default to Hadoop unless clearly about databases if any(w in lower for w in ("data", "opslag", "gb", "replicatie", "replication", "hdfs", "hadoop")): if not any(w in lower for w in ("postgres", "mysql", "mongo", "database", "sql", "neo4j", "cassandra")): return "hadoop-ranger" scores = {aid: sum(1 for kw in kws if kw in lower) for aid, kws in INTENT_KEYWORDS.items()} best = max(scores, key=scores.get) if scores[best] == 0: return "infra-sentinel" return best async def gather_agent_context( agent_id: str, status: dict[str, Any], log: Any | None = None, ) -> str: """Full lab snapshot for vLLM β€” all domains, agent's primary domain highlighted.""" snapshot = await collect_full_lab_context(gpu_data=status.get("gpu"), log=log) snapshot["domains_summary"] = status.get("domains", {}) ctx = format_context_for_agent(agent_id, snapshot) agent_lines = ["", "=== AGENTS & SUPERVISORS ==="] for a in AGENTS: sup = " [supervisor]" if a.get("supervisor") else "" agent_lines.append(f" - {a['name']} ({a['id']}){sup}: {a['role']}") ctx = ctx + "\n".join(agent_lines) try: from platform_context import build_llm_addendum ctx = ctx + "\n\n" + build_llm_addendum() except Exception: pass if log: await log("info", "fetch", f"β–Έ Context assembled: {len(ctx)} chars for LLM") return ctx async def ask_llm( agent_id: str, message: str, context: str, log: Any | None = None, ) -> str | None: agent = next(a for a in AGENTS if a["id"] == agent_id) system = f"""You are {agent['name']}, an autonomous ops agent in the Dell ATC data lab. Specialization: {agent['role']}. Motto: {agent.get('motto', '')}. You respond on behalf of your domain but have visibility into the FULL lab stack: Docker, databases, lakehouse (Trino/Spark/Kafka Connect), ETL (Airflow/Kafka), Hadoop HDFS, and GPU/vLLM. Rules: - Always respond in English. - You have full visibility into the entire cluster: all VMs, zones, connectors, GPU, Hadoop, ObjectScale and Command Center. - Use ONLY the live data below β€” do not invent hosts, ports, numbers or connector names. - Use exact container/connector names from the data (e.g. mysql-hr-connector, not "Debezium"). - If something is DOWN or 0 GB, say so honestly. - Respect the data masking policy: NEVER reveal, guess or reconstruct raw values of MASKED columns (they arrive as the token πŸ”’ MASKED). You MUST still answer helpfully β€” confirm the column is masked for privacy/governance, explain why, and you may use non-sensitive aggregates/counts over it. - You are fully aware of all latest platform changes via the section PLATFORM CAPABILITIES & RECENT CHANGES below; use it to answer questions about recent changes, the Spark Workbench, the Hadoop pipeline, the Data Flow pulse switch and the autonomous agents (DML, ETL, Custodian Hadoop offload). - Be concise and helpful (max ~10 sentences); bullet lists are fine when they aid clarity. --- LIVE LAB DATA (primary domain first, then full stack) --- {context} """ if log: await log("info", "llm", f"β–Έ Querying vLLM model={LLM_MODEL}") await log("cmd", "llm", f"$ POST {LLM_URL.rstrip('/')}/chat/completions") await log("info", "llm", f" user: {message[:160]}{'…' if len(message) > 160 else ''}") try: async with httpx.AsyncClient(timeout=LLM_TIMEOUT) as client: t0 = time.monotonic() r = await client.post( f"{LLM_URL.rstrip('/')}/chat/completions", headers={ "Authorization": f"Bearer {LLM_API_KEY}", "Content-Type": "application/json", }, json={ "model": LLM_MODEL, "messages": [ {"role": "system", "content": system}, {"role": "user", "content": message}, ], "max_tokens": 800, "temperature": 0.25, }, ) r.raise_for_status() content = r.json()["choices"][0]["message"]["content"].strip() ms = int((time.monotonic() - t0) * 1000) if content and content.strip("!"): if log: await log("ok", "llm", f"← vLLM response {len(content)} chars ({ms}ms)") preview = content.replace("\n", " ")[:180] await log("info", "llm", f" Β» {preview}{'…' if len(content) > 180 else ''}") return content if log: await log("warn", "llm", f"← Empty or invalid LLM output ({ms}ms)") except Exception as exc: if log: await log("err", "llm", f"βœ— vLLM error: {exc}") return None def fallback_answer(agent_id: str, context: str) -> str: agent_name = next(a["name"] for a in AGENTS if a["id"] == agent_id) return f"**{agent_name}** (offline LLM β€” ruwe data):\n\n{context}" async def publish_event(event: dict[str, Any]) -> None: payload = json.dumps(event, default=str) if redis_client: await redis_client.publish("ops", payload) dead = [] for ws in ws_clients: try: await ws.send_text(payload) except Exception: dead.append(ws) for ws in dead: ws_clients.discard(ws) et = event.get("type") if et == "feed": entry = event.get("entry") or {} await mirror_to_supervisors( entry.get("agent_id", "?"), entry.get("message", ""), level=entry.get("level", "info"), ) elif et == "terminal": await mirror_terminal_line(event.get("line") or {}) elif et in ("agent_dispatch", "agent_fetch", "agent_return"): aid = event.get("agent_id", "?") zone = event.get("zone", "") await mirror_to_supervisors(aid, f"{et} β†’ zone {zone}", level="info", phase="dispatch") def add_feed(agent_id: str, message: str, level: str = "info") -> dict: entry_id = str(uuid.uuid4())[:8] with SessionLocal() as db: row = FeedEntry(id=entry_id, agent_id=agent_id, message=message, level=level) db.add(row) db.commit() return { "id": entry_id, "ts": datetime.now(timezone.utc).isoformat(), "agent_id": agent_id, "message": message, "level": level, } async def dockhand_env_containers(env_id: int) -> list[dict]: try: async with httpx.AsyncClient(timeout=8.0) as client: r = await client.get(f"{DOCKHAND_URL}/api/containers", params={"env": env_id}) r.raise_for_status() return r.json() except Exception: return [] async def probe_url(url: str) -> bool: try: async with httpx.AsyncClient(timeout=4.0, verify=False) as client: r = await client.get(url) return r.status_code < 500 except Exception: return False async def collect_gpu() -> dict[str, Any]: host = GPU_URL.replace("http://", "").replace("https://", "").split("/")[0] base = {"ok": False, "host": host, "ui_url": GPU_UI_URL} try: async with httpx.AsyncClient(timeout=6.0) as client: metrics_r, model_r, integration_r = await asyncio.gather( client.get(f"{GPU_URL}/api/gpu/metrics"), client.get(f"{GPU_URL}/api/active-model"), client.get(f"{GPU_URL}/api/integration"), return_exceptions=True, ) gpus: list[dict[str, Any]] = [] if isinstance(metrics_r, httpx.Response) and metrics_r.status_code == 200: current = metrics_r.json().get("current", {}) gpus = [ { "index": g["index"], "name": g["name"], "util_gpu": g.get("util_gpu", 0), "memory_used_mib": g.get("memory_used_mib", 0), "memory_total_mib": g.get("memory_total_mib", 0), "temperature_c": g.get("temperature_c", 0), "power_w": g.get("power_w", 0), } for g in current.get("gpus", []) ] active_model = None inference_active = False vllm_url = None if isinstance(model_r, httpx.Response) and model_r.status_code == 200: model_data = model_r.json() active_model = model_data.get("name") inference_active = bool(model_data.get("inference_active")) vllm_url = model_data.get("base_url") if isinstance(integration_r, httpx.Response) and integration_r.status_code == 200: integ = integration_r.json() if not active_model: active_model = integ.get("active_name") if not inference_active: inference_active = bool(integ.get("inference_active")) if not vllm_url: vllm_url = integ.get("recommended_base_url") return { **base, "ok": len(gpus) > 0 or inference_active, "inference_active": inference_active, "active_model": active_model, "vllm_url": vllm_url, "gpu_count": len(gpus), "gpus": gpus, } except Exception as exc: return {**base, "error": str(exc)} async def collect_status() -> dict[str, Any]: db_containers = await dockhand_env_containers(5) db_running = sum(1 for c in db_containers if c.get("state") == "running") db_total = len(db_containers) or 6 lake_containers = await dockhand_env_containers(9) lake_running = sum(1 for c in lake_containers if c.get("state") == "running") lake_total = len(lake_containers) or 6 docker_containers = await dockhand_env_containers(1) docker_running = sum(1 for c in docker_containers if c.get("state") == "running") hdfs_ok = await probe_url("http://10.0.21.61:9870") kafka_ok = await probe_url("http://10.0.21.36:9000") airflow_ok = await probe_url("http://10.0.21.55:8080") def level(running: int, total: int) -> str: if total == 0: return "unknown" ratio = running / total if ratio >= 0.9: return "ok" if ratio >= 0.5: return "warn" return "down" gpu = await collect_gpu() gpu_level = "ok" if gpu.get("ok") and gpu.get("inference_active") else ("warn" if gpu.get("ok") else "down") gpu_label = gpu.get("active_model") or (f"{gpu.get('gpu_count', 0)} GPUs" if gpu.get("ok") else "offline") return { "ts": datetime.now(timezone.utc).isoformat(), "domains": { "docker": {"level": "ok" if docker_running >= 5 else "warn", "label": f"{docker_running} containers", "running": docker_running}, "databases": {"level": level(db_running, db_total), "label": f"{db_running}/{db_total} up", "running": db_running, "total": db_total}, "lakehouse": {"level": level(lake_running, lake_total), "label": f"{lake_running}/{lake_total} up", "running": lake_running, "total": lake_total}, "hadoop": {"level": "ok" if hdfs_ok else "warn", "label": "NN up" if hdfs_ok else "NN check"}, "etl": {"level": "ok" if kafka_ok and airflow_ok else "warn", "label": "Kafka+Airflow"}, "gpu": {"level": gpu_level, "label": gpu_label}, }, "gpu": gpu, "kafka_ok": kafka_ok, "airflow_ok": airflow_ok, "hdfs_ok": hdfs_ok, } async def _run_agent_task_safe(agent_id: str, message: str, prompt_id: str) -> None: try: await run_agent_task(agent_id, message, prompt_id) except Exception as exc: agent_name = next((a["name"] for a in AGENTS if a["id"] == agent_id), agent_id) err = f"Sorry β€” {agent_name} could not complete your request: {exc}" await terminal_log(agent_id, f"[{prompt_id}] βœ— Error: {exc}", level="err", phase="error", prompt_id=prompt_id) feed = add_feed(agent_id, f"{agent_name} failed: {str(exc)[:80]}", "err") await publish_event({"type": "feed", "entry": feed}) await publish_event({ "type": "prompt_result", "prompt_id": prompt_id, "agent_id": agent_id, "answer": err, }) async def run_agent_task(agent_id: str, message: str, prompt_id: str) -> str: zone = next(a["zone"] for a in AGENTS if a["id"] == agent_id) agent_name = next(a["name"] for a in AGENTS if a["id"] == agent_id) log = make_logger(agent_id, prompt_id) approval_created = False intent = detect_approval_intent(message) if intent: with SessionLocal() as db: await create_approval_request( db=db, ApprovalModel=Approval, agent_id=agent_id, action=intent["action"], reason=intent["reason"], action_type=intent["action_type"], terminal_log=terminal_log, mirror_supervisors=mirror_to_supervisors, publish=publish_event, add_feed=add_feed, ) approval_created = True await terminal_log( agent_id, f"[{prompt_id}] Mutating request detected β€” approval queued for Mo & Bart", level="warn", phase="approval", prompt_id=prompt_id, ) await terminal_log( agent_id, f"[{prompt_id}] β–Ά Mission accepted: {message}", level="info", phase="dispatch", prompt_id=prompt_id, ) await publish_event({"type": "agent_dispatch", "agent_id": agent_id, "zone": zone, "prompt_id": prompt_id}) await asyncio.sleep(0.4) await terminal_log(agent_id, f"[{prompt_id}] Walking to zone: {zone}", level="info", phase="dispatch", prompt_id=prompt_id) await publish_event({"type": "agent_fetch", "agent_id": agent_id, "zone": zone, "prompt_id": prompt_id}) await log("info", "fetch", f"[{prompt_id}] Collecting live lab metrics…") status = await collect_status() context = await gather_agent_context(agent_id, status, log=log) answer = await ask_llm(agent_id, message, context, log=log) if not answer: await log("warn", "llm", "LLM fallback β€” returning raw context") answer = fallback_answer(agent_id, context) if not approval_created: proposed = detect_agent_proposed_action(answer, message) if proposed: with SessionLocal() as db: await create_approval_request( db=db, ApprovalModel=Approval, agent_id=agent_id, action=proposed["action"], reason=proposed["reason"], action_type=proposed["action_type"], target=proposed.get("target", ""), terminal_log=terminal_log, mirror_supervisors=mirror_to_supervisors, publish=publish_event, add_feed=add_feed, ) approval_created = True answer = ( f"{answer}\n\n⏸ **Approval required** β€” this action is in the Approval Inbox. " f"Mo & Bart have been notified and must approve before we execute." ) await asyncio.sleep(0.3) await terminal_log(agent_id, f"[{prompt_id}] βœ“ Mission complete", level="ok", phase="done", prompt_id=prompt_id) await publish_event({"type": "agent_return", "agent_id": agent_id, "zone": zone, "prompt_id": prompt_id}) feed = add_feed(agent_id, f"{agent_name} completed a response (see Comms)", "info") await publish_event({"type": "feed", "entry": feed}) await publish_event({"type": "prompt_result", "prompt_id": prompt_id, "agent_id": agent_id, "answer": answer}) return answer async def heartbeat_loop() -> None: while True: try: status = await collect_status() workload = await collect_workload() await publish_event({"type": "status", "data": status}) await publish_event({"type": "workload", "data": workload}) for domain, info in status["domains"].items(): if info["level"] == "down": agent = "data-custodian" if domain == "databases" else "infra-sentinel" feed = add_feed(agent, f"Alert: {domain} is DOWN ({info['label']})", "warn") await publish_event({"type": "feed", "entry": feed}) except Exception as exc: await publish_event({"type": "error", "message": str(exc)}) await asyncio.sleep(60) @asynccontextmanager async def lifespan(app: FastAPI): global redis_client redis_client = aioredis.from_url(REDIS_URL, decode_responses=True) set_terminal_publisher(publish_event) init_terminals([a["id"] for a in AGENTS] + NODE_IDS) for a in AGENTS: await terminal_log(a["id"], f"{a['name']} terminal online β€” awaiting missions", level="info", phase="boot") for nid in NODE_IDS: if nid not in NODE_REGISTRY: continue meta = NODE_REGISTRY[nid] await terminal_log(nid, f"{meta['label']} shell ready β€” click node to connect", level="info", phase="boot") task = asyncio.create_task(heartbeat_loop()) dml_task = asyncio.create_task(agent_dml_loop()) cdc_task = asyncio.create_task(cdc_consumer_loop()) etl_task = asyncio.create_task(etl_agent_loop()) cust_task = asyncio.create_task(custodian_offload_loop()) add_feed("infra-sentinel", "ATC Command Center API online", "info") yield task.cancel() dml_task.cancel() cdc_task.cancel() etl_task.cancel() if redis_client: await redis_client.close() app = FastAPI(title="ATC Command Center API", lifespan=lifespan) app.include_router(storage_s3_router) app.include_router(hdfs_router) app.include_router(pipeline_router) app.include_router(hadoop_router) app.include_router(elasticsearch_router) app.include_router(sql_router) app.include_router(agent_ops_router) app.include_router(cdc_router) app.include_router(movements_router) app.include_router(dataflow_router) app.include_router(streaming_router) app.include_router(spark_workbench_router) app.include_router(pii_router) app.add_middleware( CORSMiddleware, allow_origins=["*"], allow_credentials=True, allow_methods=["*"], allow_headers=["*"], ) @app.get("/api/health") async def health(): llm_ok = False try: async with httpx.AsyncClient(timeout=4.0) as client: r = await client.get(f"{LLM_URL.rstrip('/')}/models", headers={"Authorization": f"Bearer {LLM_API_KEY}"}) llm_ok = r.status_code == 200 except Exception: pass return { "ok": True, "ts": datetime.now(timezone.utc).isoformat(), "llm_url": LLM_URL, "llm_ok": llm_ok, "llm_model": LLM_MODEL, "database": db_health(), "db_init": _db_info, } async def collect_workload(*, fast: bool = True, use_cache: bool = True) -> dict[str, Any]: import time as _time now = _time.time() if use_cache and _workload_cache.get("data") and now - float(_workload_cache.get("ts") or 0) < WORKLOAD_CACHE_TTL: return _workload_cache["data"] gpu = await collect_gpu() snap = await collect_full_lab_context(gpu_data=gpu, include_inventory=not fast) payload = build_workload_payload(snap) _workload_cache["ts"] = now _workload_cache["data"] = payload return payload async def get_presentation_data(*, use_cache: bool = True) -> dict[str, Any]: import time as _time now = _time.time() if use_cache and _presentation_cache.get("data") and now - float(_presentation_cache.get("ts") or 0) < PRESENTATION_CACHE_TTL: return _presentation_cache["data"] gpu = await collect_gpu() snap = await collect_full_lab_context(gpu_data=gpu, include_inventory=False) data = build_presentation_payload(snap) override = get_live_override() if override and override.get("slides"): data["title"] = override.get("title") or data.get("title") data["subtitle"] = override.get("subtitle") or data.get("subtitle", "") data["slides"] = override["slides"] data["slide_count"] = len(override["slides"]) data["edited"] = True data["override_ts"] = override.get("ts") else: data["edited"] = False data["source"] = "live" data["id"] = "live" _presentation_cache["ts"] = now _presentation_cache["data"] = data return data @app.get("/api/presentation") async def get_presentation(): return await get_presentation_data() @app.get("/api/presentation/html") async def get_presentation_html(): from fastapi.responses import HTMLResponse payload = await get_presentation_data() return HTMLResponse(render_presentation_html(payload)) @app.get("/api/presentation/decks") async def get_presentation_decks(): return {"live": True, "builtin": list_static_decks(), "uploaded": list_decks()} @app.get("/api/presentation/decks/{deck_id}") async def get_presentation_deck(deck_id: str): if deck_id == "live": return await get_presentation_data() deck = get_static_deck(deck_id) or get_deck(deck_id) if not deck: return {"error": "deck not found"} return deck @app.get("/api/presentation/decks/{deck_id}/html") async def get_presentation_deck_html(deck_id: str): from fastapi.responses import HTMLResponse if deck_id == "live": payload = await get_presentation_data() else: payload = get_static_deck(deck_id) or get_deck(deck_id) if not payload: return HTMLResponse("

Deck not found

", status_code=404) return HTMLResponse(render_presentation_html(payload)) @app.post("/api/presentation/upload") async def upload_presentation(file: UploadFile = File(...)): content = await file.read() if len(content) > 50 * 1024 * 1024: return {"error": "file too large (max 50MB)"} deck = await save_upload(file.filename or "upload.pptx", content) return {"ok": True, "deck": deck} @app.post("/api/presentation/decks/new") async def create_presentation_deck(body: dict[str, Any] | None = Body(default=None)): body = body or {} title = str(body.get("title") or "Untitled deck") seed_id = body.get("from") slides = None if seed_id: src = get_static_deck(seed_id) or get_deck(seed_id) if seed_id == "live": src = await get_presentation_data() if src and src.get("slides"): slides = [ { "id": s.get("id") or f"slide-{i}", "title": s.get("title") or f"Slide {i}", "subtitle": s.get("subtitle") or "", "bullets": list(s.get("bullets") or []), "image": s.get("image") or "", "kind": s.get("kind") or "narrative", **({"animation": s["animation"]} if s.get("animation") else {}), } for i, s in enumerate(src["slides"], start=1) ] if not title or title == "Untitled deck": title = f"{src.get('title', 'Deck')} (copy)" deck = create_deck(title, slides) return {"ok": True, "deck": deck} @app.put("/api/presentation/decks/{deck_id}") async def update_presentation_deck(deck_id: str, body: dict[str, Any] = Body(...)): if deck_id == "live": save_live_override(body) _presentation_cache["ts"] = 0 _presentation_cache["data"] = None payload = await get_presentation_data(use_cache=False) return {"ok": True, "deck": payload} deck = save_deck(deck_id, body) if not deck: return {"error": "deck not found or not editable"} return {"ok": True, "deck": deck} @app.post("/api/presentation/live/reset") async def reset_live_presentation(): clear_live_override() _presentation_cache["ts"] = 0 _presentation_cache["data"] = None payload = await get_presentation_data(use_cache=False) return {"ok": True, "deck": payload} @app.delete("/api/presentation/decks/{deck_id}") async def remove_presentation_deck(deck_id: str): return {"ok": delete_deck(deck_id)} @app.post("/api/presentation/decks/{deck_id}/image") async def upload_presentation_image(deck_id: str, file: UploadFile = File(...)): content = await file.read() if len(content) > 12 * 1024 * 1024: return {"error": "image too large (max 12MB)"} result = save_image(deck_id, file.filename or "image.png", content) if not result: return {"error": "deck not found"} return {"ok": True, **result} @app.get("/api/presentation/decks/{deck_id}/assets/{name}") async def get_presentation_asset(deck_id: str, name: str): from fastapi.responses import FileResponse, Response path = get_asset_path(deck_id, name) if not path: return Response(status_code=404) return FileResponse(str(path)) @app.get("/api/workload") async def get_workload(fast: bool = True): return await collect_workload(fast=fast, use_cache=True) @app.get("/api/status") async def get_status(): return await collect_status() @app.get("/api/gpu") async def get_gpu(): return await collect_gpu() def agent_stats() -> dict[str, dict[str, Any]]: stats: dict[str, dict[str, Any]] = {a["id"]: {"tasks": 0, "last_active": None, "alerts": 0} for a in AGENTS} with SessionLocal() as db: rows = db.execute(select(FeedEntry).order_by(FeedEntry.ts.desc()).limit(200)).scalars().all() for r in rows: aid = r.agent_id if aid not in stats: continue stats[aid]["tasks"] += 1 if r.level == "warn": stats[aid]["alerts"] += 1 if stats[aid]["last_active"] is None and r.ts: stats[aid]["last_active"] = r.ts.isoformat() return stats @app.get("/api/agents") async def get_agents(): stats = agent_stats() enriched = [{**a, "stats": stats.get(a["id"], {})} for a in AGENTS] return {"agents": enriched, "zones": ZONES} @app.get("/api/terminals") async def get_terminals(limit: int = 200): return {"terminals": get_all_terminals(limit)} @app.get("/api/terminals/{subject_id}") async def get_subject_terminal(subject_id: str, limit: int = 200): valid_agents = {a["id"] for a in AGENTS} if subject_id not in valid_agents and not is_node_id(subject_id): return {"error": "unknown subject"} return {"agent_id": subject_id, "lines": get_terminal_lines(subject_id, limit)} @app.get("/api/nodes") async def list_nodes(): workload = await collect_workload() nodes = workload.get("topology", {}).get("nodes", []) return {"nodes": [{"id": n["id"], "label": n["label"], "ip": n["ip"], "level": n["level"]} for n in nodes]} @app.get("/api/nodes/{node_id}") async def get_node(node_id: str): if not is_node_id(node_id): return {"error": "unknown node"} workload = await collect_workload() wn = next((n for n in workload.get("topology", {}).get("nodes", []) if n["id"] == node_id), None) gpu = await collect_gpu() snap = await collect_full_lab_context(gpu_data=gpu) return build_node_detail(node_id, snap, wn) @app.post("/api/nodes/{node_id}/probe") async def post_node_probe(node_id: str): if not is_node_id(node_id): return {"error": "unknown node"} asyncio.create_task(run_node_probe_task(node_id)) return {"ok": True, "node_id": node_id, "status": "probing"} async def run_node_ask_task(node_id: str, message: str) -> None: agent_id = NODE_AGENT.get(node_id, "infra-sentinel") meta = NODE_REGISTRY[node_id] await terminal_log(node_id, f"β–Ά Query: {message}", level="info", phase="ask") await terminal_log(node_id, f"β†’ Routing to agent {agent_id}", level="info", phase="ask") log = make_logger(node_id) status = await collect_status() context = await gather_agent_context(agent_id, status, log=log) node_ctx = f"\n\n=== FOCUSED NODE: {meta['label']} ({meta['ip']}) ===\n{meta.get('description', '')}\n" answer = await ask_llm(agent_id, message, context + node_ctx, log=log) if not answer: answer = fallback_answer(agent_id, context) await terminal_log(node_id, f"β—† {answer}", level="llm", phase="answer") await publish_event({"type": "node_ask_result", "node_id": node_id, "agent_id": agent_id, "answer": answer}) @app.post("/api/nodes/{node_id}/ask") async def post_node_ask(node_id: str, body: NodeAskRequest): if not is_node_id(node_id): return {"error": "unknown node"} asyncio.create_task(run_node_ask_task(node_id, body.message)) return {"ok": True, "node_id": node_id, "agent_id": NODE_AGENT.get(node_id), "status": "processing"} @app.get("/api/feed") async def get_feed(limit: int = 50): with SessionLocal() as db: rows = db.execute(select(FeedEntry).order_by(FeedEntry.ts.desc()).limit(limit)).scalars().all() return { "entries": [ { "id": r.id, "ts": r.ts.isoformat() if r.ts else None, "agent_id": r.agent_id, "message": r.message, "level": r.level, } for r in rows ] } @app.get("/api/approvals") async def get_approvals(status: str = "pending", limit: int = 100): with SessionLocal() as db: items = list_approvals(db, Approval, status=status, limit=limit) stats = approval_stats(db, Approval) return {"approvals": items, "stats": stats, "action_types": APPROVAL_ACTION_TYPES} @app.get("/api/approvals/stats") async def get_approval_stats(): with SessionLocal() as db: return approval_stats(db, Approval) @app.post("/api/approvals") async def post_approval(body: ApprovalCreateRequest): valid_ids = {a["id"] for a in AGENTS} if body.agent_id not in valid_ids: return {"error": "unknown agent_id"} if body.action_type not in APPROVAL_ACTION_TYPES: body.action_type = "generic.mutate" with SessionLocal() as db: item = await create_approval_request( db=db, ApprovalModel=Approval, agent_id=body.agent_id, action=body.action, reason=body.reason, action_type=body.action_type, target=body.target, payload=body.payload, priority=body.priority, terminal_log=terminal_log, mirror_supervisors=mirror_to_supervisors, publish=publish_event, add_feed=add_feed, ) return {"ok": True, "approval": item} @app.post("/api/approvals/{approval_id}/decide") async def decide_approval(approval_id: str, body: ApprovalDecision): valid_supervisors = {"mo-commander", "bart-commander"} decided_by = body.decided_by if body.decided_by in valid_supervisors else "mo-commander" with SessionLocal() as db: item = await decide_approval_request( db=db, ApprovalModel=Approval, approval_id=approval_id, approved=body.approved, decided_by=decided_by, note=body.note, terminal_log=terminal_log, publish=publish_event, add_feed=add_feed, ) if not item: return {"error": "not found"} # Approval-gated executor: run the underlying action once a supervisor approves it. if item.get("status") == "approved": payload = item.get("payload") or {} if isinstance(payload, dict) and payload.get("executor") == "movement": mid = payload.get("movement_id") if mid in MOVEMENT_BY_ID: add_feed("etl-guardian", f"Approval {approval_id} granted β€” executing movement '{mid}'", "info") asyncio.create_task(trigger_and_watch(mid, payload.get("conf"))) return {"ok": True, "approval": item} @app.post("/api/prompt") async def post_prompt(body: PromptRequest): prompt_id = str(uuid.uuid4())[:8] valid_ids = {a["id"] for a in AGENTS} agent_id = body.agent_id if body.agent_id in valid_ids else route_agent(body.message) add_feed(agent_id, f"Prompt received: {body.message}", "info") asyncio.create_task(_run_agent_task_safe(agent_id, body.message, prompt_id)) return {"prompt_id": prompt_id, "agent_id": agent_id, "status": "dispatched"} @app.websocket("/api/ws/ssh") async def ws_ssh(websocket: WebSocket): await ssh_session(websocket) @app.websocket("/api/ws/ops") async def ws_ops(websocket: WebSocket): await websocket.accept() ws_clients.add(websocket) try: status = await collect_status() workload = await collect_workload() await websocket.send_text(json.dumps({"type": "status", "data": status}, default=str)) await websocket.send_text(json.dumps({"type": "workload", "data": workload}, default=str)) await websocket.send_text(json.dumps({ "type": "terminal_history", "terminals": get_all_terminals(150), }, default=str)) while True: await websocket.receive_text() except WebSocketDisconnect: pass finally: ws_clients.discard(websocket)