"""ATC Command Center API — FastAPI backend.""" from __future__ import annotations import asyncio import json import os import time import uuid from contextlib import asynccontextmanager from datetime import datetime, timezone from typing import Any import httpx import redis.asyncio as aioredis from fastapi import Body, FastAPI, File, UploadFile, WebSocket, WebSocketDisconnect from fastapi.middleware.cors import CORSMiddleware import auth as cockpit_auth from agent_terminal import ( get_all_terminals, get_terminal_lines, init_terminals, make_logger, set_terminal_publisher, terminal_log, ) from lab_context import collect_full_lab_context, format_context_for_agent from presentation import build_presentation_payload, render_presentation_html from presentation_upload import ( clear_live_override, create_deck, delete_deck, get_asset_path, get_deck, get_live_override, list_decks, save_deck, save_image, save_live_override, save_upload, ) from presentation_static import get_static_deck, list_static_decks from storage_s3 import router as storage_s3_router from hdfs_api import router as hdfs_router from pipeline_ops import router as pipeline_router from hadoop_analytics import router as hadoop_router from elasticsearch_api import router as elasticsearch_router from sql_console import router as sql_router from agent_ops import router as agent_ops_router, agent_dml_loop, etl_agent_loop, custodian_offload_loop from agent_activity import agent_activity_loop from streaming_ops import connector_autoheal_loop from cdc_consumer import router as cdc_router, cdc_consumer_loop from movements import router as movements_router from movements import MOVEMENT_BY_ID, trigger_and_watch from dataflow import router as dataflow_router from streaming_ops import router as streaming_router from spark_workbench import router as spark_workbench_router from pii_catalog import router as pii_router from trino_federated import router as federated_router from etl_offload import router as etl_offload_router from lineage import router as lineage_router from dq_monitor import router as dq_router from observability import router as observability_router from catalog_governance import router as governance_router from ssh_terminal import ssh_session from node_registry import NODE_IDS, NODE_AGENT, NODE_REGISTRY, is_node_id from node_ops import build_node_detail, probe_node, run_node_probe_task from approval_service import ( APPROVAL_ACTION_TYPES, approval_stats, create_approval_request, decide_approval_request, detect_agent_proposed_action, detect_approval_intent, list_approvals, ) from db import SessionLocal, db_health, init_database from supervisor import mirror_terminal_line, mirror_to_supervisors from workload import build_workload_payload from gpu_config import ( get_gpu_config, get_gpu_config_payload, get_gpu_urls, reset_gpu_config, save_gpu_config, test_gpu_target, ) _workload_cache: dict[str, Any] = {"ts": 0.0, "data": None} _presentation_cache: dict[str, Any] = {"ts": 0.0, "data": None} WORKLOAD_CACHE_TTL = 30.0 PRESENTATION_CACHE_TTL = 45.0 from pydantic import BaseModel, Field from sqlalchemy import Column, DateTime, String, Text, select from sqlalchemy.orm import DeclarativeBase REDIS_URL = os.getenv("REDIS_URL", "redis://redis:6379/0") DOCKHAND_URL = os.getenv("DOCKHAND_URL", "http://10.0.21.45:8082") DOCKHAND_API_TOKEN = os.getenv("DOCKHAND_API_TOKEN", "") def _dockhand_headers() -> dict[str, str]: if DOCKHAND_API_TOKEN: return {"Authorization": f"Bearer {DOCKHAND_API_TOKEN}"} return {} GPU_URL = os.getenv("GPU_URL", "http://10.0.10.106:9000") GPU_UI_URL = os.getenv("GPU_UI_URL", GPU_URL) LLM_URL = os.getenv("LLM_URL", "http://10.0.10.106:8001/v1") LLM_MODEL = os.getenv("LLM_MODEL", "gpt-4o") LLM_API_KEY = os.getenv("LLM_API_KEY", "sk-local") LLM_TIMEOUT = float(os.getenv("LLM_TIMEOUT", "120")) # Llama-3-70B GPTQ on V100 is capped at 4096; keep a hard safety budget. LLM_MAX_MODEL_LEN = int(os.getenv("LLM_MAX_MODEL_LEN", "4096")) # Conservative estimate: Llama tokenizers often use ~2.2–2.8 chars/token on English+lab text. LLM_CHARS_PER_TOKEN = float(os.getenv("LLM_CHARS_PER_TOKEN", "2.4")) LLM_CONTEXT_MARGIN = int(os.getenv("LLM_CONTEXT_MARGIN", "160")) LLM_MAX_OUTPUT = int(os.getenv("LLM_MAX_OUTPUT", "256")) LLM_MAX_CONTEXT_CHARS = int(os.getenv("LLM_MAX_CONTEXT_CHARS", "5500")) def _estimate_tokens(text: str) -> int: # Slightly inflate so we never underestimate vs vLLM's tokenizer. return max(1, int(len(text) / LLM_CHARS_PER_TOKEN) + 32) def _compact_lab_context(context: str) -> str: """Keep primary section + short per-domain summaries; drop verbose inventory lines.""" lines = context.splitlines() out: list[str] = [] in_full = False detail = 0 max_detail = 4 for line in lines: if line.startswith("=== FULL LAB"): in_full = True out.append(line) continue if line.startswith("=== PRIMARY"): in_full = False detail = 0 out.append(line) continue if line.startswith("--- "): detail = 0 out.append(line) continue # Drop long platform-capabilities essays if present — keep a one-liner marker if line.startswith("=== PLATFORM CAPABILITIES"): out.append(line) out.append(" (see Command Center UI for full feature list)") continue if out and out[-1].startswith(" (see Command Center"): if line.startswith("===") or line.startswith("--- ") or line.startswith("=== AGENTS") or line.startswith("=== DATA MASKING") or line.startswith("=== PII MASKING"): pass else: continue # Always keep masking / PII evidence sections in full (demo-critical) if line.startswith("=== DATA MASKING") or line.startswith("=== PII MASKING"): # flush remaining lines of this section without detail limits by marking out.append(line) continue # Limit bullet detail in general lab dump, but keep masking evidence intact keep_full = any(s in "\n".join(out[-5:]) for s in ("=== DATA MASKING", "=== PII MASKING")) if (line.startswith(" - ") or line.startswith(" - ")) and not keep_full: detail += 1 if detail > max_detail: if detail == max_detail + 1: out.append(" …") continue out.append(line) return "\n".join(out) def _truncate_for_llm(context: str, max_chars: int) -> str: context = _compact_lab_context(context) if len(context) <= max_chars: return context # Prefer keeping PRIMARY section; cut FULL LAB first primary_end = context.find("=== FULL LAB") if primary_end > 200: head = context[:primary_end].rstrip() tail_budget = max(400, max_chars - len(head) - 80) tail = context[primary_end: primary_end + tail_budget] trimmed = head + "\n" + tail else: trimmed = context[:max_chars] if len(trimmed) > max_chars: trimmed = trimmed[: max_chars - 60].rsplit("\n", 1)[0] if len(context) > len(trimmed): trimmed += f"\n\n[… truncated for {LLM_MAX_MODEL_LEN}-token model window …]" return trimmed def _fit_llm_payload(system_rules: str, context: str, user_message: str) -> tuple[str, int]: """Fit prompt+completion into the served model length with a safety margin.""" user_tok = _estimate_tokens(user_message) rules_tok = _estimate_tokens(system_rules) budget = LLM_MAX_MODEL_LEN - LLM_CONTEXT_MARGIN max_out = min(LLM_MAX_OUTPUT, 256) # Absolute char cap first (independent of estimate errors) context = _truncate_for_llm(context, LLM_MAX_CONTEXT_CHARS) for _ in range(6): ctx_budget_tok = budget - user_tok - rules_tok - max_out if ctx_budget_tok < 200: max_out = max(64, max_out // 2) continue ctx_max_chars = max(600, int(ctx_budget_tok * LLM_CHARS_PER_TOKEN * 0.85)) fitted = _truncate_for_llm(context, ctx_max_chars) total = rules_tok + _estimate_tokens(fitted) + user_tok + max_out if total <= budget: return fitted, max_out # Still too big — shrink context harder, then output context = fitted LLM_MAX = max(800, int(len(fitted) * 0.7)) context = _truncate_for_llm(context, LLM_MAX) max_out = max(64, max_out - 32) return _truncate_for_llm(context, 800), 64 AGENTS = [ { "id": "etl-guardian", "name": "ETL Guardian", "color": "#00f0ff", "zone": "etl", "role": "Airflow, Kafka, Debezium, S3 pipeline", "icon": "⚡", "motto": "Pipelines never sleep", "capabilities": ["Airflow", "Kafka", "Debezium", "S3", "Connectors"], "suggested_prompts": [ "How is Debezium doing?", "Are all Airflow DAGs healthy?", "Kafka connector status?", ], }, { "id": "lakehouse-ops", "name": "Lakehouse Ops", "color": "#ff00aa", "zone": "lakehouse", "role": "Spark, Trino, Iceberg", "icon": "🏔️", "motto": "Query the lake, trust the table", "capabilities": ["Spark", "Trino", "Iceberg", "Delta", "SQL"], "suggested_prompts": [ "Lakehouse stack status?", "Is Trino reachable?", "How many lakehouse containers are running?", ], }, { "id": "data-custodian", "name": "Data Custodian", "color": "#ffaa00", "zone": "db", "role": "PostgreSQL, MySQL, Mongo, Cassandra, Neo4j", "icon": "🛡️", "motto": "Guardian of every row", "capabilities": ["PostgreSQL", "MySQL", "MongoDB", "Cassandra", "Neo4j"], "suggested_prompts": [ "How much data is in the databases?", "What is in PostgreSQL sales_orders?", "MongoDB supplychain overview", ], }, { "id": "hadoop-ranger", "name": "Hadoop Ranger", "color": "#39ff14", "zone": "hadoop", "role": "HDFS, YARN cluster", "icon": "🌲", "motto": "Patrol the data forest", "capabilities": ["HDFS", "YARN", "NameNode", "DataNodes"], "suggested_prompts": [ "Is HDFS NameNode up?", "Hadoop cluster status?", "YARN nodes healthy?", ], }, { "id": "infra-sentinel", "name": "Infra Sentinel", "color": "#9b72cf", "zone": "docker", "role": "Docker, Proxmox, GPU, monitoring", "icon": "👁️", "motto": "See everything, miss nothing", "capabilities": ["Docker", "Proxmox", "GPU", "vLLM", "Monitoring"], "suggested_prompts": [ "GPU status?", "Which LLM model is running?", "Docker container overview", ], }, { "id": "mo-commander", "name": "Mo · Command", "color": "#4c9aed", "zone": "command", "role": "Supervisor — full event intel, ingress, approvals", "icon": "🎯", "motto": "Nothing happens without Mo knowing", "supervisor": True, "person": "mo", "capabilities": ["Events", "Ingress", "Approvals", "Agent dispatch", "Network IN"], "suggested_prompts": [ "What happened today?", "What events came in?", "Pipeline status overview", ], }, { "id": "bart-commander", "name": "Bart · Ops", "color": "#3fb950", "zone": "command", "role": "Supervisor — egress, MCP comms, network OUT", "icon": "📡", "motto": "All traffic flows through Bart", "supervisor": True, "person": "bart", "capabilities": ["Egress", "MCP routing", "Network OUT", "GPU inference", "S3 writes"], "suggested_prompts": [ "What is leaving the cluster?", "MCP agent communication status?", "Network egress overview", ], }, { "id": "network-watcher", "name": "Network Watcher", "color": "#58a6ff", "zone": "network", "role": "VLAN 20/21 traffic, data in & out paths", "icon": "🌐", "motto": "Every packet tells a story", "capabilities": ["VLAN 20", "VLAN 21", "Ingress", "Egress", "Firewall paths"], "suggested_prompts": [ "Data ingress status?", "What leaves the cluster?", "Network path to S3?", ], }, { "id": "mcp-coordinator", "name": "MCP Coordinator", "color": "#f778ba", "zone": "mcp", "role": "MCP hub — routes all agent tool calls & comms", "icon": "🔀", "motto": "Route once, deliver everywhere", "capabilities": ["MCP servers", "Tool routing", "Agent relay", "WebSocket bus"], "suggested_prompts": [ "Which MCP agents are active?", "MCP hub route status?", "Agent communication overview", ], }, ] ZONES = [ {"id": "docker", "label": "DOCKER RACK", "x": 8, "color": "#b366ff"}, {"id": "db", "label": "DB VAULT", "x": 28, "color": "#ffaa00"}, {"id": "lakehouse", "label": "LAKEHOUSE HUB", "x": 50, "color": "#ff00aa"}, {"id": "hadoop", "label": "HADOOP CLUSTER", "x": 72, "color": "#39ff14"}, {"id": "etl", "label": "ETL PIPE", "x": 92, "color": "#00f0ff"}, ] INTENT_KEYWORDS: dict[str, list[str]] = { "data-custodian": [ "database", "postgres", "postgresql", "mysql", "mongo", "mongodb", "cassandra", "neo4j", "sql", "db ", "pii", "mask", "masked", "masking", "email", "e-mail", "phone", "iban", "address", "customer", "employee", "gdpr", "privacy", "sensitive", "personal", "name", "ssn", "national_id", ], "lakehouse-ops": ["trino", "spark", "lakehouse", "iceberg", "query", "table"], "hadoop-ranger": [ "hadoop", "hdfs", "yarn", "datanode", "namenode", "replicatie", "replication", "rf factor", "opslag", "bestanden", "blocks", "cluster opslag", "data op", ], "infra-sentinel": ["docker", "container", "vm", "proxmox", "infra", "grafana", "gpu", "vllm", "llm", "nvidia", "inference", "model"], "etl-guardian": ["airflow", "dag", "debezium", "kafka", "connector", "etl", "pipeline", "s3"], "network-watcher": ["network", "vlan", "ingress", "egress", "traffic", "packet", "firewall", "route"], "mcp-coordinator": ["mcp", "tool", "router", "relay", "websocket", "hub"], "mo-commander": ["mo", "supervisor", "events", "overzicht", "alles", "gebeurd"], "bart-commander": ["bart", "egress", "uitgaand", "communicatie", "mcp comm"], } class Base(DeclarativeBase): pass class FeedEntry(Base): __tablename__ = "feed" id = Column(String, primary_key=True) ts = Column(DateTime, default=lambda: datetime.now(timezone.utc)) agent_id = Column(String) level = Column(String, default="info") message = Column(Text) class Approval(Base): __tablename__ = "approvals" id = Column(String, primary_key=True) ts = Column(DateTime, default=lambda: datetime.now(timezone.utc)) agent_id = Column(String) action = Column(Text) reason = Column(Text) status = Column(String, default="pending") action_type = Column(String, default="generic.mutate") target = Column(Text, default="") payload = Column(Text, default="{}") decided_by = Column(String, nullable=True) decide_note = Column(Text, nullable=True) decided_at = Column(DateTime, nullable=True) priority = Column(String, default="normal") _db_info = init_database(Base) redis_client: aioredis.Redis | None = None ws_clients: set[WebSocket] = set() class PromptRequest(BaseModel): message: str = Field(min_length=1, max_length=2000) agent_id: str | None = None class NodeAskRequest(BaseModel): message: str = Field(min_length=1, max_length=2000) class ApprovalCreateRequest(BaseModel): agent_id: str = Field(min_length=1, max_length=64) action: str = Field(min_length=1, max_length=2000) reason: str = Field(min_length=1, max_length=2000) action_type: str = "generic.mutate" target: str = "" payload: dict[str, Any] | None = None priority: str = "normal" class ApprovalDecision(BaseModel): approved: bool decided_by: str = "mo-commander" note: str = "" def route_agent(message: str) -> str: lower = message.lower() # PII / privacy questions always go to Data Custodian (masking demo path) pii_words = ( "pii", "mask", "masked", "masking", "email", "e-mail", "mail adres", "mail address", "phone", "telefoon", "iban", "address", "adres", "customer name", "employee", "gdpr", "privacy", "sensitive", "personal", "national_id", "ssn", "gevoelig", ) if any(w in lower for w in pii_words): return "data-custodian" # Storage/data questions default to Hadoop unless clearly about databases if any(w in lower for w in ("opslag", "gb", "replicatie", "replication", "hdfs", "hadoop", "datanode", "namenode")): if not any(w in lower for w in ("postgres", "mysql", "mongo", "database", "sql", "neo4j", "cassandra", "pii", "email")): return "hadoop-ranger" if any(w in lower for w in ("gpu", "vllm", "llm", "nvidia", "inference", "vram", "model")): return "infra-sentinel" if any(w in lower for w in ("kafka", "airflow", "debezium", "connector", "etl", "pipeline", "dag")): return "etl-guardian" if any(w in lower for w in ("trino", "spark", "iceberg", "lakehouse")): return "lakehouse-ops" scores = {aid: sum(1 for kw in kws if kw in lower) for aid, kws in INTENT_KEYWORDS.items()} best = max(scores, key=scores.get) if scores[best] == 0: return "infra-sentinel" return best def _is_pii_question(message: str) -> bool: lower = message.lower() return any(w in lower for w in ( "pii", "mask", "masked", "masking", "unmask", "visible", "email", "e-mail", "phone", "iban", "address", "adres", "customer", "employee", "privacy", "gdpr", "sensitive", "personal", "gevoelig", "name", "telefoon", "mail", "data flow", "national_id", "ssn", "bsn", "geboorte", "birth", )) async def _pii_evidence_block(message: str, log: Any | None = None) -> str: """Live policy + samples synced with Data Flow masking toggles.""" try: from pii_catalog import build_policy_evidence return build_policy_evidence() except Exception as exc: return f"=== PII MASKING EVIDENCE ===\n(unavailable: {exc})" async def gather_agent_context( agent_id: str, status: dict[str, Any], log: Any | None = None, message: str | None = None, ) -> str: """Full lab snapshot for vLLM — all domains, agent's primary domain highlighted.""" snapshot = await collect_full_lab_context(gpu_data=status.get("gpu"), log=log, include_inventory=False) snapshot["domains_summary"] = status.get("domains", {}) ctx = format_context_for_agent(agent_id, snapshot) agent_lines = ["", "=== AGENTS & SUPERVISORS ==="] for a in AGENTS: sup = " [supervisor]" if a.get("supervisor") else "" agent_lines.append(f" - {a['name']} ({a['id']}){sup}: {a['role']}") ctx = ctx + "\n".join(agent_lines) try: from platform_context import build_masking_section # Fresh policy so chat mirrors Data Flow toggles (skip huge business catalog). ctx = ctx + "\n\n" + build_masking_section(fresh=True) except Exception: try: from platform_context import build_llm_addendum ctx = ctx + "\n\n" + build_llm_addendum() except Exception: pass if message and _is_pii_question(message): try: evidence = await _pii_evidence_block(message, log=log) ctx = ctx + "\n\n" + evidence if log: await log("ok", "fetch", "▸ PII masking evidence attached (synced with Data Flow)") except Exception as exc: if log: await log("warn", "fetch", f"▸ PII evidence skipped: {exc}") if log: await log("info", "fetch", f"▸ Context assembled: {len(ctx)} chars for LLM") return ctx async def ask_llm( agent_id: str, message: str, context: str, log: Any | None = None, ) -> str | None: agent = next(a for a in AGENTS if a["id"] == agent_id) # Deterministic PII path: always mirror Data Flow masked vs visible toggles. if _is_pii_question(message): try: from pii_catalog import format_pii_chat_answer answer = format_pii_chat_answer(message) if log: await log("ok", "pii", "▸ Returning Data Flow–synced masking answer (masked + visible)") return answer except Exception as exc: if log: await log("warn", "pii", f"▸ PII answer builder failed: {exc}") rules = f"""You are {agent['name']} ({agent['role']}) in the Dell ATC data lab. Answer in English, briefly (max ~8 sentences). Use ONLY the live data below — never invent hosts/ports/numbers. If data is missing or DOWN, say so. PII / masking rules (critical — synced with Data Flow tab): - MASKED columns: NEVER reveal raw values; quote the token 🔒 MASKED when present. - VISIBLE columns (operator opted out in Data Flow): you MAY report the real sample values and say they are visible by policy. - Never invent emails, phones, names, IBANs, or addresses that are not in the live samples. - If asked what is masked vs visible, list columns from the DATA MASKING POLICY / PII EVIDENCE sections. --- LIVE LAB DATA ---""" fitted_ctx, max_tokens = _fit_llm_payload(rules, context, message) system = rules + "\n" + fitted_ctx urls = get_gpu_urls() llm_url = urls["llm_url"] if log: est = _estimate_tokens(system) + _estimate_tokens(message) await log("info", "llm", f"▸ Querying vLLM model={LLM_MODEL} @ {urls['host']} (~{est}+{max_tokens} tok)") if len(context) > len(fitted_ctx): await log("warn", "llm", f" context trimmed {len(context)} → {len(fitted_ctx)} chars") await log("cmd", "llm", f"$ POST {llm_url.rstrip('/')}/chat/completions") await log("info", "llm", f" user: {message[:160]}{'…' if len(message) > 160 else ''}") try: async with httpx.AsyncClient(timeout=LLM_TIMEOUT) as client: t0 = time.monotonic() r = await client.post( f"{llm_url.rstrip('/')}/chat/completions", headers={ "Authorization": f"Bearer {LLM_API_KEY}", "Content-Type": "application/json", }, json={ "model": LLM_MODEL, "messages": [ {"role": "system", "content": system}, {"role": "user", "content": message}, ], "max_tokens": max_tokens, "temperature": 0.25, }, ) r.raise_for_status() content = r.json()["choices"][0]["message"]["content"].strip() ms = int((time.monotonic() - t0) * 1000) if content and content.strip("!"): if log: await log("ok", "llm", f"← vLLM response {len(content)} chars ({ms}ms)") preview = content.replace("\n", " ")[:180] await log("info", "llm", f" » {preview}{'…' if len(content) > 180 else ''}") return content if log: await log("warn", "llm", f"← Empty or invalid LLM output ({ms}ms)") except httpx.HTTPStatusError as exc: detail = exc.response.text[:200] if exc.response is not None else str(exc) if log: await log("err", "llm", f"✗ vLLM HTTP {exc.response.status_code}: {detail}") # One hard retry with a minimal context if we blew the window. if exc.response is not None and exc.response.status_code == 400 and "maximum context length" in detail: tiny = _truncate_for_llm(context, 1200) system2 = rules + "\n" + tiny max2 = 128 if log: await log("warn", "llm", f" retry with tiny context ({len(tiny)} chars, max_tokens={max2})") try: async with httpx.AsyncClient(timeout=LLM_TIMEOUT) as client: r2 = await client.post( f"{llm_url.rstrip('/')}/chat/completions", headers={ "Authorization": f"Bearer {LLM_API_KEY}", "Content-Type": "application/json", }, json={ "model": LLM_MODEL, "messages": [ {"role": "system", "content": system2}, {"role": "user", "content": message}, ], "max_tokens": max2, "temperature": 0.25, }, ) r2.raise_for_status() content2 = r2.json()["choices"][0]["message"]["content"].strip() if content2: if log: await log("ok", "llm", f"← vLLM retry OK {len(content2)} chars") return content2 except Exception as exc2: if log: await log("err", "llm", f"✗ vLLM retry failed: {exc2}") except Exception as exc: if log: await log("err", "llm", f"✗ vLLM error: {exc}") return None def fallback_answer(agent_id: str, context: str, user_message: str = "") -> str: agent_name = next(a["name"] for a in AGENTS if a["id"] == agent_id) if user_message and _is_pii_question(user_message): try: from pii_catalog import format_pii_chat_answer return f"**{agent_name}**\n\n" + format_pii_chat_answer(user_message) except Exception: marker = "=== PII MASKING EVIDENCE" if marker in context: return ( f"**{agent_name}** — masking policy (synced with Data Flow):\n\n" + context[context.index(marker):].strip() ) preview_lines: list[str] = [] for line in context.splitlines(): if line.startswith(("=== PRIMARY", "Health summary", "ATC Lab", "--- ")): preview_lines.append(line) if len(preview_lines) >= 14: break hint = "\n".join(preview_lines) if preview_lines else "Lab snapshot collected; LLM unavailable." q = f"\n\nYour question: _{user_message[:200]}_" if user_message else "" return ( f"**{agent_name}** — I could not get a reply from the GPU LLM " f"(context window or vLLM error).{q}\n\n" "Try a short, specific question " "(e.g. *How many GPUs are online?* or *Is Kafka healthy?*).\n\n" f"Quick snapshot:\n{hint}" ) async def publish_event(event: dict[str, Any]) -> None: payload = json.dumps(event, default=str) if redis_client: await redis_client.publish("ops", payload) dead = [] for ws in ws_clients: try: await ws.send_text(payload) except Exception: dead.append(ws) for ws in dead: ws_clients.discard(ws) et = event.get("type") if et == "feed": entry = event.get("entry") or {} await mirror_to_supervisors( entry.get("agent_id", "?"), entry.get("message", ""), level=entry.get("level", "info"), ) elif et == "terminal": await mirror_terminal_line(event.get("line") or {}) elif et in ("agent_dispatch", "agent_fetch", "agent_return"): aid = event.get("agent_id", "?") zone = event.get("zone", "") await mirror_to_supervisors(aid, f"{et} → zone {zone}", level="info", phase="dispatch") def add_feed(agent_id: str, message: str, level: str = "info") -> dict: entry_id = str(uuid.uuid4())[:8] with SessionLocal() as db: row = FeedEntry(id=entry_id, agent_id=agent_id, message=message, level=level) db.add(row) db.commit() return { "id": entry_id, "ts": datetime.now(timezone.utc).isoformat(), "agent_id": agent_id, "message": message, "level": level, } async def dockhand_env_containers(env_id: int) -> list[dict]: try: async with httpx.AsyncClient(timeout=8.0) as client: r = await client.get( f"{DOCKHAND_URL}/api/containers", params={"env": env_id}, headers=_dockhand_headers(), ) r.raise_for_status() return r.json() except Exception: return [] async def probe_url(url: str) -> bool: try: async with httpx.AsyncClient(timeout=4.0, verify=False) as client: r = await client.get(url) return r.status_code < 500 except Exception: return False async def collect_gpu() -> dict[str, Any]: urls = get_gpu_urls() cfg = get_gpu_config() gpu_url = urls["gpu_url"] host = urls["host"] base = { "ok": False, "host": host, "ip": host, "ui_url": urls["gpu_ui_url"], "config_source": cfg.get("source", "env"), "preset_id": cfg.get("preset_id"), "config_label": cfg.get("label"), } try: async with httpx.AsyncClient(timeout=6.0) as client: metrics_r, model_r, integration_r = await asyncio.gather( client.get(f"{gpu_url}/api/gpu/metrics"), client.get(f"{gpu_url}/api/active-model"), client.get(f"{gpu_url}/api/integration"), return_exceptions=True, ) gpus: list[dict[str, Any]] = [] if isinstance(metrics_r, httpx.Response) and metrics_r.status_code == 200: current = metrics_r.json().get("current", {}) gpus = [ { "index": g["index"], "name": g["name"], "util_gpu": g.get("util_gpu", 0), "memory_used_mib": g.get("memory_used_mib", 0), "memory_total_mib": g.get("memory_total_mib", 0), "temperature_c": g.get("temperature_c", 0), "power_w": g.get("power_w", 0), } for g in current.get("gpus", []) ] active_model = None inference_active = False vllm_url = None if isinstance(model_r, httpx.Response) and model_r.status_code == 200: model_data = model_r.json() active_model = model_data.get("name") inference_active = bool(model_data.get("inference_active")) vllm_url = model_data.get("base_url") if isinstance(integration_r, httpx.Response) and integration_r.status_code == 200: integ = integration_r.json() if not active_model: active_model = integ.get("active_name") if not inference_active: inference_active = bool(integ.get("inference_active")) if not vllm_url: vllm_url = integ.get("recommended_base_url") return { **base, "ok": len(gpus) > 0 or inference_active, "inference_active": inference_active, "active_model": active_model, "vllm_url": vllm_url, "gpu_count": len(gpus), "gpus": gpus, } except Exception as exc: return {**base, "error": str(exc)} async def collect_status() -> dict[str, Any]: db_containers = await dockhand_env_containers(5) db_running = sum(1 for c in db_containers if c.get("state") == "running") db_total = len(db_containers) or 6 lake_containers = await dockhand_env_containers(9) lake_running = sum(1 for c in lake_containers if c.get("state") == "running") lake_total = len(lake_containers) or 6 docker_containers = await dockhand_env_containers(1) docker_running = sum(1 for c in docker_containers if c.get("state") == "running") hdfs_ok = await probe_url("http://10.0.21.61:9870") kafka_ok = await probe_url("http://10.0.21.36:9000") airflow_ok = await probe_url("http://10.0.21.55:8080") def level(running: int, total: int) -> str: if total == 0: return "unknown" ratio = running / total if ratio >= 0.9: return "ok" if ratio >= 0.5: return "warn" return "down" gpu = await collect_gpu() gpu_level = "ok" if gpu.get("ok") and gpu.get("inference_active") else ("warn" if gpu.get("ok") else "down") gpu_label = gpu.get("active_model") or (f"{gpu.get('gpu_count', 0)} GPUs" if gpu.get("ok") else "offline") return { "ts": datetime.now(timezone.utc).isoformat(), "domains": { "docker": {"level": "ok" if docker_running >= 5 else "warn", "label": f"{docker_running} containers", "running": docker_running}, "databases": {"level": level(db_running, db_total), "label": f"{db_running}/{db_total} up", "running": db_running, "total": db_total}, "lakehouse": {"level": level(lake_running, lake_total), "label": f"{lake_running}/{lake_total} up", "running": lake_running, "total": lake_total}, "hadoop": {"level": "ok" if hdfs_ok else "warn", "label": "NN up" if hdfs_ok else "NN check"}, "etl": {"level": "ok" if kafka_ok and airflow_ok else "warn", "label": "Kafka+Airflow"}, "gpu": {"level": gpu_level, "label": gpu_label}, }, "gpu": gpu, "kafka_ok": kafka_ok, "airflow_ok": airflow_ok, "hdfs_ok": hdfs_ok, } async def _run_agent_task_safe(agent_id: str, message: str, prompt_id: str) -> None: try: await run_agent_task(agent_id, message, prompt_id) except Exception as exc: agent_name = next((a["name"] for a in AGENTS if a["id"] == agent_id), agent_id) err = f"Sorry — {agent_name} could not complete your request: {exc}" await terminal_log(agent_id, f"[{prompt_id}] ✗ Error: {exc}", level="err", phase="error", prompt_id=prompt_id) feed = add_feed(agent_id, f"{agent_name} failed: {str(exc)[:80]}", "err") await publish_event({"type": "feed", "entry": feed}) await publish_event({ "type": "prompt_result", "prompt_id": prompt_id, "agent_id": agent_id, "answer": err, }) async def run_agent_task(agent_id: str, message: str, prompt_id: str) -> str: zone = next(a["zone"] for a in AGENTS if a["id"] == agent_id) agent_name = next(a["name"] for a in AGENTS if a["id"] == agent_id) log = make_logger(agent_id, prompt_id) approval_created = False intent = detect_approval_intent(message) if intent: with SessionLocal() as db: await create_approval_request( db=db, ApprovalModel=Approval, agent_id=agent_id, action=intent["action"], reason=intent["reason"], action_type=intent["action_type"], terminal_log=terminal_log, mirror_supervisors=mirror_to_supervisors, publish=publish_event, add_feed=add_feed, ) approval_created = True await terminal_log( agent_id, f"[{prompt_id}] Mutating request detected — approval queued for Mo & Bart", level="warn", phase="approval", prompt_id=prompt_id, ) await terminal_log( agent_id, f"[{prompt_id}] ▶ Mission accepted: {message}", level="info", phase="dispatch", prompt_id=prompt_id, ) await publish_event({"type": "agent_dispatch", "agent_id": agent_id, "zone": zone, "prompt_id": prompt_id}) await asyncio.sleep(0.4) await terminal_log(agent_id, f"[{prompt_id}] Walking to zone: {zone}", level="info", phase="dispatch", prompt_id=prompt_id) await publish_event({"type": "agent_fetch", "agent_id": agent_id, "zone": zone, "prompt_id": prompt_id}) await log("info", "fetch", f"[{prompt_id}] Collecting live lab metrics…") status = await collect_status() context = await gather_agent_context(agent_id, status, log=log, message=message) answer = await ask_llm(agent_id, message, context, log=log) if not answer: await log("warn", "llm", "LLM fallback — returning raw context") answer = fallback_answer(agent_id, context, message) if not approval_created: proposed = detect_agent_proposed_action(answer, message) if proposed: with SessionLocal() as db: await create_approval_request( db=db, ApprovalModel=Approval, agent_id=agent_id, action=proposed["action"], reason=proposed["reason"], action_type=proposed["action_type"], target=proposed.get("target", ""), terminal_log=terminal_log, mirror_supervisors=mirror_to_supervisors, publish=publish_event, add_feed=add_feed, ) approval_created = True answer = ( f"{answer}\n\n⏸ **Approval required** — this action is in the Approval Inbox. " f"Mo & Bart have been notified and must approve before we execute." ) await asyncio.sleep(0.3) await terminal_log(agent_id, f"[{prompt_id}] ✓ Mission complete", level="ok", phase="done", prompt_id=prompt_id) await publish_event({"type": "agent_return", "agent_id": agent_id, "zone": zone, "prompt_id": prompt_id}) feed = add_feed(agent_id, f"{agent_name} completed a response (see Comms)", "info") await publish_event({"type": "feed", "entry": feed}) await publish_event({"type": "prompt_result", "prompt_id": prompt_id, "agent_id": agent_id, "answer": answer}) return answer async def heartbeat_loop() -> None: while True: try: status = await collect_status() workload = await collect_workload() await publish_event({"type": "status", "data": status}) await publish_event({"type": "workload", "data": workload}) for domain, info in status["domains"].items(): if info["level"] == "down": agent = "data-custodian" if domain == "databases" else "infra-sentinel" feed = add_feed(agent, f"Alert: {domain} is DOWN ({info['label']})", "warn") await publish_event({"type": "feed", "entry": feed}) except Exception as exc: await publish_event({"type": "error", "message": str(exc)}) await asyncio.sleep(60) @asynccontextmanager async def lifespan(app: FastAPI): global redis_client redis_client = aioredis.from_url(REDIS_URL, decode_responses=True) set_terminal_publisher(publish_event) init_terminals([a["id"] for a in AGENTS] + NODE_IDS) for a in AGENTS: await terminal_log(a["id"], f"{a['name']} terminal online — awaiting missions", level="info", phase="boot") for nid in NODE_IDS: if nid not in NODE_REGISTRY: continue meta = NODE_REGISTRY[nid] await terminal_log(nid, f"{meta['label']} shell ready — click node to connect", level="info", phase="boot") task = asyncio.create_task(heartbeat_loop()) dml_task = asyncio.create_task(agent_dml_loop()) cdc_task = asyncio.create_task(cdc_consumer_loop()) etl_task = asyncio.create_task(etl_agent_loop()) cust_task = asyncio.create_task(custodian_offload_loop()) act_task = asyncio.create_task(agent_activity_loop()) heal_task = asyncio.create_task(connector_autoheal_loop()) add_feed("infra-sentinel", "ATC Command Center API online", "info") yield task.cancel() dml_task.cancel() cdc_task.cancel() etl_task.cancel() cust_task.cancel() act_task.cancel() heal_task.cancel() if redis_client: await redis_client.close() app = FastAPI(title="ATC Command Center API", lifespan=lifespan) app.include_router(storage_s3_router) app.include_router(hdfs_router) app.include_router(pipeline_router) app.include_router(hadoop_router) app.include_router(elasticsearch_router) app.include_router(sql_router) app.include_router(agent_ops_router) app.include_router(cdc_router) app.include_router(movements_router) app.include_router(dataflow_router) app.include_router(streaming_router) app.include_router(spark_workbench_router) app.include_router(pii_router) app.include_router(federated_router) app.include_router(etl_offload_router) app.include_router(lineage_router) app.include_router(dq_router) app.include_router(observability_router) app.include_router(governance_router) app.add_middleware( CORSMiddleware, allow_origins=["*"], allow_credentials=True, allow_methods=["*"], allow_headers=["*"], ) # Authentik OIDC session + API guard cockpit_auth.init_auth_middleware(app) cockpit_auth.setup_auth(app) @app.get("/api/health") async def health(): llm_ok = False try: async with httpx.AsyncClient(timeout=4.0) as client: r = await client.get(f"{LLM_URL.rstrip('/')}/models", headers={"Authorization": f"Bearer {LLM_API_KEY}"}) llm_ok = r.status_code == 200 except Exception: pass return { "ok": True, "ts": datetime.now(timezone.utc).isoformat(), "llm_url": LLM_URL, "llm_ok": llm_ok, "llm_model": LLM_MODEL, "database": db_health(), "db_init": _db_info, } async def collect_workload(*, fast: bool = True, use_cache: bool = True) -> dict[str, Any]: import time as _time now = _time.time() if use_cache and _workload_cache.get("data") and now - float(_workload_cache.get("ts") or 0) < WORKLOAD_CACHE_TTL: return _workload_cache["data"] gpu = await collect_gpu() snap = await collect_full_lab_context(gpu_data=gpu, include_inventory=not fast) payload = build_workload_payload(snap) _workload_cache["ts"] = now _workload_cache["data"] = payload return payload async def get_presentation_data(*, use_cache: bool = True) -> dict[str, Any]: import time as _time now = _time.time() if use_cache and _presentation_cache.get("data") and now - float(_presentation_cache.get("ts") or 0) < PRESENTATION_CACHE_TTL: return _presentation_cache["data"] gpu = await collect_gpu() snap = await collect_full_lab_context(gpu_data=gpu, include_inventory=False) data = build_presentation_payload(snap) override = get_live_override() if override and override.get("slides"): data["title"] = override.get("title") or data.get("title") data["subtitle"] = override.get("subtitle") or data.get("subtitle", "") data["slides"] = override["slides"] data["slide_count"] = len(override["slides"]) data["edited"] = True data["override_ts"] = override.get("ts") else: data["edited"] = False data["source"] = "live" data["id"] = "live" _presentation_cache["ts"] = now _presentation_cache["data"] = data return data @app.get("/api/presentation") async def get_presentation(): return await get_presentation_data() @app.get("/api/presentation/html") async def get_presentation_html(): from fastapi.responses import HTMLResponse payload = await get_presentation_data() return HTMLResponse(render_presentation_html(payload)) @app.get("/api/presentation/decks") async def get_presentation_decks(): return {"live": True, "builtin": list_static_decks(), "uploaded": list_decks()} @app.get("/api/presentation/decks/{deck_id}") async def get_presentation_deck(deck_id: str): if deck_id == "live": return await get_presentation_data() deck = get_static_deck(deck_id) or get_deck(deck_id) if not deck: return {"error": "deck not found"} return deck @app.get("/api/presentation/decks/{deck_id}/html") async def get_presentation_deck_html(deck_id: str): from fastapi.responses import HTMLResponse if deck_id == "live": payload = await get_presentation_data() else: payload = get_static_deck(deck_id) or get_deck(deck_id) if not payload: return HTMLResponse("

Deck not found

", status_code=404) return HTMLResponse(render_presentation_html(payload)) @app.post("/api/presentation/upload") async def upload_presentation(file: UploadFile = File(...)): content = await file.read() if len(content) > 50 * 1024 * 1024: return {"error": "file too large (max 50MB)"} deck = await save_upload(file.filename or "upload.pptx", content) return {"ok": True, "deck": deck} @app.post("/api/presentation/decks/new") async def create_presentation_deck(body: dict[str, Any] | None = Body(default=None)): body = body or {} title = str(body.get("title") or "Untitled deck") seed_id = body.get("from") slides = None if seed_id: src = get_static_deck(seed_id) or get_deck(seed_id) if seed_id == "live": src = await get_presentation_data() if src and src.get("slides"): slides = [ { "id": s.get("id") or f"slide-{i}", "title": s.get("title") or f"Slide {i}", "subtitle": s.get("subtitle") or "", "bullets": list(s.get("bullets") or []), "image": s.get("image") or "", "kind": s.get("kind") or "narrative", **({"animation": s["animation"]} if s.get("animation") else {}), } for i, s in enumerate(src["slides"], start=1) ] if not title or title == "Untitled deck": title = f"{src.get('title', 'Deck')} (copy)" deck = create_deck(title, slides) return {"ok": True, "deck": deck} @app.put("/api/presentation/decks/{deck_id}") async def update_presentation_deck(deck_id: str, body: dict[str, Any] = Body(...)): if deck_id == "live": save_live_override(body) _presentation_cache["ts"] = 0 _presentation_cache["data"] = None payload = await get_presentation_data(use_cache=False) return {"ok": True, "deck": payload} deck = save_deck(deck_id, body) if not deck: return {"error": "deck not found or not editable"} return {"ok": True, "deck": deck} @app.post("/api/presentation/live/reset") async def reset_live_presentation(): clear_live_override() _presentation_cache["ts"] = 0 _presentation_cache["data"] = None payload = await get_presentation_data(use_cache=False) return {"ok": True, "deck": payload} @app.delete("/api/presentation/decks/{deck_id}") async def remove_presentation_deck(deck_id: str): return {"ok": delete_deck(deck_id)} @app.post("/api/presentation/decks/{deck_id}/image") async def upload_presentation_image(deck_id: str, file: UploadFile = File(...)): content = await file.read() if len(content) > 12 * 1024 * 1024: return {"error": "image too large (max 12MB)"} result = save_image(deck_id, file.filename or "image.png", content) if not result: return {"error": "deck not found"} return {"ok": True, **result} @app.get("/api/presentation/decks/{deck_id}/assets/{name}") async def get_presentation_asset(deck_id: str, name: str): from fastapi.responses import FileResponse, Response path = get_asset_path(deck_id, name) if not path: return Response(status_code=404) return FileResponse(str(path)) @app.get("/api/workload") async def get_workload(fast: bool = True): return await collect_workload(fast=fast, use_cache=True) @app.get("/api/status") async def get_status(): return await collect_status() @app.get("/api/gpu") async def get_gpu(): return await collect_gpu() @app.get("/api/gpu/config") async def get_gpu_config_endpoint(): return get_gpu_config_payload() @app.post("/api/gpu/config") async def post_gpu_config(body: dict[str, Any]): try: saved = save_gpu_config( preset_id=body.get("preset_id"), host=body.get("host"), gpu_ui_port=body.get("gpu_ui_port"), llm_port=body.get("llm_port"), ) return {"ok": True, "active": saved, "presets": get_gpu_config_payload()["presets"]} except ValueError as exc: return JSONResponse({"ok": False, "detail": str(exc)}, status_code=400) @app.post("/api/gpu/config/test") async def post_gpu_config_test(body: dict[str, Any]): return await test_gpu_target( preset_id=body.get("preset_id"), host=body.get("host"), gpu_ui_port=body.get("gpu_ui_port"), llm_port=body.get("llm_port"), ) @app.delete("/api/gpu/config") async def delete_gpu_config(): active = reset_gpu_config() return {"ok": True, "active": active} def agent_stats() -> dict[str, dict[str, Any]]: stats: dict[str, dict[str, Any]] = {a["id"]: {"tasks": 0, "last_active": None, "alerts": 0} for a in AGENTS} with SessionLocal() as db: rows = db.execute(select(FeedEntry).order_by(FeedEntry.ts.desc()).limit(200)).scalars().all() for r in rows: aid = r.agent_id if aid not in stats: continue stats[aid]["tasks"] += 1 if r.level == "warn": stats[aid]["alerts"] += 1 if stats[aid]["last_active"] is None and r.ts: stats[aid]["last_active"] = r.ts.isoformat() return stats @app.get("/api/agents") async def get_agents(): stats = agent_stats() enriched = [{**a, "stats": stats.get(a["id"], {})} for a in AGENTS] return {"agents": enriched, "zones": ZONES} @app.get("/api/terminals") async def get_terminals(limit: int = 200): return {"terminals": get_all_terminals(limit)} @app.get("/api/terminals/{subject_id}") async def get_subject_terminal(subject_id: str, limit: int = 200): valid_agents = {a["id"] for a in AGENTS} if subject_id not in valid_agents and not is_node_id(subject_id): return {"error": "unknown subject"} return {"agent_id": subject_id, "lines": get_terminal_lines(subject_id, limit)} @app.get("/api/nodes") async def list_nodes(): workload = await collect_workload() nodes = workload.get("topology", {}).get("nodes", []) return {"nodes": [{"id": n["id"], "label": n["label"], "ip": n["ip"], "level": n["level"]} for n in nodes]} @app.get("/api/nodes/{node_id}") async def get_node(node_id: str): if not is_node_id(node_id): return {"error": "unknown node"} workload = await collect_workload() wn = next((n for n in workload.get("topology", {}).get("nodes", []) if n["id"] == node_id), None) gpu = await collect_gpu() snap = await collect_full_lab_context(gpu_data=gpu) return build_node_detail(node_id, snap, wn) @app.post("/api/nodes/{node_id}/probe") async def post_node_probe(node_id: str): if not is_node_id(node_id): return {"error": "unknown node"} asyncio.create_task(run_node_probe_task(node_id)) return {"ok": True, "node_id": node_id, "status": "probing"} async def run_node_ask_task(node_id: str, message: str) -> None: agent_id = NODE_AGENT.get(node_id, "infra-sentinel") meta = NODE_REGISTRY[node_id] await terminal_log(node_id, f"▶ Query: {message}", level="info", phase="ask") await terminal_log(node_id, f"→ Routing to agent {agent_id}", level="info", phase="ask") log = make_logger(node_id) status = await collect_status() context = await gather_agent_context(agent_id, status, log=log, message=message) node_ctx = f"\n\n=== FOCUSED NODE: {meta['label']} ({meta['ip']}) ===\n{meta.get('description', '')}\n" answer = await ask_llm(agent_id, message, context + node_ctx, log=log) if not answer: answer = fallback_answer(agent_id, context, message) await terminal_log(node_id, f"◆ {answer}", level="llm", phase="answer") await publish_event({"type": "node_ask_result", "node_id": node_id, "agent_id": agent_id, "answer": answer}) @app.post("/api/nodes/{node_id}/ask") async def post_node_ask(node_id: str, body: NodeAskRequest): if not is_node_id(node_id): return {"error": "unknown node"} asyncio.create_task(run_node_ask_task(node_id, body.message)) return {"ok": True, "node_id": node_id, "agent_id": NODE_AGENT.get(node_id), "status": "processing"} @app.get("/api/feed") async def get_feed(limit: int = 50): with SessionLocal() as db: rows = db.execute(select(FeedEntry).order_by(FeedEntry.ts.desc()).limit(limit)).scalars().all() return { "entries": [ { "id": r.id, "ts": r.ts.isoformat() if r.ts else None, "agent_id": r.agent_id, "message": r.message, "level": r.level, } for r in rows ] } @app.get("/api/approvals") async def get_approvals(status: str = "pending", limit: int = 100): with SessionLocal() as db: items = list_approvals(db, Approval, status=status, limit=limit) stats = approval_stats(db, Approval) return {"approvals": items, "stats": stats, "action_types": APPROVAL_ACTION_TYPES} @app.get("/api/approvals/stats") async def get_approval_stats(): with SessionLocal() as db: return approval_stats(db, Approval) @app.post("/api/approvals") async def post_approval(body: ApprovalCreateRequest): valid_ids = {a["id"] for a in AGENTS} if body.agent_id not in valid_ids: return {"error": "unknown agent_id"} if body.action_type not in APPROVAL_ACTION_TYPES: body.action_type = "generic.mutate" with SessionLocal() as db: item = await create_approval_request( db=db, ApprovalModel=Approval, agent_id=body.agent_id, action=body.action, reason=body.reason, action_type=body.action_type, target=body.target, payload=body.payload, priority=body.priority, terminal_log=terminal_log, mirror_supervisors=mirror_to_supervisors, publish=publish_event, add_feed=add_feed, ) return {"ok": True, "approval": item} @app.post("/api/approvals/{approval_id}/decide") async def decide_approval(approval_id: str, body: ApprovalDecision): valid_supervisors = {"mo-commander", "bart-commander"} decided_by = body.decided_by if body.decided_by in valid_supervisors else "mo-commander" with SessionLocal() as db: item = await decide_approval_request( db=db, ApprovalModel=Approval, approval_id=approval_id, approved=body.approved, decided_by=decided_by, note=body.note, terminal_log=terminal_log, publish=publish_event, add_feed=add_feed, ) if not item: return {"error": "not found"} # Approval-gated executor: run the underlying action once a supervisor approves it. if item.get("status") == "approved": payload = item.get("payload") or {} if isinstance(payload, dict) and payload.get("executor") == "movement": mid = payload.get("movement_id") if mid in MOVEMENT_BY_ID: add_feed("etl-guardian", f"Approval {approval_id} granted — executing movement '{mid}'", "info") asyncio.create_task(trigger_and_watch(mid, payload.get("conf"))) return {"ok": True, "approval": item} @app.post("/api/prompt") async def post_prompt(body: PromptRequest): prompt_id = str(uuid.uuid4())[:8] valid_ids = {a["id"] for a in AGENTS} agent_id = body.agent_id if body.agent_id in valid_ids else route_agent(body.message) add_feed(agent_id, f"Prompt received: {body.message}", "info") asyncio.create_task(_run_agent_task_safe(agent_id, body.message, prompt_id)) return {"prompt_id": prompt_id, "agent_id": agent_id, "status": "dispatched"} @app.websocket("/api/ws/ssh") async def ws_ssh(websocket: WebSocket): await ssh_session(websocket) @app.websocket("/api/ws/ops") async def ws_ops(websocket: WebSocket): await websocket.accept() ws_clients.add(websocket) try: status = await collect_status() workload = await collect_workload() await websocket.send_text(json.dumps({"type": "status", "data": status}, default=str)) await websocket.send_text(json.dumps({"type": "workload", "data": workload}, default=str)) await websocket.send_text(json.dumps({ "type": "terminal_history", "terminals": get_all_terminals(150), }, default=str)) while True: await websocket.receive_text() except WebSocketDisconnect: pass finally: ws_clients.discard(websocket)