mo
9008fbd512
Add OIDC auth for Command Center and runtime GPU endpoint selection pointed at atc-gpu-prod (10.0.10.106), matching what is currently deployed.
1503 lines
57 KiB
Python
1503 lines
57 KiB
Python
"""ATC Command Center API — FastAPI backend."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import json
|
||
import os
|
||
import time
|
||
import uuid
|
||
from contextlib import asynccontextmanager
|
||
from datetime import datetime, timezone
|
||
from typing import Any
|
||
|
||
import httpx
|
||
import redis.asyncio as aioredis
|
||
from fastapi import Body, FastAPI, File, UploadFile, WebSocket, WebSocketDisconnect
|
||
from fastapi.middleware.cors import CORSMiddleware
|
||
import auth as cockpit_auth
|
||
from agent_terminal import (
|
||
get_all_terminals,
|
||
get_terminal_lines,
|
||
init_terminals,
|
||
make_logger,
|
||
set_terminal_publisher,
|
||
terminal_log,
|
||
)
|
||
from lab_context import collect_full_lab_context, format_context_for_agent
|
||
from presentation import build_presentation_payload, render_presentation_html
|
||
from presentation_upload import (
|
||
clear_live_override,
|
||
create_deck,
|
||
delete_deck,
|
||
get_asset_path,
|
||
get_deck,
|
||
get_live_override,
|
||
list_decks,
|
||
save_deck,
|
||
save_image,
|
||
save_live_override,
|
||
save_upload,
|
||
)
|
||
from presentation_static import get_static_deck, list_static_decks
|
||
from storage_s3 import router as storage_s3_router
|
||
from hdfs_api import router as hdfs_router
|
||
from pipeline_ops import router as pipeline_router
|
||
from hadoop_analytics import router as hadoop_router
|
||
from elasticsearch_api import router as elasticsearch_router
|
||
from sql_console import router as sql_router
|
||
from agent_ops import router as agent_ops_router, agent_dml_loop, etl_agent_loop, custodian_offload_loop
|
||
from agent_activity import agent_activity_loop
|
||
from streaming_ops import connector_autoheal_loop
|
||
from cdc_consumer import router as cdc_router, cdc_consumer_loop
|
||
from movements import router as movements_router
|
||
from movements import MOVEMENT_BY_ID, trigger_and_watch
|
||
from dataflow import router as dataflow_router
|
||
from streaming_ops import router as streaming_router
|
||
from spark_workbench import router as spark_workbench_router
|
||
from pii_catalog import router as pii_router
|
||
from trino_federated import router as federated_router
|
||
from etl_offload import router as etl_offload_router
|
||
from lineage import router as lineage_router
|
||
from dq_monitor import router as dq_router
|
||
from observability import router as observability_router
|
||
from catalog_governance import router as governance_router
|
||
from ssh_terminal import ssh_session
|
||
from node_registry import NODE_IDS, NODE_AGENT, NODE_REGISTRY, is_node_id
|
||
from node_ops import build_node_detail, probe_node, run_node_probe_task
|
||
from approval_service import (
|
||
APPROVAL_ACTION_TYPES,
|
||
approval_stats,
|
||
create_approval_request,
|
||
decide_approval_request,
|
||
detect_agent_proposed_action,
|
||
detect_approval_intent,
|
||
list_approvals,
|
||
)
|
||
from db import SessionLocal, db_health, init_database
|
||
from supervisor import mirror_terminal_line, mirror_to_supervisors
|
||
|
||
from workload import build_workload_payload
|
||
from gpu_config import (
|
||
get_gpu_config,
|
||
get_gpu_config_payload,
|
||
get_gpu_urls,
|
||
reset_gpu_config,
|
||
save_gpu_config,
|
||
test_gpu_target,
|
||
)
|
||
|
||
_workload_cache: dict[str, Any] = {"ts": 0.0, "data": None}
|
||
_presentation_cache: dict[str, Any] = {"ts": 0.0, "data": None}
|
||
WORKLOAD_CACHE_TTL = 30.0
|
||
PRESENTATION_CACHE_TTL = 45.0
|
||
|
||
from pydantic import BaseModel, Field
|
||
from sqlalchemy import Column, DateTime, String, Text, select
|
||
from sqlalchemy.orm import DeclarativeBase
|
||
|
||
REDIS_URL = os.getenv("REDIS_URL", "redis://redis:6379/0")
|
||
DOCKHAND_URL = os.getenv("DOCKHAND_URL", "http://10.0.21.45:8082")
|
||
DOCKHAND_API_TOKEN = os.getenv("DOCKHAND_API_TOKEN", "")
|
||
|
||
|
||
def _dockhand_headers() -> dict[str, str]:
|
||
if DOCKHAND_API_TOKEN:
|
||
return {"Authorization": f"Bearer {DOCKHAND_API_TOKEN}"}
|
||
return {}
|
||
|
||
GPU_URL = os.getenv("GPU_URL", "http://10.0.10.106:9000")
|
||
GPU_UI_URL = os.getenv("GPU_UI_URL", GPU_URL)
|
||
LLM_URL = os.getenv("LLM_URL", "http://10.0.10.106:8001/v1")
|
||
LLM_MODEL = os.getenv("LLM_MODEL", "gpt-4o")
|
||
LLM_API_KEY = os.getenv("LLM_API_KEY", "sk-local")
|
||
LLM_TIMEOUT = float(os.getenv("LLM_TIMEOUT", "120"))
|
||
# Llama-3-70B GPTQ on V100 is capped at 4096; keep a hard safety budget.
|
||
LLM_MAX_MODEL_LEN = int(os.getenv("LLM_MAX_MODEL_LEN", "4096"))
|
||
# Conservative estimate: Llama tokenizers often use ~2.2–2.8 chars/token on English+lab text.
|
||
LLM_CHARS_PER_TOKEN = float(os.getenv("LLM_CHARS_PER_TOKEN", "2.4"))
|
||
LLM_CONTEXT_MARGIN = int(os.getenv("LLM_CONTEXT_MARGIN", "160"))
|
||
LLM_MAX_OUTPUT = int(os.getenv("LLM_MAX_OUTPUT", "256"))
|
||
LLM_MAX_CONTEXT_CHARS = int(os.getenv("LLM_MAX_CONTEXT_CHARS", "5500"))
|
||
|
||
|
||
def _estimate_tokens(text: str) -> int:
|
||
# Slightly inflate so we never underestimate vs vLLM's tokenizer.
|
||
return max(1, int(len(text) / LLM_CHARS_PER_TOKEN) + 32)
|
||
|
||
|
||
def _compact_lab_context(context: str) -> str:
|
||
"""Keep primary section + short per-domain summaries; drop verbose inventory lines."""
|
||
lines = context.splitlines()
|
||
out: list[str] = []
|
||
in_full = False
|
||
detail = 0
|
||
max_detail = 4
|
||
for line in lines:
|
||
if line.startswith("=== FULL LAB"):
|
||
in_full = True
|
||
out.append(line)
|
||
continue
|
||
if line.startswith("=== PRIMARY"):
|
||
in_full = False
|
||
detail = 0
|
||
out.append(line)
|
||
continue
|
||
if line.startswith("--- "):
|
||
detail = 0
|
||
out.append(line)
|
||
continue
|
||
# Drop long platform-capabilities essays if present — keep a one-liner marker
|
||
if line.startswith("=== PLATFORM CAPABILITIES"):
|
||
out.append(line)
|
||
out.append(" (see Command Center UI for full feature list)")
|
||
continue
|
||
if out and out[-1].startswith(" (see Command Center"):
|
||
if line.startswith("===") or line.startswith("--- ") or line.startswith("=== AGENTS") or line.startswith("=== DATA MASKING") or line.startswith("=== PII MASKING"):
|
||
pass
|
||
else:
|
||
continue
|
||
# Always keep masking / PII evidence sections in full (demo-critical)
|
||
if line.startswith("=== DATA MASKING") or line.startswith("=== PII MASKING"):
|
||
# flush remaining lines of this section without detail limits by marking
|
||
out.append(line)
|
||
continue
|
||
# Limit bullet detail in general lab dump, but keep masking evidence intact
|
||
keep_full = any(s in "\n".join(out[-5:]) for s in ("=== DATA MASKING", "=== PII MASKING"))
|
||
if (line.startswith(" - ") or line.startswith(" - ")) and not keep_full:
|
||
detail += 1
|
||
if detail > max_detail:
|
||
if detail == max_detail + 1:
|
||
out.append(" …")
|
||
continue
|
||
out.append(line)
|
||
return "\n".join(out)
|
||
|
||
|
||
def _truncate_for_llm(context: str, max_chars: int) -> str:
|
||
context = _compact_lab_context(context)
|
||
if len(context) <= max_chars:
|
||
return context
|
||
# Prefer keeping PRIMARY section; cut FULL LAB first
|
||
primary_end = context.find("=== FULL LAB")
|
||
if primary_end > 200:
|
||
head = context[:primary_end].rstrip()
|
||
tail_budget = max(400, max_chars - len(head) - 80)
|
||
tail = context[primary_end: primary_end + tail_budget]
|
||
trimmed = head + "\n" + tail
|
||
else:
|
||
trimmed = context[:max_chars]
|
||
if len(trimmed) > max_chars:
|
||
trimmed = trimmed[: max_chars - 60].rsplit("\n", 1)[0]
|
||
if len(context) > len(trimmed):
|
||
trimmed += f"\n\n[… truncated for {LLM_MAX_MODEL_LEN}-token model window …]"
|
||
return trimmed
|
||
|
||
|
||
def _fit_llm_payload(system_rules: str, context: str, user_message: str) -> tuple[str, int]:
|
||
"""Fit prompt+completion into the served model length with a safety margin."""
|
||
user_tok = _estimate_tokens(user_message)
|
||
rules_tok = _estimate_tokens(system_rules)
|
||
budget = LLM_MAX_MODEL_LEN - LLM_CONTEXT_MARGIN
|
||
max_out = min(LLM_MAX_OUTPUT, 256)
|
||
|
||
# Absolute char cap first (independent of estimate errors)
|
||
context = _truncate_for_llm(context, LLM_MAX_CONTEXT_CHARS)
|
||
|
||
for _ in range(6):
|
||
ctx_budget_tok = budget - user_tok - rules_tok - max_out
|
||
if ctx_budget_tok < 200:
|
||
max_out = max(64, max_out // 2)
|
||
continue
|
||
ctx_max_chars = max(600, int(ctx_budget_tok * LLM_CHARS_PER_TOKEN * 0.85))
|
||
fitted = _truncate_for_llm(context, ctx_max_chars)
|
||
total = rules_tok + _estimate_tokens(fitted) + user_tok + max_out
|
||
if total <= budget:
|
||
return fitted, max_out
|
||
# Still too big — shrink context harder, then output
|
||
context = fitted
|
||
LLM_MAX = max(800, int(len(fitted) * 0.7))
|
||
context = _truncate_for_llm(context, LLM_MAX)
|
||
max_out = max(64, max_out - 32)
|
||
|
||
return _truncate_for_llm(context, 800), 64
|
||
|
||
|
||
AGENTS = [
|
||
{
|
||
"id": "etl-guardian",
|
||
"name": "ETL Guardian",
|
||
"color": "#00f0ff",
|
||
"zone": "etl",
|
||
"role": "Airflow, Kafka, Debezium, S3 pipeline",
|
||
"icon": "⚡",
|
||
"motto": "Pipelines never sleep",
|
||
"capabilities": ["Airflow", "Kafka", "Debezium", "S3", "Connectors"],
|
||
"suggested_prompts": [
|
||
"How is Debezium doing?",
|
||
"Are all Airflow DAGs healthy?",
|
||
"Kafka connector status?",
|
||
],
|
||
},
|
||
{
|
||
"id": "lakehouse-ops",
|
||
"name": "Lakehouse Ops",
|
||
"color": "#ff00aa",
|
||
"zone": "lakehouse",
|
||
"role": "Spark, Trino, Iceberg",
|
||
"icon": "🏔️",
|
||
"motto": "Query the lake, trust the table",
|
||
"capabilities": ["Spark", "Trino", "Iceberg", "Delta", "SQL"],
|
||
"suggested_prompts": [
|
||
"Lakehouse stack status?",
|
||
"Is Trino reachable?",
|
||
"How many lakehouse containers are running?",
|
||
],
|
||
},
|
||
{
|
||
"id": "data-custodian",
|
||
"name": "Data Custodian",
|
||
"color": "#ffaa00",
|
||
"zone": "db",
|
||
"role": "PostgreSQL, MySQL, Mongo, Cassandra, Neo4j",
|
||
"icon": "🛡️",
|
||
"motto": "Guardian of every row",
|
||
"capabilities": ["PostgreSQL", "MySQL", "MongoDB", "Cassandra", "Neo4j"],
|
||
"suggested_prompts": [
|
||
"How much data is in the databases?",
|
||
"What is in PostgreSQL sales_orders?",
|
||
"MongoDB supplychain overview",
|
||
],
|
||
},
|
||
{
|
||
"id": "hadoop-ranger",
|
||
"name": "Hadoop Ranger",
|
||
"color": "#39ff14",
|
||
"zone": "hadoop",
|
||
"role": "HDFS, YARN cluster",
|
||
"icon": "🌲",
|
||
"motto": "Patrol the data forest",
|
||
"capabilities": ["HDFS", "YARN", "NameNode", "DataNodes"],
|
||
"suggested_prompts": [
|
||
"Is HDFS NameNode up?",
|
||
"Hadoop cluster status?",
|
||
"YARN nodes healthy?",
|
||
],
|
||
},
|
||
{
|
||
"id": "infra-sentinel",
|
||
"name": "Infra Sentinel",
|
||
"color": "#9b72cf",
|
||
"zone": "docker",
|
||
"role": "Docker, Proxmox, GPU, monitoring",
|
||
"icon": "👁️",
|
||
"motto": "See everything, miss nothing",
|
||
"capabilities": ["Docker", "Proxmox", "GPU", "vLLM", "Monitoring"],
|
||
"suggested_prompts": [
|
||
"GPU status?",
|
||
"Which LLM model is running?",
|
||
"Docker container overview",
|
||
],
|
||
},
|
||
{
|
||
"id": "mo-commander",
|
||
"name": "Mo · Command",
|
||
"color": "#4c9aed",
|
||
"zone": "command",
|
||
"role": "Supervisor — full event intel, ingress, approvals",
|
||
"icon": "🎯",
|
||
"motto": "Nothing happens without Mo knowing",
|
||
"supervisor": True,
|
||
"person": "mo",
|
||
"capabilities": ["Events", "Ingress", "Approvals", "Agent dispatch", "Network IN"],
|
||
"suggested_prompts": [
|
||
"What happened today?",
|
||
"What events came in?",
|
||
"Pipeline status overview",
|
||
],
|
||
},
|
||
{
|
||
"id": "bart-commander",
|
||
"name": "Bart · Ops",
|
||
"color": "#3fb950",
|
||
"zone": "command",
|
||
"role": "Supervisor — egress, MCP comms, network OUT",
|
||
"icon": "📡",
|
||
"motto": "All traffic flows through Bart",
|
||
"supervisor": True,
|
||
"person": "bart",
|
||
"capabilities": ["Egress", "MCP routing", "Network OUT", "GPU inference", "S3 writes"],
|
||
"suggested_prompts": [
|
||
"What is leaving the cluster?",
|
||
"MCP agent communication status?",
|
||
"Network egress overview",
|
||
],
|
||
},
|
||
{
|
||
"id": "network-watcher",
|
||
"name": "Network Watcher",
|
||
"color": "#58a6ff",
|
||
"zone": "network",
|
||
"role": "VLAN 20/21 traffic, data in & out paths",
|
||
"icon": "🌐",
|
||
"motto": "Every packet tells a story",
|
||
"capabilities": ["VLAN 20", "VLAN 21", "Ingress", "Egress", "Firewall paths"],
|
||
"suggested_prompts": [
|
||
"Data ingress status?",
|
||
"What leaves the cluster?",
|
||
"Network path to S3?",
|
||
],
|
||
},
|
||
{
|
||
"id": "mcp-coordinator",
|
||
"name": "MCP Coordinator",
|
||
"color": "#f778ba",
|
||
"zone": "mcp",
|
||
"role": "MCP hub — routes all agent tool calls & comms",
|
||
"icon": "🔀",
|
||
"motto": "Route once, deliver everywhere",
|
||
"capabilities": ["MCP servers", "Tool routing", "Agent relay", "WebSocket bus"],
|
||
"suggested_prompts": [
|
||
"Which MCP agents are active?",
|
||
"MCP hub route status?",
|
||
"Agent communication overview",
|
||
],
|
||
},
|
||
]
|
||
|
||
ZONES = [
|
||
{"id": "docker", "label": "DOCKER RACK", "x": 8, "color": "#b366ff"},
|
||
{"id": "db", "label": "DB VAULT", "x": 28, "color": "#ffaa00"},
|
||
{"id": "lakehouse", "label": "LAKEHOUSE HUB", "x": 50, "color": "#ff00aa"},
|
||
{"id": "hadoop", "label": "HADOOP CLUSTER", "x": 72, "color": "#39ff14"},
|
||
{"id": "etl", "label": "ETL PIPE", "x": 92, "color": "#00f0ff"},
|
||
]
|
||
|
||
INTENT_KEYWORDS: dict[str, list[str]] = {
|
||
"data-custodian": [
|
||
"database", "postgres", "postgresql", "mysql", "mongo", "mongodb", "cassandra", "neo4j", "sql", "db ",
|
||
"pii", "mask", "masked", "masking", "email", "e-mail", "phone", "iban", "address", "customer",
|
||
"employee", "gdpr", "privacy", "sensitive", "personal", "name", "ssn", "national_id",
|
||
],
|
||
"lakehouse-ops": ["trino", "spark", "lakehouse", "iceberg", "query", "table"],
|
||
"hadoop-ranger": [
|
||
"hadoop", "hdfs", "yarn", "datanode", "namenode", "replicatie", "replication",
|
||
"rf factor", "opslag", "bestanden", "blocks", "cluster opslag", "data op",
|
||
],
|
||
"infra-sentinel": ["docker", "container", "vm", "proxmox", "infra", "grafana", "gpu", "vllm", "llm", "nvidia", "inference", "model"],
|
||
"etl-guardian": ["airflow", "dag", "debezium", "kafka", "connector", "etl", "pipeline", "s3"],
|
||
"network-watcher": ["network", "vlan", "ingress", "egress", "traffic", "packet", "firewall", "route"],
|
||
"mcp-coordinator": ["mcp", "tool", "router", "relay", "websocket", "hub"],
|
||
"mo-commander": ["mo", "supervisor", "events", "overzicht", "alles", "gebeurd"],
|
||
"bart-commander": ["bart", "egress", "uitgaand", "communicatie", "mcp comm"],
|
||
}
|
||
|
||
|
||
class Base(DeclarativeBase):
|
||
pass
|
||
|
||
|
||
class FeedEntry(Base):
|
||
__tablename__ = "feed"
|
||
|
||
id = Column(String, primary_key=True)
|
||
ts = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
||
agent_id = Column(String)
|
||
level = Column(String, default="info")
|
||
message = Column(Text)
|
||
|
||
|
||
class Approval(Base):
|
||
__tablename__ = "approvals"
|
||
|
||
id = Column(String, primary_key=True)
|
||
ts = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
||
agent_id = Column(String)
|
||
action = Column(Text)
|
||
reason = Column(Text)
|
||
status = Column(String, default="pending")
|
||
action_type = Column(String, default="generic.mutate")
|
||
target = Column(Text, default="")
|
||
payload = Column(Text, default="{}")
|
||
decided_by = Column(String, nullable=True)
|
||
decide_note = Column(Text, nullable=True)
|
||
decided_at = Column(DateTime, nullable=True)
|
||
priority = Column(String, default="normal")
|
||
|
||
|
||
_db_info = init_database(Base)
|
||
|
||
redis_client: aioredis.Redis | None = None
|
||
ws_clients: set[WebSocket] = set()
|
||
|
||
|
||
class PromptRequest(BaseModel):
|
||
message: str = Field(min_length=1, max_length=2000)
|
||
agent_id: str | None = None
|
||
|
||
|
||
class NodeAskRequest(BaseModel):
|
||
message: str = Field(min_length=1, max_length=2000)
|
||
|
||
|
||
class ApprovalCreateRequest(BaseModel):
|
||
agent_id: str = Field(min_length=1, max_length=64)
|
||
action: str = Field(min_length=1, max_length=2000)
|
||
reason: str = Field(min_length=1, max_length=2000)
|
||
action_type: str = "generic.mutate"
|
||
target: str = ""
|
||
payload: dict[str, Any] | None = None
|
||
priority: str = "normal"
|
||
|
||
|
||
class ApprovalDecision(BaseModel):
|
||
approved: bool
|
||
decided_by: str = "mo-commander"
|
||
note: str = ""
|
||
|
||
|
||
def route_agent(message: str) -> str:
|
||
lower = message.lower()
|
||
# PII / privacy questions always go to Data Custodian (masking demo path)
|
||
pii_words = (
|
||
"pii", "mask", "masked", "masking", "email", "e-mail", "mail adres", "mail address",
|
||
"phone", "telefoon", "iban", "address", "adres", "customer name", "employee",
|
||
"gdpr", "privacy", "sensitive", "personal", "national_id", "ssn", "gevoelig",
|
||
)
|
||
if any(w in lower for w in pii_words):
|
||
return "data-custodian"
|
||
# Storage/data questions default to Hadoop unless clearly about databases
|
||
if any(w in lower for w in ("opslag", "gb", "replicatie", "replication", "hdfs", "hadoop", "datanode", "namenode")):
|
||
if not any(w in lower for w in ("postgres", "mysql", "mongo", "database", "sql", "neo4j", "cassandra", "pii", "email")):
|
||
return "hadoop-ranger"
|
||
if any(w in lower for w in ("gpu", "vllm", "llm", "nvidia", "inference", "vram", "model")):
|
||
return "infra-sentinel"
|
||
if any(w in lower for w in ("kafka", "airflow", "debezium", "connector", "etl", "pipeline", "dag")):
|
||
return "etl-guardian"
|
||
if any(w in lower for w in ("trino", "spark", "iceberg", "lakehouse")):
|
||
return "lakehouse-ops"
|
||
scores = {aid: sum(1 for kw in kws if kw in lower) for aid, kws in INTENT_KEYWORDS.items()}
|
||
best = max(scores, key=scores.get)
|
||
if scores[best] == 0:
|
||
return "infra-sentinel"
|
||
return best
|
||
|
||
|
||
|
||
def _is_pii_question(message: str) -> bool:
|
||
lower = message.lower()
|
||
return any(w in lower for w in (
|
||
"pii", "mask", "masked", "masking", "unmask", "visible", "email", "e-mail", "phone",
|
||
"iban", "address", "adres", "customer", "employee", "privacy", "gdpr", "sensitive",
|
||
"personal", "gevoelig", "name", "telefoon", "mail", "data flow", "national_id",
|
||
"ssn", "bsn", "geboorte", "birth",
|
||
))
|
||
|
||
|
||
async def _pii_evidence_block(message: str, log: Any | None = None) -> str:
|
||
"""Live policy + samples synced with Data Flow masking toggles."""
|
||
try:
|
||
from pii_catalog import build_policy_evidence
|
||
return build_policy_evidence()
|
||
except Exception as exc:
|
||
return f"=== PII MASKING EVIDENCE ===\n(unavailable: {exc})"
|
||
|
||
|
||
async def gather_agent_context(
|
||
agent_id: str,
|
||
status: dict[str, Any],
|
||
log: Any | None = None,
|
||
message: str | None = None,
|
||
) -> str:
|
||
"""Full lab snapshot for vLLM — all domains, agent's primary domain highlighted."""
|
||
snapshot = await collect_full_lab_context(gpu_data=status.get("gpu"), log=log, include_inventory=False)
|
||
snapshot["domains_summary"] = status.get("domains", {})
|
||
ctx = format_context_for_agent(agent_id, snapshot)
|
||
agent_lines = ["", "=== AGENTS & SUPERVISORS ==="]
|
||
for a in AGENTS:
|
||
sup = " [supervisor]" if a.get("supervisor") else ""
|
||
agent_lines.append(f" - {a['name']} ({a['id']}){sup}: {a['role']}")
|
||
ctx = ctx + "\n".join(agent_lines)
|
||
try:
|
||
from platform_context import build_masking_section
|
||
# Fresh policy so chat mirrors Data Flow toggles (skip huge business catalog).
|
||
ctx = ctx + "\n\n" + build_masking_section(fresh=True)
|
||
except Exception:
|
||
try:
|
||
from platform_context import build_llm_addendum
|
||
ctx = ctx + "\n\n" + build_llm_addendum()
|
||
except Exception:
|
||
pass
|
||
if message and _is_pii_question(message):
|
||
try:
|
||
evidence = await _pii_evidence_block(message, log=log)
|
||
ctx = ctx + "\n\n" + evidence
|
||
if log:
|
||
await log("ok", "fetch", "▸ PII masking evidence attached (synced with Data Flow)")
|
||
except Exception as exc:
|
||
if log:
|
||
await log("warn", "fetch", f"▸ PII evidence skipped: {exc}")
|
||
if log:
|
||
await log("info", "fetch", f"▸ Context assembled: {len(ctx)} chars for LLM")
|
||
return ctx
|
||
|
||
|
||
async def ask_llm(
|
||
agent_id: str,
|
||
message: str,
|
||
context: str,
|
||
log: Any | None = None,
|
||
) -> str | None:
|
||
agent = next(a for a in AGENTS if a["id"] == agent_id)
|
||
# Deterministic PII path: always mirror Data Flow masked vs visible toggles.
|
||
if _is_pii_question(message):
|
||
try:
|
||
from pii_catalog import format_pii_chat_answer
|
||
answer = format_pii_chat_answer(message)
|
||
if log:
|
||
await log("ok", "pii", "▸ Returning Data Flow–synced masking answer (masked + visible)")
|
||
return answer
|
||
except Exception as exc:
|
||
if log:
|
||
await log("warn", "pii", f"▸ PII answer builder failed: {exc}")
|
||
rules = f"""You are {agent['name']} ({agent['role']}) in the Dell ATC data lab.
|
||
Answer in English, briefly (max ~8 sentences). Use ONLY the live data below — never invent hosts/ports/numbers.
|
||
If data is missing or DOWN, say so.
|
||
|
||
PII / masking rules (critical — synced with Data Flow tab):
|
||
- MASKED columns: NEVER reveal raw values; quote the token 🔒 MASKED when present.
|
||
- VISIBLE columns (operator opted out in Data Flow): you MAY report the real sample values and say they are visible by policy.
|
||
- Never invent emails, phones, names, IBANs, or addresses that are not in the live samples.
|
||
- If asked what is masked vs visible, list columns from the DATA MASKING POLICY / PII EVIDENCE sections.
|
||
|
||
--- LIVE LAB DATA ---"""
|
||
fitted_ctx, max_tokens = _fit_llm_payload(rules, context, message)
|
||
system = rules + "\n" + fitted_ctx
|
||
urls = get_gpu_urls()
|
||
llm_url = urls["llm_url"]
|
||
if log:
|
||
est = _estimate_tokens(system) + _estimate_tokens(message)
|
||
await log("info", "llm", f"▸ Querying vLLM model={LLM_MODEL} @ {urls['host']} (~{est}+{max_tokens} tok)")
|
||
if len(context) > len(fitted_ctx):
|
||
await log("warn", "llm", f" context trimmed {len(context)} → {len(fitted_ctx)} chars")
|
||
await log("cmd", "llm", f"$ POST {llm_url.rstrip('/')}/chat/completions")
|
||
await log("info", "llm", f" user: {message[:160]}{'…' if len(message) > 160 else ''}")
|
||
try:
|
||
async with httpx.AsyncClient(timeout=LLM_TIMEOUT) as client:
|
||
t0 = time.monotonic()
|
||
r = await client.post(
|
||
f"{llm_url.rstrip('/')}/chat/completions",
|
||
headers={
|
||
"Authorization": f"Bearer {LLM_API_KEY}",
|
||
"Content-Type": "application/json",
|
||
},
|
||
json={
|
||
"model": LLM_MODEL,
|
||
"messages": [
|
||
{"role": "system", "content": system},
|
||
{"role": "user", "content": message},
|
||
],
|
||
"max_tokens": max_tokens,
|
||
"temperature": 0.25,
|
||
},
|
||
)
|
||
r.raise_for_status()
|
||
content = r.json()["choices"][0]["message"]["content"].strip()
|
||
ms = int((time.monotonic() - t0) * 1000)
|
||
if content and content.strip("!"):
|
||
if log:
|
||
await log("ok", "llm", f"← vLLM response {len(content)} chars ({ms}ms)")
|
||
preview = content.replace("\n", " ")[:180]
|
||
await log("info", "llm", f" » {preview}{'…' if len(content) > 180 else ''}")
|
||
return content
|
||
if log:
|
||
await log("warn", "llm", f"← Empty or invalid LLM output ({ms}ms)")
|
||
except httpx.HTTPStatusError as exc:
|
||
detail = exc.response.text[:200] if exc.response is not None else str(exc)
|
||
if log:
|
||
await log("err", "llm", f"✗ vLLM HTTP {exc.response.status_code}: {detail}")
|
||
# One hard retry with a minimal context if we blew the window.
|
||
if exc.response is not None and exc.response.status_code == 400 and "maximum context length" in detail:
|
||
tiny = _truncate_for_llm(context, 1200)
|
||
system2 = rules + "\n" + tiny
|
||
max2 = 128
|
||
if log:
|
||
await log("warn", "llm", f" retry with tiny context ({len(tiny)} chars, max_tokens={max2})")
|
||
try:
|
||
async with httpx.AsyncClient(timeout=LLM_TIMEOUT) as client:
|
||
r2 = await client.post(
|
||
f"{llm_url.rstrip('/')}/chat/completions",
|
||
headers={
|
||
"Authorization": f"Bearer {LLM_API_KEY}",
|
||
"Content-Type": "application/json",
|
||
},
|
||
json={
|
||
"model": LLM_MODEL,
|
||
"messages": [
|
||
{"role": "system", "content": system2},
|
||
{"role": "user", "content": message},
|
||
],
|
||
"max_tokens": max2,
|
||
"temperature": 0.25,
|
||
},
|
||
)
|
||
r2.raise_for_status()
|
||
content2 = r2.json()["choices"][0]["message"]["content"].strip()
|
||
if content2:
|
||
if log:
|
||
await log("ok", "llm", f"← vLLM retry OK {len(content2)} chars")
|
||
return content2
|
||
except Exception as exc2:
|
||
if log:
|
||
await log("err", "llm", f"✗ vLLM retry failed: {exc2}")
|
||
except Exception as exc:
|
||
if log:
|
||
await log("err", "llm", f"✗ vLLM error: {exc}")
|
||
return None
|
||
|
||
|
||
def fallback_answer(agent_id: str, context: str, user_message: str = "") -> str:
|
||
agent_name = next(a["name"] for a in AGENTS if a["id"] == agent_id)
|
||
if user_message and _is_pii_question(user_message):
|
||
try:
|
||
from pii_catalog import format_pii_chat_answer
|
||
return f"**{agent_name}**\n\n" + format_pii_chat_answer(user_message)
|
||
except Exception:
|
||
marker = "=== PII MASKING EVIDENCE"
|
||
if marker in context:
|
||
return (
|
||
f"**{agent_name}** — masking policy (synced with Data Flow):\n\n"
|
||
+ context[context.index(marker):].strip()
|
||
)
|
||
preview_lines: list[str] = []
|
||
for line in context.splitlines():
|
||
if line.startswith(("=== PRIMARY", "Health summary", "ATC Lab", "--- ")):
|
||
preview_lines.append(line)
|
||
if len(preview_lines) >= 14:
|
||
break
|
||
hint = "\n".join(preview_lines) if preview_lines else "Lab snapshot collected; LLM unavailable."
|
||
q = f"\n\nYour question: _{user_message[:200]}_" if user_message else ""
|
||
return (
|
||
f"**{agent_name}** — I could not get a reply from the GPU LLM "
|
||
f"(context window or vLLM error).{q}\n\n"
|
||
"Try a short, specific question "
|
||
"(e.g. *How many GPUs are online?* or *Is Kafka healthy?*).\n\n"
|
||
f"Quick snapshot:\n{hint}"
|
||
)
|
||
|
||
|
||
async def publish_event(event: dict[str, Any]) -> None:
|
||
payload = json.dumps(event, default=str)
|
||
if redis_client:
|
||
await redis_client.publish("ops", payload)
|
||
dead = []
|
||
for ws in ws_clients:
|
||
try:
|
||
await ws.send_text(payload)
|
||
except Exception:
|
||
dead.append(ws)
|
||
for ws in dead:
|
||
ws_clients.discard(ws)
|
||
|
||
et = event.get("type")
|
||
if et == "feed":
|
||
entry = event.get("entry") or {}
|
||
await mirror_to_supervisors(
|
||
entry.get("agent_id", "?"),
|
||
entry.get("message", ""),
|
||
level=entry.get("level", "info"),
|
||
)
|
||
elif et == "terminal":
|
||
await mirror_terminal_line(event.get("line") or {})
|
||
elif et in ("agent_dispatch", "agent_fetch", "agent_return"):
|
||
aid = event.get("agent_id", "?")
|
||
zone = event.get("zone", "")
|
||
await mirror_to_supervisors(aid, f"{et} → zone {zone}", level="info", phase="dispatch")
|
||
|
||
|
||
def add_feed(agent_id: str, message: str, level: str = "info") -> dict:
|
||
entry_id = str(uuid.uuid4())[:8]
|
||
with SessionLocal() as db:
|
||
row = FeedEntry(id=entry_id, agent_id=agent_id, message=message, level=level)
|
||
db.add(row)
|
||
db.commit()
|
||
return {
|
||
"id": entry_id,
|
||
"ts": datetime.now(timezone.utc).isoformat(),
|
||
"agent_id": agent_id,
|
||
"message": message,
|
||
"level": level,
|
||
}
|
||
|
||
|
||
async def dockhand_env_containers(env_id: int) -> list[dict]:
|
||
try:
|
||
async with httpx.AsyncClient(timeout=8.0) as client:
|
||
r = await client.get(
|
||
f"{DOCKHAND_URL}/api/containers",
|
||
params={"env": env_id},
|
||
headers=_dockhand_headers(),
|
||
)
|
||
r.raise_for_status()
|
||
return r.json()
|
||
except Exception:
|
||
return []
|
||
|
||
|
||
async def probe_url(url: str) -> bool:
|
||
try:
|
||
async with httpx.AsyncClient(timeout=4.0, verify=False) as client:
|
||
r = await client.get(url)
|
||
return r.status_code < 500
|
||
except Exception:
|
||
return False
|
||
|
||
|
||
async def collect_gpu() -> dict[str, Any]:
|
||
urls = get_gpu_urls()
|
||
cfg = get_gpu_config()
|
||
gpu_url = urls["gpu_url"]
|
||
host = urls["host"]
|
||
base = {
|
||
"ok": False,
|
||
"host": host,
|
||
"ip": host,
|
||
"ui_url": urls["gpu_ui_url"],
|
||
"config_source": cfg.get("source", "env"),
|
||
"preset_id": cfg.get("preset_id"),
|
||
"config_label": cfg.get("label"),
|
||
}
|
||
|
||
try:
|
||
async with httpx.AsyncClient(timeout=6.0) as client:
|
||
metrics_r, model_r, integration_r = await asyncio.gather(
|
||
client.get(f"{gpu_url}/api/gpu/metrics"),
|
||
client.get(f"{gpu_url}/api/active-model"),
|
||
client.get(f"{gpu_url}/api/integration"),
|
||
return_exceptions=True,
|
||
)
|
||
|
||
gpus: list[dict[str, Any]] = []
|
||
if isinstance(metrics_r, httpx.Response) and metrics_r.status_code == 200:
|
||
current = metrics_r.json().get("current", {})
|
||
gpus = [
|
||
{
|
||
"index": g["index"],
|
||
"name": g["name"],
|
||
"util_gpu": g.get("util_gpu", 0),
|
||
"memory_used_mib": g.get("memory_used_mib", 0),
|
||
"memory_total_mib": g.get("memory_total_mib", 0),
|
||
"temperature_c": g.get("temperature_c", 0),
|
||
"power_w": g.get("power_w", 0),
|
||
}
|
||
for g in current.get("gpus", [])
|
||
]
|
||
|
||
active_model = None
|
||
inference_active = False
|
||
vllm_url = None
|
||
|
||
if isinstance(model_r, httpx.Response) and model_r.status_code == 200:
|
||
model_data = model_r.json()
|
||
active_model = model_data.get("name")
|
||
inference_active = bool(model_data.get("inference_active"))
|
||
vllm_url = model_data.get("base_url")
|
||
|
||
if isinstance(integration_r, httpx.Response) and integration_r.status_code == 200:
|
||
integ = integration_r.json()
|
||
if not active_model:
|
||
active_model = integ.get("active_name")
|
||
if not inference_active:
|
||
inference_active = bool(integ.get("inference_active"))
|
||
if not vllm_url:
|
||
vllm_url = integ.get("recommended_base_url")
|
||
|
||
return {
|
||
**base,
|
||
"ok": len(gpus) > 0 or inference_active,
|
||
"inference_active": inference_active,
|
||
"active_model": active_model,
|
||
"vllm_url": vllm_url,
|
||
"gpu_count": len(gpus),
|
||
"gpus": gpus,
|
||
}
|
||
except Exception as exc:
|
||
return {**base, "error": str(exc)}
|
||
|
||
|
||
async def collect_status() -> dict[str, Any]:
|
||
db_containers = await dockhand_env_containers(5)
|
||
db_running = sum(1 for c in db_containers if c.get("state") == "running")
|
||
db_total = len(db_containers) or 6
|
||
|
||
lake_containers = await dockhand_env_containers(9)
|
||
lake_running = sum(1 for c in lake_containers if c.get("state") == "running")
|
||
lake_total = len(lake_containers) or 6
|
||
|
||
docker_containers = await dockhand_env_containers(1)
|
||
docker_running = sum(1 for c in docker_containers if c.get("state") == "running")
|
||
|
||
hdfs_ok = await probe_url("http://10.0.21.61:9870")
|
||
kafka_ok = await probe_url("http://10.0.21.36:9000")
|
||
airflow_ok = await probe_url("http://10.0.21.55:8080")
|
||
|
||
def level(running: int, total: int) -> str:
|
||
if total == 0:
|
||
return "unknown"
|
||
ratio = running / total
|
||
if ratio >= 0.9:
|
||
return "ok"
|
||
if ratio >= 0.5:
|
||
return "warn"
|
||
return "down"
|
||
|
||
gpu = await collect_gpu()
|
||
gpu_level = "ok" if gpu.get("ok") and gpu.get("inference_active") else ("warn" if gpu.get("ok") else "down")
|
||
gpu_label = gpu.get("active_model") or (f"{gpu.get('gpu_count', 0)} GPUs" if gpu.get("ok") else "offline")
|
||
|
||
return {
|
||
"ts": datetime.now(timezone.utc).isoformat(),
|
||
"domains": {
|
||
"docker": {"level": "ok" if docker_running >= 5 else "warn", "label": f"{docker_running} containers", "running": docker_running},
|
||
"databases": {"level": level(db_running, db_total), "label": f"{db_running}/{db_total} up", "running": db_running, "total": db_total},
|
||
"lakehouse": {"level": level(lake_running, lake_total), "label": f"{lake_running}/{lake_total} up", "running": lake_running, "total": lake_total},
|
||
"hadoop": {"level": "ok" if hdfs_ok else "warn", "label": "NN up" if hdfs_ok else "NN check"},
|
||
"etl": {"level": "ok" if kafka_ok and airflow_ok else "warn", "label": "Kafka+Airflow"},
|
||
"gpu": {"level": gpu_level, "label": gpu_label},
|
||
},
|
||
"gpu": gpu,
|
||
"kafka_ok": kafka_ok,
|
||
"airflow_ok": airflow_ok,
|
||
"hdfs_ok": hdfs_ok,
|
||
}
|
||
|
||
|
||
|
||
|
||
async def _run_agent_task_safe(agent_id: str, message: str, prompt_id: str) -> None:
|
||
try:
|
||
await run_agent_task(agent_id, message, prompt_id)
|
||
except Exception as exc:
|
||
agent_name = next((a["name"] for a in AGENTS if a["id"] == agent_id), agent_id)
|
||
err = f"Sorry — {agent_name} could not complete your request: {exc}"
|
||
await terminal_log(agent_id, f"[{prompt_id}] ✗ Error: {exc}", level="err", phase="error", prompt_id=prompt_id)
|
||
feed = add_feed(agent_id, f"{agent_name} failed: {str(exc)[:80]}", "err")
|
||
await publish_event({"type": "feed", "entry": feed})
|
||
await publish_event({
|
||
"type": "prompt_result",
|
||
"prompt_id": prompt_id,
|
||
"agent_id": agent_id,
|
||
"answer": err,
|
||
})
|
||
|
||
|
||
async def run_agent_task(agent_id: str, message: str, prompt_id: str) -> str:
|
||
zone = next(a["zone"] for a in AGENTS if a["id"] == agent_id)
|
||
agent_name = next(a["name"] for a in AGENTS if a["id"] == agent_id)
|
||
log = make_logger(agent_id, prompt_id)
|
||
approval_created = False
|
||
|
||
intent = detect_approval_intent(message)
|
||
if intent:
|
||
with SessionLocal() as db:
|
||
await create_approval_request(
|
||
db=db,
|
||
ApprovalModel=Approval,
|
||
agent_id=agent_id,
|
||
action=intent["action"],
|
||
reason=intent["reason"],
|
||
action_type=intent["action_type"],
|
||
terminal_log=terminal_log,
|
||
mirror_supervisors=mirror_to_supervisors,
|
||
publish=publish_event,
|
||
add_feed=add_feed,
|
||
)
|
||
approval_created = True
|
||
await terminal_log(
|
||
agent_id,
|
||
f"[{prompt_id}] Mutating request detected — approval queued for Mo & Bart",
|
||
level="warn",
|
||
phase="approval",
|
||
prompt_id=prompt_id,
|
||
)
|
||
|
||
await terminal_log(
|
||
agent_id,
|
||
f"[{prompt_id}] ▶ Mission accepted: {message}",
|
||
level="info",
|
||
phase="dispatch",
|
||
prompt_id=prompt_id,
|
||
)
|
||
await publish_event({"type": "agent_dispatch", "agent_id": agent_id, "zone": zone, "prompt_id": prompt_id})
|
||
await asyncio.sleep(0.4)
|
||
await terminal_log(agent_id, f"[{prompt_id}] Walking to zone: {zone}", level="info", phase="dispatch", prompt_id=prompt_id)
|
||
await publish_event({"type": "agent_fetch", "agent_id": agent_id, "zone": zone, "prompt_id": prompt_id})
|
||
|
||
await log("info", "fetch", f"[{prompt_id}] Collecting live lab metrics…")
|
||
status = await collect_status()
|
||
context = await gather_agent_context(agent_id, status, log=log, message=message)
|
||
|
||
answer = await ask_llm(agent_id, message, context, log=log)
|
||
if not answer:
|
||
await log("warn", "llm", "LLM fallback — returning raw context")
|
||
answer = fallback_answer(agent_id, context, message)
|
||
|
||
if not approval_created:
|
||
proposed = detect_agent_proposed_action(answer, message)
|
||
if proposed:
|
||
with SessionLocal() as db:
|
||
await create_approval_request(
|
||
db=db,
|
||
ApprovalModel=Approval,
|
||
agent_id=agent_id,
|
||
action=proposed["action"],
|
||
reason=proposed["reason"],
|
||
action_type=proposed["action_type"],
|
||
target=proposed.get("target", ""),
|
||
terminal_log=terminal_log,
|
||
mirror_supervisors=mirror_to_supervisors,
|
||
publish=publish_event,
|
||
add_feed=add_feed,
|
||
)
|
||
approval_created = True
|
||
answer = (
|
||
f"{answer}\n\n⏸ **Approval required** — this action is in the Approval Inbox. "
|
||
f"Mo & Bart have been notified and must approve before we execute."
|
||
)
|
||
|
||
await asyncio.sleep(0.3)
|
||
await terminal_log(agent_id, f"[{prompt_id}] ✓ Mission complete", level="ok", phase="done", prompt_id=prompt_id)
|
||
await publish_event({"type": "agent_return", "agent_id": agent_id, "zone": zone, "prompt_id": prompt_id})
|
||
feed = add_feed(agent_id, f"{agent_name} completed a response (see Comms)", "info")
|
||
await publish_event({"type": "feed", "entry": feed})
|
||
await publish_event({"type": "prompt_result", "prompt_id": prompt_id, "agent_id": agent_id, "answer": answer})
|
||
return answer
|
||
|
||
|
||
async def heartbeat_loop() -> None:
|
||
while True:
|
||
try:
|
||
status = await collect_status()
|
||
workload = await collect_workload()
|
||
await publish_event({"type": "status", "data": status})
|
||
await publish_event({"type": "workload", "data": workload})
|
||
for domain, info in status["domains"].items():
|
||
if info["level"] == "down":
|
||
agent = "data-custodian" if domain == "databases" else "infra-sentinel"
|
||
feed = add_feed(agent, f"Alert: {domain} is DOWN ({info['label']})", "warn")
|
||
await publish_event({"type": "feed", "entry": feed})
|
||
except Exception as exc:
|
||
await publish_event({"type": "error", "message": str(exc)})
|
||
await asyncio.sleep(60)
|
||
|
||
|
||
@asynccontextmanager
|
||
async def lifespan(app: FastAPI):
|
||
global redis_client
|
||
redis_client = aioredis.from_url(REDIS_URL, decode_responses=True)
|
||
set_terminal_publisher(publish_event)
|
||
init_terminals([a["id"] for a in AGENTS] + NODE_IDS)
|
||
for a in AGENTS:
|
||
await terminal_log(a["id"], f"{a['name']} terminal online — awaiting missions", level="info", phase="boot")
|
||
for nid in NODE_IDS:
|
||
if nid not in NODE_REGISTRY:
|
||
continue
|
||
meta = NODE_REGISTRY[nid]
|
||
await terminal_log(nid, f"{meta['label']} shell ready — click node to connect", level="info", phase="boot")
|
||
task = asyncio.create_task(heartbeat_loop())
|
||
dml_task = asyncio.create_task(agent_dml_loop())
|
||
cdc_task = asyncio.create_task(cdc_consumer_loop())
|
||
etl_task = asyncio.create_task(etl_agent_loop())
|
||
cust_task = asyncio.create_task(custodian_offload_loop())
|
||
act_task = asyncio.create_task(agent_activity_loop())
|
||
heal_task = asyncio.create_task(connector_autoheal_loop())
|
||
add_feed("infra-sentinel", "ATC Command Center API online", "info")
|
||
yield
|
||
task.cancel()
|
||
dml_task.cancel()
|
||
cdc_task.cancel()
|
||
etl_task.cancel()
|
||
cust_task.cancel()
|
||
act_task.cancel()
|
||
heal_task.cancel()
|
||
if redis_client:
|
||
await redis_client.close()
|
||
|
||
|
||
app = FastAPI(title="ATC Command Center API", lifespan=lifespan)
|
||
app.include_router(storage_s3_router)
|
||
app.include_router(hdfs_router)
|
||
app.include_router(pipeline_router)
|
||
app.include_router(hadoop_router)
|
||
app.include_router(elasticsearch_router)
|
||
app.include_router(sql_router)
|
||
app.include_router(agent_ops_router)
|
||
app.include_router(cdc_router)
|
||
app.include_router(movements_router)
|
||
app.include_router(dataflow_router)
|
||
app.include_router(streaming_router)
|
||
app.include_router(spark_workbench_router)
|
||
app.include_router(pii_router)
|
||
app.include_router(federated_router)
|
||
app.include_router(etl_offload_router)
|
||
app.include_router(lineage_router)
|
||
app.include_router(dq_router)
|
||
app.include_router(observability_router)
|
||
app.include_router(governance_router)
|
||
app.add_middleware(
|
||
CORSMiddleware,
|
||
allow_origins=["*"],
|
||
allow_credentials=True,
|
||
allow_methods=["*"],
|
||
allow_headers=["*"],
|
||
)
|
||
|
||
# Authentik OIDC session + API guard
|
||
cockpit_auth.init_auth_middleware(app)
|
||
cockpit_auth.setup_auth(app)
|
||
|
||
|
||
@app.get("/api/health")
|
||
async def health():
|
||
llm_ok = False
|
||
try:
|
||
async with httpx.AsyncClient(timeout=4.0) as client:
|
||
r = await client.get(f"{LLM_URL.rstrip('/')}/models", headers={"Authorization": f"Bearer {LLM_API_KEY}"})
|
||
llm_ok = r.status_code == 200
|
||
except Exception:
|
||
pass
|
||
return {
|
||
"ok": True,
|
||
"ts": datetime.now(timezone.utc).isoformat(),
|
||
"llm_url": LLM_URL,
|
||
"llm_ok": llm_ok,
|
||
"llm_model": LLM_MODEL,
|
||
"database": db_health(),
|
||
"db_init": _db_info,
|
||
}
|
||
|
||
|
||
async def collect_workload(*, fast: bool = True, use_cache: bool = True) -> dict[str, Any]:
|
||
import time as _time
|
||
now = _time.time()
|
||
if use_cache and _workload_cache.get("data") and now - float(_workload_cache.get("ts") or 0) < WORKLOAD_CACHE_TTL:
|
||
return _workload_cache["data"]
|
||
gpu = await collect_gpu()
|
||
snap = await collect_full_lab_context(gpu_data=gpu, include_inventory=not fast)
|
||
payload = build_workload_payload(snap)
|
||
_workload_cache["ts"] = now
|
||
_workload_cache["data"] = payload
|
||
return payload
|
||
|
||
|
||
|
||
|
||
async def get_presentation_data(*, use_cache: bool = True) -> dict[str, Any]:
|
||
import time as _time
|
||
now = _time.time()
|
||
if use_cache and _presentation_cache.get("data") and now - float(_presentation_cache.get("ts") or 0) < PRESENTATION_CACHE_TTL:
|
||
return _presentation_cache["data"]
|
||
gpu = await collect_gpu()
|
||
snap = await collect_full_lab_context(gpu_data=gpu, include_inventory=False)
|
||
data = build_presentation_payload(snap)
|
||
override = get_live_override()
|
||
if override and override.get("slides"):
|
||
data["title"] = override.get("title") or data.get("title")
|
||
data["subtitle"] = override.get("subtitle") or data.get("subtitle", "")
|
||
data["slides"] = override["slides"]
|
||
data["slide_count"] = len(override["slides"])
|
||
data["edited"] = True
|
||
data["override_ts"] = override.get("ts")
|
||
else:
|
||
data["edited"] = False
|
||
data["source"] = "live"
|
||
data["id"] = "live"
|
||
_presentation_cache["ts"] = now
|
||
_presentation_cache["data"] = data
|
||
return data
|
||
|
||
|
||
@app.get("/api/presentation")
|
||
async def get_presentation():
|
||
return await get_presentation_data()
|
||
|
||
|
||
@app.get("/api/presentation/html")
|
||
async def get_presentation_html():
|
||
from fastapi.responses import HTMLResponse
|
||
payload = await get_presentation_data()
|
||
return HTMLResponse(render_presentation_html(payload))
|
||
|
||
|
||
@app.get("/api/presentation/decks")
|
||
async def get_presentation_decks():
|
||
return {"live": True, "builtin": list_static_decks(), "uploaded": list_decks()}
|
||
|
||
|
||
@app.get("/api/presentation/decks/{deck_id}")
|
||
async def get_presentation_deck(deck_id: str):
|
||
if deck_id == "live":
|
||
return await get_presentation_data()
|
||
deck = get_static_deck(deck_id) or get_deck(deck_id)
|
||
if not deck:
|
||
return {"error": "deck not found"}
|
||
return deck
|
||
|
||
|
||
@app.get("/api/presentation/decks/{deck_id}/html")
|
||
async def get_presentation_deck_html(deck_id: str):
|
||
from fastapi.responses import HTMLResponse
|
||
if deck_id == "live":
|
||
payload = await get_presentation_data()
|
||
else:
|
||
payload = get_static_deck(deck_id) or get_deck(deck_id)
|
||
if not payload:
|
||
return HTMLResponse("<h1>Deck not found</h1>", status_code=404)
|
||
return HTMLResponse(render_presentation_html(payload))
|
||
|
||
|
||
@app.post("/api/presentation/upload")
|
||
async def upload_presentation(file: UploadFile = File(...)):
|
||
content = await file.read()
|
||
if len(content) > 50 * 1024 * 1024:
|
||
return {"error": "file too large (max 50MB)"}
|
||
deck = await save_upload(file.filename or "upload.pptx", content)
|
||
return {"ok": True, "deck": deck}
|
||
|
||
|
||
@app.post("/api/presentation/decks/new")
|
||
async def create_presentation_deck(body: dict[str, Any] | None = Body(default=None)):
|
||
body = body or {}
|
||
title = str(body.get("title") or "Untitled deck")
|
||
seed_id = body.get("from")
|
||
slides = None
|
||
if seed_id:
|
||
src = get_static_deck(seed_id) or get_deck(seed_id)
|
||
if seed_id == "live":
|
||
src = await get_presentation_data()
|
||
if src and src.get("slides"):
|
||
slides = [
|
||
{
|
||
"id": s.get("id") or f"slide-{i}",
|
||
"title": s.get("title") or f"Slide {i}",
|
||
"subtitle": s.get("subtitle") or "",
|
||
"bullets": list(s.get("bullets") or []),
|
||
"image": s.get("image") or "",
|
||
"kind": s.get("kind") or "narrative",
|
||
**({"animation": s["animation"]} if s.get("animation") else {}),
|
||
}
|
||
for i, s in enumerate(src["slides"], start=1)
|
||
]
|
||
if not title or title == "Untitled deck":
|
||
title = f"{src.get('title', 'Deck')} (copy)"
|
||
deck = create_deck(title, slides)
|
||
return {"ok": True, "deck": deck}
|
||
|
||
|
||
@app.put("/api/presentation/decks/{deck_id}")
|
||
async def update_presentation_deck(deck_id: str, body: dict[str, Any] = Body(...)):
|
||
if deck_id == "live":
|
||
save_live_override(body)
|
||
_presentation_cache["ts"] = 0
|
||
_presentation_cache["data"] = None
|
||
payload = await get_presentation_data(use_cache=False)
|
||
return {"ok": True, "deck": payload}
|
||
deck = save_deck(deck_id, body)
|
||
if not deck:
|
||
return {"error": "deck not found or not editable"}
|
||
return {"ok": True, "deck": deck}
|
||
|
||
|
||
@app.post("/api/presentation/live/reset")
|
||
async def reset_live_presentation():
|
||
clear_live_override()
|
||
_presentation_cache["ts"] = 0
|
||
_presentation_cache["data"] = None
|
||
payload = await get_presentation_data(use_cache=False)
|
||
return {"ok": True, "deck": payload}
|
||
|
||
|
||
@app.delete("/api/presentation/decks/{deck_id}")
|
||
async def remove_presentation_deck(deck_id: str):
|
||
return {"ok": delete_deck(deck_id)}
|
||
|
||
|
||
@app.post("/api/presentation/decks/{deck_id}/image")
|
||
async def upload_presentation_image(deck_id: str, file: UploadFile = File(...)):
|
||
content = await file.read()
|
||
if len(content) > 12 * 1024 * 1024:
|
||
return {"error": "image too large (max 12MB)"}
|
||
result = save_image(deck_id, file.filename or "image.png", content)
|
||
if not result:
|
||
return {"error": "deck not found"}
|
||
return {"ok": True, **result}
|
||
|
||
|
||
@app.get("/api/presentation/decks/{deck_id}/assets/{name}")
|
||
async def get_presentation_asset(deck_id: str, name: str):
|
||
from fastapi.responses import FileResponse, Response
|
||
path = get_asset_path(deck_id, name)
|
||
if not path:
|
||
return Response(status_code=404)
|
||
return FileResponse(str(path))
|
||
|
||
@app.get("/api/workload")
|
||
async def get_workload(fast: bool = True):
|
||
return await collect_workload(fast=fast, use_cache=True)
|
||
|
||
|
||
@app.get("/api/status")
|
||
async def get_status():
|
||
return await collect_status()
|
||
|
||
|
||
@app.get("/api/gpu")
|
||
async def get_gpu():
|
||
return await collect_gpu()
|
||
|
||
@app.get("/api/gpu/config")
|
||
async def get_gpu_config_endpoint():
|
||
return get_gpu_config_payload()
|
||
|
||
|
||
@app.post("/api/gpu/config")
|
||
async def post_gpu_config(body: dict[str, Any]):
|
||
try:
|
||
saved = save_gpu_config(
|
||
preset_id=body.get("preset_id"),
|
||
host=body.get("host"),
|
||
gpu_ui_port=body.get("gpu_ui_port"),
|
||
llm_port=body.get("llm_port"),
|
||
)
|
||
return {"ok": True, "active": saved, "presets": get_gpu_config_payload()["presets"]}
|
||
except ValueError as exc:
|
||
return JSONResponse({"ok": False, "detail": str(exc)}, status_code=400)
|
||
|
||
|
||
@app.post("/api/gpu/config/test")
|
||
async def post_gpu_config_test(body: dict[str, Any]):
|
||
return await test_gpu_target(
|
||
preset_id=body.get("preset_id"),
|
||
host=body.get("host"),
|
||
gpu_ui_port=body.get("gpu_ui_port"),
|
||
llm_port=body.get("llm_port"),
|
||
)
|
||
|
||
|
||
@app.delete("/api/gpu/config")
|
||
async def delete_gpu_config():
|
||
active = reset_gpu_config()
|
||
return {"ok": True, "active": active}
|
||
|
||
|
||
|
||
def agent_stats() -> dict[str, dict[str, Any]]:
|
||
stats: dict[str, dict[str, Any]] = {a["id"]: {"tasks": 0, "last_active": None, "alerts": 0} for a in AGENTS}
|
||
with SessionLocal() as db:
|
||
rows = db.execute(select(FeedEntry).order_by(FeedEntry.ts.desc()).limit(200)).scalars().all()
|
||
for r in rows:
|
||
aid = r.agent_id
|
||
if aid not in stats:
|
||
continue
|
||
stats[aid]["tasks"] += 1
|
||
if r.level == "warn":
|
||
stats[aid]["alerts"] += 1
|
||
if stats[aid]["last_active"] is None and r.ts:
|
||
stats[aid]["last_active"] = r.ts.isoformat()
|
||
return stats
|
||
|
||
|
||
@app.get("/api/agents")
|
||
async def get_agents():
|
||
stats = agent_stats()
|
||
enriched = [{**a, "stats": stats.get(a["id"], {})} for a in AGENTS]
|
||
return {"agents": enriched, "zones": ZONES}
|
||
|
||
|
||
@app.get("/api/terminals")
|
||
async def get_terminals(limit: int = 200):
|
||
return {"terminals": get_all_terminals(limit)}
|
||
|
||
|
||
@app.get("/api/terminals/{subject_id}")
|
||
async def get_subject_terminal(subject_id: str, limit: int = 200):
|
||
valid_agents = {a["id"] for a in AGENTS}
|
||
if subject_id not in valid_agents and not is_node_id(subject_id):
|
||
return {"error": "unknown subject"}
|
||
return {"agent_id": subject_id, "lines": get_terminal_lines(subject_id, limit)}
|
||
|
||
|
||
@app.get("/api/nodes")
|
||
async def list_nodes():
|
||
workload = await collect_workload()
|
||
nodes = workload.get("topology", {}).get("nodes", [])
|
||
return {"nodes": [{"id": n["id"], "label": n["label"], "ip": n["ip"], "level": n["level"]} for n in nodes]}
|
||
|
||
|
||
@app.get("/api/nodes/{node_id}")
|
||
async def get_node(node_id: str):
|
||
if not is_node_id(node_id):
|
||
return {"error": "unknown node"}
|
||
workload = await collect_workload()
|
||
wn = next((n for n in workload.get("topology", {}).get("nodes", []) if n["id"] == node_id), None)
|
||
gpu = await collect_gpu()
|
||
snap = await collect_full_lab_context(gpu_data=gpu)
|
||
return build_node_detail(node_id, snap, wn)
|
||
|
||
|
||
@app.post("/api/nodes/{node_id}/probe")
|
||
async def post_node_probe(node_id: str):
|
||
if not is_node_id(node_id):
|
||
return {"error": "unknown node"}
|
||
asyncio.create_task(run_node_probe_task(node_id))
|
||
return {"ok": True, "node_id": node_id, "status": "probing"}
|
||
|
||
|
||
async def run_node_ask_task(node_id: str, message: str) -> None:
|
||
agent_id = NODE_AGENT.get(node_id, "infra-sentinel")
|
||
meta = NODE_REGISTRY[node_id]
|
||
await terminal_log(node_id, f"▶ Query: {message}", level="info", phase="ask")
|
||
await terminal_log(node_id, f"→ Routing to agent {agent_id}", level="info", phase="ask")
|
||
log = make_logger(node_id)
|
||
status = await collect_status()
|
||
context = await gather_agent_context(agent_id, status, log=log, message=message)
|
||
node_ctx = f"\n\n=== FOCUSED NODE: {meta['label']} ({meta['ip']}) ===\n{meta.get('description', '')}\n"
|
||
answer = await ask_llm(agent_id, message, context + node_ctx, log=log)
|
||
if not answer:
|
||
answer = fallback_answer(agent_id, context, message)
|
||
await terminal_log(node_id, f"◆ {answer}", level="llm", phase="answer")
|
||
await publish_event({"type": "node_ask_result", "node_id": node_id, "agent_id": agent_id, "answer": answer})
|
||
|
||
|
||
@app.post("/api/nodes/{node_id}/ask")
|
||
async def post_node_ask(node_id: str, body: NodeAskRequest):
|
||
if not is_node_id(node_id):
|
||
return {"error": "unknown node"}
|
||
asyncio.create_task(run_node_ask_task(node_id, body.message))
|
||
return {"ok": True, "node_id": node_id, "agent_id": NODE_AGENT.get(node_id), "status": "processing"}
|
||
|
||
|
||
@app.get("/api/feed")
|
||
async def get_feed(limit: int = 50):
|
||
with SessionLocal() as db:
|
||
rows = db.execute(select(FeedEntry).order_by(FeedEntry.ts.desc()).limit(limit)).scalars().all()
|
||
return {
|
||
"entries": [
|
||
{
|
||
"id": r.id,
|
||
"ts": r.ts.isoformat() if r.ts else None,
|
||
"agent_id": r.agent_id,
|
||
"message": r.message,
|
||
"level": r.level,
|
||
}
|
||
for r in rows
|
||
]
|
||
}
|
||
|
||
|
||
@app.get("/api/approvals")
|
||
async def get_approvals(status: str = "pending", limit: int = 100):
|
||
with SessionLocal() as db:
|
||
items = list_approvals(db, Approval, status=status, limit=limit)
|
||
stats = approval_stats(db, Approval)
|
||
return {"approvals": items, "stats": stats, "action_types": APPROVAL_ACTION_TYPES}
|
||
|
||
|
||
@app.get("/api/approvals/stats")
|
||
async def get_approval_stats():
|
||
with SessionLocal() as db:
|
||
return approval_stats(db, Approval)
|
||
|
||
|
||
@app.post("/api/approvals")
|
||
async def post_approval(body: ApprovalCreateRequest):
|
||
valid_ids = {a["id"] for a in AGENTS}
|
||
if body.agent_id not in valid_ids:
|
||
return {"error": "unknown agent_id"}
|
||
if body.action_type not in APPROVAL_ACTION_TYPES:
|
||
body.action_type = "generic.mutate"
|
||
with SessionLocal() as db:
|
||
item = await create_approval_request(
|
||
db=db,
|
||
ApprovalModel=Approval,
|
||
agent_id=body.agent_id,
|
||
action=body.action,
|
||
reason=body.reason,
|
||
action_type=body.action_type,
|
||
target=body.target,
|
||
payload=body.payload,
|
||
priority=body.priority,
|
||
terminal_log=terminal_log,
|
||
mirror_supervisors=mirror_to_supervisors,
|
||
publish=publish_event,
|
||
add_feed=add_feed,
|
||
)
|
||
return {"ok": True, "approval": item}
|
||
|
||
|
||
@app.post("/api/approvals/{approval_id}/decide")
|
||
async def decide_approval(approval_id: str, body: ApprovalDecision):
|
||
valid_supervisors = {"mo-commander", "bart-commander"}
|
||
decided_by = body.decided_by if body.decided_by in valid_supervisors else "mo-commander"
|
||
with SessionLocal() as db:
|
||
item = await decide_approval_request(
|
||
db=db,
|
||
ApprovalModel=Approval,
|
||
approval_id=approval_id,
|
||
approved=body.approved,
|
||
decided_by=decided_by,
|
||
note=body.note,
|
||
terminal_log=terminal_log,
|
||
publish=publish_event,
|
||
add_feed=add_feed,
|
||
)
|
||
if not item:
|
||
return {"error": "not found"}
|
||
|
||
# Approval-gated executor: run the underlying action once a supervisor approves it.
|
||
if item.get("status") == "approved":
|
||
payload = item.get("payload") or {}
|
||
if isinstance(payload, dict) and payload.get("executor") == "movement":
|
||
mid = payload.get("movement_id")
|
||
if mid in MOVEMENT_BY_ID:
|
||
add_feed("etl-guardian", f"Approval {approval_id} granted — executing movement '{mid}'", "info")
|
||
asyncio.create_task(trigger_and_watch(mid, payload.get("conf")))
|
||
|
||
return {"ok": True, "approval": item}
|
||
|
||
|
||
@app.post("/api/prompt")
|
||
async def post_prompt(body: PromptRequest):
|
||
prompt_id = str(uuid.uuid4())[:8]
|
||
valid_ids = {a["id"] for a in AGENTS}
|
||
agent_id = body.agent_id if body.agent_id in valid_ids else route_agent(body.message)
|
||
add_feed(agent_id, f"Prompt received: {body.message}", "info")
|
||
asyncio.create_task(_run_agent_task_safe(agent_id, body.message, prompt_id))
|
||
return {"prompt_id": prompt_id, "agent_id": agent_id, "status": "dispatched"}
|
||
|
||
|
||
@app.websocket("/api/ws/ssh")
|
||
async def ws_ssh(websocket: WebSocket):
|
||
await ssh_session(websocket)
|
||
|
||
|
||
@app.websocket("/api/ws/ops")
|
||
async def ws_ops(websocket: WebSocket):
|
||
await websocket.accept()
|
||
ws_clients.add(websocket)
|
||
try:
|
||
status = await collect_status()
|
||
workload = await collect_workload()
|
||
await websocket.send_text(json.dumps({"type": "status", "data": status}, default=str))
|
||
await websocket.send_text(json.dumps({"type": "workload", "data": workload}, default=str))
|
||
await websocket.send_text(json.dumps({
|
||
"type": "terminal_history",
|
||
"terminals": get_all_terminals(150),
|
||
}, default=str))
|
||
while True:
|
||
await websocket.receive_text()
|
||
except WebSocketDisconnect:
|
||
pass
|
||
finally:
|
||
ws_clients.discard(websocket)
|