Add ATC Command Center v1 with light UI theme.
Agent hub dashboard, FastAPI backend, and Docker stack for VM 304 MCP. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
mo
+385
@@ -0,0 +1,385 @@
|
||||
"""ATC Command Center API — FastAPI backend."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import uuid
|
||||
from contextlib import asynccontextmanager
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
import redis.asyncio as aioredis
|
||||
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy import Column, DateTime, String, Text, create_engine, select
|
||||
from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker
|
||||
|
||||
REDIS_URL = os.getenv("REDIS_URL", "redis://redis:6379/0")
|
||||
DOCKHAND_URL = os.getenv("DOCKHAND_URL", "http://10.0.21.45:8082")
|
||||
DATABASE_URL = os.getenv("DATABASE_URL", "sqlite:////data/atc-agents.db")
|
||||
|
||||
AGENTS = [
|
||||
{
|
||||
"id": "etl-guardian",
|
||||
"name": "ETL Guardian",
|
||||
"color": "#00f0ff",
|
||||
"zone": "etl",
|
||||
"role": "Airflow, Kafka, Debezium, S3 pipeline",
|
||||
},
|
||||
{
|
||||
"id": "lakehouse-ops",
|
||||
"name": "Lakehouse Ops",
|
||||
"color": "#ff00aa",
|
||||
"zone": "lakehouse",
|
||||
"role": "Spark, Trino, Iceberg",
|
||||
},
|
||||
{
|
||||
"id": "data-custodian",
|
||||
"name": "Data Custodian",
|
||||
"color": "#ffaa00",
|
||||
"zone": "db",
|
||||
"role": "PostgreSQL, MySQL, Mongo, Cassandra, Neo4j",
|
||||
},
|
||||
{
|
||||
"id": "hadoop-ranger",
|
||||
"name": "Hadoop Ranger",
|
||||
"color": "#39ff14",
|
||||
"zone": "hadoop",
|
||||
"role": "HDFS, YARN cluster",
|
||||
},
|
||||
{
|
||||
"id": "infra-sentinel",
|
||||
"name": "Infra Sentinel",
|
||||
"color": "#b366ff",
|
||||
"zone": "docker",
|
||||
"role": "Docker, Proxmox, monitoring",
|
||||
},
|
||||
]
|
||||
|
||||
ZONES = [
|
||||
{"id": "docker", "label": "DOCKER RACK", "x": 8, "color": "#b366ff"},
|
||||
{"id": "db", "label": "DB VAULT", "x": 28, "color": "#ffaa00"},
|
||||
{"id": "lakehouse", "label": "LAKEHOUSE HUB", "x": 50, "color": "#ff00aa"},
|
||||
{"id": "hadoop", "label": "HADOOP CLUSTER", "x": 72, "color": "#39ff14"},
|
||||
{"id": "etl", "label": "ETL PIPE", "x": 92, "color": "#00f0ff"},
|
||||
]
|
||||
|
||||
INTENT_KEYWORDS: dict[str, list[str]] = {
|
||||
"data-custodian": ["database", "db", "postgres", "mysql", "mongo", "cassandra", "neo4j", "sql"],
|
||||
"lakehouse-ops": ["trino", "spark", "lakehouse", "iceberg", "query"],
|
||||
"hadoop-ranger": ["hadoop", "hdfs", "yarn", "datanode"],
|
||||
"infra-sentinel": ["docker", "container", "vm", "proxmox", "infra", "grafana"],
|
||||
"etl-guardian": ["airflow", "dag", "debezium", "kafka", "connector", "etl", "pipeline", "s3"],
|
||||
}
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
pass
|
||||
|
||||
|
||||
class FeedEntry(Base):
|
||||
__tablename__ = "feed"
|
||||
|
||||
id = Column(String, primary_key=True)
|
||||
ts = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
||||
agent_id = Column(String)
|
||||
level = Column(String, default="info")
|
||||
message = Column(Text)
|
||||
|
||||
|
||||
class Approval(Base):
|
||||
__tablename__ = "approvals"
|
||||
|
||||
id = Column(String, primary_key=True)
|
||||
ts = Column(DateTime, default=lambda: datetime.now(timezone.utc))
|
||||
agent_id = Column(String)
|
||||
action = Column(Text)
|
||||
reason = Column(Text)
|
||||
status = Column(String, default="pending")
|
||||
|
||||
|
||||
engine = create_engine(DATABASE_URL, connect_args={"check_same_thread": False})
|
||||
SessionLocal = sessionmaker(bind=engine)
|
||||
Base.metadata.create_all(engine)
|
||||
|
||||
redis_client: aioredis.Redis | None = None
|
||||
ws_clients: set[WebSocket] = set()
|
||||
|
||||
|
||||
class PromptRequest(BaseModel):
|
||||
message: str = Field(min_length=1, max_length=2000)
|
||||
|
||||
|
||||
class ApprovalDecision(BaseModel):
|
||||
approved: bool
|
||||
|
||||
|
||||
def route_agent(message: str) -> str:
|
||||
lower = message.lower()
|
||||
scores = {aid: sum(1 for kw in kws if kw in lower) for aid, kws in INTENT_KEYWORDS.items()}
|
||||
best = max(scores, key=scores.get)
|
||||
if scores[best] == 0:
|
||||
return "infra-sentinel"
|
||||
return best
|
||||
|
||||
|
||||
async def publish_event(event: dict[str, Any]) -> None:
|
||||
payload = json.dumps(event, default=str)
|
||||
if redis_client:
|
||||
await redis_client.publish("ops", payload)
|
||||
dead = []
|
||||
for ws in ws_clients:
|
||||
try:
|
||||
await ws.send_text(payload)
|
||||
except Exception:
|
||||
dead.append(ws)
|
||||
for ws in dead:
|
||||
ws_clients.discard(ws)
|
||||
|
||||
|
||||
def add_feed(agent_id: str, message: str, level: str = "info") -> dict:
|
||||
entry_id = str(uuid.uuid4())[:8]
|
||||
with SessionLocal() as db:
|
||||
row = FeedEntry(id=entry_id, agent_id=agent_id, message=message, level=level)
|
||||
db.add(row)
|
||||
db.commit()
|
||||
return {
|
||||
"id": entry_id,
|
||||
"ts": datetime.now(timezone.utc).isoformat(),
|
||||
"agent_id": agent_id,
|
||||
"message": message,
|
||||
"level": level,
|
||||
}
|
||||
|
||||
|
||||
async def dockhand_env_containers(env_id: int) -> list[dict]:
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=8.0) as client:
|
||||
r = await client.get(f"{DOCKHAND_URL}/api/containers", params={"env": env_id})
|
||||
r.raise_for_status()
|
||||
return r.json()
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
async def probe_url(url: str) -> bool:
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=4.0, verify=False) as client:
|
||||
r = await client.get(url)
|
||||
return r.status_code < 500
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
async def collect_status() -> dict[str, Any]:
|
||||
db_containers = await dockhand_env_containers(5)
|
||||
db_running = sum(1 for c in db_containers if c.get("state") == "running")
|
||||
db_total = len(db_containers) or 6
|
||||
|
||||
lake_containers = await dockhand_env_containers(9)
|
||||
lake_running = sum(1 for c in lake_containers if c.get("state") == "running")
|
||||
lake_total = len(lake_containers) or 6
|
||||
|
||||
docker_containers = await dockhand_env_containers(1)
|
||||
docker_running = sum(1 for c in docker_containers if c.get("state") == "running")
|
||||
|
||||
hdfs_ok = await probe_url("http://10.0.21.61:9870")
|
||||
kafka_ok = await probe_url("http://10.0.21.36:9000")
|
||||
airflow_ok = await probe_url("http://10.0.21.55:8080")
|
||||
|
||||
def level(running: int, total: int) -> str:
|
||||
if total == 0:
|
||||
return "unknown"
|
||||
ratio = running / total
|
||||
if ratio >= 0.9:
|
||||
return "ok"
|
||||
if ratio >= 0.5:
|
||||
return "warn"
|
||||
return "down"
|
||||
|
||||
return {
|
||||
"ts": datetime.now(timezone.utc).isoformat(),
|
||||
"domains": {
|
||||
"docker": {"level": "ok" if docker_running >= 5 else "warn", "label": f"{docker_running} containers", "running": docker_running},
|
||||
"databases": {"level": level(db_running, db_total), "label": f"{db_running}/{db_total} up", "running": db_running, "total": db_total},
|
||||
"lakehouse": {"level": level(lake_running, lake_total), "label": f"{lake_running}/{lake_total} up", "running": lake_running, "total": lake_total},
|
||||
"hadoop": {"level": "ok" if hdfs_ok else "warn", "label": "NN up" if hdfs_ok else "NN check"},
|
||||
"etl": {"level": "ok" if kafka_ok and airflow_ok else "warn", "label": "Kafka+Airflow"},
|
||||
},
|
||||
"kafka_ok": kafka_ok,
|
||||
"airflow_ok": airflow_ok,
|
||||
"hdfs_ok": hdfs_ok,
|
||||
}
|
||||
|
||||
|
||||
async def run_agent_task(agent_id: str, message: str, prompt_id: str) -> str:
|
||||
zone = next(a["zone"] for a in AGENTS if a["id"] == agent_id)
|
||||
await publish_event({"type": "agent_dispatch", "agent_id": agent_id, "zone": zone, "prompt_id": prompt_id})
|
||||
await asyncio.sleep(0.8)
|
||||
await publish_event({"type": "agent_fetch", "agent_id": agent_id, "zone": zone, "prompt_id": prompt_id})
|
||||
|
||||
status = await collect_status()
|
||||
answer_parts = [f"**{next(a['name'] for a in AGENTS if a['id'] == agent_id)}** reporting:"]
|
||||
|
||||
if agent_id == "data-custodian":
|
||||
db = status["domains"]["databases"]
|
||||
containers = await dockhand_env_containers(5)
|
||||
names = ", ".join(f"{c['name']}:{c.get('state','?')}" for c in containers[:8])
|
||||
answer_parts.append(f"Databases {db['label']}. Containers: {names or 'unreachable'}.")
|
||||
elif agent_id == "etl-guardian":
|
||||
answer_parts.append(
|
||||
f"Kafka UI: {'OK' if status['kafka_ok'] else 'DOWN'}. "
|
||||
f"Airflow: {'OK' if status['airflow_ok'] else 'DOWN'}. "
|
||||
f"Lakehouse {status['domains']['lakehouse']['label']}."
|
||||
)
|
||||
elif agent_id == "lakehouse-ops":
|
||||
answer_parts.append(f"Lakehouse stack {status['domains']['lakehouse']['label']}. Trino at 10.0.21.50:8089.")
|
||||
elif agent_id == "hadoop-ranger":
|
||||
answer_parts.append(f"HDFS NameNode: {'reachable' if status['hdfs_ok'] else 'unreachable'} on 10.0.21.61:9870.")
|
||||
else:
|
||||
answer_parts.append(
|
||||
f"Docker {status['domains']['docker']['label']}. "
|
||||
f"Overall lab health snapshot collected."
|
||||
)
|
||||
|
||||
answer = " ".join(answer_parts)
|
||||
await asyncio.sleep(0.6)
|
||||
await publish_event({"type": "agent_return", "agent_id": agent_id, "zone": zone, "prompt_id": prompt_id})
|
||||
feed = add_feed(agent_id, f"Prompt answered: {message[:80]}", "info")
|
||||
await publish_event({"type": "feed", "entry": feed})
|
||||
await publish_event({"type": "prompt_result", "prompt_id": prompt_id, "agent_id": agent_id, "answer": answer})
|
||||
return answer
|
||||
|
||||
|
||||
async def heartbeat_loop() -> None:
|
||||
while True:
|
||||
try:
|
||||
status = await collect_status()
|
||||
await publish_event({"type": "status", "data": status})
|
||||
for domain, info in status["domains"].items():
|
||||
if info["level"] == "down":
|
||||
agent = "data-custodian" if domain == "databases" else "infra-sentinel"
|
||||
feed = add_feed(agent, f"Alert: {domain} is DOWN ({info['label']})", "warn")
|
||||
await publish_event({"type": "feed", "entry": feed})
|
||||
except Exception as exc:
|
||||
await publish_event({"type": "error", "message": str(exc)})
|
||||
await asyncio.sleep(60)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
global redis_client
|
||||
redis_client = aioredis.from_url(REDIS_URL, decode_responses=True)
|
||||
task = asyncio.create_task(heartbeat_loop())
|
||||
add_feed("infra-sentinel", "ATC Command Center API online", "info")
|
||||
yield
|
||||
task.cancel()
|
||||
if redis_client:
|
||||
await redis_client.close()
|
||||
|
||||
|
||||
app = FastAPI(title="ATC Command Center API", lifespan=lifespan)
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"],
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
|
||||
@app.get("/api/health")
|
||||
async def health():
|
||||
return {"ok": True, "ts": datetime.now(timezone.utc).isoformat()}
|
||||
|
||||
|
||||
@app.get("/api/status")
|
||||
async def get_status():
|
||||
return await collect_status()
|
||||
|
||||
|
||||
@app.get("/api/agents")
|
||||
async def get_agents():
|
||||
return {"agents": AGENTS, "zones": ZONES}
|
||||
|
||||
|
||||
@app.get("/api/feed")
|
||||
async def get_feed(limit: int = 50):
|
||||
with SessionLocal() as db:
|
||||
rows = db.execute(select(FeedEntry).order_by(FeedEntry.ts.desc()).limit(limit)).scalars().all()
|
||||
return {
|
||||
"entries": [
|
||||
{
|
||||
"id": r.id,
|
||||
"ts": r.ts.isoformat() if r.ts else None,
|
||||
"agent_id": r.agent_id,
|
||||
"message": r.message,
|
||||
"level": r.level,
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/approvals")
|
||||
async def get_approvals():
|
||||
with SessionLocal() as db:
|
||||
rows = db.execute(select(Approval).where(Approval.status == "pending")).scalars().all()
|
||||
return {
|
||||
"approvals": [
|
||||
{
|
||||
"id": r.id,
|
||||
"ts": r.ts.isoformat() if r.ts else None,
|
||||
"agent_id": r.agent_id,
|
||||
"action": r.action,
|
||||
"reason": r.reason,
|
||||
"status": r.status,
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
@app.post("/api/approvals/{approval_id}/decide")
|
||||
async def decide_approval(approval_id: str, body: ApprovalDecision):
|
||||
with SessionLocal() as db:
|
||||
row = db.get(Approval, approval_id)
|
||||
if not row:
|
||||
return {"error": "not found"}
|
||||
row.status = "approved" if body.approved else "denied"
|
||||
db.commit()
|
||||
agent_id = row.agent_id
|
||||
action = row.action
|
||||
msg = f"Approval {'approved' if body.approved else 'denied'}: {action}"
|
||||
feed = add_feed(agent_id, msg, "info" if body.approved else "warn")
|
||||
await publish_event({"type": "feed", "entry": feed})
|
||||
await publish_event({"type": "approval_update", "id": approval_id, "status": row.status})
|
||||
return {"ok": True, "status": row.status}
|
||||
|
||||
|
||||
@app.post("/api/prompt")
|
||||
async def post_prompt(body: PromptRequest):
|
||||
prompt_id = str(uuid.uuid4())[:8]
|
||||
agent_id = route_agent(body.message)
|
||||
add_feed(agent_id, f"Prompt received: {body.message}", "info")
|
||||
asyncio.create_task(run_agent_task(agent_id, body.message, prompt_id))
|
||||
return {"prompt_id": prompt_id, "agent_id": agent_id, "status": "dispatched"}
|
||||
|
||||
|
||||
@app.websocket("/api/ws/ops")
|
||||
async def ws_ops(websocket: WebSocket):
|
||||
await websocket.accept()
|
||||
ws_clients.add(websocket)
|
||||
try:
|
||||
status = await collect_status()
|
||||
await websocket.send_text(json.dumps({"type": "status", "data": status}, default=str))
|
||||
while True:
|
||||
await websocket.receive_text()
|
||||
except WebSocketDisconnect:
|
||||
pass
|
||||
finally:
|
||||
ws_clients.discard(websocket)
|
||||
Reference in New Issue
Block a user