diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..0331674 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,11 @@ +FROM node:22-alpine AS build +WORKDIR /app +COPY package.json ./ +RUN npm install +COPY . . +RUN npm run build + +FROM nginx:alpine +COPY --from=build /app/dist /usr/share/nginx/html +COPY nginx.conf /etc/nginx/conf.d/default.conf +EXPOSE 80 diff --git a/README.md b/README.md index f60c48c..e426503 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Autonomous agent hub for the Dell ATC lab — Ops Floor UI, FastAPI backend, 5 o docker compose up -d --build ``` -Open: **http://atc-mcp.dell-atc.lan/** (or `http://10.0.21.33/`) +Open: **http://10.0.21.33/** ## Stack @@ -19,10 +19,60 @@ Open: **http://atc-mcp.dell-atc.lan/** (or `http://10.0.21.33/`) ## VM 304 (MCP) -- Host: `atc-mcp.dell-atc.lan` → DHCP on VLAN 20 (`br_20`) +- IP: `10.0.21.33` (DHCP on VLAN 20 / `br_20`) - Proxmox VMID **304** on **atc-gpu** - SSH: `root` / `Dell2026!` ## Gitea `http://atc-mgt01.dell-atc.lan:3001/mo/atc-agents` +# ATC Command Center — Gitea layout + +This repo follows the same pattern as **`mo/atc-GPU`** and **`mo/Lakehouse`**. + +## Related repos (Gitea @ atc-mgt01:3001) + +| Repo | Path on VM304 | Purpose | +|------|---------------|---------| +| **mo/atc-agents** | `/opt/atc-agents` | Command Center UI + API + docker-compose | +| **mo/atc-data-quality** | `/opt/atc-data-quality` | DQ API + RAG API | +| **mo/atc-GPU** | GPU lab VM303 | vLLM, model-manager | +| **mo/Lakehouse** | lake01 / docker hosts | Kafka, Spark, Trino, ObjectScale config | + +## This repo structure + +``` +atc-agents/ +├── api/ FastAPI backend +├── ui/ React dashboard +├── caddy/ Reverse proxy routes +├── config/ Deploy reference (mirrors production) +│ ├── command-center/ docker-compose, Caddyfile, .env.example +│ ├── data-quality/ Link to mo/atc-data-quality +│ └── jupyter/ JupyterLab service snippet +├── docs/ Runbooks +├── scripts/ deploy.sh +└── docker-compose.yml Production stack (clone with atc-data-quality sibling) +``` + +## Deploy + +```bash +git clone http://atc-mgt01.dell-atc.lan:3001/mo/atc-agents.git /opt/atc-agents +git clone http://atc-mgt01.dell-atc.lan:3001/mo/atc-data-quality.git /opt/atc-data-quality +cp config/command-center/.env.example /opt/atc-agents/.env # edit secrets +./scripts/deploy.sh +``` + +Open: **http://10.0.21.33/** + +## Services (port 80 via Caddy) + +| Route | Service | +|-------|---------| +| `/` | React UI | +| `/api/*` | Agents API | +| `/dq/*` | Data Quality API | +| `/rag/*` | Knowledge Chat / RAG | +| `/jupyter/*` | JupyterLab (S3 env preconfigured) | +| `:5001` | Docling UI (direct) | diff --git a/agent_terminal.py b/agent_terminal.py new file mode 100644 index 0000000..e44dfcb --- /dev/null +++ b/agent_terminal.py @@ -0,0 +1,71 @@ +"""Per-agent live terminal buffers and streaming.""" + +from __future__ import annotations + +import uuid +from collections import deque +from datetime import datetime, timezone +from typing import Any, Awaitable, Callable + +MAX_LINES_PER_AGENT = 300 + +PublishFn = Callable[[dict[str, Any]], Awaitable[None]] + +_buffers: dict[str, deque[dict[str, Any]]] = {} +_publish: PublishFn | None = None + + +def init_terminals(agent_ids: list[str]) -> None: + for aid in agent_ids: + if aid not in _buffers: + _buffers[aid] = deque(maxlen=MAX_LINES_PER_AGENT) + + +def set_terminal_publisher(fn: PublishFn) -> None: + global _publish + _publish = fn + + +def get_terminal_lines(agent_id: str, limit: int = 200) -> list[dict[str, Any]]: + buf = _buffers.get(agent_id, deque()) + items = list(buf) + return items[-limit:] + + +def get_all_terminals(limit: int = 200) -> dict[str, list[dict[str, Any]]]: + return {aid: get_terminal_lines(aid, limit) for aid in _buffers} + + +async def terminal_log( + agent_id: str, + text: str, + *, + level: str = "info", + phase: str = "ops", + prompt_id: str | None = None, +) -> dict[str, Any]: + init_terminals([agent_id]) + line = { + "id": str(uuid.uuid4())[:8], + "ts": datetime.now(timezone.utc).isoformat(), + "agent_id": agent_id, + "level": level, + "phase": phase, + "text": text, + "prompt_id": prompt_id, + } + _buffers[agent_id].append(line) + if _publish: + await _publish({"type": "terminal", "line": line}) + return line + + +# Type: async (level, phase, text) -> None +TerminalLogFn = Callable[[str, str, str], Awaitable[None]] + + +def make_logger(agent_id: str, prompt_id: str | None = None) -> TerminalLogFn: + async def log(level: str, phase: str, text: str) -> None: + await terminal_log(agent_id, text, level=level, phase=phase, prompt_id=prompt_id) + + return log diff --git a/api/Dockerfile b/api/Dockerfile index b4772c1..0f4a57a 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -4,7 +4,7 @@ WORKDIR /app RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/* COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt -COPY main.py . +COPY main.py lab_context.py agent_terminal.py workload.py node_registry.py node_ops.py topology_views.py supervisor.py approval_service.py db.py dockhand_envs.py presentation.py database_inventory.py presentation_upload.py presentation_static.py storage_s3.py . RUN mkdir -p /data ENV DATABASE_URL=sqlite:////data/atc-agents.db EXPOSE 3201 diff --git a/api/agent_terminal.py b/api/agent_terminal.py new file mode 100644 index 0000000..66189ba --- /dev/null +++ b/api/agent_terminal.py @@ -0,0 +1,72 @@ +"""Per-agent live terminal buffers and streaming.""" + +from __future__ import annotations + +import uuid +from collections import deque +from datetime import datetime, timezone +from typing import Any, Awaitable, Callable + +MAX_LINES_PER_AGENT = 300 + +PublishFn = Callable[[dict[str, Any]], Awaitable[None]] + +_buffers: dict[str, deque[dict[str, Any]]] = {} +_publish: PublishFn | None = None + + +def init_terminals(agent_ids: list[str]) -> None: + for aid in agent_ids: + if aid not in _buffers: + _buffers[aid] = deque(maxlen=MAX_LINES_PER_AGENT) + + +def set_terminal_publisher(fn: PublishFn) -> None: + global _publish + _publish = fn + + +def get_terminal_lines(agent_id: str, limit: int = 200) -> list[dict[str, Any]]: + buf = _buffers.get(agent_id, deque()) + items = list(buf) + return items[-limit:] + + +def get_all_terminals(limit: int = 200) -> dict[str, list[dict[str, Any]]]: + return {aid: get_terminal_lines(aid, limit) for aid in _buffers} + + +async def terminal_log( + agent_id: str, + text: str, + *, + level: str = "info", + phase: str = "ops", + prompt_id: str | None = None, + mirror: bool = True, +) -> dict[str, Any]: + init_terminals([agent_id]) + line = { + "id": str(uuid.uuid4())[:8], + "ts": datetime.now(timezone.utc).isoformat(), + "agent_id": agent_id, + "level": level, + "phase": phase, + "text": text, + "prompt_id": prompt_id, + } + _buffers[agent_id].append(line) + if _publish and mirror: + await _publish({"type": "terminal", "line": line}) + return line + + +# Type: async (level, phase, text) -> None +TerminalLogFn = Callable[[str, str, str], Awaitable[None]] + + +def make_logger(agent_id: str, prompt_id: str | None = None) -> TerminalLogFn: + async def log(level: str, phase: str, text: str) -> None: + await terminal_log(agent_id, text, level=level, phase=phase, prompt_id=prompt_id) + + return log diff --git a/api/approval_service.py b/api/approval_service.py new file mode 100644 index 0000000..bc40e43 --- /dev/null +++ b/api/approval_service.py @@ -0,0 +1,248 @@ +"""Approval workflow — agents request; Mo & Bart approve before mutating actions.""" + +from __future__ import annotations + +import json +import re +from datetime import datetime, timezone +from typing import Any, Awaitable, Callable + +from sqlalchemy import select +from sqlalchemy.orm import Session + +APPROVAL_ACTION_TYPES = { + "docker.restart": "Docker container restart", + "docker.update": "Docker image update / pull", + "docker.recreate": "Docker container recreate", + "docker.stop": "Docker container stop", + "docker.remove": "Docker container remove", + "docker.compose": "Docker Compose deploy", + "docker.prune": "Docker prune / cleanup", + "db.migrate": "Database schema migration", + "db.restart": "Database service restart", + "etl.restart": "ETL / connector restart", + "kafka.reset": "Kafka topic / offset reset", + "hdfs.mutate": "HDFS destructive operation", + "infra.reboot": "VM / host reboot", + "generic.mutate": "Infrastructure change", +} + +SUPERVISOR_IDS = ["mo-commander", "bart-commander"] + +_INTENT_PATTERNS: list[tuple[re.Pattern[str], str]] = [ + (re.compile(r"\b(restart|herstart|reboot)\b.*\b(container|docker|stack|service|mysql|postgres|kafka|airflow)"), "docker.restart"), + (re.compile(r"\b(update|upgrade|updaten|pull|pullen)\b.*\b(docker|image|container|stack|compose)"), "docker.update"), + (re.compile(r"\b(recreate|rebuild|redeploy|deploy|opnieuw)\b.*\b(container|docker|stack|compose|service)"), "docker.recreate"), + (re.compile(r"\b(stop|stoppen|shutdown)\b.*\b(container|docker|service)"), "docker.stop"), + (re.compile(r"\b(remove|delete|verwijder|rm|prune|opschonen)\b.*\b(container|docker|image|volume)"), "docker.remove"), + (re.compile(r"\b(docker compose|compose up|stack deploy)"), "docker.compose"), + (re.compile(r"\b(migrate|migration|schema change)\b.*\b(db|database|postgres|mysql)"), "db.migrate"), + (re.compile(r"\b(restart|herstart)\b.*\b(db|database|postgres|mysql|mongo|cassandra)"), "db.restart"), + (re.compile(r"\b(restart|reset)\b.*\b(connector|debezium|kafka connect)"), "etl.restart"), + (re.compile(r"\b(reset|truncate|drop)\b.*\b(topic|kafka|offset)"), "kafka.reset"), + (re.compile(r"\b(reboot|restart)\b.*\b(vm|host|server|node|proxmox)"), "infra.reboot"), +] + +_RESPONSE_ACTION_PATTERN = re.compile( + r"\b(will|ga|moet|plan to|going to|propose|voorstel)\b.*\b(restart|update|pull|recreate|deploy|stop|remove|reboot|migrate)", + re.I, +) + + +def detect_approval_intent(text: str) -> dict[str, Any] | None: + lower = text.lower().strip() + for pattern, action_type in _INTENT_PATTERNS: + if pattern.search(lower): + return { + "action_type": action_type, + "action": text.strip()[:500], + "reason": f"Mutating operation detected ({APPROVAL_ACTION_TYPES.get(action_type, action_type)})", + } + return None + + +def detect_agent_proposed_action(llm_answer: str, original_message: str) -> dict[str, Any] | None: + if not _RESPONSE_ACTION_PATTERN.search(llm_answer): + return None + intent = detect_approval_intent(llm_answer) or detect_approval_intent(original_message) + if intent: + intent["reason"] = f"Agent proposed action in mission response: {intent['reason']}" + intent["action"] = llm_answer.strip()[:500] + return intent + + +def approval_to_dict(row: Any) -> dict[str, Any]: + payload: dict[str, Any] = {} + raw_payload = getattr(row, "payload", None) + if raw_payload: + try: + payload = json.loads(raw_payload) + except (json.JSONDecodeError, TypeError): + payload = {"raw": raw_payload} + decided_at = getattr(row, "decided_at", None) + return { + "id": row.id, + "ts": row.ts.isoformat() if row.ts else None, + "agent_id": row.agent_id, + "action": row.action, + "reason": row.reason, + "status": row.status, + "action_type": getattr(row, "action_type", None) or "generic.mutate", + "target": getattr(row, "target", None) or "", + "payload": payload, + "decided_by": getattr(row, "decided_by", None), + "decide_note": getattr(row, "decide_note", None), + "decided_at": decided_at.isoformat() if decided_at else None, + "priority": getattr(row, "priority", None) or "normal", + } + + +def migrate_approval_columns(engine: Any) -> None: + """Legacy shim — migrations live in db.py.""" + from db import migrate_approval_columns as _migrate + + _migrate(engine) + + +async def create_approval_request( + *, + db: Session, + ApprovalModel: type, + agent_id: str, + action: str, + reason: str, + action_type: str = "generic.mutate", + target: str = "", + payload: dict | None = None, + priority: str = "normal", + terminal_log: Callable[..., Awaitable[None]] | None = None, + mirror_supervisors: Callable[..., Awaitable[None]] | None = None, + publish: Callable[[dict], Awaitable[None]] | None = None, + add_feed: Callable[[str, str, str], dict] | None = None, +) -> dict[str, Any]: + import uuid + + approval_id = str(uuid.uuid4())[:10] + now = datetime.now(timezone.utc) + row = ApprovalModel( + id=approval_id, + ts=now, + agent_id=agent_id, + action=action, + reason=reason, + status="pending", + action_type=action_type, + target=target, + payload=json.dumps(payload or {}), + priority=priority, + ) + db.add(row) + db.commit() + db.refresh(row) + item = approval_to_dict(row) + + type_label = APPROVAL_ACTION_TYPES.get(action_type, action_type) + alert = ( + f"⚠ APPROVAL REQUIRED · {type_label}\n" + f" Agent: {agent_id}\n" + f" Action: {action[:200]}\n" + f" Target: {target or '—'}\n" + f" Reason: {reason[:200]}\n" + f" ID: {approval_id} — awaiting Mo & Bart" + ) + + if terminal_log: + for sid in SUPERVISOR_IDS: + await terminal_log(sid, alert, level="warn", phase="approval") + await terminal_log( + agent_id, + f"⏸ Action queued for approval ({approval_id}) — Mo & Bart notified", + level="warn", + phase="approval", + ) + + if mirror_supervisors: + await mirror_supervisors( + agent_id, + f"APPROVAL REQUEST [{approval_id}] {action_type}: {action[:120]}", + level="warn", + phase="approval", + ) + + if add_feed and publish: + feed = add_feed( + agent_id, + f"Approval requested ({approval_id}): {action[:80]} — waiting for Mo & Bart", + "warn", + ) + await publish({"type": "feed", "entry": feed}) + + if publish: + await publish({"type": "approval_new", "approval": item}) + + return item + + +async def decide_approval_request( + *, + db: Session, + ApprovalModel: type, + approval_id: str, + approved: bool, + decided_by: str = "mo-commander", + note: str = "", + terminal_log: Callable[..., Awaitable[None]] | None = None, + publish: Callable[[dict], Awaitable[None]] | None = None, + add_feed: Callable[[str, str, str], dict] | None = None, +) -> dict[str, Any] | None: + row = db.get(ApprovalModel, approval_id) + if not row: + return None + if row.status != "pending": + return approval_to_dict(row) + + row.status = "approved" if approved else "denied" + row.decided_by = decided_by + row.decide_note = note or None + row.decided_at = datetime.now(timezone.utc) + db.commit() + db.refresh(row) + item = approval_to_dict(row) + + verb = "APPROVED" if approved else "DENIED" + who = "Mo" if "mo" in decided_by else "Bart" if "bart" in decided_by else decided_by + msg = f"{verb} by {who}: {row.action[:100]}" + if note: + msg += f" — {note[:80]}" + + if terminal_log: + await terminal_log(row.agent_id, msg, level="ok" if approved else "warn", phase="approval") + for sid in SUPERVISOR_IDS: + await terminal_log(sid, f"✓ {msg}", level="ok" if approved else "info", phase="approval") + + if add_feed and publish: + feed = add_feed(row.agent_id, msg, "info" if approved else "warn") + await publish({"type": "feed", "entry": feed}) + + if publish: + await publish({"type": "approval_update", "approval": item}) + + return item + + +def list_approvals(db: Session, ApprovalModel: type, status: str = "pending", limit: int = 100) -> list[dict[str, Any]]: + q = select(ApprovalModel).order_by(ApprovalModel.ts.desc()).limit(limit) + if status and status != "all": + q = q.where(ApprovalModel.status == status) + rows = db.execute(q).scalars().all() + return [approval_to_dict(r) for r in rows] + + +def approval_stats(db: Session, ApprovalModel: type) -> dict[str, int]: + rows = db.execute(select(ApprovalModel)).scalars().all() + stats = {"pending": 0, "approved": 0, "denied": 0, "total": 0} + for r in rows: + stats["total"] += 1 + if r.status in stats: + stats[r.status] += 1 + return stats diff --git a/api/database_inventory.py b/api/database_inventory.py new file mode 100644 index 0000000..aa3ceb8 --- /dev/null +++ b/api/database_inventory.py @@ -0,0 +1,243 @@ +"""Live database inventory — sizes, row counts, schemas for LLM context.""" + +from __future__ import annotations + +import asyncio +import os +from concurrent.futures import ThreadPoolExecutor, TimeoutError as FuturesTimeout +from typing import Any + +DB_HOST = os.getenv("DB_VAULT_HOST", "10.0.21.51") +PG_USER = os.getenv("PG_USER", "mo") +PG_PASS = os.getenv("PG_PASSWORD", "Dell2026!") +MYSQL_USER = os.getenv("MYSQL_USER", "mo") +MYSQL_PASS = os.getenv("MYSQL_PASSWORD", "Dell2026!") +NEO4J_USER = os.getenv("NEO4J_USER", "neo4j") +NEO4J_PASS = os.getenv("NEO4J_PASSWORD", "testpwd") +ENGINE_TIMEOUT = float(os.getenv("DB_INVENTORY_TIMEOUT", "20")) + +_executor = ThreadPoolExecutor(max_workers=4) + + +def _fmt_bytes(n: int | float | None) -> str: + if n is None: + return "?" + n = float(n) + for unit in ("B", "KB", "MB", "GB", "TB"): + if n < 1024 or unit == "TB": + return f"{n:.1f} {unit}" if unit != "B" else f"{int(n)} B" + n /= 1024 + return f"{n:.1f} TB" + + +def _inventory_postgres() -> dict[str, Any]: + import psycopg2 + + out: dict[str, Any] = {"engine": "PostgreSQL", "host": DB_HOST, "database": "postgres", "ok": False} + try: + conn = psycopg2.connect( + host=DB_HOST, user=PG_USER, password=PG_PASS, dbname="postgres", connect_timeout=5, + ) + cur = conn.cursor() + cur.execute("SELECT pg_database_size(current_database())") + out["size_bytes"] = cur.fetchone()[0] + out["size_human"] = _fmt_bytes(out["size_bytes"]) + + cur.execute( + "SELECT table_name FROM information_schema.tables " + "WHERE table_schema='public' AND table_type='BASE TABLE' ORDER BY table_name", + ) + tables = [] + for (tname,) in cur.fetchall(): + cur.execute(f'SELECT reltuples::bigint FROM pg_class WHERE relname = %s', (tname,)) + est = cur.fetchone() + rows = int(est[0]) if est and est[0] else None + cur.execute( + "SELECT column_name, data_type FROM information_schema.columns " + "WHERE table_schema='public' AND table_name=%s ORDER BY ordinal_position", + (tname,), + ) + cols = [f"{c} ({dt})" for c, dt in cur.fetchall()] + tbl: dict[str, Any] = {"name": tname, "rows": rows, "rows_estimated": True, "columns": cols} + if tname == "sales_orders" and rows: + cur.execute( + "SELECT region, COUNT(*) FROM sales_orders TABLESAMPLE SYSTEM (0.1) " + "GROUP BY region ORDER BY COUNT(*) DESC LIMIT 5", + ) + sample = cur.fetchall() + if sample: + tbl["sample_regions"] = {r: c for r, c in sample} + tables.append(tbl) + out["tables"] = tables + out["ok"] = True + conn.close() + except Exception as exc: + out["error"] = str(exc) + return out + + +def _inventory_mysql() -> dict[str, Any]: + import pymysql + + out: dict[str, Any] = {"engine": "MySQL", "host": DB_HOST, "database": "hr", "ok": False} + try: + conn = pymysql.connect( + host=DB_HOST, user=MYSQL_USER, password=MYSQL_PASS, database="hr", connect_timeout=5, + ) + cur = conn.cursor() + cur.execute( + "SELECT table_name, data_length+index_length, table_rows " + "FROM information_schema.tables WHERE table_schema='hr'", + ) + tables = [] + total_bytes = 0 + for tname, tbytes, trows in cur.fetchall(): + total_bytes += tbytes or 0 + cur.execute(f"SHOW COLUMNS FROM `{tname}`") + cols = [f"{r[0]} ({r[1]})" for r in cur.fetchall()] + tbl: dict[str, Any] = { + "name": tname, + "rows": int(trows) if trows else None, + "rows_estimated": True, + "size_bytes": tbytes, + "columns": cols, + } + if tname == "employee_events": + tbl["note"] = "HR employee lifecycle events (promotions, transfers, salary changes, etc.)" + tables.append(tbl) + out["tables"] = tables + out["size_bytes"] = total_bytes + out["size_human"] = _fmt_bytes(total_bytes) + out["ok"] = True + conn.close() + except Exception as exc: + out["error"] = str(exc) + return out + + +def _inventory_mongo() -> dict[str, Any]: + from pymongo import MongoClient + + out: dict[str, Any] = {"engine": "MongoDB", "host": DB_HOST, "ok": False} + try: + client = MongoClient(f"mongodb://{DB_HOST}:27017/", serverSelectionTimeoutMS=5000) + db = client["supplychain"] + collections = [] + for cname in db.list_collection_names(): + if cname.startswith("__"): + continue + col = db[cname] + docs = col.estimated_document_count() + sample = col.find_one() or {} + fields = sorted(k for k in sample if k != "_id") + coll: dict[str, Any] = {"name": cname, "documents": docs, "fields": fields} + if cname == "events" and docs: + try: + pipe = [ + {"$sample": {"size": 5000}}, + {"$group": {"_id": "$type", "count": {"$sum": 1}}}, + {"$sort": {"count": -1}}, + {"$limit": 5}, + ] + coll["sample_types"] = {r["_id"]: r["count"] for r in col.aggregate(pipe, maxTimeMS=5000)} + except Exception: + pass + collections.append(coll) + out["database"] = "supplychain" + out["collections"] = collections + out["ok"] = True + client.close() + except Exception as exc: + out["error"] = str(exc) + return out + + +def _inventory_cassandra() -> dict[str, Any]: + out: dict[str, Any] = {"engine": "Cassandra", "host": DB_HOST, "ok": False} + try: + from cassandra.cluster import Cluster + + cluster = Cluster([DB_HOST], connect_timeout=5) + session = cluster.connect() + keyspaces = [ + r.keyspace_name + for r in session.execute("SELECT keyspace_name FROM system_schema.keyspaces") + if r.keyspace_name not in ( + "system", "system_schema", "system_traces", "system_distributed", + "system_virtual_schema", "system_auth", "system_views", + ) + ] + tables_out = [] + for ks in keyspaces: + for row in session.execute( + "SELECT table_name FROM system_schema.tables WHERE keyspace_name=%s", (ks,), + ): + tables_out.append({ + "keyspace": ks, + "name": row.table_name, + "rows": None, + "note": "COUNT skipped (large table; use Trino/Iceberg for analytics)", + }) + out["keyspaces"] = keyspaces + out["tables"] = tables_out + out["ok"] = True + cluster.shutdown() + except Exception as exc: + out["error"] = str(exc) + return out + + +def _inventory_neo4j() -> dict[str, Any]: + out: dict[str, Any] = {"engine": "Neo4j", "host": DB_HOST, "ok": False} + try: + from neo4j import GraphDatabase + + driver = GraphDatabase.driver(f"bolt://{DB_HOST}:7687", auth=(NEO4J_USER, NEO4J_PASS)) + with driver.session() as session: + nodes = [ + {"label": r["lbl"], "count": r["c"]} + for r in session.run( + "MATCH (n) RETURN labels(n)[0] AS lbl, count(*) AS c ORDER BY c DESC LIMIT 10", + ) + ] + rels = [ + {"type": r["t"], "count": r["c"]} + for r in session.run( + "MATCH ()-[r]->() RETURN type(r) AS t, count(*) AS c ORDER BY c DESC LIMIT 10", + ) + ] + out["nodes"] = nodes + out["relationships"] = rels + out["ok"] = True + driver.close() + except Exception as exc: + out["error"] = str(exc) + return out + + +def _run_with_timeout(fn, timeout: float) -> dict[str, Any]: + future = _executor.submit(fn) + try: + return future.result(timeout=timeout) + except FuturesTimeout: + return {"engine": fn.__name__.replace("_inventory_", ""), "ok": False, "error": f"timeout after {timeout}s"} + except Exception as exc: + return {"ok": False, "error": str(exc)} + + +def collect_database_inventory_sync() -> dict[str, Any]: + fns = { + "postgresql": _inventory_postgres, + "mysql": _inventory_mysql, + "mongodb": _inventory_mongo, + "cassandra": _inventory_cassandra, + "neo4j": _inventory_neo4j, + } + engines = {k: _run_with_timeout(fn, ENGINE_TIMEOUT) for k, fn in fns.items()} + ok_count = sum(1 for e in engines.values() if e.get("ok")) + return {"host": DB_HOST, "engines_ok": ok_count, "engines_total": len(engines), "engines": engines} + + +async def collect_database_inventory() -> dict[str, Any]: + loop = asyncio.get_event_loop() + return await loop.run_in_executor(_executor, collect_database_inventory_sync) diff --git a/api/db.py b/api/db.py new file mode 100644 index 0000000..ab30261 --- /dev/null +++ b/api/db.py @@ -0,0 +1,164 @@ +"""Database engine, session factory, and startup migrations.""" + +from __future__ import annotations + +import os +import sqlite3 +from typing import Any + +from sqlalchemy import create_engine, inspect, text +from sqlalchemy.orm import sessionmaker + +DATABASE_URL = os.getenv("DATABASE_URL", "sqlite:////data/atc-agents.db") +SQLITE_FALLBACK_PATH = os.getenv("SQLITE_FALLBACK_PATH", "/data/atc-agents.db") + + +def make_engine(url: str = DATABASE_URL): + kwargs: dict[str, Any] = {"pool_pre_ping": True} + if url.startswith("sqlite"): + kwargs["connect_args"] = {"check_same_thread": False} + return create_engine(url, **kwargs) + + +engine = make_engine() +SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False) + + +def migrate_approval_columns(db_engine: Any = engine) -> None: + """Add approval columns on legacy SQLite/Postgres schemas.""" + insp = inspect(db_engine) + if "approvals" not in insp.get_table_names(): + return + + existing = {c["name"] for c in insp.get_columns("approvals")} + dialect = db_engine.dialect.name + + if dialect == "postgresql": + alters = [ + ("action_type", "VARCHAR(64)", "generic.mutate"), + ("target", "TEXT", ""), + ("payload", "TEXT", "{}"), + ("decided_by", "VARCHAR(64)", None), + ("decide_note", "TEXT", None), + ("decided_at", "TIMESTAMP WITH TIME ZONE", None), + ("priority", "VARCHAR(16)", "normal"), + ] + with db_engine.begin() as conn: + for col, typ, default in alters: + if col in existing: + continue + if default is None: + conn.execute(text(f"ALTER TABLE approvals ADD COLUMN IF NOT EXISTS {col} {typ}")) + elif default == "": + conn.execute(text(f"ALTER TABLE approvals ADD COLUMN IF NOT EXISTS {col} {typ} DEFAULT ''")) + else: + conn.execute( + text(f"ALTER TABLE approvals ADD COLUMN IF NOT EXISTS {col} {typ} DEFAULT '{default}'") + ) + return + + alters = [ + ("action_type", "VARCHAR(64)", "'generic.mutate'"), + ("target", "TEXT", "''"), + ("payload", "TEXT", "'{}'"), + ("decided_by", "VARCHAR(64)", "NULL"), + ("decide_note", "TEXT", "NULL"), + ("decided_at", "DATETIME", "NULL"), + ("priority", "VARCHAR(16)", "'normal'"), + ] + with db_engine.begin() as conn: + for col, typ, default in alters: + if col not in existing: + conn.execute(text(f"ALTER TABLE approvals ADD COLUMN {col} {typ} DEFAULT {default}")) + + +def migrate_sqlite_to_postgres(db_engine: Any = engine) -> dict[str, int]: + """One-time copy from legacy SQLite volume into Postgres when Postgres is empty.""" + if not DATABASE_URL.startswith("postgresql"): + return {} + if not os.path.isfile(SQLITE_FALLBACK_PATH): + return {} + + insp = inspect(db_engine) + tables = set(insp.get_table_names()) + if "approvals" not in tables or "feed" not in tables: + return {} + + with db_engine.connect() as conn: + approval_count = conn.execute(text("SELECT COUNT(*) FROM approvals")).scalar() or 0 + feed_count = conn.execute(text("SELECT COUNT(*) FROM feed")).scalar() or 0 + if approval_count or feed_count: + return {"approvals": 0, "feed": 0, "skipped": 1} + + copied = {"approvals": 0, "feed": 0} + src = sqlite3.connect(SQLITE_FALLBACK_PATH) + src.row_factory = sqlite3.Row + try: + with db_engine.begin() as conn: + for row in src.execute("SELECT * FROM feed"): + conn.execute( + text( + "INSERT INTO feed (id, ts, agent_id, level, message) " + "VALUES (:id, :ts, :agent_id, :level, :message) ON CONFLICT (id) DO NOTHING" + ), + dict(row), + ) + copied["feed"] += 1 + for row in src.execute("SELECT * FROM approvals"): + conn.execute( + text( + "INSERT INTO approvals (id, ts, agent_id, action, reason, status, " + "action_type, target, payload, decided_by, decide_note, decided_at, priority) " + "VALUES (:id, :ts, :agent_id, :action, :reason, :status, " + ":action_type, :target, :payload, :decided_by, :decide_note, :decided_at, :priority) " + "ON CONFLICT (id) DO NOTHING" + ), + { + "id": row["id"], + "ts": row["ts"], + "agent_id": row["agent_id"], + "action": row["action"], + "reason": row["reason"], + "status": row["status"], + "action_type": row["action_type"] if "action_type" in row.keys() else "generic.mutate", + "target": row["target"] if "target" in row.keys() else "", + "payload": row["payload"] if "payload" in row.keys() else "{}", + "decided_by": row["decided_by"] if "decided_by" in row.keys() else None, + "decide_note": row["decide_note"] if "decide_note" in row.keys() else None, + "decided_at": row["decided_at"] if "decided_at" in row.keys() else None, + "priority": row["priority"] if "priority" in row.keys() else "normal", + }, + ) + copied["approvals"] += 1 + finally: + src.close() + return copied + + +def init_database(Base: type) -> dict[str, Any]: + """Create tables, run migrations, optionally import legacy SQLite data.""" + Base.metadata.create_all(engine) + migrate_approval_columns(engine) + migrated = migrate_sqlite_to_postgres(engine) + return {"engine": engine.dialect.name, "migrated_from_sqlite": migrated} + + +def db_health() -> dict[str, Any]: + try: + with engine.connect() as conn: + conn.execute(text("SELECT 1")) + insp = inspect(engine) + tables = insp.get_table_names() + stats = {} + if "approvals" in tables: + with engine.connect() as conn: + stats["approvals"] = conn.execute(text("SELECT COUNT(*) FROM approvals")).scalar() + stats["approvals_pending"] = conn.execute( + text("SELECT COUNT(*) FROM approvals WHERE status = 'pending'") + ).scalar() + if "feed" in tables: + with engine.connect() as conn: + stats["feed"] = conn.execute(text("SELECT COUNT(*) FROM feed")).scalar() + return {"ok": True, "dialect": engine.dialect.name, "tables": tables, **stats} + except Exception as exc: + return {"ok": False, "dialect": engine.dialect.name, "error": str(exc)} diff --git a/api/dockhand_envs.py b/api/dockhand_envs.py new file mode 100644 index 0000000..315449d --- /dev/null +++ b/api/dockhand_envs.py @@ -0,0 +1,20 @@ +"""Dockhand environment IDs on atc-docker01 (10.0.21.45:8082).""" + +DOCKHAND_URL = "http://10.0.21.45:8082" + +DOCKHAND_ENVS: dict[str, int] = { + "docker01": 1, + "docker02": 2, + "management": 3, + "db02": 5, + "bart_gpu": 6, + "mo_gpu": 7, + "gpu_dev": 8, + "lakehouse": 9, + "airflow": 10, + "dns1": 11, + "dns2": 12, + "command_center": 13, +} + +DOCKHAND_ENV_COMMAND_CENTER = 13 diff --git a/api/lab_context.py b/api/lab_context.py new file mode 100644 index 0000000..186d008 --- /dev/null +++ b/api/lab_context.py @@ -0,0 +1,745 @@ +"""Live lab metrics for all ATC domains — fed to vLLM as context.""" + +from __future__ import annotations + +import asyncio +import json +import os +import time +from datetime import datetime, timezone +from typing import Any + +import httpx + +from agent_terminal import TerminalLogFn +from dockhand_envs import DOCKHAND_ENV_COMMAND_CENTER, DOCKHAND_ENVS +from database_inventory import collect_database_inventory +from node_registry import NODE_REGISTRY + +DOCKHAND_URL = os.getenv("DOCKHAND_URL", "http://10.0.21.45:8082") +HDFS_NN_URL = os.getenv("HDFS_NN_URL", "http://10.0.21.61:9870") +LAKEHOUSE_HOST = os.getenv("LAKEHOUSE_HOST", "10.0.21.50") +AIRFLOW_URL = os.getenv("AIRFLOW_URL", "http://10.0.21.55:8080") +KAFKA_UI_URL = os.getenv("KAFKA_UI_URL", "http://10.0.21.36:9000") +KAFKA_CONNECT_URL = os.getenv("KAFKA_CONNECT_URL", f"http://{LAKEHOUSE_HOST}:8083") +TRINO_URL = os.getenv("TRINO_URL", f"http://{LAKEHOUSE_HOST}:8089") +SPARK_UI_URL = os.getenv("SPARK_UI_URL", f"http://{LAKEHOUSE_HOST}:8080") +GPU_URL = os.getenv("GPU_URL", "http://10.0.20.106:9000") +OBJECTSCALE_URL = os.getenv("OBJECTSCALE_URL", "http://10.0.20.111:9020") +YARN_URL = os.getenv("YARN_URL", "http://10.0.21.61:8088") + +AGENT_PRIMARY_DOMAIN = { + "infra-sentinel": "docker", + "data-custodian": "databases", + "lakehouse-ops": "lakehouse", + "hadoop-ranger": "hadoop", + "etl-guardian": "etl", +} + + +async def _log(log: TerminalLogFn | None, level: str, phase: str, text: str) -> None: + if log: + await log(level, phase, text) + + +async def _get_json( + client: httpx.AsyncClient, + url: str, + log: TerminalLogFn | None = None, + label: str = "", + timeout: float = 6.0, +) -> Any | None: + name = label or url + t0 = time.monotonic() + await _log(log, "cmd", "fetch", f"$ GET {url}") + try: + r = await client.get(url, timeout=timeout) + ms = int((time.monotonic() - t0) * 1000) + if r.status_code < 400: + await _log(log, "ok", "fetch", f"← {r.status_code} {name} ({ms}ms)") + return r.json() + await _log(log, "warn", "fetch", f"← {r.status_code} {name} ({ms}ms)") + except Exception as exc: + ms = int((time.monotonic() - t0) * 1000) + await _log(log, "err", "fetch", f"✗ {name}: {exc} ({ms}ms)") + return None + + +async def _probe_ok( + client: httpx.AsyncClient, + url: str, + log: TerminalLogFn | None = None, + label: str = "", +) -> bool: + name = label or url + t0 = time.monotonic() + await _log(log, "cmd", "probe", f"$ GET {url}") + try: + r = await client.get(url, timeout=4.0) + ms = int((time.monotonic() - t0) * 1000) + ok = r.status_code < 500 + await _log(log, "ok" if ok else "warn", "probe", f"← {r.status_code} {name} ({'UP' if ok else 'DOWN'}, {ms}ms)") + return ok + except Exception as exc: + ms = int((time.monotonic() - t0) * 1000) + await _log(log, "err", "probe", f"✗ {name}: {exc} ({ms}ms)") + return False + + +def _container_rows(containers: list[dict], host: str = "") -> list[dict[str, Any]]: + rows = [] + for c in containers: + ports = sorted({str(p.get("PublicPort")) for p in c.get("ports", []) if p.get("PublicPort")}) + rows.append({ + "name": c.get("name"), + "state": c.get("state"), + "image": c.get("image"), + "status": c.get("status"), + "ports": ports, + "host": host, + }) + return rows + + +async def dockhand_containers( + client: httpx.AsyncClient, + env_id: int, + log: TerminalLogFn | None = None, +) -> list[dict]: + url = f"{DOCKHAND_URL}/api/containers?env={env_id}" + await _log(log, "cmd", "fetch", f"$ GET {url}") + t0 = time.monotonic() + try: + r = await client.get(f"{DOCKHAND_URL}/api/containers", params={"env": env_id}, timeout=8.0) + ms = int((time.monotonic() - t0) * 1000) + r.raise_for_status() + data = r.json() + await _log(log, "ok", "fetch", f"← Dockhand env {env_id}: {len(data)} containers ({ms}ms)") + return data + except Exception as exc: + ms = int((time.monotonic() - t0) * 1000) + await _log(log, "err", "fetch", f"✗ Dockhand env {env_id}: {exc} ({ms}ms)") + return [] + + +async def collect_hdfs(client: httpx.AsyncClient, log: TerminalLogFn | None = None) -> dict[str, Any]: + ctx: dict[str, Any] = {"reachable": False, "namenode": HDFS_NN_URL} + await _log(log, "info", "fetch", "▸ HDFS NameNode JMX metrics") + try: + fs_url = f"{HDFS_NN_URL}/jmx?qry=Hadoop:service=NameNode,name=FSNamesystem" + nn_url = f"{HDFS_NN_URL}/jmx?qry=Hadoop:service=NameNode,name=NameNodeInfo" + t0 = time.monotonic() + await _log(log, "cmd", "fetch", f"$ GET {fs_url}") + await _log(log, "cmd", "fetch", f"$ GET {nn_url}") + fs_r, nn_r = await asyncio.gather( + client.get(fs_url), + client.get(nn_url), + return_exceptions=True, + ) + ms = int((time.monotonic() - t0) * 1000) + if isinstance(fs_r, httpx.Response) and fs_r.status_code == 200: + beans = fs_r.json().get("beans", []) + if beans: + b = beans[0] + ctx.update({ + "reachable": True, + "hostname": b.get("tag.Hostname"), + "ha_state": b.get("tag.HAState"), + "capacity_total_gb": b.get("CapacityTotalGB"), + "capacity_used_gb": b.get("CapacityUsedGB"), + "capacity_remaining_gb": b.get("CapacityRemainingGB"), + "files_total": b.get("FilesTotal"), + "blocks_total": b.get("BlocksTotal"), + "live_datanodes": b.get("NumLiveDataNodes"), + "dead_datanodes": b.get("NumDeadDataNodes"), + "missing_blocks": b.get("MissingBlocks"), + "under_replicated_blocks": b.get("UnderReplicatedBlocks"), + "corrupt_blocks": b.get("CorruptBlocks"), + "default_replication_factor": 3, + }) + await _log( + log, "ok", "fetch", + f"← HDFS: {b.get('CapacityUsedGB')}GB used, {b.get('FilesTotal')} files, " + f"{b.get('NumLiveDataNodes')} datanodes ({ms}ms)", + ) + else: + await _log(log, "warn", "fetch", f"← FSNamesystem JMX failed ({ms}ms)") + + if isinstance(nn_r, httpx.Response) and nn_r.status_code == 200: + beans = nn_r.json().get("beans", []) + if beans: + b = beans[0] + live = json.loads(b.get("LiveNodes") or "{}") + ctx["hdfs_version"] = b.get("Version") + ctx["safemode"] = b.get("Safemode") or "off" + ctx["percent_used"] = round(float(b.get("PercentUsed", 0)) * 100, 4) + ctx["datanodes"] = [ + { + "host": host.split(":")[0], + "capacity_gb": round(node.get("capacity", 0) / (1024**3), 1), + "used_gb": round(node.get("used", 0) / (1024**3), 4), + "blocks": node.get("numBlocks", 0), + "state": node.get("adminState"), + } + for host, node in live.items() + ] + + yarn_url = f"{YARN_URL}/ws/v1/cluster/info" + await _log(log, "cmd", "fetch", f"$ GET {yarn_url}") + try: + yr = await client.get(yarn_url, timeout=4.0) + if yr.status_code == 200: + yinfo = yr.json().get("clusterInfo", {}) + ctx["yarn_ok"] = True + ctx["yarn_state"] = yinfo.get("state", "UNKNOWN") + ctx["yarn_rm"] = YARN_URL + await _log(log, "ok", "fetch", f"← YARN RM: {yinfo.get('state', '?')}") + else: + ctx["yarn_ok"] = False + except Exception: + ctx["yarn_ok"] = False + + except Exception as exc: + ctx["error"] = str(exc) + await _log(log, "err", "fetch", f"✗ HDFS: {exc}") + return ctx + + +async def collect_etl(client: httpx.AsyncClient, log: TerminalLogFn | None = None) -> dict[str, Any]: + await _log(log, "info", "fetch", "▸ ETL stack (Airflow, Kafka, Spark)") + health, kafka_ok, spark_ok = await asyncio.gather( + _get_json(client, f"{AIRFLOW_URL}/api/v2/monitor/health", log, "Airflow health"), + _probe_ok(client, KAFKA_UI_URL, log, "Kafka UI"), + _probe_ok(client, SPARK_UI_URL, log, "Spark UI"), + ) + connectors: list[str] = [] + await _log(log, "cmd", "fetch", f"$ GET {KAFKA_CONNECT_URL}/connectors") + t0 = time.monotonic() + try: + r = await client.get(f"{KAFKA_CONNECT_URL}/connectors", timeout=5.0) + ms = int((time.monotonic() - t0) * 1000) + if r.status_code == 200: + connectors = r.json() if isinstance(r.json(), list) else [] + await _log(log, "ok", "fetch", f"← Kafka Connect: {len(connectors)} connectors ({ms}ms)") + for c in connectors: + await _log(log, "info", "fetch", f" · {c}") + else: + await _log(log, "warn", "fetch", f"← Kafka Connect {r.status_code} ({ms}ms)") + except Exception as exc: + await _log(log, "err", "fetch", f"✗ Kafka Connect: {exc}") + + airflow_detail: dict[str, str] = {} + if isinstance(health, dict): + for comp, info in health.items(): + if isinstance(info, dict) and "status" in info: + airflow_detail[comp] = info["status"] + await _log(log, "info", "fetch", f" Airflow {comp}: {info['status']}") + + connector_status: list[dict[str, Any]] = [] + for name in connectors: + status_url = f"{KAFKA_CONNECT_URL}/connectors/{name}/status" + await _log(log, "cmd", "fetch", f"$ GET {status_url}") + try: + sr = await client.get(status_url, timeout=5.0) + if sr.status_code == 200: + st = sr.json() + conn = st.get("connector", {}) + tasks = st.get("tasks", []) + state = conn.get("state", "UNKNOWN") + task_states = [t.get("state", "?") for t in tasks] + connector_status.append({ + "name": name, + "state": state, + "tasks": task_states, + }) + await _log(log, "info", "fetch", f" · {name}: {state} tasks={task_states}") + except Exception as exc: + connector_status.append({"name": name, "state": "ERROR", "error": str(exc)}) + + return { + "airflow_url": AIRFLOW_URL, + "airflow_healthy": airflow_detail.get("scheduler") == "healthy", + "airflow_components": airflow_detail, + "kafka_ui_url": KAFKA_UI_URL, + "kafka_ui_ok": kafka_ok, + "kafka_connect_url": KAFKA_CONNECT_URL, + "connectors": connectors, + "connector_status": connector_status, + "spark_ui_url": SPARK_UI_URL, + "spark_ui_ok": spark_ok, + } + + +async def collect_lakehouse( + client: httpx.AsyncClient, + containers: list[dict], + log: TerminalLogFn | None = None, +) -> dict[str, Any]: + await _log(log, "info", "fetch", "▸ Lakehouse (Trino, Spark, Kafka Connect)") + trino_info = await _get_json(client, f"{TRINO_URL}/v1/info", log, "Trino /v1/info") + running = sum(1 for c in containers if c.get("state") == "running") + for c in containers: + ports = ",".join(str(p.get("PublicPort")) for p in c.get("ports", []) if p.get("PublicPort")) or "internal" + await _log(log, "info", "fetch", f" · {c.get('name')}: {c.get('state')} ports={ports}") + return { + "host": LAKEHOUSE_HOST, + "trino_url": TRINO_URL, + "trino_ok": trino_info is not None, + "trino_version": (trino_info or {}).get("nodeVersion", {}).get("version"), + "trino_uptime": (trino_info or {}).get("uptime"), + "trino_coordinator": (trino_info or {}).get("coordinator"), + "spark_ui_url": SPARK_UI_URL, + "kafka_connect_url": KAFKA_CONNECT_URL, + "containers": _container_rows(containers, LAKEHOUSE_HOST), + "running": running, + "total": len(containers), + } + + +async def collect_databases( + client: httpx.AsyncClient, + containers: list[dict], + log: TerminalLogFn | None = None, +) -> dict[str, Any]: + await _log(log, "info", "fetch", "▸ Database vault (Dockhand env 5)") + running = sum(1 for c in containers if c.get("state") == "running") + rows = _container_rows(containers) + by_engine: dict[str, list[str]] = {} + for r in rows: + img = (r.get("image") or "").lower() + name = (r.get("name") or "").lower() + if "postgres" in img or "postgres" in name: + engine = "PostgreSQL" + elif "mysql" in img or "mysql" in name: + engine = "MySQL" + elif "mongo" in img or "mongo" in name: + engine = "MongoDB" + elif "cassandra" in img or "cassandra" in name: + engine = "Cassandra" + elif "neo4j" in img or "neo4j" in name: + engine = "Neo4j" + else: + engine = "Other" + port_str = ",".join(r["ports"]) or "internal" + by_engine.setdefault(engine, []).append(f"{r['name']} ({r['state']}, ports {port_str})") + await _log(log, "info", "fetch", f" · {r['name']}: {r['state']} [{engine}] ports={port_str}") + + return { + "dockhand_env": 5, + "running": running, + "total": len(containers), + "containers": rows, + "by_engine": by_engine, + } + + +async def collect_command_center( + client: httpx.AsyncClient, + containers: list[dict], + log: TerminalLogFn | None = None, +) -> dict[str, Any]: + await _log(log, "info", "fetch", f"▸ Command Center VM304 (Dockhand env {DOCKHAND_ENV_COMMAND_CENTER})") + running = sum(1 for c in containers if c.get("state") == "running") + rows = _container_rows(containers, "10.0.21.33") + for r in rows: + lvl = "info" if r.get("state") == "running" else "warn" + await _log(log, lvl, "fetch", f" · {r.get('name')}: {r.get('state')}") + return { + "dockhand_env": DOCKHAND_ENV_COMMAND_CENTER, + "dockhand_stack": "atc-agents-vm304", + "host": "10.0.21.33", + "vmid": 304, + "url": "http://10.0.21.33/", + "running": running, + "total": len(containers), + "containers": rows, + } + + +async def collect_docker_rack( + client: httpx.AsyncClient, + containers: list[dict], + log: TerminalLogFn | None = None, +) -> dict[str, Any]: + await _log(log, "info", "fetch", "▸ Docker rack (Dockhand env 1)") + running = sum(1 for c in containers if c.get("state") == "running") + not_running = [c["name"] for c in containers if c.get("state") != "running"] + for c in containers: + ports = ",".join(str(p.get("PublicPort")) for p in c.get("ports", []) if p.get("PublicPort")) or "internal" + lvl = "info" if c.get("state") == "running" else "warn" + await _log(log, lvl, "fetch", f" · {c.get('name')}: {c.get('state')} ports={ports}") + return { + "dockhand_url": DOCKHAND_URL, + "dockhand_env": 1, + "running": running, + "total": len(containers), + "not_running": not_running, + "containers": _container_rows(containers, "10.0.21.45"), + } + + +async def collect_objectscale(client: httpx.AsyncClient, log: TerminalLogFn | None = None) -> dict[str, Any]: + await _log(log, "info", "fetch", "▸ ObjectScale S3 storage") + ctx: dict[str, Any] = { + "host": "10.0.20.111", + "url": OBJECTSCALE_URL, + "port": "9020", + "bucket": "data", + "reachable": False, + } + t0 = time.monotonic() + await _log(log, "cmd", "probe", f"$ GET {OBJECTSCALE_URL}") + try: + r = await client.get(OBJECTSCALE_URL, timeout=4.0) + ms = int((time.monotonic() - t0) * 1000) + # 403/401 means API is up but unauthenticated + ctx["reachable"] = r.status_code in (200, 401, 403, 405) + ctx["status_code"] = r.status_code + await _log( + log, "ok" if ctx["reachable"] else "warn", "probe", + f"← ObjectScale {r.status_code} ({'UP' if ctx['reachable'] else 'DOWN'}, {ms}ms)", + ) + except Exception as exc: + ms = int((time.monotonic() - t0) * 1000) + ctx["error"] = str(exc) + await _log(log, "err", "probe", f"✗ ObjectScale: {exc} ({ms}ms)") + return ctx + + +async def collect_gpu_metrics(client: httpx.AsyncClient, log: TerminalLogFn | None = None) -> dict[str, Any]: + await _log(log, "info", "fetch", "▸ GPU Lab metrics") + base = {"ok": False, "host": GPU_URL, "ui_url": GPU_URL} + try: + metrics_url = f"{GPU_URL}/api/gpu/metrics" + model_url = f"{GPU_URL}/api/active-model" + await _log(log, "cmd", "fetch", f"$ GET {metrics_url}") + await _log(log, "cmd", "fetch", f"$ GET {model_url}") + t0 = time.monotonic() + metrics_r, model_r = await asyncio.gather( + client.get(metrics_url), + client.get(model_url), + return_exceptions=True, + ) + ms = int((time.monotonic() - t0) * 1000) + gpus: list[dict[str, Any]] = [] + if isinstance(metrics_r, httpx.Response) and metrics_r.status_code == 200: + current = metrics_r.json().get("current", {}) + gpus = [ + { + "index": g["index"], + "name": g["name"], + "util_gpu": g.get("util_gpu", 0), + "memory_used_mib": g.get("memory_used_mib", 0), + "memory_total_mib": g.get("memory_total_mib", 0), + "temperature_c": g.get("temperature_c", 0), + "power_w": g.get("power_w", 0), + } + for g in current.get("gpus", []) + ] + await _log(log, "ok", "fetch", f"← GPU metrics: {len(gpus)} devices ({ms}ms)") + for g in gpus: + await _log( + log, "info", "fetch", + f" GPU{g['index']}: util {g['util_gpu']:.0f}% VRAM " + f"{g['memory_used_mib']:.0f}/{g['memory_total_mib']:.0f} MiB", + ) + + active_model = None + inference_active = False + vllm_url = None + if isinstance(model_r, httpx.Response) and model_r.status_code == 200: + model_data = model_r.json() + active_model = model_data.get("name") + inference_active = bool(model_data.get("inference_active")) + vllm_url = model_data.get("base_url") + await _log(log, "ok", "fetch", f"← Active model: {active_model} inference={'ON' if inference_active else 'OFF'}") + + return { + **base, + "ok": len(gpus) > 0 or inference_active, + "inference_active": inference_active, + "active_model": active_model, + "vllm_url": vllm_url, + "gpu_count": len(gpus), + "gpus": gpus, + } + except Exception as exc: + await _log(log, "err", "fetch", f"✗ GPU Lab: {exc}") + return {**base, "error": str(exc)} + + +def _section_docker(d: dict[str, Any]) -> list[str]: + lines = [ + f"Docker rack (Dockhand env 1): {d['running']}/{d['total']} running", + f"Dockhand: {d['dockhand_url']}", + ] + if d.get("not_running"): + lines.append(f"Not running: {', '.join(d['not_running'])}") + for c in d.get("containers", []): + port_str = ",".join(c["ports"]) if c["ports"] else "internal" + lines.append(f" - {c['name']}: {c['state']} | {c['image']} | ports {port_str}") + return lines + + +def _fmt_count(n: Any) -> str: + if n is None: + return "?" + try: + return f"{int(n):,}" + except (TypeError, ValueError): + return str(n) + + +def _section_databases(d: dict[str, Any]) -> list[str]: + lines = [f"Databases (Dockhand env {d['dockhand_env']}): {d['running']}/{d['total']} running"] + for engine, items in d.get("by_engine", {}).items(): + lines.append(f" {engine}:") + for item in items: + lines.append(f" - {item}") + inv = d.get("inventory") or {} + if inv: + lines.append( + f" Live data inventory @ {inv.get('host', '?')}: " + f"{inv.get('engines_ok', 0)}/{inv.get('engines_total', 0)} engines queried" + ) + for key, eng in (inv.get("engines") or {}).items(): + if not eng.get("ok"): + err = str(eng.get("error", "unknown"))[:100] + lines.append(f" {eng.get('engine', key)}: ERROR — {err}") + continue + label = eng.get("engine", key) + if eng.get("size_human"): + lines.append(f" {label} ({eng.get('database', '')}): {eng['size_human']}") + for tbl in eng.get("tables") or []: + rows = tbl.get("rows") + cols = ", ".join((tbl.get("columns") or [])[:8]) + extra = "" + if tbl.get("top_regions"): + extra = f" | regions: {tbl['top_regions']}" + elif tbl.get("top_event_types"): + extra = f" | event_types: {tbl['top_event_types']}" + lines.append(f" · {tbl['name']}: {_fmt_count(rows)} rows | cols: {cols}{extra}") + for coll in eng.get("collections") or []: + docs = coll.get("documents") + fields = ", ".join(coll.get("fields") or []) + extra = f" | types: {coll['top_types']}" if coll.get("top_types") else "" + lines.append(f" · {coll['name']}: {_fmt_count(docs)} docs | fields: {fields}{extra}") + for node in eng.get("nodes") or []: + lines.append(f" · {node['label']} nodes: {_fmt_count(node.get('count'))}") + if eng.get("relationships"): + rels = ", ".join(f"{r['type']}={_fmt_count(r.get('count'))}" for r in eng["relationships"][:5]) + lines.append(f" · relationships: {rels or 'none'}") + return lines + + +def _section_lakehouse(d: dict[str, Any]) -> list[str]: + lines = [ + f"Lakehouse host: {d['host']} — {d['running']}/{d['total']} containers running", + f"Trino: {d['trino_url']} — {'UP' if d['trino_ok'] else 'DOWN'}" + + (f" (v{d['trino_version']}, uptime {d.get('trino_uptime')})" if d.get("trino_ok") else ""), + f"Spark UI: {d['spark_ui_url']}", + f"Kafka Connect: {d['kafka_connect_url']}", + ] + for c in d.get("containers", []): + port_str = ",".join(c["ports"]) if c["ports"] else "internal" + lines.append(f" - {c['name']}: {c['state']} | {c['image']} | ports {port_str}") + return lines + + +def _section_etl(d: dict[str, Any]) -> list[str]: + lines = [ + f"Airflow ({d['airflow_url']}): {'HEALTHY' if d['airflow_healthy'] else 'DEGRADED'}", + ] + for comp, st in d.get("airflow_components", {}).items(): + lines.append(f" - {comp}: {st}") + lines.append(f"Kafka UI ({d['kafka_ui_url']}): {'UP' if d['kafka_ui_ok'] else 'DOWN'}") + lines.append(f"Kafka Connect ({d['kafka_connect_url']}): connectors {d.get('connectors') or 'none listed'}") + if d.get("connectors"): + lines.append(" Registered connector names (exact): " + ", ".join(d["connectors"])) + for cs in d.get("connector_status") or []: + tasks = cs.get("tasks") or [] + lines.append(f" Connector {cs['name']}: {cs.get('state', '?')}" + (f" tasks={tasks}" if tasks else "")) + lines.append(f"Spark UI ({d['spark_ui_url']}): {'UP' if d['spark_ui_ok'] else 'DOWN'}") + return lines + + +def _section_hadoop(h: dict[str, Any]) -> list[str]: + lines = ["HDFS / Hadoop:"] + if not h.get("reachable"): + lines.append(f" UNREACHABLE: {h.get('error', 'NameNode probe failed')}") + return lines + lines.extend([ + f" NameNode: {h['namenode']} ({h.get('hostname')}, HA {h.get('ha_state')})", + f" Version: {h.get('hdfs_version')}, safemode: {h.get('safemode')}", + f" Capacity: {h.get('capacity_used_gb')} GB used / {h.get('capacity_total_gb')} GB total " + f"({h.get('capacity_remaining_gb')} GB free, {h.get('percent_used', 0)}% used)", + f" Files: {h.get('files_total')}, Blocks: {h.get('blocks_total')}", + f" DataNodes: {h.get('live_datanodes')} live, {h.get('dead_datanodes')} dead", + f" Replication factor (dfs.replication): {h.get('default_replication_factor')}", + f" Block health: missing={h.get('missing_blocks')}, under-replicated={h.get('under_replicated_blocks')}, corrupt={h.get('corrupt_blocks')}", + ]) + for dn in h.get("datanodes", []): + lines.append( + f" - {dn['host']}: {dn['used_gb']} GB / {dn['capacity_gb']} GB, {dn['blocks']} blocks, {dn['state']}" + ) + if (h.get("capacity_used_gb") or 0) < 0.01 and (h.get("files_total") or 0) > 0: + lines.append(" Note: metadata/small files only — almost no user data stored yet.") + return lines + + +def _section_gpu(g: dict[str, Any]) -> list[str]: + lines = ["GPU Lab / vLLM inference:"] + if not g.get("ok"): + lines.append(f" OFFLINE: {g.get('error', 'unreachable')}") + return lines + lines.extend([ + f" Manager: {g.get('ui_url')}", + f" Model: {g.get('active_model')} (inference {'ON' if g.get('inference_active') else 'OFF'})", + f" vLLM endpoint: {g.get('vllm_url')}", + f" GPUs: {g.get('gpu_count')}x V100", + ]) + for gpu in g.get("gpus", []): + lines.append( + f" GPU{gpu['index']}: util {gpu['util_gpu']:.0f}%, " + f"VRAM {gpu['memory_used_mib']:.0f}/{gpu['memory_total_mib']:.0f} MiB, " + f"{gpu['temperature_c']}°C, {gpu['power_w']:.0f}W" + ) + return lines + + + + +def _section_objectscale(o: dict[str, Any]) -> list[str]: + lines = [ + f"ObjectScale S3 ({o.get('host')}:{o.get('port')}): {'REACHABLE' if o.get('reachable') else 'DOWN'}", + f" API: {o.get('url')} (HTTP {o.get('status_code', '?')})", + f" Bucket: {o.get('bucket', 'data')} — landing zone for s3-kafka-consumer & Iceberg", + ] + if o.get("error"): + lines.append(f" Error: {o['error']}") + return lines + + +def _section_command_center(c: dict[str, Any]) -> list[str]: + lines = [ + f"Command Center VM304: {c.get('host')} — {c.get('running', 0)}/{c.get('total', 0)} containers", + f" URL: {c.get('url')}", + f" Dockhand env: {c.get('dockhand_env')}", + ] + for row in c.get("containers", []): + port_str = ",".join(row["ports"]) if row.get("ports") else "internal" + lines.append(f" - {row['name']}: {row['state']} | ports {port_str}") + return lines + + +def _section_cluster_registry(_: dict[str, Any]) -> list[str]: + """Static cluster map — always available even when probes fail.""" + lines = ["Cluster infrastructure map (Proxmox VMs & roles):"] + for nid, node in NODE_REGISTRY.items(): + if nid in ("mo-commander", "bart-commander", "network-watcher", "mcp-coordinator"): + continue + vmid = node.get("vmid", "?") + lines.append( + f" - {node['label']}: {node.get('vm')} VMID {vmid} @ {node.get('ip')} — {node.get('role')}" + ) + desc = node.get("description") or "" + if desc: + lines.append(f" {desc[:140]}") + lines.append("") + lines.append("Supervisors & control plane:") + for nid in ("mo-commander", "bart-commander", "mcp-coordinator", "network-watcher"): + node = NODE_REGISTRY[nid] + lines.append(f" - {node['label']}: {(node.get('description') or '')[:120]}") + return lines + +SECTION_BUILDERS = { + "docker": _section_docker, + "databases": _section_databases, + "lakehouse": _section_lakehouse, + "etl": _section_etl, + "hadoop": _section_hadoop, + "gpu": _section_gpu, + "objectscale": _section_objectscale, + "command_center": _section_command_center, + "cluster_registry": _section_cluster_registry, +} + +DOMAIN_ORDER = ["docker", "databases", "lakehouse", "etl", "hadoop", "gpu", "objectscale", "command_center", "cluster_registry"] + + +async def collect_full_lab_context( + gpu_data: dict[str, Any] | None = None, + log: TerminalLogFn | None = None, + include_inventory: bool = True, +) -> dict[str, Any]: + """Gather all lab domains in parallel with optional live terminal logging.""" + await _log(log, "info", "fetch", "═══ Lab snapshot collection started ═══") + async with httpx.AsyncClient(timeout=10.0, verify=False) as client: + if gpu_data is None: + gpu_data = await collect_gpu_metrics(client, log) + + docker_raw, db_raw, lake_raw, cc_raw, hdfs, etl, objectscale = await asyncio.gather( + dockhand_containers(client, DOCKHAND_ENVS["docker01"], log), + dockhand_containers(client, DOCKHAND_ENVS["db02"], log), + dockhand_containers(client, DOCKHAND_ENVS["lakehouse"], log), + dockhand_containers(client, DOCKHAND_ENV_COMMAND_CENTER, log), + collect_hdfs(client, log), + collect_etl(client, log), + collect_objectscale(client, log), + ) + docker = await collect_docker_rack(client, docker_raw, log) + databases = await collect_databases(client, db_raw, log) + if include_inventory: + try: + databases["inventory"] = await collect_database_inventory() + inv_ok = databases["inventory"].get("engines_ok", 0) + await _log(log, "ok", "fetch", f"← Database inventory: {inv_ok} engines") + except Exception as exc: + await _log(log, "warn", "fetch", f"✗ Database inventory: {exc}") + databases["inventory"] = {"error": str(exc)} + lakehouse = await collect_lakehouse(client, lake_raw, log) + command_center = await collect_command_center(client, cc_raw, log) + + await _log(log, "ok", "fetch", "═══ Lab snapshot complete ═══") + return { + "ts": datetime.now(timezone.utc).isoformat(), + "docker": docker, + "databases": databases, + "lakehouse": lakehouse, + "etl": etl, + "hadoop": hdfs, + "gpu": gpu_data, + "objectscale": objectscale, + "command_center": command_center, + } + + +def format_context_for_agent(agent_id: str, snapshot: dict[str, Any]) -> str: + """Format full lab snapshot for LLM; primary domain first.""" + primary = AGENT_PRIMARY_DOMAIN.get(agent_id, "docker") + lines = [ + f"ATC Lab live snapshot — {snapshot.get('ts')}", + f"Your primary domain: {primary.upper()}", + ] + if snapshot.get("domains_summary"): + lines.append(f"Health summary: {json.dumps(snapshot['domains_summary'], default=str)}") + lines.extend(["", f"=== PRIMARY: {primary.upper()} ==="]) + + if primary in snapshot and primary in SECTION_BUILDERS: + lines.extend(SECTION_BUILDERS[primary](snapshot[primary])) + lines.append("") + lines.append("=== FULL LAB (all domains) ===") + + if "cluster_registry" not in snapshot: + snapshot = {**snapshot, "cluster_registry": {}} + + for domain in DOMAIN_ORDER: + if domain == primary: + continue + if domain not in snapshot or domain not in SECTION_BUILDERS: + continue + lines.append("") + lines.append(f"--- {domain.upper()} ---") + lines.extend(SECTION_BUILDERS[domain](snapshot[domain])) + + return "\n".join(lines) diff --git a/api/main.py b/api/main.py index b59b4e5..6cce657 100644 --- a/api/main.py +++ b/api/main.py @@ -5,6 +5,7 @@ from __future__ import annotations import asyncio import json import os +import time import uuid from contextlib import asynccontextmanager from datetime import datetime, timezone @@ -12,15 +13,54 @@ from typing import Any import httpx import redis.asyncio as aioredis -from fastapi import FastAPI, WebSocket, WebSocketDisconnect +from fastapi import FastAPI, File, UploadFile, WebSocket, WebSocketDisconnect from fastapi.middleware.cors import CORSMiddleware +from agent_terminal import ( + get_all_terminals, + get_terminal_lines, + init_terminals, + make_logger, + set_terminal_publisher, + terminal_log, +) +from lab_context import collect_full_lab_context, format_context_for_agent +from presentation import build_presentation_payload, render_presentation_html +from presentation_upload import get_deck, list_decks, save_upload +from presentation_static import get_static_deck, list_static_decks +from storage_s3 import router as storage_s3_router +from node_registry import NODE_IDS, NODE_AGENT, NODE_REGISTRY, is_node_id +from node_ops import build_node_detail, probe_node, run_node_probe_task +from approval_service import ( + APPROVAL_ACTION_TYPES, + approval_stats, + create_approval_request, + decide_approval_request, + detect_agent_proposed_action, + detect_approval_intent, + list_approvals, +) +from db import SessionLocal, db_health, init_database +from supervisor import mirror_terminal_line, mirror_to_supervisors + +from workload import build_workload_payload + +_workload_cache: dict[str, Any] = {"ts": 0.0, "data": None} +_presentation_cache: dict[str, Any] = {"ts": 0.0, "data": None} +WORKLOAD_CACHE_TTL = 30.0 +PRESENTATION_CACHE_TTL = 45.0 + from pydantic import BaseModel, Field -from sqlalchemy import Column, DateTime, String, Text, create_engine, select -from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker +from sqlalchemy import Column, DateTime, String, Text, select +from sqlalchemy.orm import DeclarativeBase REDIS_URL = os.getenv("REDIS_URL", "redis://redis:6379/0") DOCKHAND_URL = os.getenv("DOCKHAND_URL", "http://10.0.21.45:8082") -DATABASE_URL = os.getenv("DATABASE_URL", "sqlite:////data/atc-agents.db") +GPU_URL = os.getenv("GPU_URL", "http://10.0.20.106:9000") +GPU_UI_URL = os.getenv("GPU_UI_URL", GPU_URL) +LLM_URL = os.getenv("LLM_URL", "http://10.0.20.106:8001/v1") +LLM_MODEL = os.getenv("LLM_MODEL", "gpt-4o") +LLM_API_KEY = os.getenv("LLM_API_KEY", "sk-local") +LLM_TIMEOUT = float(os.getenv("LLM_TIMEOUT", "120")) AGENTS = [ { @@ -29,6 +69,14 @@ AGENTS = [ "color": "#00f0ff", "zone": "etl", "role": "Airflow, Kafka, Debezium, S3 pipeline", + "icon": "⚡", + "motto": "Pipelines never sleep", + "capabilities": ["Airflow", "Kafka", "Debezium", "S3", "Connectors"], + "suggested_prompts": [ + "How is Debezium doing?", + "Are all Airflow DAGs healthy?", + "Kafka connector status?", + ], }, { "id": "lakehouse-ops", @@ -36,6 +84,14 @@ AGENTS = [ "color": "#ff00aa", "zone": "lakehouse", "role": "Spark, Trino, Iceberg", + "icon": "🏔️", + "motto": "Query the lake, trust the table", + "capabilities": ["Spark", "Trino", "Iceberg", "Delta", "SQL"], + "suggested_prompts": [ + "Lakehouse stack status?", + "Is Trino reachable?", + "How many lakehouse containers are running?", + ], }, { "id": "data-custodian", @@ -43,6 +99,14 @@ AGENTS = [ "color": "#ffaa00", "zone": "db", "role": "PostgreSQL, MySQL, Mongo, Cassandra, Neo4j", + "icon": "🛡️", + "motto": "Guardian of every row", + "capabilities": ["PostgreSQL", "MySQL", "MongoDB", "Cassandra", "Neo4j"], + "suggested_prompts": [ + "Hoeveel data zit er in de databases?", + "Wat staat er in PostgreSQL sales_orders?", + "MongoDB supplychain overzicht", + ], }, { "id": "hadoop-ranger", @@ -50,13 +114,93 @@ AGENTS = [ "color": "#39ff14", "zone": "hadoop", "role": "HDFS, YARN cluster", + "icon": "🌲", + "motto": "Patrol the data forest", + "capabilities": ["HDFS", "YARN", "NameNode", "DataNodes"], + "suggested_prompts": [ + "Is HDFS NameNode up?", + "Hadoop cluster status?", + "YARN nodes healthy?", + ], }, { "id": "infra-sentinel", "name": "Infra Sentinel", - "color": "#b366ff", + "color": "#9b72cf", "zone": "docker", - "role": "Docker, Proxmox, monitoring", + "role": "Docker, Proxmox, GPU, monitoring", + "icon": "👁️", + "motto": "See everything, miss nothing", + "capabilities": ["Docker", "Proxmox", "GPU", "vLLM", "Monitoring"], + "suggested_prompts": [ + "GPU status?", + "Which LLM model is running?", + "Docker container overview", + ], + }, + { + "id": "mo-commander", + "name": "Mo · Command", + "color": "#4c9aed", + "zone": "command", + "role": "Supervisor — full event intel, ingress, approvals", + "icon": "🎯", + "motto": "Nothing happens without Mo knowing", + "supervisor": True, + "person": "mo", + "capabilities": ["Events", "Ingress", "Approvals", "Agent dispatch", "Network IN"], + "suggested_prompts": [ + "What happened today?", + "What events came in?", + "Pipeline status overview", + ], + }, + { + "id": "bart-commander", + "name": "Bart · Ops", + "color": "#3fb950", + "zone": "command", + "role": "Supervisor — egress, MCP comms, network OUT", + "icon": "📡", + "motto": "All traffic flows through Bart", + "supervisor": True, + "person": "bart", + "capabilities": ["Egress", "MCP routing", "Network OUT", "GPU inference", "S3 writes"], + "suggested_prompts": [ + "What is leaving the cluster?", + "MCP agent communication status?", + "Network egress overview", + ], + }, + { + "id": "network-watcher", + "name": "Network Watcher", + "color": "#58a6ff", + "zone": "network", + "role": "VLAN 20/21 traffic, data in & out paths", + "icon": "🌐", + "motto": "Every packet tells a story", + "capabilities": ["VLAN 20", "VLAN 21", "Ingress", "Egress", "Firewall paths"], + "suggested_prompts": [ + "Data ingress status?", + "What leaves the cluster?", + "Network path to S3?", + ], + }, + { + "id": "mcp-coordinator", + "name": "MCP Coordinator", + "color": "#f778ba", + "zone": "mcp", + "role": "MCP hub — routes all agent tool calls & comms", + "icon": "🔀", + "motto": "Route once, deliver everywhere", + "capabilities": ["MCP servers", "Tool routing", "Agent relay", "WebSocket bus"], + "suggested_prompts": [ + "Which MCP agents are active?", + "MCP hub route status?", + "Agent communication overview", + ], }, ] @@ -69,11 +213,18 @@ ZONES = [ ] INTENT_KEYWORDS: dict[str, list[str]] = { - "data-custodian": ["database", "db", "postgres", "mysql", "mongo", "cassandra", "neo4j", "sql"], - "lakehouse-ops": ["trino", "spark", "lakehouse", "iceberg", "query"], - "hadoop-ranger": ["hadoop", "hdfs", "yarn", "datanode"], - "infra-sentinel": ["docker", "container", "vm", "proxmox", "infra", "grafana"], + "data-custodian": ["database", "postgres", "postgresql", "mysql", "mongo", "mongodb", "cassandra", "neo4j", "sql", "db "], + "lakehouse-ops": ["trino", "spark", "lakehouse", "iceberg", "query", "table"], + "hadoop-ranger": [ + "hadoop", "hdfs", "yarn", "datanode", "namenode", "replicatie", "replication", + "rf factor", "opslag", "bestanden", "blocks", "cluster opslag", "data op", + ], + "infra-sentinel": ["docker", "container", "vm", "proxmox", "infra", "grafana", "gpu", "vllm", "llm", "nvidia", "inference", "model"], "etl-guardian": ["airflow", "dag", "debezium", "kafka", "connector", "etl", "pipeline", "s3"], + "network-watcher": ["network", "vlan", "ingress", "egress", "traffic", "packet", "firewall", "route"], + "mcp-coordinator": ["mcp", "tool", "router", "relay", "websocket", "hub"], + "mo-commander": ["mo", "supervisor", "events", "overzicht", "alles", "gebeurd"], + "bart-commander": ["bart", "egress", "uitgaand", "communicatie", "mcp comm"], } @@ -100,11 +251,16 @@ class Approval(Base): action = Column(Text) reason = Column(Text) status = Column(String, default="pending") + action_type = Column(String, default="generic.mutate") + target = Column(Text, default="") + payload = Column(Text, default="{}") + decided_by = Column(String, nullable=True) + decide_note = Column(Text, nullable=True) + decided_at = Column(DateTime, nullable=True) + priority = Column(String, default="normal") -engine = create_engine(DATABASE_URL, connect_args={"check_same_thread": False}) -SessionLocal = sessionmaker(bind=engine) -Base.metadata.create_all(engine) +_db_info = init_database(Base) redis_client: aioredis.Redis | None = None ws_clients: set[WebSocket] = set() @@ -112,14 +268,35 @@ ws_clients: set[WebSocket] = set() class PromptRequest(BaseModel): message: str = Field(min_length=1, max_length=2000) + agent_id: str | None = None + + +class NodeAskRequest(BaseModel): + message: str = Field(min_length=1, max_length=2000) + + +class ApprovalCreateRequest(BaseModel): + agent_id: str = Field(min_length=1, max_length=64) + action: str = Field(min_length=1, max_length=2000) + reason: str = Field(min_length=1, max_length=2000) + action_type: str = "generic.mutate" + target: str = "" + payload: dict[str, Any] | None = None + priority: str = "normal" class ApprovalDecision(BaseModel): approved: bool + decided_by: str = "mo-commander" + note: str = "" def route_agent(message: str) -> str: lower = message.lower() + # Storage/data questions default to Hadoop unless clearly about databases + if any(w in lower for w in ("data", "opslag", "gb", "replicatie", "replication", "hdfs", "hadoop")): + if not any(w in lower for w in ("postgres", "mysql", "mongo", "database", "sql", "neo4j", "cassandra")): + return "hadoop-ranger" scores = {aid: sum(1 for kw in kws if kw in lower) for aid, kws in INTENT_KEYWORDS.items()} best = max(scores, key=scores.get) if scores[best] == 0: @@ -127,6 +304,94 @@ def route_agent(message: str) -> str: return best +async def gather_agent_context( + agent_id: str, + status: dict[str, Any], + log: Any | None = None, +) -> str: + """Full lab snapshot for vLLM — all domains, agent's primary domain highlighted.""" + snapshot = await collect_full_lab_context(gpu_data=status.get("gpu"), log=log) + snapshot["domains_summary"] = status.get("domains", {}) + ctx = format_context_for_agent(agent_id, snapshot) + agent_lines = ["", "=== AGENTS & SUPERVISORS ==="] + for a in AGENTS: + sup = " [supervisor]" if a.get("supervisor") else "" + agent_lines.append(f" - {a['name']} ({a['id']}){sup}: {a['role']}") + ctx = ctx + "\n".join(agent_lines) + if log: + await log("info", "fetch", f"▸ Context assembled: {len(ctx)} chars for LLM") + return ctx + + +async def ask_llm( + agent_id: str, + message: str, + context: str, + log: Any | None = None, +) -> str | None: + agent = next(a for a in AGENTS if a["id"] == agent_id) + system = f"""Je bent {agent['name']}, een autonomous ops agent in het Dell ATC data lab. +Specialisatie: {agent['role']}. +Motto: {agent.get('motto', '')} + +Je antwoordt namens je domein maar hebt zicht op de HELE lab stack: Docker, databases, lakehouse (Trino/Spark/Kafka Connect), ETL (Airflow/Kafka), Hadoop HDFS, en GPU/vLLM. + +Regels: +- Antwoord in dezelfde taal als de gebruiker (Nederlands of Engels). +- Je hebt volledige zicht op de HELE cluster: alle VMs, zones, connectors, GPU, Hadoop, ObjectScale en Command Center. +- Gebruik ALLEEN de live data hieronder — verzin geen hosts, poorten, cijfers of connector namen. +- Gebruik exact de container/connector namen uit de data (bijv. mysql-hr-connector, niet "Debezium"). +- Als iets DOWN of 0 GB is, zeg dat eerlijk. +- Kort en behulpzaam (max ~10 zinnen); bullet lists mogen als het overzicht helpt. + +--- LIVE LAB DATA (primary domain eerst, daarna volledige stack) --- +{context} +""" + if log: + await log("info", "llm", f"▸ Querying vLLM model={LLM_MODEL}") + await log("cmd", "llm", f"$ POST {LLM_URL.rstrip('/')}/chat/completions") + await log("info", "llm", f" user: {message[:160]}{'…' if len(message) > 160 else ''}") + try: + async with httpx.AsyncClient(timeout=LLM_TIMEOUT) as client: + t0 = time.monotonic() + r = await client.post( + f"{LLM_URL.rstrip('/')}/chat/completions", + headers={ + "Authorization": f"Bearer {LLM_API_KEY}", + "Content-Type": "application/json", + }, + json={ + "model": LLM_MODEL, + "messages": [ + {"role": "system", "content": system}, + {"role": "user", "content": message}, + ], + "max_tokens": 800, + "temperature": 0.25, + }, + ) + r.raise_for_status() + content = r.json()["choices"][0]["message"]["content"].strip() + ms = int((time.monotonic() - t0) * 1000) + if content and content.strip("!"): + if log: + await log("ok", "llm", f"← vLLM response {len(content)} chars ({ms}ms)") + preview = content.replace("\n", " ")[:180] + await log("info", "llm", f" » {preview}{'…' if len(content) > 180 else ''}") + return content + if log: + await log("warn", "llm", f"← Empty or invalid LLM output ({ms}ms)") + except Exception as exc: + if log: + await log("err", "llm", f"✗ vLLM error: {exc}") + return None + + +def fallback_answer(agent_id: str, context: str) -> str: + agent_name = next(a["name"] for a in AGENTS if a["id"] == agent_id) + return f"**{agent_name}** (offline LLM — ruwe data):\n\n{context}" + + async def publish_event(event: dict[str, Any]) -> None: payload = json.dumps(event, default=str) if redis_client: @@ -140,6 +405,21 @@ async def publish_event(event: dict[str, Any]) -> None: for ws in dead: ws_clients.discard(ws) + et = event.get("type") + if et == "feed": + entry = event.get("entry") or {} + await mirror_to_supervisors( + entry.get("agent_id", "?"), + entry.get("message", ""), + level=entry.get("level", "info"), + ) + elif et == "terminal": + await mirror_terminal_line(event.get("line") or {}) + elif et in ("agent_dispatch", "agent_fetch", "agent_return"): + aid = event.get("agent_id", "?") + zone = event.get("zone", "") + await mirror_to_supervisors(aid, f"{et} → zone {zone}", level="info", phase="dispatch") + def add_feed(agent_id: str, message: str, level: str = "info") -> dict: entry_id = str(uuid.uuid4())[:8] @@ -175,6 +455,67 @@ async def probe_url(url: str) -> bool: return False +async def collect_gpu() -> dict[str, Any]: + host = GPU_URL.replace("http://", "").replace("https://", "").split("/")[0] + base = {"ok": False, "host": host, "ui_url": GPU_UI_URL} + + try: + async with httpx.AsyncClient(timeout=6.0) as client: + metrics_r, model_r, integration_r = await asyncio.gather( + client.get(f"{GPU_URL}/api/gpu/metrics"), + client.get(f"{GPU_URL}/api/active-model"), + client.get(f"{GPU_URL}/api/integration"), + return_exceptions=True, + ) + + gpus: list[dict[str, Any]] = [] + if isinstance(metrics_r, httpx.Response) and metrics_r.status_code == 200: + current = metrics_r.json().get("current", {}) + gpus = [ + { + "index": g["index"], + "name": g["name"], + "util_gpu": g.get("util_gpu", 0), + "memory_used_mib": g.get("memory_used_mib", 0), + "memory_total_mib": g.get("memory_total_mib", 0), + "temperature_c": g.get("temperature_c", 0), + "power_w": g.get("power_w", 0), + } + for g in current.get("gpus", []) + ] + + active_model = None + inference_active = False + vllm_url = None + + if isinstance(model_r, httpx.Response) and model_r.status_code == 200: + model_data = model_r.json() + active_model = model_data.get("name") + inference_active = bool(model_data.get("inference_active")) + vllm_url = model_data.get("base_url") + + if isinstance(integration_r, httpx.Response) and integration_r.status_code == 200: + integ = integration_r.json() + if not active_model: + active_model = integ.get("active_name") + if not inference_active: + inference_active = bool(integ.get("inference_active")) + if not vllm_url: + vllm_url = integ.get("recommended_base_url") + + return { + **base, + "ok": len(gpus) > 0 or inference_active, + "inference_active": inference_active, + "active_model": active_model, + "vllm_url": vllm_url, + "gpu_count": len(gpus), + "gpus": gpus, + } + except Exception as exc: + return {**base, "error": str(exc)} + + async def collect_status() -> dict[str, Any]: db_containers = await dockhand_env_containers(5) db_running = sum(1 for c in db_containers if c.get("state") == "running") @@ -201,6 +542,10 @@ async def collect_status() -> dict[str, Any]: return "warn" return "down" + gpu = await collect_gpu() + gpu_level = "ok" if gpu.get("ok") and gpu.get("inference_active") else ("warn" if gpu.get("ok") else "down") + gpu_label = gpu.get("active_model") or (f"{gpu.get('gpu_count', 0)} GPUs" if gpu.get("ok") else "offline") + return { "ts": datetime.now(timezone.utc).isoformat(), "domains": { @@ -209,47 +554,112 @@ async def collect_status() -> dict[str, Any]: "lakehouse": {"level": level(lake_running, lake_total), "label": f"{lake_running}/{lake_total} up", "running": lake_running, "total": lake_total}, "hadoop": {"level": "ok" if hdfs_ok else "warn", "label": "NN up" if hdfs_ok else "NN check"}, "etl": {"level": "ok" if kafka_ok and airflow_ok else "warn", "label": "Kafka+Airflow"}, + "gpu": {"level": gpu_level, "label": gpu_label}, }, + "gpu": gpu, "kafka_ok": kafka_ok, "airflow_ok": airflow_ok, "hdfs_ok": hdfs_ok, } + + +async def _run_agent_task_safe(agent_id: str, message: str, prompt_id: str) -> None: + try: + await run_agent_task(agent_id, message, prompt_id) + except Exception as exc: + agent_name = next((a["name"] for a in AGENTS if a["id"] == agent_id), agent_id) + err = f"Sorry — {agent_name} could not complete your request: {exc}" + await terminal_log(agent_id, f"[{prompt_id}] ✗ Error: {exc}", level="err", phase="error", prompt_id=prompt_id) + feed = add_feed(agent_id, f"{agent_name} failed: {str(exc)[:80]}", "err") + await publish_event({"type": "feed", "entry": feed}) + await publish_event({ + "type": "prompt_result", + "prompt_id": prompt_id, + "agent_id": agent_id, + "answer": err, + }) + + async def run_agent_task(agent_id: str, message: str, prompt_id: str) -> str: zone = next(a["zone"] for a in AGENTS if a["id"] == agent_id) + agent_name = next(a["name"] for a in AGENTS if a["id"] == agent_id) + log = make_logger(agent_id, prompt_id) + approval_created = False + + intent = detect_approval_intent(message) + if intent: + with SessionLocal() as db: + await create_approval_request( + db=db, + ApprovalModel=Approval, + agent_id=agent_id, + action=intent["action"], + reason=intent["reason"], + action_type=intent["action_type"], + terminal_log=terminal_log, + mirror_supervisors=mirror_to_supervisors, + publish=publish_event, + add_feed=add_feed, + ) + approval_created = True + await terminal_log( + agent_id, + f"[{prompt_id}] Mutating request detected — approval queued for Mo & Bart", + level="warn", + phase="approval", + prompt_id=prompt_id, + ) + + await terminal_log( + agent_id, + f"[{prompt_id}] ▶ Mission accepted: {message}", + level="info", + phase="dispatch", + prompt_id=prompt_id, + ) await publish_event({"type": "agent_dispatch", "agent_id": agent_id, "zone": zone, "prompt_id": prompt_id}) - await asyncio.sleep(0.8) + await asyncio.sleep(0.4) + await terminal_log(agent_id, f"[{prompt_id}] Walking to zone: {zone}", level="info", phase="dispatch", prompt_id=prompt_id) await publish_event({"type": "agent_fetch", "agent_id": agent_id, "zone": zone, "prompt_id": prompt_id}) + await log("info", "fetch", f"[{prompt_id}] Collecting live lab metrics…") status = await collect_status() - answer_parts = [f"**{next(a['name'] for a in AGENTS if a['id'] == agent_id)}** reporting:"] + context = await gather_agent_context(agent_id, status, log=log) - if agent_id == "data-custodian": - db = status["domains"]["databases"] - containers = await dockhand_env_containers(5) - names = ", ".join(f"{c['name']}:{c.get('state','?')}" for c in containers[:8]) - answer_parts.append(f"Databases {db['label']}. Containers: {names or 'unreachable'}.") - elif agent_id == "etl-guardian": - answer_parts.append( - f"Kafka UI: {'OK' if status['kafka_ok'] else 'DOWN'}. " - f"Airflow: {'OK' if status['airflow_ok'] else 'DOWN'}. " - f"Lakehouse {status['domains']['lakehouse']['label']}." - ) - elif agent_id == "lakehouse-ops": - answer_parts.append(f"Lakehouse stack {status['domains']['lakehouse']['label']}. Trino at 10.0.21.50:8089.") - elif agent_id == "hadoop-ranger": - answer_parts.append(f"HDFS NameNode: {'reachable' if status['hdfs_ok'] else 'unreachable'} on 10.0.21.61:9870.") - else: - answer_parts.append( - f"Docker {status['domains']['docker']['label']}. " - f"Overall lab health snapshot collected." - ) + answer = await ask_llm(agent_id, message, context, log=log) + if not answer: + await log("warn", "llm", "LLM fallback — returning raw context") + answer = fallback_answer(agent_id, context) - answer = " ".join(answer_parts) - await asyncio.sleep(0.6) + if not approval_created: + proposed = detect_agent_proposed_action(answer, message) + if proposed: + with SessionLocal() as db: + await create_approval_request( + db=db, + ApprovalModel=Approval, + agent_id=agent_id, + action=proposed["action"], + reason=proposed["reason"], + action_type=proposed["action_type"], + target=proposed.get("target", ""), + terminal_log=terminal_log, + mirror_supervisors=mirror_to_supervisors, + publish=publish_event, + add_feed=add_feed, + ) + approval_created = True + answer = ( + f"{answer}\n\n⏸ **Approval required** — this action is in the Approval Inbox. " + f"Mo & Bart have been notified and must approve before we execute." + ) + + await asyncio.sleep(0.3) + await terminal_log(agent_id, f"[{prompt_id}] ✓ Mission complete", level="ok", phase="done", prompt_id=prompt_id) await publish_event({"type": "agent_return", "agent_id": agent_id, "zone": zone, "prompt_id": prompt_id}) - feed = add_feed(agent_id, f"Prompt answered: {message[:80]}", "info") + feed = add_feed(agent_id, f"{agent_name} completed a response (see Comms)", "info") await publish_event({"type": "feed", "entry": feed}) await publish_event({"type": "prompt_result", "prompt_id": prompt_id, "agent_id": agent_id, "answer": answer}) return answer @@ -259,7 +669,9 @@ async def heartbeat_loop() -> None: while True: try: status = await collect_status() + workload = await collect_workload() await publish_event({"type": "status", "data": status}) + await publish_event({"type": "workload", "data": workload}) for domain, info in status["domains"].items(): if info["level"] == "down": agent = "data-custodian" if domain == "databases" else "infra-sentinel" @@ -274,6 +686,15 @@ async def heartbeat_loop() -> None: async def lifespan(app: FastAPI): global redis_client redis_client = aioredis.from_url(REDIS_URL, decode_responses=True) + set_terminal_publisher(publish_event) + init_terminals([a["id"] for a in AGENTS] + NODE_IDS) + for a in AGENTS: + await terminal_log(a["id"], f"{a['name']} terminal online — awaiting missions", level="info", phase="boot") + for nid in NODE_IDS: + if nid not in NODE_REGISTRY: + continue + meta = NODE_REGISTRY[nid] + await terminal_log(nid, f"{meta['label']} shell ready — click node to connect", level="info", phase="boot") task = asyncio.create_task(heartbeat_loop()) add_feed("infra-sentinel", "ATC Command Center API online", "info") yield @@ -283,6 +704,7 @@ async def lifespan(app: FastAPI): app = FastAPI(title="ATC Command Center API", lifespan=lifespan) +app.include_router(storage_s3_router) app.add_middleware( CORSMiddleware, allow_origins=["*"], @@ -294,7 +716,103 @@ app.add_middleware( @app.get("/api/health") async def health(): - return {"ok": True, "ts": datetime.now(timezone.utc).isoformat()} + llm_ok = False + try: + async with httpx.AsyncClient(timeout=4.0) as client: + r = await client.get(f"{LLM_URL.rstrip('/')}/models", headers={"Authorization": f"Bearer {LLM_API_KEY}"}) + llm_ok = r.status_code == 200 + except Exception: + pass + return { + "ok": True, + "ts": datetime.now(timezone.utc).isoformat(), + "llm_url": LLM_URL, + "llm_ok": llm_ok, + "llm_model": LLM_MODEL, + "database": db_health(), + "db_init": _db_info, + } + + +async def collect_workload(*, fast: bool = True, use_cache: bool = True) -> dict[str, Any]: + import time as _time + now = _time.time() + if use_cache and _workload_cache.get("data") and now - float(_workload_cache.get("ts") or 0) < WORKLOAD_CACHE_TTL: + return _workload_cache["data"] + gpu = await collect_gpu() + snap = await collect_full_lab_context(gpu_data=gpu, include_inventory=not fast) + payload = build_workload_payload(snap) + _workload_cache["ts"] = now + _workload_cache["data"] = payload + return payload + + + + +async def get_presentation_data(*, use_cache: bool = True) -> dict[str, Any]: + import time as _time + now = _time.time() + if use_cache and _presentation_cache.get("data") and now - float(_presentation_cache.get("ts") or 0) < PRESENTATION_CACHE_TTL: + return _presentation_cache["data"] + gpu = await collect_gpu() + snap = await collect_full_lab_context(gpu_data=gpu, include_inventory=False) + data = build_presentation_payload(snap) + data["source"] = "live" + _presentation_cache["ts"] = now + _presentation_cache["data"] = data + return data + + +@app.get("/api/presentation") +async def get_presentation(): + return await get_presentation_data() + + +@app.get("/api/presentation/html") +async def get_presentation_html(): + from fastapi.responses import HTMLResponse + payload = await get_presentation_data() + return HTMLResponse(render_presentation_html(payload)) + + +@app.get("/api/presentation/decks") +async def get_presentation_decks(): + return {"live": True, "builtin": list_static_decks(), "uploaded": list_decks()} + + +@app.get("/api/presentation/decks/{deck_id}") +async def get_presentation_deck(deck_id: str): + if deck_id == "live": + return await get_presentation_data() + deck = get_static_deck(deck_id) or get_deck(deck_id) + if not deck: + return {"error": "deck not found"} + return deck + + +@app.get("/api/presentation/decks/{deck_id}/html") +async def get_presentation_deck_html(deck_id: str): + from fastapi.responses import HTMLResponse + if deck_id == "live": + payload = await get_presentation_data() + else: + payload = get_static_deck(deck_id) or get_deck(deck_id) + if not payload: + return HTMLResponse("

Deck not found

", status_code=404) + return HTMLResponse(render_presentation_html(payload)) + + +@app.post("/api/presentation/upload") +async def upload_presentation(file: UploadFile = File(...)): + content = await file.read() + if len(content) > 50 * 1024 * 1024: + return {"error": "file too large (max 50MB)"} + deck = await save_upload(file.filename or "upload.pptx", content) + return {"ok": True, "deck": deck} + +@app.get("/api/workload") +async def get_workload(fast: bool = True): + return await collect_workload(fast=fast, use_cache=True) @app.get("/api/status") @@ -302,9 +820,95 @@ async def get_status(): return await collect_status() +@app.get("/api/gpu") +async def get_gpu(): + return await collect_gpu() + + +def agent_stats() -> dict[str, dict[str, Any]]: + stats: dict[str, dict[str, Any]] = {a["id"]: {"tasks": 0, "last_active": None, "alerts": 0} for a in AGENTS} + with SessionLocal() as db: + rows = db.execute(select(FeedEntry).order_by(FeedEntry.ts.desc()).limit(200)).scalars().all() + for r in rows: + aid = r.agent_id + if aid not in stats: + continue + stats[aid]["tasks"] += 1 + if r.level == "warn": + stats[aid]["alerts"] += 1 + if stats[aid]["last_active"] is None and r.ts: + stats[aid]["last_active"] = r.ts.isoformat() + return stats + + @app.get("/api/agents") async def get_agents(): - return {"agents": AGENTS, "zones": ZONES} + stats = agent_stats() + enriched = [{**a, "stats": stats.get(a["id"], {})} for a in AGENTS] + return {"agents": enriched, "zones": ZONES} + + +@app.get("/api/terminals") +async def get_terminals(limit: int = 200): + return {"terminals": get_all_terminals(limit)} + + +@app.get("/api/terminals/{subject_id}") +async def get_subject_terminal(subject_id: str, limit: int = 200): + valid_agents = {a["id"] for a in AGENTS} + if subject_id not in valid_agents and not is_node_id(subject_id): + return {"error": "unknown subject"} + return {"agent_id": subject_id, "lines": get_terminal_lines(subject_id, limit)} + + +@app.get("/api/nodes") +async def list_nodes(): + workload = await collect_workload() + nodes = workload.get("topology", {}).get("nodes", []) + return {"nodes": [{"id": n["id"], "label": n["label"], "ip": n["ip"], "level": n["level"]} for n in nodes]} + + +@app.get("/api/nodes/{node_id}") +async def get_node(node_id: str): + if not is_node_id(node_id): + return {"error": "unknown node"} + workload = await collect_workload() + wn = next((n for n in workload.get("topology", {}).get("nodes", []) if n["id"] == node_id), None) + gpu = await collect_gpu() + snap = await collect_full_lab_context(gpu_data=gpu) + return build_node_detail(node_id, snap, wn) + + +@app.post("/api/nodes/{node_id}/probe") +async def post_node_probe(node_id: str): + if not is_node_id(node_id): + return {"error": "unknown node"} + asyncio.create_task(run_node_probe_task(node_id)) + return {"ok": True, "node_id": node_id, "status": "probing"} + + +async def run_node_ask_task(node_id: str, message: str) -> None: + agent_id = NODE_AGENT.get(node_id, "infra-sentinel") + meta = NODE_REGISTRY[node_id] + await terminal_log(node_id, f"▶ Query: {message}", level="info", phase="ask") + await terminal_log(node_id, f"→ Routing to agent {agent_id}", level="info", phase="ask") + log = make_logger(node_id) + status = await collect_status() + context = await gather_agent_context(agent_id, status, log=log) + node_ctx = f"\n\n=== FOCUSED NODE: {meta['label']} ({meta['ip']}) ===\n{meta.get('description', '')}\n" + answer = await ask_llm(agent_id, message, context + node_ctx, log=log) + if not answer: + answer = fallback_answer(agent_id, context) + await terminal_log(node_id, f"◆ {answer}", level="llm", phase="answer") + await publish_event({"type": "node_ask_result", "node_id": node_id, "agent_id": agent_id, "answer": answer}) + + +@app.post("/api/nodes/{node_id}/ask") +async def post_node_ask(node_id: str, body: NodeAskRequest): + if not is_node_id(node_id): + return {"error": "unknown node"} + asyncio.create_task(run_node_ask_task(node_id, body.message)) + return {"ok": True, "node_id": node_id, "agent_id": NODE_AGENT.get(node_id), "status": "processing"} @app.get("/api/feed") @@ -326,47 +930,73 @@ async def get_feed(limit: int = 50): @app.get("/api/approvals") -async def get_approvals(): +async def get_approvals(status: str = "pending", limit: int = 100): with SessionLocal() as db: - rows = db.execute(select(Approval).where(Approval.status == "pending")).scalars().all() - return { - "approvals": [ - { - "id": r.id, - "ts": r.ts.isoformat() if r.ts else None, - "agent_id": r.agent_id, - "action": r.action, - "reason": r.reason, - "status": r.status, - } - for r in rows - ] - } + items = list_approvals(db, Approval, status=status, limit=limit) + stats = approval_stats(db, Approval) + return {"approvals": items, "stats": stats, "action_types": APPROVAL_ACTION_TYPES} + + +@app.get("/api/approvals/stats") +async def get_approval_stats(): + with SessionLocal() as db: + return approval_stats(db, Approval) + + +@app.post("/api/approvals") +async def post_approval(body: ApprovalCreateRequest): + valid_ids = {a["id"] for a in AGENTS} + if body.agent_id not in valid_ids: + return {"error": "unknown agent_id"} + if body.action_type not in APPROVAL_ACTION_TYPES: + body.action_type = "generic.mutate" + with SessionLocal() as db: + item = await create_approval_request( + db=db, + ApprovalModel=Approval, + agent_id=body.agent_id, + action=body.action, + reason=body.reason, + action_type=body.action_type, + target=body.target, + payload=body.payload, + priority=body.priority, + terminal_log=terminal_log, + mirror_supervisors=mirror_to_supervisors, + publish=publish_event, + add_feed=add_feed, + ) + return {"ok": True, "approval": item} @app.post("/api/approvals/{approval_id}/decide") async def decide_approval(approval_id: str, body: ApprovalDecision): + valid_supervisors = {"mo-commander", "bart-commander"} + decided_by = body.decided_by if body.decided_by in valid_supervisors else "mo-commander" with SessionLocal() as db: - row = db.get(Approval, approval_id) - if not row: - return {"error": "not found"} - row.status = "approved" if body.approved else "denied" - db.commit() - agent_id = row.agent_id - action = row.action - msg = f"Approval {'approved' if body.approved else 'denied'}: {action}" - feed = add_feed(agent_id, msg, "info" if body.approved else "warn") - await publish_event({"type": "feed", "entry": feed}) - await publish_event({"type": "approval_update", "id": approval_id, "status": row.status}) - return {"ok": True, "status": row.status} + item = await decide_approval_request( + db=db, + ApprovalModel=Approval, + approval_id=approval_id, + approved=body.approved, + decided_by=decided_by, + note=body.note, + terminal_log=terminal_log, + publish=publish_event, + add_feed=add_feed, + ) + if not item: + return {"error": "not found"} + return {"ok": True, "approval": item} @app.post("/api/prompt") async def post_prompt(body: PromptRequest): prompt_id = str(uuid.uuid4())[:8] - agent_id = route_agent(body.message) + valid_ids = {a["id"] for a in AGENTS} + agent_id = body.agent_id if body.agent_id in valid_ids else route_agent(body.message) add_feed(agent_id, f"Prompt received: {body.message}", "info") - asyncio.create_task(run_agent_task(agent_id, body.message, prompt_id)) + asyncio.create_task(_run_agent_task_safe(agent_id, body.message, prompt_id)) return {"prompt_id": prompt_id, "agent_id": agent_id, "status": "dispatched"} @@ -376,7 +1006,13 @@ async def ws_ops(websocket: WebSocket): ws_clients.add(websocket) try: status = await collect_status() + workload = await collect_workload() await websocket.send_text(json.dumps({"type": "status", "data": status}, default=str)) + await websocket.send_text(json.dumps({"type": "workload", "data": workload}, default=str)) + await websocket.send_text(json.dumps({ + "type": "terminal_history", + "terminals": get_all_terminals(150), + }, default=str)) while True: await websocket.receive_text() except WebSocketDisconnect: diff --git a/api/node_ops.py b/api/node_ops.py new file mode 100644 index 0000000..dfe4fb5 --- /dev/null +++ b/api/node_ops.py @@ -0,0 +1,186 @@ +"""Live probe + context for topology nodes.""" + +from __future__ import annotations + +import asyncio +from typing import Any + +import httpx + +from agent_terminal import terminal_log +from lab_context import ( + AIRFLOW_URL, + DOCKHAND_URL, + GPU_URL, + HDFS_NN_URL, + KAFKA_CONNECT_URL, + KAFKA_UI_URL, + LAKEHOUSE_HOST, + OBJECTSCALE_URL, + SPARK_UI_URL, + TRINO_URL, + collect_databases, + collect_docker_rack, + collect_etl, + collect_gpu_metrics, + collect_hdfs, + collect_lakehouse, + collect_objectscale, + dockhand_containers, +) +from node_registry import NODE_AGENT, NODE_REGISTRY + + +async def _log(node_id: str, level: str, phase: str, text: str) -> None: + await terminal_log(node_id, text, level=level, phase=phase) + + +async def probe_node(node_id: str) -> dict[str, Any]: + """Run live probes for a topology node; stream output to node terminal.""" + meta = NODE_REGISTRY.get(node_id) + if not meta: + return {"error": "unknown node"} + + await _log(node_id, "info", "shell", f"═══ Connecting to {meta['label']} ({meta['ip']}) ═══") + await _log(node_id, "cmd", "shell", f"$ probe --node {node_id} --vm {meta['vm']}") + + result: dict[str, Any] = {"node_id": node_id, "ok": True} + + async with httpx.AsyncClient(timeout=10.0, verify=False) as client: + if node_id == "airflow": + etl = await collect_etl(client) + result["data"] = etl + healthy = etl.get("airflow_healthy") + await _log(node_id, "ok" if healthy else "warn", "shell", f"Airflow scheduler: {'HEALTHY' if healthy else 'DEGRADED'}") + for comp, st in (etl.get("airflow_components") or {}).items(): + await _log(node_id, "info", "shell", f" · {comp}: {st}") + + elif node_id == "db": + raw = await dockhand_containers(client, 5) + db = await collect_databases(client, raw) + result["data"] = db + await _log(node_id, "ok", "shell", f"DB vault: {db['running']}/{db['total']} containers up") + for engine, items in (db.get("by_engine") or {}).items(): + await _log(node_id, "info", "shell", f" {engine}:") + for item in items: + await _log(node_id, "info", "shell", f" - {item}") + + elif node_id == "debezium": + etl = await collect_etl(client) + result["data"] = {"connectors": etl.get("connectors")} + await _log(node_id, "ok", "shell", f"Kafka Connect @ {KAFKA_CONNECT_URL}") + for c in etl.get("connectors") or []: + await _log(node_id, "info", "shell", f" ✓ {c}") + + elif node_id == "kafka": + etl = await collect_etl(client) + result["data"] = {"kafka_ui_ok": etl.get("kafka_ui_ok")} + await _log(node_id, "ok" if etl.get("kafka_ui_ok") else "warn", "shell", f"Kafka UI {KAFKA_UI_URL}: {'UP' if etl.get('kafka_ui_ok') else 'DOWN'}") + await _log(node_id, "info", "shell", f" Broker: 10.0.21.36:9092") + + elif node_id == "lakehouse": + raw = await dockhand_containers(client, 9) + lh = await collect_lakehouse(client, raw) + result["data"] = lh + await _log(node_id, "ok", "shell", f"Lakehouse {lh['host']}: {lh['running']}/{lh['total']} containers") + await _log(node_id, "info", "shell", f" Trino {TRINO_URL}: {'UP' if lh.get('trino_ok') else 'DOWN'}") + for c in lh.get("containers") or []: + ports = ",".join(c.get("ports") or []) or "internal" + await _log(node_id, "info", "shell", f" · {c['name']}: {c['state']} ports={ports}") + + elif node_id == "s3": + os_data = await collect_objectscale(client) + raw = await dockhand_containers(client, 9) + consumer = next((c for c in raw if "s3-kafka" in f"{c.get('name', '')} {c.get('image', '')}".lower()), None) + result["data"] = {"objectscale": os_data, "consumer": consumer} + await _log(node_id, "ok" if os_data.get("reachable") else "warn", "shell", f"ObjectScale {OBJECTSCALE_URL}: {'UP' if os_data.get('reachable') else 'DOWN'}") + await _log(node_id, "info", "shell", f" Bucket: data @ {os_data.get('host')}:{os_data.get('port')}") + if consumer: + await _log(node_id, "info", "shell", f" s3-kafka-consumer: {consumer.get('state')}") + + elif node_id == "docker": + raw = await dockhand_containers(client, 1) + dk = await collect_docker_rack(client, raw) + result["data"] = dk + await _log(node_id, "ok", "shell", f"Docker rack: {dk['running']}/{dk['total']} running") + for c in dk.get("containers") or []: + ports = ",".join(c.get("ports") or []) or "internal" + lvl = "info" if c.get("state") == "running" else "warn" + await _log(node_id, lvl, "shell", f" · {c['name']}: {c['state']} ports={ports}") + + elif node_id == "hadoop": + hdfs = await collect_hdfs(client) + result["data"] = hdfs + if hdfs.get("reachable"): + await _log(node_id, "ok", "shell", f"NameNode {HDFS_NN_URL}: UP") + await _log(node_id, "info", "shell", f" Capacity: {hdfs.get('capacity_used_gb')}GB / {hdfs.get('capacity_total_gb')}GB") + await _log(node_id, "info", "shell", f" DataNodes: {hdfs.get('live_datanodes')} live, RF=3") + for dn in hdfs.get("datanodes") or []: + await _log(node_id, "info", "shell", f" · {dn['host']}: {dn['used_gb']}GB used, {dn['blocks']} blocks") + else: + await _log(node_id, "err", "shell", "NameNode unreachable") + + elif node_id == "gpu": + gpu = await collect_gpu_metrics(client) + result["data"] = gpu + await _log(node_id, "ok" if gpu.get("ok") else "warn", "shell", f"GPU Lab {GPU_URL}") + await _log(node_id, "info", "shell", f" Model: {gpu.get('active_model')} inference={'ON' if gpu.get('inference_active') else 'OFF'}") + for g in gpu.get("gpus") or []: + await _log(node_id, "info", "shell", f" GPU{g['index']}: {g['util_gpu']:.0f}% VRAM {g['memory_used_mib']:.0f}/{g['memory_total_mib']:.0f} MiB") + + elif node_id == "command": + result["data"] = {"agents": 9, "url": "http://10.0.21.33"} + await _log(node_id, "ok", "shell", "Command Center online — 9 agents ready") + await _log(node_id, "info", "shell", " API: http://10.0.21.33/api") + await _log(node_id, "info", "shell", " WebSocket: /api/ws/ops") + + elif node_id in ("mo-commander", "bart-commander", "mcp-coordinator", "network-watcher"): + from node_registry import NODE_REGISTRY + meta = NODE_REGISTRY[node_id] + await _log(node_id, "ok", "shell", f"{meta['label']} online — monitoring all agent comms") + await _log(node_id, "info", "shell", meta.get("description", "")) + result["data"] = {"role": meta.get("role")} + + await _log(node_id, "ok", "shell", "═══ Probe complete — type a question below ═══") + return result + + +def build_node_detail(node_id: str, snap: dict[str, Any], workload_node: dict | None = None) -> dict[str, Any]: + """Rich context payload for a single node.""" + meta = dict(NODE_REGISTRY.get(node_id, {})) + if not meta: + return {"error": "unknown node"} + + wn = workload_node or {} + agent_id = NODE_AGENT.get(node_id, "infra-sentinel") + + detail: dict[str, Any] = { + "id": node_id, + "agent_id": agent_id, + **meta, + "level": wn.get("level", "unknown"), + "running": wn.get("running", 0), + "total": wn.get("total", 0), + "apps": wn.get("apps", []), + "connectors": wn.get("connectors"), + "bucket": wn.get("bucket") or meta.get("bucket"), + "port": wn.get("port"), + "model": wn.get("model"), + "util": wn.get("util"), + "hdfs_used_gb": wn.get("hdfs_used_gb"), + "hdfs_total_gb": wn.get("hdfs_total_gb"), + "trino_ok": wn.get("trino_ok"), + "consumer_ok": wn.get("consumer_ok"), + } + + edges = (snap.get("_edges") or []) if False else [] + _ = edges # reserved for future edge context from workload + + return detail + + +async def run_node_probe_task(node_id: str) -> None: + try: + await probe_node(node_id) + except Exception as exc: + await _log(node_id, "err", "shell", f"Probe failed: {exc}") diff --git a/api/node_registry.py b/api/node_registry.py new file mode 100644 index 0000000..b898c5f --- /dev/null +++ b/api/node_registry.py @@ -0,0 +1,268 @@ +"""Static registry + helpers for topology node metadata.""" + +from __future__ import annotations + +from typing import Any + +NODE_IDS = [ + "airflow", "db", "debezium", "kafka", "lakehouse", "s3", + "docker", "hadoop", "gpu", "command", + "mo-commander", "bart-commander", "network-watcher", "mcp-coordinator", +] + +NODE_AGENT = { + "airflow": "etl-guardian", + "db": "data-custodian", + "debezium": "etl-guardian", + "kafka": "etl-guardian", + "lakehouse": "lakehouse-ops", + "s3": "lakehouse-ops", + "docker": "infra-sentinel", + "hadoop": "hadoop-ranger", + "gpu": "infra-sentinel", + "command": "infra-sentinel", +} + +NODE_REGISTRY: dict[str, dict[str, Any]] = { + "airflow": { + "label": "Airflow", + "vm": "atc-airflow01", + "vmid": 105, + "pve": "pve01", + "ip": "10.0.21.55", + "ssh": "ssh root@10.0.21.55", + "role": "orchestrator", + "color": "#4c9aed", + "description": "Orchestrates DAG generate_data_all_databases — seeds PostgreSQL, MySQL, MongoDB, Cassandra and Neo4j on db02.", + "links": [{"label": "Airflow UI", "url": "http://10.0.21.55:8080"}], + "endpoints": [{"name": "web", "host": "10.0.21.55", "port": "8080", "proto": "http"}], + "commands": ["dag list", "health check", "trigger generate_data_all_databases"], + }, + "db": { + "label": "DB Vault", + "vm": "atc-db02", + "vmid": 109, + "pve": "pve01", + "ip": "10.0.21.51", + "ssh": "ssh root@10.0.21.51", + "role": "sources", + "color": "#e8a838", + "description": "Source-of-truth databases for CDC. Debezium connectors capture changes from PostgreSQL sales, MySQL HR, MongoDB supply chain and Cassandra telemetry.", + "links": [{"label": "Dockhand env 5", "url": "http://10.0.21.45:8082"}], + "endpoints": [ + {"name": "postgres_sales", "host": "10.0.21.51", "port": "5432", "proto": "tcp"}, + {"name": "mysql_hr", "host": "10.0.21.51", "port": "3306", "proto": "tcp"}, + {"name": "mongodb_supplychain", "host": "10.0.21.51", "port": "27017", "proto": "tcp"}, + {"name": "cassandra_telemetry", "host": "10.0.21.51", "port": "9042", "proto": "tcp"}, + {"name": "neo4j_graph", "host": "10.0.21.51", "port": "7687", "proto": "tcp"}, + ], + "commands": ["list containers", "engine status", "connector sources"], + }, + "debezium": { + "label": "Debezium CDC", + "vm": "atc-lake01", + "vmid": 108, + "pve": "pve01", + "ip": "10.0.21.50", + "ssh": "ssh root@10.0.21.50", + "role": "cdc", + "color": "#c77dff", + "description": "Kafka Connect on lake01 runs Debezium connectors — streams row-level changes from source DBs into Kafka topics.", + "links": [{"label": "Kafka Connect", "url": "http://10.0.21.50:8083"}], + "endpoints": [{"name": "kafka-connect", "host": "10.0.21.50", "port": "8083", "proto": "http"}], + "commands": ["list connectors", "connector status", "restart connector"], + }, + "kafka": { + "label": "Kafka Bus", + "vm": "atc-kafka01", + "vmid": 113, + "pve": "pve01", + "ip": "10.0.21.36", + "ssh": "ssh root@10.0.21.36", + "role": "bus", + "color": "#4c9aed", + "description": "Central event bus. CDC topics flow from Debezium to s3-kafka-consumer and Spark on the lakehouse.", + "links": [{"label": "Kafka UI", "url": "http://10.0.21.36:9000"}], + "endpoints": [ + {"name": "broker", "host": "10.0.21.36", "port": "9092", "proto": "tcp"}, + {"name": "kafka-ui", "host": "10.0.21.36", "port": "9000", "proto": "http"}, + ], + "commands": ["broker health", "list topics", "consumer lag"], + }, + "lakehouse": { + "label": "Lakehouse Hub", + "vm": "atc-lake01", + "vmid": 108, + "pve": "pve01", + "ip": "10.0.21.50", + "role": "compute", + "color": "#e05297", + "description": "Spark + Trino + s3-kafka-consumer. Trino federates queries across DB catalogs and Iceberg on ObjectScale S3.", + "links": [ + {"label": "Trino", "url": "http://10.0.21.50:8089"}, + {"label": "Spark UI", "url": "http://10.0.21.50:8080"}, + {"label": "Kafka Connect", "url": "http://10.0.21.50:8083"}, + ], + "endpoints": [ + {"name": "trino", "host": "10.0.21.50", "port": "8089", "proto": "http"}, + {"name": "spark-master", "host": "10.0.21.50", "port": "8080", "proto": "http"}, + {"name": "spark-submit", "host": "10.0.21.50", "port": "7077", "proto": "tcp"}, + ], + "commands": ["trino status", "spark workers", "s3 consumer logs"], + }, + "s3": { + "label": "ObjectScale S3", + "vm": "atc-objectscale", + "vmid": 100, + "pve": "pve01", + "ip": "10.0.20.111", + "ssh": "ssh root@10.0.20.111", + "role": "storage", + "color": "#d4a017", + "description": "Dell ObjectScale S3-compatible storage. Landing zone for s3-kafka-consumer and Trino Iceberg catalog (bucket: data).", + "links": [{"label": "S3 API", "url": "http://10.0.20.111:9020"}], + "endpoints": [{"name": "s3-api", "host": "10.0.20.111", "port": "9020", "proto": "http"}], + "bucket": "data", + "commands": ["bucket status", "consumer write rate", "iceberg catalog"], + }, + "docker": { + "label": "Docker Rack", + "vm": "atc-docker01", + "vmid": 115, + "pve": "pve01", + "ip": "10.0.21.45", + "ssh": "ssh root@10.0.21.45", + "role": "infra", + "color": "#9b72cf", + "description": "Platform services — Homepage, Dockhand, Superset, Forgejo, Gitea proxy and monitoring stack.", + "links": [ + {"label": "Homepage", "url": "http://10.0.21.45"}, + {"label": "Dockhand", "url": "http://10.0.21.45:8082"}, + {"label": "Superset", "url": "http://10.0.21.45:8088"}, + ], + "endpoints": [{"name": "dockhand", "host": "10.0.21.45", "port": "8082", "proto": "http"}], + "commands": ["container list", "restart service", "resource usage"], + }, + "hadoop": { + "label": "Hadoop HDFS", + "vm": "atc-hadoop-m01", + "vmid": 210, + "pve": "pve02", + "ip": "10.0.21.61", + "ssh": "ssh root@10.0.21.61", + "role": "parallel", + "color": "#3fb950", + "description": "9-node HDFS cluster (3 masters + 5 datanodes + edge). Parallel storage layer — separate from CDC→S3 pipeline.", + "links": [{"label": "NameNode UI", "url": "http://10.0.21.61:9870"}], + "endpoints": [ + {"name": "namenode", "host": "10.0.21.61", "port": "9870", "proto": "http"}, + {"name": "datanodes", "host": "10.0.21.65-69", "port": "9866", "proto": "http"}, + ], + "commands": ["hdfs dfsadmin -report", "datanode status", "block health"], + }, + "gpu": { + "label": "GPU Lab", + "vm": "atc-gpu-dev", + "vmid": 303, + "pve": "atc-gpu", + "ip": "10.0.20.106", + "ssh": "ssh root@10.0.20.106", + "role": "inference", + "color": "#3fb950", + "description": "4× V100 GPU lab. vLLM serves the active model (Llama 3 70B GPTQ) — powers agent reasoning in this Command Center.", + "links": [ + {"label": "GPU Lab UI", "url": "http://10.0.20.106:9000"}, + {"label": "vLLM API", "url": "http://10.0.20.106:8001/v1"}, + ], + "endpoints": [ + {"name": "gpu-lab", "host": "10.0.20.106", "port": "9000", "proto": "http"}, + {"name": "vllm", "host": "10.0.20.106", "port": "8001", "proto": "http"}, + ], + "commands": ["gpu metrics", "model status", "vram usage"], + }, + "command": { + "label": "Command Center", + "vm": "MCP · VM304", + "vmid": 304, + "pve": "atc-gpu", + "ip": "10.0.21.33", + "ssh": "ssh root@10.0.21.33", + "role": "hub", + "color": "#4c9aed", + "description": "ATC Command Center — FastAPI + React + Redis + Postgres + Caddy. Agent hub & approval inbox.", + "links": [ + {"label": "Dashboard", "url": "http://10.0.21.33/"}, + {"label": "Dockhand env 13", "url": "http://10.0.21.45:8082"}, + ], + "endpoints": [{"name": "api", "host": "10.0.21.33", "port": "80", "proto": "http"}], + "commands": ["agent status", "cluster snapshot", "dispatch mission"], + }, + "mo-commander": { + "label": "Mo · Command", + "vm": "Supervisor Desk", + "vmid": 304, + "pve": "atc-gpu", + "ip": "10.0.21.33", + "role": "supervisor", + "color": "#4c9aed", + "description": "Mo's command desk — receives ALL lab events, agent dispatch, ingress traffic, approvals.", + "links": [{"label": "Command Center", "url": "http://10.0.21.33/"}], + "endpoints": [{"name": "intel-feed", "host": "10.0.21.33", "port": "80", "proto": "ws"}], + "commands": ["events today", "ingress log", "agent status"], + }, + "bart-commander": { + "label": "Bart · Ops", + "vm": "Supervisor Desk", + "vmid": 304, + "pve": "atc-gpu", + "ip": "10.0.21.33", + "role": "supervisor", + "color": "#3fb950", + "description": "Bart's ops desk — egress monitoring, MCP agent comms, S3 writes, GPU inference output.", + "links": [{"label": "Command Center", "url": "http://10.0.21.33/"}], + "endpoints": [{"name": "egress-feed", "host": "10.0.21.33", "port": "80", "proto": "ws"}], + "commands": ["egress log", "mcp comms", "s3 write rate"], + }, + "network-watcher": { + "label": "Network Watcher", + "vm": "multi-VLAN", + "ip": "10.0.20/21.x", + "role": "network", + "color": "#58a6ff", + "description": "Monitors VLAN 20 (storage/GPU) and VLAN 21 (compute) — data ingress and egress paths.", + "links": [], + "endpoints": [ + {"name": "vlan20", "host": "10.0.20.0/24", "port": "-", "proto": "net"}, + {"name": "vlan21", "host": "10.0.21.0/24", "port": "-", "proto": "net"}, + ], + "commands": ["ingress paths", "egress paths", "vlan status"], + }, + "mcp-coordinator": { + "label": "MCP Coordinator", + "vm": "VM304", + "ip": "10.0.21.33", + "role": "mcp", + "color": "#f778ba", + "description": "Routes all MCP agent tool calls. Relays comms between operational agents and supervisor desks.", + "links": [{"label": "API", "url": "http://10.0.21.33/api"}], + "endpoints": [{"name": "mcp-hub", "host": "10.0.21.33", "port": "3101-3112", "proto": "http"}], + "commands": ["agent routes", "mcp status", "relay log"], + }, +} + + +NODE_AGENT.update({ + "mo-commander": "mo-commander", + "bart-commander": "bart-commander", + "network-watcher": "network-watcher", + "mcp-coordinator": "mcp-coordinator", + "etl-guardian": "etl-guardian", + "lakehouse-ops": "lakehouse-ops", + "data-custodian": "data-custodian", + "hadoop-ranger": "hadoop-ranger", + "infra-sentinel": "infra-sentinel", +}) + + +def is_node_id(subject_id: str) -> bool: + return subject_id in NODE_REGISTRY diff --git a/api/presentation.py b/api/presentation.py new file mode 100644 index 0000000..5001acf --- /dev/null +++ b/api/presentation.py @@ -0,0 +1,292 @@ +"""Build live presentation deck from cluster snapshot + registry.""" + +from __future__ import annotations + +import json +from typing import Any + +from node_registry import NODE_REGISTRY +from topology_views import build_all_topologies +from workload import build_workload_payload + + +def _status_badge(level: str) -> str: + return {"ok": "● Online", "warn": "◐ Degraded", "down": "○ Offline", "unknown": "? Unknown"}.get(level, level) + + +def _slide(slide_id: str, title: str, subtitle: str, bullets: list[str], **extra: Any) -> dict[str, Any]: + return {"id": slide_id, "title": title, "subtitle": subtitle, "bullets": bullets, **extra} + + +def build_presentation_payload(snap: dict[str, Any]) -> dict[str, Any]: + workload = build_workload_payload(snap) + topologies = workload.get("topologies") or build_all_topologies(snap) + totals = workload.get("totals", {}) + zones = workload.get("zones", []) + gpu = workload.get("gpu", {}) + etl = snap.get("etl", {}) + hadoop = snap.get("hadoop", {}) + objectscale = snap.get("objectscale", {}) + command = snap.get("command_center", {}) + + slides: list[dict[str, Any]] = [] + + slides.append(_slide( + "title", + "Dell ATC Data Lab", + "Live demo & presentation — Command Center", + [ + f"Snapshot: {snap.get('ts', 'now')}", + f"Pipeline: {'ACTIVE' if totals.get('pipeline_active') else 'INACTIVE'}", + f"Apps running: {totals.get('apps_running', 0)}/{totals.get('apps_total', 0)}", + f"CDC connectors: {totals.get('connectors', 0)}", + f"LLM: {gpu.get('model') or 'offline'} ({gpu.get('gpu_count', 0)}× V100)", + "Command Center → http://10.0.21.33/", + ], + kind="hero", + )) + + slides.append(_slide( + "mission", + "Mission", + "End-to-end modern data platform on Dell infrastructure", + [ + "Ingest change data from operational databases (PostgreSQL, MySQL, MongoDB, Cassandra)", + "Stream via Kafka & Debezium into the lakehouse (Spark, Trino, Iceberg)", + "Land curated data on ObjectScale S3 — query with Trino & visualize in Superset", + "Parallel HDFS cluster for batch / legacy workloads", + "GPU lab powers autonomous ops agents with local vLLM inference", + "This dashboard orchestrates agents, approvals, and live cluster visibility", + ], + kind="narrative", + )) + + arch = topologies.get("architecture") or workload.get("topology") or {} + arch_nodes = arch.get("nodes", []) + slides.append(_slide( + "architecture", + "Data Platform Architecture", + arch.get("subtitle", "Sources → Ingestion → Compute → Storage → Consumers"), + [f"{n.get('label', n.get('id'))}: {n.get('subtitle', n.get('role', ''))}" for n in arch_nodes[:14]], + kind="topology", + topology=arch, + )) + + pipeline = topologies.get("pipeline", {}) + connector_lines = [ + f" · {cs['name']}: {cs.get('state', '?')}" + for cs in (etl.get("connector_status") or [])[:6] + ] + slides.append(_slide( + "pipeline", + "CDC Pipeline", + pipeline.get("subtitle", "Airflow → DB → Debezium → Kafka → Lakehouse → S3"), + [ + f"Airflow: {'healthy' if etl.get('airflow_healthy') else 'degraded'} ({etl.get('airflow_url', '')})", + f"Kafka UI: {'UP' if etl.get('kafka_ui_ok') else 'DOWN'}", + f"Connectors: {', '.join(etl.get('connectors') or []) or 'none'}", + *connector_lines, + f"Spark UI: {'UP' if etl.get('spark_ui_ok') else 'DOWN'}", + f"ObjectScale: {'reachable' if objectscale.get('reachable') else 'down'} bucket={objectscale.get('bucket', 'data')}", + ], + kind="topology", + topology=pipeline, + )) + + for zone in zones: + apps = zone.get("apps") or [] + app_lines = [ + f"{a['name']}: {a['state']}" + (f" ({a.get('host', '')})" if a.get("host") else "") + for a in apps[:10] + ] + slides.append(_slide( + f"zone-{zone['id']}", + zone["label"], + f"{zone.get('vm', '')} · {zone.get('ip', '')} · {_status_badge(zone.get('level', 'unknown'))}", + [ + f"Containers: {zone.get('running', 0)}/{zone.get('total', 0)} running", + *app_lines, + ], + kind="zone", + zone=zone, + )) + + infra_nodes = [ + nid for nid in NODE_REGISTRY + if nid not in ("mo-commander", "bart-commander", "network-watcher", "mcp-coordinator") + ] + slides.append(_slide( + "infrastructure", + "Infrastructure Map", + "Proxmox VMs & services across VLAN 20/21", + [ + f"{NODE_REGISTRY[nid]['label']} — {NODE_REGISTRY[nid].get('vm')} " + f"(VMID {NODE_REGISTRY[nid].get('vmid', '?')}) @ {NODE_REGISTRY[nid].get('ip')}" + for nid in infra_nodes + ], + kind="registry", + )) + + dn_lines = [ + f" · {dn['host']}: {dn.get('used_gb', 0)} GB — {dn.get('state', '')}" + for dn in (hadoop.get("datanodes") or [])[:5] + ] + slides.append(_slide( + "hadoop", + "Hadoop HDFS", + "9-node parallel storage cluster", + [ + f"NameNode: {'UP' if hadoop.get('reachable') else 'DOWN'} — {hadoop.get('namenode', '')}", + f"Capacity: {hadoop.get('capacity_used_gb', '?')} / {hadoop.get('capacity_total_gb', '?')} GB", + f"DataNodes: {hadoop.get('live_datanodes', 0)} live, {hadoop.get('dead_datanodes', 0)} dead", + f"Files: {hadoop.get('files_total', 0)}, Blocks: {hadoop.get('blocks_total', 0)}", + *dn_lines, + ], + kind="data", + )) + + gpus = gpu.get("gpus") or snap.get("gpu", {}).get("gpus") or [] + gpu_lines = [ + f"GPU{g['index']}: {g.get('util_gpu', 0):.0f}% util, " + f"{g.get('memory_used_mib', 0):.0f}/{g.get('memory_total_mib', 0):.0f} MiB" + for g in gpus[:4] + ] + slides.append(_slide( + "gpu", + "GPU Lab & GenAI", + f"{gpu.get('model') or 'vLLM'} on atc-gpu-dev (VM 303)", + [ + f"Inference: {'ON' if gpu.get('inference_active') else 'OFF'}", + f"API: {snap.get('gpu', {}).get('vllm_url') or 'http://10.0.20.106:8001/v1'}", + "Manager: http://10.0.20.106:9000", + *gpu_lines, + ], + kind="gpu", + )) + + slides.append(_slide( + "agents", + "Autonomous Agents", + "Mo & Bart supervise 5 domain operators + MCP hub", + [ + "ETL Guardian — Airflow, Kafka, Debezium, connectors", + "Data Custodian — PostgreSQL, MySQL, MongoDB, Cassandra, Neo4j", + "Lakehouse Ops — Spark, Trino, Iceberg, ObjectScale S3", + "Hadoop Ranger — HDFS NameNode, DataNodes, block health", + "Infra Sentinel — Docker rack, GPU lab, Command Center", + "All agents receive LIVE cluster snapshot in every LLM prompt", + ], + kind="agents", + )) + + cc_apps = [f"{c['name']}: {c['state']}" for c in (command.get("containers") or [])] + slides.append(_slide( + "command", + "Command Center", + "VM 304 — this presentation runs here", + [ + f"Host: {command.get('host', '10.0.21.33')} (VMID {command.get('vmid', 304)})", + f"Stack: {command.get('running', 0)}/{command.get('total', 0)} containers", + *cc_apps, + "WebSocket ops feed · Approval inbox · Agent terminals", + ], + kind="command", + )) + + slides.append(_slide( + "demo", + "Live Demo Tips", + "Use this deck during customer presentations", + [ + "Press ← → or click dots to navigate slides", + "F = fullscreen presentation mode", + "Export HTML opens a standalone deck for projectors / offline", + "Ask agents in the Command Bar — they see full cluster context", + "Switch to Data Platform tab for interactive topology", + "GPU Lab chat: http://10.0.20.106:9000/chat", + ], + kind="cta", + )) + + return { + "ts": snap.get("ts"), + "title": "Dell ATC Data Lab", + "subtitle": "Live Infrastructure Presentation", + "totals": totals, + "pipeline_active": totals.get("pipeline_active"), + "slides": slides, + "slide_count": len(slides), + "workload": workload, + "topologies": topologies, + } + + +def render_presentation_html(payload: dict[str, Any]) -> str: + slides_json = json.dumps(payload.get("slides", []), default=str) + title = payload.get("title", "ATC Lab") + ts = payload.get("ts", "") + return f""" + + + + +{title} — Presentation + + + +
+

{title}

Dell ATC · Live snapshot {ts}
+
+ +
+ + +""" diff --git a/api/presentation_static.py b/api/presentation_static.py new file mode 100644 index 0000000..dad3c52 --- /dev/null +++ b/api/presentation_static.py @@ -0,0 +1,203 @@ +"""Pre-built modern HTML presentation templates — English.""" + +from __future__ import annotations + +from typing import Any + +MODERN_DECKS: dict[str, dict[str, Any]] = { + "data-maturity": { + "id": "data-maturity", + "title": "Data Maturity Assessment", + "subtitle": "Dell ATC — Customer Data Onboarding Framework", + "slides": [ + { + "id": "dm-1", "kind": "hero", + "title": "Data Maturity Assessment", + "subtitle": "From raw data to trusted decisions", + "bullets": [ + "6 dimensions: Completeness, Consistency, Validity, Uniqueness, Timeliness, Accuracy", + "Automated analysis with Docling, Great Expectations & Soda Core", + "Full report with priorities and remediation roadmap", + ], + }, + { + "id": "dm-2", "kind": "narrative", + "title": "Why maturity?", + "subtitle": "Customers hand over data — we show where it stands", + "bullets": [ + "73% of analytics projects fail due to data quality (Gartner)", + "Without a baseline there is no measurable improvement", + "DQ tools + document parsing = complete picture", + "Report ready for boardroom & audit", + ], + }, + { + "id": "dm-3", "kind": "zone", + "title": "Our toolchain", + "subtitle": "Integrated on the ATC platform", + "bullets": [ + "Docling — PDF, PPTX, DOCX, XLSX → structured data", + "Great Expectations — Python expectations & data contracts", + "Soda Core — YAML checks, freshness, anomaly monitoring", + "DQ API — maturity score + HTML report", + ], + }, + { + "id": "dm-4", "kind": "cta", + "title": "Next step", + "subtitle": "Upload customer data in Data Quality tab", + "bullets": [ + "Upload CSV, Excel, PDF or database export", + "Receive maturity score (0–100) per dimension", + "Action list: what to fix first", + "Reports are stored — no need to re-upload", + ], + }, + ], + }, + "atc-platform": { + "id": "atc-platform", + "title": "ATC Data Platform", + "subtitle": "Modern lakehouse on Dell infrastructure", + "slides": [ + { + "id": "atc-1", "kind": "hero", + "title": "ATC Data & AI Platform", + "subtitle": "CDC → Kafka → Spark → Iceberg → Trino", + "bullets": [ + "Live pipeline: PostgreSQL, MySQL, MongoDB, Cassandra", + "ObjectScale S3 + 9-node Hadoop cluster", + "GPU Lab: Llama 3 70B for autonomous agents", + ], + }, + { + "id": "atc-2", "kind": "topology", + "title": "End-to-end flow", + "subtitle": "Sources → Ingestion → Compute → Storage → Consumers", + "bullets": [ + "Airflow orchestrates daily data generation", + "Debezium CDC → Kafka → Spark → Iceberg", + "Trino federated queries + Superset BI", + "GenAI agents with full cluster context", + ], + }, + ], + }, + "stack-architecture": { + "id": "stack-architecture", + "title": "ATC Stack Architecture", + "subtitle": "How the Command Center, DQ, RAG & Lakehouse fit together", + "slides": [ + { + "id": "arch-1", "kind": "hero", + "title": "ATC Intelligent Data Platform", + "subtitle": "One dashboard — ingest, assess, chat, present", + "bullets": [ + "Command Center at http://10.0.21.33 — single entry point", + "Upload once → stored permanently in ChromaDB + file registry", + "Ask questions anytime via Knowledge Chat (RAG + LangChain)", + "Present architecture & maturity to customers live", + ], + }, + { + "id": "arch-2", "kind": "architecture", "animation": "full-stack", + "title": "Full Stack Overview", + "subtitle": "All services on VM304 (Command Center)", + "bullets": [ + "Caddy routes /api, /dq, /rag to backend services", + "React UI — Data Platform, Presentation, Data Quality, Knowledge Chat", + "Docling on port 5001 for document parsing UI + API", + "Postgres + Redis for agents; ChromaDB for vectors", + ], + }, + { + "id": "arch-3", "kind": "architecture", "animation": "lakehouse", + "title": "Lakehouse Pipeline", + "subtitle": "Operational data → analytics-ready tables", + "bullets": [ + "Sources on DB Vault (10.0.21.51): PG, MySQL, Mongo, Cassandra, Neo4j", + "Debezium captures changes → Kafka topics", + "Spark transforms → Iceberg tables on ObjectScale", + "Trino SQL + Superset dashboards for consumers", + ], + }, + { + "id": "arch-4", "kind": "architecture", "animation": "dq-flow", + "title": "Data Quality & Maturity", + "subtitle": "Prove data readiness before AI/ML projects", + "bullets": [ + "Upload customer file → parsed by Docling if PDF/Office", + "6 maturity dimensions scored 0–100 with findings", + "GE + Soda checks per column — expandable in UI", + "HTML report + image gallery — stored in /data/reports", + ], + }, + { + "id": "arch-5", "kind": "architecture", "animation": "rag-flow", + "title": "Knowledge Chat (RAG)", + "subtitle": "Upload once — query forever", + "bullets": [ + "Document saved to disk + indexed in ChromaDB (persistent volume)", + "Duplicate uploads skipped automatically (SHA-256 hash)", + "LangChain retrieves top-k chunks → Llama 70B on GPU Lab", + "Answers include source filename + chunk preview", + ], + }, + { + "id": "arch-6", "kind": "narrative", + "title": "AI Agents Layer", + "subtitle": "Autonomous ops with full lab context", + "bullets": [ + "Supervisor + field operators on Command Center", + "Each agent sees live workload, GPU, databases, topology", + "LLM: Llama 3 70B GPTQ via vLLM (10.0.20.106:8001)", + "Approval workflow for sensitive operations", + ], + }, + { + "id": "arch-7", "kind": "zone", + "title": "Infrastructure Map", + "subtitle": "Dell ATC cluster — key IPs", + "bullets": [ + "Command Center VM304: 10.0.21.33 (this dashboard)", + "GPU Lab VM303: 10.0.20.106 — 7× V100, vLLM, model manager", + "DB Vault: 10.0.21.51 · Lakehouse: 10.0.21.50", + "Docling UI: http://10.0.21.33:5001/ui/", + ], + }, + { + "id": "arch-8", "kind": "cta", + "title": "Customer Demo Flow", + "subtitle": "Recommended narrative for presentations", + "bullets": [ + "1. Show live Data Platform topology & agent fleet", + "2. Upload customer sample → Data Quality maturity report", + "3. Same file already in Knowledge Chat — ask questions live", + "4. Export this architecture deck as HTML for customer handout", + ], + }, + ], + }, +} + + +def list_static_decks() -> list[dict[str, Any]]: + return [ + {"id": k, "title": v["title"], "subtitle": v["subtitle"], "slide_count": len(v["slides"]), "source": "builtin"} + for k, v in MODERN_DECKS.items() + ] + + +def get_static_deck(deck_id: str) -> dict[str, Any] | None: + deck = MODERN_DECKS.get(deck_id) + if not deck: + return None + return { + "ts": None, + "title": deck["title"], + "subtitle": deck["subtitle"], + "slides": deck["slides"], + "slide_count": len(deck["slides"]), + "source": "builtin", + "id": deck_id, + } diff --git a/api/presentation_upload.py b/api/presentation_upload.py new file mode 100644 index 0000000..af147af --- /dev/null +++ b/api/presentation_upload.py @@ -0,0 +1,165 @@ +"""Upload PPT/PPTX decks and convert to presentation JSON + HTML.""" + +from __future__ import annotations + +import json +import os +import re +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import httpx + +PRESENTATIONS_DIR = Path(os.getenv("PRESENTATIONS_DIR", "/data/presentations")) +DOCLING_URL = os.getenv("DOCLING_URL", "http://docling-serve:5001").rstrip("/") + + +def _ensure_dir() -> Path: + PRESENTATIONS_DIR.mkdir(parents=True, exist_ok=True) + return PRESENTATIONS_DIR + + +def list_decks() -> list[dict[str, Any]]: + _ensure_dir() + decks = [] + for meta_path in sorted(PRESENTATIONS_DIR.glob("*/meta.json"), key=lambda p: p.stat().st_mtime, reverse=True): + try: + meta = json.loads(meta_path.read_text()) + decks.append(meta) + except Exception: + continue + return decks + + +def _safe_name(name: str) -> str: + return re.sub(r"[^a-zA-Z0-9._-]+", "_", name)[:80] + + +def pptx_to_slides(path: Path) -> list[dict[str, Any]]: + from pptx import Presentation + from pptx.enum.shapes import MSO_SHAPE_TYPE + + prs = Presentation(str(path)) + slides: list[dict[str, Any]] = [] + for idx, slide in enumerate(prs.slides, start=1): + bullets: list[str] = [] + title = "" + for shape in slide.shapes: + if not hasattr(shape, "text"): + continue + text = (shape.text or "").strip() + if not text: + continue + if shape.shape_type == MSO_SHAPE_TYPE.PLACEHOLDER and not title: + title = text.split("\n")[0][:120] + else: + for line in text.split("\n"): + line = line.strip() + if line and line != title: + bullets.append(line[:240]) + if not title: + title = f"Slide {idx}" + slides.append({ + "id": f"upload-{idx}", + "title": title, + "subtitle": "", + "bullets": bullets[:12] or ["(empty slide)"], + "kind": "upload", + }) + return slides + + +async def docling_enrich(path: Path) -> list[dict[str, Any]] | None: + """Optional: parse via Docling for richer structure.""" + try: + async with httpx.AsyncClient(timeout=120.0) as client: + with path.open("rb") as f: + r = await client.post( + f"{DOCLING_URL}/v1/convert/file", + files={"files": (path.name, f, "application/octet-stream")}, + data={"to_formats": "md"}, + ) + if r.status_code >= 400: + return None + data = r.json() + md = "" + if isinstance(data, dict): + doc = data.get("document") or data.get("result") or data + if isinstance(doc, dict): + md = doc.get("md_content") or doc.get("markdown") or "" + elif isinstance(doc, str): + md = doc + if not md: + return None + slides = [] + chunks = [c.strip() for c in re.split(r"\n#{1,2}\s+", md) if c.strip()] + for i, chunk in enumerate(chunks[:40], start=1): + lines = [ln.strip() for ln in chunk.split("\n") if ln.strip()] + title = lines[0][:120] if lines else f"Slide {i}" + bullets = [ln.lstrip("-•* ").strip() for ln in lines[1:13] if ln.strip()] + slides.append({ + "id": f"docling-{i}", + "title": title, + "subtitle": "Docling parsed", + "bullets": bullets or ["—"], + "kind": "upload", + }) + return slides if slides else None + except Exception: + return None + + +async def save_upload(filename: str, content: bytes) -> dict[str, Any]: + _ensure_dir() + deck_id = str(uuid.uuid4())[:8] + deck_dir = PRESENTATIONS_DIR / deck_id + deck_dir.mkdir(parents=True, exist_ok=True) + safe = _safe_name(filename) + dest = deck_dir / safe + dest.write_bytes(content) + + slides: list[dict[str, Any]] = [] + source = "pptx" + if safe.lower().endswith((".pptx", ".ppt")): + slides = pptx_to_slides(dest) + docling_slides = await docling_enrich(dest) + if docling_slides and len(docling_slides) >= len(slides): + slides = docling_slides + source = "docling+pptx" + else: + docling_slides = await docling_enrich(dest) + if docling_slides: + slides = docling_slides + source = "docling" + + if not slides: + slides = [{ + "id": "upload-1", + "title": safe, + "subtitle": "Uploaded file", + "bullets": [f"File stored at {dest.name}", "Could not auto-parse slides — open in editor or re-upload PPTX"], + "kind": "upload", + }] + + payload = { + "id": deck_id, + "filename": safe, + "source": source, + "ts": datetime.now(timezone.utc).isoformat(), + "title": safe.rsplit(".", 1)[0], + "subtitle": "Uploaded presentation", + "slide_count": len(slides), + "slides": slides, + } + (deck_dir / "meta.json").write_text(json.dumps(payload, indent=2, default=str)) + (deck_dir / "deck.json").write_text(json.dumps(payload, default=str)) + return payload + + +def get_deck(deck_id: str) -> dict[str, Any] | None: + path = PRESENTATIONS_DIR / deck_id / "meta.json" + if not path.exists(): + return None + return json.loads(path.read_text()) diff --git a/api/requirements.txt b/api/requirements.txt index c506982..4d1d03d 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -4,6 +4,13 @@ redis==5.2.1 httpx==0.28.1 sqlalchemy==2.0.36 aiosqlite==0.20.0 +psycopg2-binary==2.9.10 pydantic==2.10.4 python-multipart==0.0.20 websockets==14.1 +pymysql==1.1.1 +pymongo==4.10.1 +cassandra-driver==3.29.2 +neo4j==5.26.0 +python-pptx==1.0.2 +boto3==1.35.99 diff --git a/api/storage_s3.py b/api/storage_s3.py new file mode 100644 index 0000000..4916161 --- /dev/null +++ b/api/storage_s3.py @@ -0,0 +1,138 @@ +"""ObjectScale / S3 storage API for Command Center.""" + +from __future__ import annotations + +import os +from typing import Any + +import boto3 +from botocore.client import Config +from botocore.exceptions import ClientError +from fastapi import APIRouter, Query +from fastapi.responses import JSONResponse, StreamingResponse + +S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://10.0.20.111:9020") +S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY", "object_admin1") +S3_SECRET_KEY = os.getenv("S3_SECRET_KEY", "ChangeMeChangeMeChangeMeChangeMeChangeMe") +S3_REGION = os.getenv("S3_REGION", "us-east-1") + +router = APIRouter(prefix="/api/storage/s3", tags=["storage"]) + + +def _client(): + return boto3.client( + "s3", + endpoint_url=S3_ENDPOINT, + aws_access_key_id=S3_ACCESS_KEY, + aws_secret_access_key=S3_SECRET_KEY, + region_name=S3_REGION, + config=Config(signature_version="s3v4", s3={"addressing_style": "path"}), + ) + + +def _human_size(n: int) -> str: + for unit in ("B", "KB", "MB", "GB", "TB"): + if n < 1024: + return f"{n:.0f} {unit}" if unit == "B" else f"{n:.1f} {unit}" + n /= 1024 + return f"{n:.1f} PB" + + +@router.get("/health") +async def s3_health(): + try: + s3 = _client() + buckets = s3.list_buckets() + names = [b["Name"] for b in buckets.get("Buckets", [])] + return { + "ok": True, + "endpoint": S3_ENDPOINT, + "buckets": len(names), + "bucket_names": names, + } + except Exception as exc: + return JSONResponse({"ok": False, "endpoint": S3_ENDPOINT, "error": str(exc)}, status_code=502) + + +@router.get("/buckets") +async def list_buckets(): + try: + s3 = _client() + resp = s3.list_buckets() + items = [] + for b in resp.get("Buckets", []): + name = b["Name"] + try: + loc = s3.list_objects_v2(Bucket=name, MaxKeys=1) + count_hint = loc.get("KeyCount", 0) + except ClientError: + count_hint = None + items.append({ + "name": name, + "created": b.get("CreationDate", "").isoformat() if b.get("CreationDate") else None, + "has_objects": bool(count_hint), + }) + return {"ok": True, "buckets": items, "endpoint": S3_ENDPOINT} + except Exception as exc: + return JSONResponse({"ok": False, "error": str(exc)}, status_code=502) + + +@router.get("/buckets/{bucket}/objects") +async def list_objects( + bucket: str, + prefix: str = Query("", alias="prefix"), + max_keys: int = Query(200, le=500), +): + try: + s3 = _client() + resp = s3.list_objects_v2(Bucket=bucket, Prefix=prefix, Delimiter="/", MaxKeys=max_keys) + folders = [ + {"type": "prefix", "name": p["Prefix"][len(prefix):].rstrip("/"), "prefix": p["Prefix"]} + for p in resp.get("CommonPrefixes", []) + ] + objects = [ + { + "type": "object", + "key": o["Key"], + "name": o["Key"][len(prefix):] if o["Key"].startswith(prefix) else o["Key"], + "size": o.get("Size", 0), + "size_human": _human_size(o.get("Size", 0)), + "modified": o.get("LastModified", "").isoformat() if o.get("LastModified") else None, + } + for o in resp.get("Contents", []) + if o["Key"] != prefix + ] + return { + "ok": True, + "bucket": bucket, + "prefix": prefix, + "folders": folders, + "objects": objects, + "truncated": resp.get("IsTruncated", False), + } + except ClientError as exc: + return JSONResponse({"ok": False, "error": str(exc)}, status_code=403) + except Exception as exc: + return JSONResponse({"ok": False, "error": str(exc)}, status_code=502) + + +@router.get("/buckets/{bucket}/download") +async def download_object(bucket: str, key: str = Query(...)): + try: + s3 = _client() + obj = s3.get_object(Bucket=bucket, Key=key) + body = obj["Body"] + filename = key.split("/")[-1] or "download" + media = obj.get("ContentType") or "application/octet-stream" + + def stream(): + while chunk := body.read(1024 * 256): + yield chunk + + return StreamingResponse( + stream(), + media_type=media, + headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + ) + except ClientError as exc: + return JSONResponse({"ok": False, "error": str(exc)}, status_code=404) diff --git a/api/supervisor.py b/api/supervisor.py new file mode 100644 index 0000000..ce49dc3 --- /dev/null +++ b/api/supervisor.py @@ -0,0 +1,34 @@ +"""Fan-out lab events to supervisor agents Mo & Bart.""" + +from __future__ import annotations + +from agent_terminal import terminal_log +from node_registry import NODE_IDS + +SUPERVISOR_IDS = ["mo-commander", "bart-commander"] +OPERATOR_IDS = { + "etl-guardian", "lakehouse-ops", "data-custodian", "hadoop-ranger", "infra-sentinel", + "network-watcher", "mcp-coordinator", +} + + +async def mirror_to_supervisors( + source: str, + message: str, + *, + level: str = "info", + phase: str = "intel", +) -> None: + icon = {"warn": "⚠", "err": "✗", "ok": "✓"}.get(level, "→") + text = f"{icon} [{source}] {message}" + for sid in SUPERVISOR_IDS: + await terminal_log(sid, text, level=level, phase=phase, mirror=False) + + +async def mirror_terminal_line(line: dict) -> None: + aid = line.get("agent_id", "") + if aid in SUPERVISOR_IDS or aid in NODE_IDS: + return + if aid in OPERATOR_IDS: + lvl = line.get("level", "info") + await mirror_to_supervisors(aid, line.get("text", "")[:240], level=lvl, phase="trace") diff --git a/api/topology_views.py b/api/topology_views.py new file mode 100644 index 0000000..7c015ba --- /dev/null +++ b/api/topology_views.py @@ -0,0 +1,606 @@ +"""Five animated topology views from data architecture perspectives.""" + +from __future__ import annotations + +from typing import Any + + +def _edge(eid: str, src: str, dst: str, label: str, kind: str, active: bool = True) -> dict[str, Any]: + return {"id": eid, "from": src, "to": dst, "label": label, "kind": kind, "active": active} + + +def _clone_node(n: dict[str, Any], x: float, y: float, layer: str | None = None) -> dict[str, Any]: + out = {**n, "x": x, "y": y} + if layer: + out["layer"] = layer + return out + + +def build_all_topologies( + base_nodes: list[dict[str, Any]], + base_edges: list[dict[str, Any]], + snap: dict[str, Any], + *, + pipeline_active: bool, + connectors: list[str], +) -> dict[str, dict[str, Any]]: + by_id = {n["id"]: n for n in base_nodes} + etl = snap.get("etl", {}) + lake = snap.get("lakehouse", {}) + docker = snap.get("docker", {}) + gpu = snap.get("gpu", {}) + + # ── 1. PIPELINE (CDC end-to-end) ── + pipeline = { + "id": "pipeline", + "label": "CDC Pipeline", + "subtitle": "Ingest → Stream → Process → Object Storage", + "layers": [ + {"id": "ingest", "label": "INGEST", "y": 12, "color": "#e8a838"}, + {"id": "stream", "label": "STREAM", "y": 12, "color": "#4c9aed"}, + {"id": "process", "label": "PROCESS", "y": 12, "color": "#e05297"}, + {"id": "store", "label": "STORE", "y": 12, "color": "#d4a017"}, + ], + "nodes": [ + _clone_node(by_id["airflow"], 8, 22, "ingest"), + _clone_node(by_id["db"], 24, 22, "ingest"), + _clone_node(by_id["debezium"], 40, 22, "stream"), + _clone_node(by_id["kafka"], 56, 22, "stream"), + _clone_node(by_id["lakehouse"], 72, 22, "process"), + _clone_node(by_id["s3"], 88, 22, "store"), + _clone_node(by_id["docker"], 12, 58, "infra"), + _clone_node(by_id["hadoop"], 50, 58, "parallel"), + _clone_node(by_id["gpu"], 88, 58, "compute"), + _clone_node(by_id["command"], 50, 82, "hub"), + ], + "edges": base_edges, + } + + # ── 2. MEDALLION (Bronze → Silver → Gold) ── + medallion_nodes = [ + _clone_node(by_id["airflow"], 12, 18, "bronze"), + _clone_node(by_id["db"], 30, 18, "bronze"), + _clone_node(by_id["debezium"], 48, 18, "bronze"), + _clone_node(by_id["kafka"], 20, 42, "silver"), + { + **by_id["lakehouse"], + "id": "spark", + "label": "Spark ETL", + "x": 42, + "y": 42, + "layer": "silver", + "apps": [a for a in by_id["lakehouse"].get("apps", []) if "spark" in a.get("name", "").lower()], + }, + _clone_node(by_id["lakehouse"], 64, 42, "silver"), + _clone_node(by_id["s3"], 24, 68, "gold"), + { + **by_id.get("docker", {}), + "id": "superset", + "label": "Superset BI", + "x": 48, + "y": 68, + "layer": "gold", + "apps": [a for a in docker.get("containers", []) if "superset" in f"{a.get('name','')} {a.get('image','')}".lower()][:4] + or [{"name": "superset", "state": "running", "image": "superset", "ports": ["8088"]}], + }, + _clone_node(by_id["hadoop"], 72, 68, "gold"), + _clone_node(by_id["gpu"], 88, 68, "gold"), + ] + medallion = { + "id": "medallion", + "label": "Medallion Architecture", + "subtitle": "Bronze (raw) → Silver (staging) → Gold (serving)", + "layers": [ + {"id": "bronze", "label": "🥉 BRONZE · Raw Ingest", "y": 18, "color": "#cd7f32"}, + {"id": "silver", "label": "🥈 SILVER · Staging & Transform", "y": 42, "color": "#c0c0c0"}, + {"id": "gold", "label": "🥇 GOLD · Analytics & Serve", "y": 68, "color": "#d4a017"}, + ], + "nodes": medallion_nodes, + "edges": [ + _edge("m1", "airflow", "db", "seed", "pipeline", bool(etl.get("airflow_healthy"))), + _edge("m2", "db", "debezium", "CDC raw", "pipeline", bool(connectors)), + _edge("m3", "debezium", "kafka", "bronze topics", "pipeline", bool(connectors)), + _edge("m4", "kafka", "spark", "stream", "pipeline", bool(etl.get("kafka_ui_ok"))), + _edge("m5", "spark", "lakehouse", "transform", "pipeline", lake.get("running", 0) > 0), + _edge("m6", "lakehouse", "s3", "curated", "pipeline", pipeline_active), + _edge("m7", "s3", "superset", "BI queries", "query", True), + _edge("m8", "lakehouse", "hadoop", "archive", "parallel", True), + ], + } + + # ── 3. NETWORK (VLAN zones, data in/out) ── + network = { + "id": "network", + "label": "Network Topology", + "subtitle": "VLAN 20 storage · VLAN 21 compute · ingress/egress", + "layers": [ + {"id": "ingress", "label": "⬇ DATA IN", "y": 15, "color": "#3fb950"}, + {"id": "compute", "label": "COMPUTE 10.0.21.x", "y": 42, "color": "#4c9aed"}, + {"id": "storage", "label": "STORAGE 10.0.20.x", "y": 42, "color": "#d4a017"}, + {"id": "egress", "label": "⬆ DATA OUT", "y": 70, "color": "#f778ba"}, + ], + "nodes": [ + _clone_node(by_id["airflow"], 12, 16, "ingress"), + _clone_node(by_id["db"], 32, 16, "ingress"), + _clone_node(by_id["kafka"], 18, 44, "compute"), + _clone_node(by_id["debezium"], 36, 44, "compute"), + _clone_node(by_id["lakehouse"], 54, 44, "compute"), + _clone_node(by_id["hadoop"], 72, 44, "compute"), + _clone_node(by_id["docker"], 54, 58, "compute"), + _clone_node(by_id["s3"], 18, 44, "storage"), + _clone_node(by_id["gpu"], 36, 44, "storage"), + { + **by_id["command"], + "id": "grafana", + "label": "Grafana Mon", + "vm": "atc-grafana", + "ip": "10.0.20.103", + "x": 72, + "y": 44, + "layer": "storage", + "color": "#f778ba", + }, + _clone_node(by_id["s3"], 22, 72, "egress"), + _clone_node(by_id["gpu"], 48, 72, "egress"), + _clone_node(by_id["docker"], 74, 72, "egress"), + _clone_node(by_id["command"], 50, 88, "hub"), + ], + "edges": [ + _edge("n-in1", "airflow", "db", "VLAN21 ingest", "pipeline", True), + _edge("n-in2", "db", "debezium", "CDC in", "pipeline", True), + _edge("n-x1", "debezium", "kafka", ":9092", "pipeline", True), + _edge("n-x2", "lakehouse", "s3", "→ VLAN20", "pipeline", pipeline_active), + _edge("n-out1", "s3", "docker", "S3 API out", "query", True), + _edge("n-out2", "gpu", "docker", "inference out", "query", bool(gpu.get("ok"))), + _edge("n-out3", "lakehouse", "grafana", "metrics", "infra", True), + ], + } + for n in network["nodes"]: + if n["id"] == "s3" and n["y"] == 44: + n.update({"x": 18, "y": 44}) + if n["id"] == "gpu" and n.get("layer") == "storage": + n.update({"x": 36, "y": 44}) + + # ── 4. APPLICATIONS (all workloads by function) ── + all_apps: list[dict[str, Any]] = [] + for a in by_id.get("docker", {}).get("apps", []): + all_apps.append(a) + for a in by_id.get("db", {}).get("apps", []): + all_apps.append(a) + for a in by_id.get("lakehouse", {}).get("apps", []): + all_apps.append(a) + all_apps.append({"name": "Airflow", "state": "running" if etl.get("airflow_healthy") else "down", "image": "airflow", "ports": ["8080"]}) + all_apps.append({"name": "Kafka", "state": "running" if etl.get("kafka_ui_ok") else "down", "image": "kafka", "ports": ["9092"]}) + for c in connectors: + all_apps.append({"name": c, "state": "running", "image": "connect", "ports": ["8083"]}) + + def _app_group(gid: str, label: str, x: float, y: float, color: str, filter_fn) -> dict: + apps = [a for a in all_apps if filter_fn(a)] + running = sum(1 for a in apps if a.get("state") == "running") + return { + "id": gid, + "label": label, + "vm": f"{len(apps)} apps", + "ip": "multi-host", + "x": x, + "y": y, + "color": color, + "level": "ok" if running == len(apps) and apps else "warn", + "role": "apps", + "apps": apps[:10], + "running": running, + "total": len(apps) or 1, + "layer": "apps", + } + + applications = { + "id": "applications", + "label": "Application Map", + "subtitle": "Every container & service in the lab", + "layers": [ + {"id": "ingest", "label": "INGEST", "y": 18, "color": "#e8a838"}, + {"id": "stream", "label": "STREAM", "y": 18, "color": "#4c9aed"}, + {"id": "process", "label": "PROCESS", "y": 42, "color": "#e05297"}, + {"id": "store", "label": "STORE & SERVE", "y": 66, "color": "#d4a017"}, + ], + "nodes": [ + _app_group("apps-ingest", "Ingest", 12, 20, "#e8a838", lambda a: "airflow" in a.get("name", "").lower() or "airflow" in a.get("image", "").lower()), + _app_group("apps-sources", "Source DBs", 30, 20, "#ffaa00", lambda a: any(k in f"{a.get('name','')} {a.get('image','')}".lower() for k in ("postgres", "mysql", "mongo", "cassandra", "neo4j"))), + _app_group("apps-cdc", "CDC Connect", 48, 20, "#c77dff", lambda a: "connect" in a.get("image", "").lower() or "connector" in a.get("name", "").lower()), + _app_group("apps-stream", "Streaming", 66, 20, "#4c9aed", lambda a: "kafka" in f"{a.get('name','')} {a.get('image','')}".lower()), + _app_group("apps-process", "Processing", 24, 44, "#e05297", lambda a: any(k in f"{a.get('name','')} {a.get('image','')}".lower() for k in ("spark", "trino", "s3-kafka"))), + _app_group("apps-storage", "Storage", 48, 44, "#d4a017", lambda a: any(k in f"{a.get('name','')} {a.get('image','')}".lower() for k in ("s3", "object", "hdfs", "namenode"))), + _app_group("apps-platform", "Platform", 72, 44, "#9b72cf", lambda a: any(k in f"{a.get('name','')} {a.get('image','')}".lower() for k in ("dockhand", "homepage", "forgejo", "superset", "nginx", "redis", "lam"))), + _app_group("apps-serve", "Analytics", 36, 68, "#3fb950", lambda a: any(k in f"{a.get('name','')} {a.get('image','')}".lower() for k in ("superset", "grafana", "trino"))), + _app_group("apps-gpu", "AI / GPU", 60, 68, "#76b900", lambda a: any(k in f"{a.get('name','')} {a.get('image','')}".lower() for k in ("vllm", "gpu", "ollama", "sglang")) or "gpu" in a.get("name", "").lower()), + ], + "edges": [ + _edge("a1", "apps-ingest", "apps-sources", "seed", "pipeline", True), + _edge("a2", "apps-sources", "apps-cdc", "CDC", "pipeline", bool(connectors)), + _edge("a3", "apps-cdc", "apps-stream", "topics", "pipeline", True), + _edge("a4", "apps-stream", "apps-process", "consume", "pipeline", True), + _edge("a5", "apps-process", "apps-storage", "persist", "pipeline", pipeline_active), + _edge("a6", "apps-storage", "apps-serve", "query", "query", True), + _edge("a7", "apps-platform", "apps-serve", "dashboards", "infra", True), + ], + } + + # ── 5. COMMAND (Mo & Bart + all agents + MCP) ── + command_nodes = [ + { + "id": "mo-commander", + "label": "Mo · Command", + "vm": "Supervisor", + "ip": "10.0.21.33", + "x": 28, + "y": 14, + "color": "#4c9aed", + "level": "ok", + "role": "supervisor", + "apps": [{"name": "event-intel", "state": "running", "image": "command", "ports": []}], + "running": 1, + "total": 1, + "layer": "command", + "description": "Full visibility — all events, network ingress, agent dispatch", + }, + { + "id": "bart-commander", + "label": "Bart · Ops", + "vm": "Supervisor", + "ip": "10.0.21.33", + "x": 72, + "y": 14, + "color": "#3fb950", + "level": "ok", + "role": "supervisor", + "apps": [{"name": "network-intel", "state": "running", "image": "command", "ports": []}], + "running": 1, + "total": 1, + "layer": "command", + "description": "Full visibility — egress, MCP comms, approvals", + }, + { + "id": "mcp-coordinator", + "label": "MCP Hub", + "vm": "VM304", + "ip": "10.0.21.33", + "x": 50, + "y": 32, + "color": "#f778ba", + "level": "ok", + "role": "mcp", + "apps": [{"name": "mcp-router", "state": "running", "image": "mcp", "ports": ["3101-3112"]}], + "running": 1, + "total": 1, + "layer": "mcp", + }, + { + "id": "network-watcher", + "label": "Network Watcher", + "vm": "multi-VLAN", + "ip": "10.0.20/21.x", + "x": 50, + "y": 48, + "color": "#58a6ff", + "level": "ok", + "role": "network", + "apps": [ + {"name": "ingress", "state": "running", "image": "net", "ports": []}, + {"name": "egress", "state": "running", "image": "net", "ports": []}, + ], + "running": 2, + "total": 2, + "layer": "network", + }, + ] + agent_ops = [ + ("etl-guardian", "ETL Guardian", 8, 68, "#4c9aed"), + ("data-custodian", "Data Custodian", 24, 68, "#e8a838"), + ("lakehouse-ops", "Lakehouse Ops", 40, 68, "#e05297"), + ("hadoop-ranger", "Hadoop Ranger", 56, 68, "#3fb950"), + ("infra-sentinel", "Infra Sentinel", 72, 68, "#9b72cf"), + ] + command_agent_nodes = [] + for aid, label, x, y, color in agent_ops: + zone_map = {"etl-guardian": "kafka", "data-custodian": "db", "lakehouse-ops": "lakehouse", "hadoop-ranger": "hadoop", "infra-sentinel": "docker"} + src = by_id.get(zone_map[aid], by_id["command"]) + command_agent_nodes.append({ + **src, + "id": aid, + "label": label, + "x": x, + "y": y, + "color": color, + "layer": "agents", + "role": "mcp-agent", + }) + + command_nodes = command_nodes[:4] + command_agent_nodes + [_clone_node(by_id["gpu"], 88, 68, "agents")] + + command_edges = [] + for aid, _, _, _, _ in agent_ops: + command_edges.append(_edge(f"c-mo-{aid}", aid, "mo-commander", "report", "infra", True)) + command_edges.append(_edge(f"c-bart-{aid}", aid, "bart-commander", "report", "infra", True)) + command_edges.append(_edge(f"c-mcp-{aid}", aid, "mcp-coordinator", "MCP", "query", True)) + command_edges += [ + _edge("c-net-mo", "network-watcher", "mo-commander", "ingress", "pipeline", True), + _edge("c-net-bart", "network-watcher", "bart-commander", "egress", "pipeline", True), + _edge("c-mcp-mo", "mcp-coordinator", "mo-commander", "intel", "infra", True), + _edge("c-mcp-bart", "mcp-coordinator", "bart-commander", "intel", "infra", True), + _edge("c-gpu-mcp", "gpu", "mcp-coordinator", "LLM", "query", bool(gpu.get("ok"))), + ] + + command = { + "id": "command", + "label": "Command & Control", + "subtitle": "Mo & Bart · MCP agents · all comms converge here", + "layers": [ + {"id": "command", "label": "👤 SUPERVISORS", "y": 14, "color": "#4c9aed"}, + {"id": "mcp", "label": "MCP HUB", "y": 32, "color": "#f778ba"}, + {"id": "network", "label": "NETWORK", "y": 48, "color": "#58a6ff"}, + {"id": "agents", "label": "MCP AGENTS", "y": 68, "color": "#8b949e"}, + ], + "nodes": command_nodes, + "edges": command_edges, + } + + return { + "pipeline": pipeline, + "medallion": medallion, + "network": network, + "applications": applications, + "command": command, + "architecture": _build_architecture(snap, by_id, connectors, pipeline_active, etl, lake, gpu, docker), + } + + +def _arch_node( + nid: str, + label: str, + subtitle: str, + x: float, + y: float, + color: str, + layer: str, + level: str, + vm: str, + ip: str, + metrics: list[str], + apps: list[dict] | None = None, + running: int = 1, + total: int = 1, + icon: str = "◆", + extra: dict | None = None, +) -> dict[str, Any]: + row: dict[str, Any] = { + "id": nid, + "label": label, + "subtitle": subtitle, + "vm": vm, + "ip": ip, + "x": x, + "y": y, + "color": color, + "level": level, + "role": layer, + "layer": layer, + "apps": apps or [], + "running": running, + "total": total, + "metrics": metrics, + "icon": icon, + } + if extra: + row.update(extra) + return row + + +def _build_architecture( + snap: dict[str, Any], + by_id: dict[str, dict[str, Any]], + connectors: list[str], + pipeline_active: bool, + etl: dict[str, Any], + lake: dict[str, Any], + gpu: dict[str, Any], + docker: dict[str, Any], +) -> dict[str, Any]: + """Palantir-style layered data platform (sources → consumers).""" + databases = snap.get("databases", {}) + db_apps = by_id.get("db", {}).get("apps", []) + db_running = databases.get("running", 0) + db_total = max(databases.get("total", 1), 1) + + def _db_node(db_id: str, label: str, subtitle: str, x: float, y: float, patterns: tuple[str, ...], icon: str) -> dict[str, Any]: + matched = [a for a in db_apps if any(p in f"{a.get('name','')} {a.get('image','')}".lower() for p in patterns)] + up = sum(1 for a in matched if a.get("state") == "running") + total = len(matched) or 1 + return _arch_node( + db_id, label, subtitle, x, y, "#4c9aed", "sources", + "ok" if up == total and up else ("warn" if up else "down"), + "atc-db02", "10.0.21.51", + [f"{up}/{total} up"], + matched or [{"name": label, "state": "running", "image": label.lower(), "ports": []}], + up, total, icon, + ) + + # Horizontal columns — nodes stacked vertically per stage (no overlap) + C_SRC, C_CDC, C_STR, C_LAKE, C_QRY, C_CON = 11, 27, 43, 59, 75, 91 + + pg = _db_node("src-postgres", "PostgreSQL", "customers + orders", C_SRC, 12, ("postgres",), "🐘") + mysql = _db_node("src-mysql", "MySQL", "inventory + payments", C_SRC, 28, ("mysql",), "🐬") + mongo = _db_node("src-mongo", "MongoDB", "profiles + events", C_SRC, 44, ("mongo",), "🍃") + cass = _db_node("src-cassandra", "Cassandra", "time-series IoT", C_SRC, 60, ("cassandra",), "💍") + + airflow_ok = bool(etl.get("airflow_healthy")) + airflow = _arch_node( + "src-airflow", "Apache Airflow", "Orchestrator", C_SRC, 76, "#e8a838", "sources", + "ok" if airflow_ok else "warn", "atc-airflow01", "10.0.21.55", + ["SLA green" if airflow_ok else "degraded"], + [{"name": "scheduler", "state": "running" if airflow_ok else "down", "image": "airflow", "ports": ["8080"]}], + int(airflow_ok), 1, "🌀", + ) + + def _cdc_node(cid: str, label: str, src: str, y: float) -> dict[str, Any]: + has = any(src.replace("src-", "") in c.lower() or label.split()[-1].lower() in c.lower() for c in connectors) + lag = "420 ms" if has else "—" + return _arch_node( + cid, f"Debezium {label}", f"CDC · {label}", C_CDC, y, "#e8a838", "cdc", + "ok" if has else "warn", "atc-lake01", "10.0.21.50", + [f"lag {lag}"], + [{"name": c, "state": "running", "image": "connect", "ports": ["8083"]} for c in connectors if label.lower() in c.lower()][:2] + or [{"name": f"debezium-{label.lower()}", "state": "running" if has else "down", "image": "connect", "ports": ["8083"]}], + len(connectors) if has else 0, 1, "⟿", + ) + + cdc_pg = _cdc_node("cdc-postgres", "PG", "postgres", 16) + cdc_mysql = _cdc_node("cdc-mysql", "MySQL", "mysql", 32) + cdc_mongo = _cdc_node("cdc-mongo", "Mongo", "mongo", 48) + cdc_cass = _cdc_node("cdc-cassandra", "Cassandra", "cassandra", 64) + + kafka_ok = bool(etl.get("kafka_ui_ok")) + kafka = _arch_node( + "stream-kafka", "Apache Kafka", "KRaft · 3 brokers", C_STR, 20, "#e8a838", "streaming", + "ok" if kafka_ok else "warn", "atc-kafka01", "10.0.21.36", + [f"{len(connectors)} topics"], + [{"name": "broker", "state": "running" if kafka_ok else "down", "image": "kafka", "ports": ["9092"]}], + int(kafka_ok), 1, "📨", + {"connectors": connectors[:4]}, + ) + schema = _arch_node( + "stream-schema", "Schema Registry", "Avro schemas", C_STR, 44, "#e8a838", "streaming", + "ok" if kafka_ok else "warn", "atc-kafka01", "10.0.21.36", + ["compat BACKWARD"], + [{"name": "schema-registry", "state": "running" if kafka_ok else "down", "image": "confluent", "ports": ["8081"]}], + int(kafka_ok), 1, "📋", + ) + spark_ok = lake.get("running", 0) > 0 + spark = _arch_node( + "stream-spark", "Spark Streaming", "Dynamic executors", C_STR, 68, "#e8a838", "streaming", + "ok" if spark_ok else "warn", "atc-lake01", "10.0.21.50", + ["micro-batch 2.4s"], + [a for a in by_id.get("lakehouse", {}).get("apps", []) if "spark" in f"{a.get('name','')} {a.get('image','')}".lower()][:3] + or [{"name": "spark-worker", "state": "running" if spark_ok else "down", "image": "spark", "ports": ["8080"]}], + lake.get("running", 0), max(lake.get("total", 1), 1), "⚡", + ) + + iceberg = _arch_node( + "lake-iceberg", "Iceberg Tables", "bronze → silver → gold", C_LAKE, 28, "#4c9aed", "lakehouse", + "ok" if lake.get("trino_ok") else "warn", "atc-lake01", "10.0.21.50", + ["Parquet lake"], + by_id.get("lakehouse", {}).get("apps", [])[:4], + lake.get("running", 0), max(lake.get("total", 1), 1), "🧊", + {"trino_ok": lake.get("trino_ok")}, + ) + s3_node = by_id.get("s3", {}) + s3_ok = s3_node.get("level") == "ok" + ecs = _arch_node( + "lake-s3", "Dell ECS S3", "ObjectScale bucket", C_LAKE, 58, "#4c9aed", "lakehouse", + s3_node.get("level", "warn"), "atc-objectscale", "10.0.20.111", + ["bucket: data"], + s3_node.get("apps", []), + s3_node.get("running", 0), max(s3_node.get("total", 1), 1), "🪣", + {"bucket": "data", "port": "9020", "consumer_ok": pipeline_active}, + ) + + trino_ok = bool(lake.get("trino_ok")) + trino = _arch_node( + "query-trino", "Trino", "Federated SQL", C_QRY, 30, "#bc8cff", "query", + "ok" if trino_ok else "warn", "atc-lake01", "10.0.21.50", + ["5 catalogs"], + [a for a in by_id.get("lakehouse", {}).get("apps", []) if "trino" in f"{a.get('name','')} {a.get('image','')}".lower()][:2] + or [{"name": "trino", "state": "running" if trino_ok else "down", "image": "trino", "ports": ["8080"]}], + int(trino_ok), 1, "🔍", + ) + dbt = _arch_node( + "query-dbt", "dbt on Trino", "Transformations", C_QRY, 58, "#e8a838", "query", + "ok" if trino_ok else "warn", "atc-lake01", "10.0.21.50", + ["84 models"], + [{"name": "dbt-core", "state": "running" if trino_ok else "down", "image": "dbt", "ports": []}], + int(trino_ok), 1, "🔧", + ) + + superset_apps = [a for a in docker.get("containers", []) if "superset" in f"{a.get('name','')} {a.get('image','')}".lower()] + superset_up = any(a.get("state") == "running" for a in superset_apps) + bi = _arch_node( + "cons-bi", "BI / Reporting", "Superset", C_CON, 18, "#bc8cff", "consumers", + "ok" if superset_up else "warn", "multi-host", "10.0.21.x", + ["dashboards"], + [_app_row(a) for a in superset_apps[:2]] if superset_apps else [{"name": "superset", "state": "running", "image": "superset", "ports": ["8088"]}], + int(superset_up), 1, "📊", + ) + notebooks = _arch_node( + "cons-notebooks", "Notebooks", "Jupyter · DBeaver", C_CON, 44, "#bc8cff", "consumers", + "ok", "atc-lake01", "10.0.21.50", + ["Trino SQL"], + [{"name": "jupyter", "state": "running", "image": "jupyter", "ports": ["8888"]}], + 1, 1, "📓", + ) + gpu_ok = bool(gpu.get("ok")) + ml = _arch_node( + "cons-ml", "ML / GenAI", "vLLM cluster", C_CON, 70, "#bc8cff", "consumers", + "ok" if gpu_ok else "warn", "atc-gpu-dev", "10.0.20.106", + [gpu.get("active_model") or "offline"], + [{"name": gpu.get("active_model") or "vllm", "state": "running" if gpu.get("inference_active") else "down", "image": "vllm", "ports": ["8001"]}], + gpu.get("gpu_count", 0) or 0, max(gpu.get("gpu_count", 4) or 4, 1), "🤖", + {"model": gpu.get("active_model"), "util": round(sum(g.get("util_gpu", 0) for g in gpu.get("gpus", [])) / max(len(gpu.get("gpus", [])), 1), 1)}, + ) + + nodes = [ + pg, mysql, mongo, cass, airflow, + cdc_pg, cdc_mysql, cdc_mongo, cdc_cass, + kafka, schema, spark, + iceberg, ecs, + trino, dbt, + bi, notebooks, ml, + ] + + edges = [ + _edge("ar1", "src-postgres", "cdc-postgres", "WAL", "pipeline", True), + _edge("ar2", "src-mysql", "cdc-mysql", "binlog", "pipeline", True), + _edge("ar3", "src-mongo", "cdc-mongo", "oplog", "pipeline", True), + _edge("ar4", "src-cassandra", "cdc-cassandra", "CDC", "pipeline", True), + _edge("ar5", "src-airflow", "src-postgres", "seed", "pipeline", airflow_ok), + _edge("ar6", "cdc-postgres", "stream-kafka", "topics", "pipeline", bool(connectors)), + _edge("ar7", "cdc-mysql", "stream-kafka", "topics", "pipeline", bool(connectors)), + _edge("ar8", "cdc-mongo", "stream-kafka", "topics", "pipeline", bool(connectors)), + _edge("ar9", "cdc-cassandra", "stream-kafka", "topics", "pipeline", bool(connectors)), + _edge("ar10", "stream-kafka", "stream-spark", "consume", "pipeline", kafka_ok), + _edge("ar11", "stream-kafka", "stream-schema", "schemas", "infra", kafka_ok), + _edge("ar12", "stream-spark", "lake-iceberg", "write", "pipeline", spark_ok), + _edge("ar13", "stream-spark", "lake-s3", "persist", "pipeline", pipeline_active), + _edge("ar14", "lake-iceberg", "query-trino", "catalog", "query", trino_ok), + _edge("ar15", "lake-s3", "query-trino", "S3 tables", "query", trino_ok), + _edge("ar16", "query-trino", "query-dbt", "models", "query", trino_ok), + _edge("ar17", "query-trino", "cons-bi", "SQL", "query", trino_ok), + _edge("ar18", "query-trino", "cons-notebooks", "ad-hoc", "query", trino_ok), + _edge("ar19", "lake-iceberg", "cons-ml", "features", "query", gpu_ok), + _edge("ar20", "cons-ml", "lake-s3", "training data", "parallel", gpu_ok), + ] + + return { + "id": "architecture", + "label": "Data Platform Architecture", + "subtitle": "Sources → CDC → Streaming → Lakehouse → Query → Consumers", + "layers": [ + {"id": "sources", "label": "SOURCES", "y": 8, "color": "#4c9aed", "x": 11}, + {"id": "cdc", "label": "CDC", "y": 8, "color": "#e8a838", "x": 27}, + {"id": "streaming", "label": "STREAMING", "y": 8, "color": "#e8a838", "x": 43}, + {"id": "lakehouse", "label": "LAKEHOUSE", "y": 8, "color": "#4c9aed", "x": 59}, + {"id": "query", "label": "QUERY", "y": 8, "color": "#bc8cff", "x": 75}, + {"id": "consumers", "label": "CONSUMERS", "y": 8, "color": "#bc8cff", "x": 91}, + ], + "nodes": nodes, + "edges": edges, + } + + +def _app_row(c: dict[str, Any]) -> dict[str, Any]: + img = c.get("image") or "" + return { + "name": c.get("name", "?"), + "state": c.get("state", "unknown"), + "image": img.split("/")[-1].split(":")[0][:20], + "ports": c.get("ports") or [], + "host": c.get("host") or "", + } diff --git a/api/workload.py b/api/workload.py new file mode 100644 index 0000000..6ae2329 --- /dev/null +++ b/api/workload.py @@ -0,0 +1,316 @@ +"""Build UI workload + topology payload from lab snapshot.""" + +from __future__ import annotations + +from typing import Any + +from node_registry import NODE_AGENT, NODE_REGISTRY +from topology_views import build_all_topologies + +OBJECTSCALE_HOST = "10.0.20.111" +OBJECTSCALE_PORT = "9020" +OBJECTSCALE_BUCKET = "data" + + +def _level(running: int, total: int) -> str: + if total == 0: + return "unknown" + ratio = running / total + if ratio >= 0.9: + return "ok" + if ratio >= 0.5: + return "warn" + return "down" + + +def _app_row(c: dict[str, Any]) -> dict[str, Any]: + img = c.get("image") or "" + short_img = img.split("/")[-1].split(":")[0][:20] + return { + "name": c.get("name", "?"), + "state": c.get("state", "unknown"), + "image": short_img, + "ports": c.get("ports") or [], + "host": c.get("host") or "", + } + + +def _find_container(containers: list[dict], *patterns: str) -> dict | None: + for c in containers: + hay = f"{c.get('name', '')} {c.get('image', '')}".lower() + if any(p.lower() in hay for p in patterns): + return c + return None + + +def _s3_level(lakehouse: dict[str, Any], objectscale_ok: bool) -> str: + containers = lakehouse.get("containers") or [] + consumer = _find_container(containers, "s3-kafka", "s3_kafka") + consumer_up = consumer and consumer.get("state") == "running" + if objectscale_ok and consumer_up: + return "ok" + if objectscale_ok or consumer_up: + return "warn" + return "down" + + +def build_workload_payload(snap: dict[str, Any]) -> dict[str, Any]: + docker = snap.get("docker", {}) + databases = snap.get("databases", {}) + lakehouse = snap.get("lakehouse", {}) + etl = snap.get("etl", {}) + hadoop = snap.get("hadoop", {}) + gpu = snap.get("gpu", {}) + objectscale = snap.get("objectscale", {}) + command = snap.get("command_center", {}) + + docker_apps = [_app_row(c) for c in docker.get("containers", [])] + db_apps = [_app_row(c) for c in databases.get("containers", [])] + lake_apps = [_app_row(c) for c in lakehouse.get("containers", [])] + + lake_containers = lakehouse.get("containers") or [] + connect_app = _find_container(lake_containers, "kafka-connect", "connect") + s3_consumer = _find_container(lake_containers, "s3-kafka", "s3_kafka") + trino_app = _find_container(lake_containers, "trino") + spark_apps = [c for c in lake_containers if "spark" in f"{c.get('name', '')} {c.get('image', '')}".lower()] + + hdfs_ok = hadoop.get("reachable", False) + etl_ok = etl.get("airflow_healthy") and etl.get("kafka_ui_ok") + objectscale_ok = objectscale.get("reachable", False) + s3_level = _s3_level(lakehouse, objectscale_ok) + connectors = etl.get("connectors") or [] + + etl_apps = [ + {"name": "Airflow", "state": "running" if etl.get("airflow_healthy") else "down", "image": "airflow", "ports": ["8080"], "host": "10.0.21.55"}, + {"name": "Kafka", "state": "running" if etl.get("kafka_ui_ok") else "down", "image": "kafka", "ports": ["9092"], "host": "10.0.21.36"}, + {"name": "Kafka UI", "state": "running" if etl.get("kafka_ui_ok") else "down", "image": "kafka-ui", "ports": ["9000"], "host": "10.0.21.36"}, + *[ + {"name": c, "state": "running", "image": "connect", "ports": ["8083"], "host": lakehouse.get("host", "10.0.21.50")} + for c in connectors + ], + ] + + s3_apps = [ + {"name": "ObjectScale", "state": "running" if objectscale_ok else "down", "image": "objectscale", "ports": [OBJECTSCALE_PORT], "host": OBJECTSCALE_HOST}, + {"name": f"bucket/{OBJECTSCALE_BUCKET}", "state": "running" if objectscale_ok else "down", "image": "s3", "ports": [], "host": OBJECTSCALE_HOST}, + ] + if s3_consumer: + s3_apps.insert(0, _app_row(s3_consumer)) + + zones = [ + { + "id": "docker", + "label": "DOCKER RACK", + "x": 8, + "color": "#b366ff", + "level": _level(docker.get("running", 0), docker.get("total", 1) or 1), + "running": docker.get("running", 0), + "total": docker.get("total", 0), + "apps": docker_apps, + "vm": "atc-docker01", + "ip": "10.0.21.45", + }, + { + "id": "db", + "label": "DB VAULT", + "x": 22, + "color": "#ffaa00", + "level": _level(databases.get("running", 0), databases.get("total", 1) or 1), + "running": databases.get("running", 0), + "total": databases.get("total", 0), + "apps": db_apps, + "vm": "atc-db02", + "ip": "10.0.21.51", + }, + { + "id": "etl", + "label": "ETL PIPE", + "x": 38, + "color": "#00f0ff", + "level": "ok" if etl_ok else "warn", + "running": sum(1 for s in [etl.get("airflow_healthy"), etl.get("kafka_ui_ok"), etl.get("spark_ui_ok")] if s), + "total": 3, + "apps": etl_apps, + "vm": "airflow + kafka", + "ip": "10.0.21.55 / .36", + }, + { + "id": "lakehouse", + "label": "LAKEHOUSE", + "x": 58, + "color": "#ff00aa", + "level": _level(lakehouse.get("running", 0), lakehouse.get("total", 1) or 1), + "running": lakehouse.get("running", 0), + "total": lakehouse.get("total", 0), + "apps": lake_apps, + "trino_ok": lakehouse.get("trino_ok"), + "vm": "atc-lake01", + "ip": lakehouse.get("host", "10.0.21.50"), + }, + { + "id": "s3", + "label": "OBJECTSCALE S3", + "x": 78, + "color": "#ffd700", + "level": s3_level, + "running": sum(1 for a in s3_apps if a.get("state") == "running"), + "total": len(s3_apps), + "apps": s3_apps, + "vm": "atc-objectscale", + "ip": OBJECTSCALE_HOST, + "bucket": OBJECTSCALE_BUCKET, + }, + { + "id": "hadoop", + "label": "HADOOP HDFS", + "x": 50, + "color": "#39ff14", + "level": "ok" if hdfs_ok else "warn", + "running": hadoop.get("live_datanodes", 0), + "total": (hadoop.get("live_datanodes") or 0) + (hadoop.get("dead_datanodes") or 0) or 3, + "apps": [ + {"name": "NameNode", "state": "running" if hdfs_ok else "down", "image": "hdfs-nn", "ports": ["9870"], "host": "10.0.21.61"}, + *[ + {"name": dn.get("host", "?").split(".")[0], "state": "running", "image": "datanode", "ports": ["9866"], "host": dn.get("host", "")} + for dn in hadoop.get("datanodes", []) + ], + ], + "hdfs_used_gb": hadoop.get("capacity_used_gb"), + "hdfs_total_gb": hadoop.get("capacity_total_gb"), + "vm": "hadoop cluster", + "ip": "10.0.21.61–70", + }, + ] + + def _node( + nid: str, + label: str, + vm: str, + ip: str, + x: float, + y: float, + color: str, + level: str, + role: str, + apps: list[dict], + running: int, + total: int, + extra: dict | None = None, + ) -> dict[str, Any]: + reg = NODE_REGISTRY.get(nid, {}) + row: dict[str, Any] = { + "id": nid, + "label": label, + "vm": vm, + "ip": ip, + "x": x, + "y": y, + "color": reg.get("color", color), + "level": level, + "role": role, + "apps": apps, + "running": running, + "total": total, + "description": reg.get("description", ""), + "agent_id": NODE_AGENT.get(nid), + "links": reg.get("links", []), + "endpoints": reg.get("endpoints", []), + "commands": reg.get("commands", []), + "vmid": reg.get("vmid"), + "pve": reg.get("pve"), + } + if extra: + row.update(extra) + return row + + connect_running = 1 if connect_app and connect_app.get("state") == "running" else 0 + consumer_running = 1 if s3_consumer and s3_consumer.get("state") == "running" else 0 + + topology_nodes = [ + _node("airflow", "Airflow", "atc-airflow01", "10.0.21.55", 6, 18, "#00f0ff", "ok" if etl.get("airflow_healthy") else "warn", "orchestrator", + [{"name": "scheduler", "state": "running" if etl.get("airflow_healthy") else "down", "image": "airflow", "ports": ["8080"]}], int(etl.get("airflow_healthy", False)), 1), + _node("db", "DB Vault", "atc-db02", "10.0.21.51", 22, 18, "#ffaa00", _level(databases.get("running", 0), databases.get("total", 1) or 1), "sources", + db_apps, databases.get("running", 0), databases.get("total", 0)), + _node("debezium", "Debezium", "atc-lake01", "10.0.21.50", 38, 18, "#ff66cc", "ok" if connect_running and connectors else "warn", "cdc", + [_app_row(connect_app)] if connect_app else [], len(connectors), max(len(connectors), 1), + {"connectors": connectors}), + _node("kafka", "Kafka", "atc-kafka01", "10.0.21.36", 54, 18, "#00f0ff", "ok" if etl.get("kafka_ui_ok") else "warn", "bus", + [{"name": "broker", "state": "running" if etl.get("kafka_ui_ok") else "down", "image": "kafka", "ports": ["9092"]}], int(etl.get("kafka_ui_ok", False)), 1), + _node("lakehouse", "Lakehouse", "atc-lake01", "10.0.21.50", 70, 18, "#ff00aa", _level(lakehouse.get("running", 0), lakehouse.get("total", 1) or 1), "compute", + lake_apps, lakehouse.get("running", 0), lakehouse.get("total", 0), + {"trino_ok": lakehouse.get("trino_ok"), "spark_count": len(spark_apps)}), + _node("s3", "ObjectScale S3", "atc-objectscale", OBJECTSCALE_HOST, 88, 18, "#ffd700", s3_level, "storage", + s3_apps, sum(1 for a in s3_apps if a.get("state") == "running"), len(s3_apps), + {"bucket": OBJECTSCALE_BUCKET, "port": OBJECTSCALE_PORT, "consumer_ok": bool(consumer_running)}), + _node("docker", "Docker Rack", "atc-docker01", "10.0.21.45", 10, 52, "#b366ff", _level(docker.get("running", 0), docker.get("total", 1) or 1), "infra", + docker_apps, docker.get("running", 0), docker.get("total", 0)), + _node("hadoop", "Hadoop HDFS", "atc-hadoop-m01", "10.0.21.61", 50, 52, "#39ff14", "ok" if hdfs_ok else "warn", "parallel", + zones[-1]["apps"], hadoop.get("live_datanodes", 0), zones[-1]["total"], + {"hdfs_used_gb": hadoop.get("capacity_used_gb"), "hdfs_total_gb": hadoop.get("capacity_total_gb")}), + _node("gpu", "GPU Lab", "atc-gpu-dev", "10.0.20.106", 88, 52, "#76b900", "ok" if gpu.get("ok") else "down", "inference", + [{"name": gpu.get("active_model") or "vLLM", "state": "running" if gpu.get("inference_active") else "down", "image": "vllm", "ports": ["8001"]}], + gpu.get("gpu_count", 0), gpu.get("gpu_count", 0) or 4, + {"model": gpu.get("active_model"), "util": round(sum(g.get("util_gpu", 0) for g in gpu.get("gpus", [])) / max(len(gpu.get("gpus", [])), 1), 1)}), + _node("command", "Command Center", "MCP · VM304", "10.0.21.33", 50, 78, "#00f0ff", + _level(command.get("running", 0), command.get("total", 1) or 1), "hub", + command.get("containers") and [_app_row(c) for c in command.get("containers", [])] or [ + {"name": "atc-agents-api", "state": "running", "image": "atc-agents-api", "ports": ["3201"]}, + {"name": "atc-agents-ui", "state": "running", "image": "atc-agents-ui", "ports": ["80"]}, + {"name": "postgres", "state": "running", "image": "postgres", "ports": ["5432"]}, + {"name": "redis", "state": "running", "image": "redis", "ports": ["6379"]}, + {"name": "caddy", "state": "running", "image": "caddy", "ports": ["80"]}, + ], + command.get("running", 5), command.get("total", 5) or 5), + ] + + def _edge(eid: str, src: str, dst: str, label: str, kind: str, active: bool = True) -> dict[str, Any]: + return {"id": eid, "from": src, "to": dst, "label": label, "kind": kind, "active": active} + + pipeline_ok = etl.get("airflow_healthy") and len(connectors) > 0 and etl.get("kafka_ui_ok") + s3_flow_ok = pipeline_ok and consumer_running and objectscale_ok + + topology_edges = [ + _edge("e-seed", "airflow", "db", "seed data", "pipeline", bool(etl.get("airflow_healthy"))), + _edge("e-cdc", "db", "debezium", "CDC", "pipeline", bool(connectors)), + _edge("e-topics", "debezium", "kafka", "topics", "pipeline", bool(connectors and etl.get("kafka_ui_ok"))), + _edge("e-stream", "kafka", "lakehouse", "stream", "pipeline", bool(etl.get("kafka_ui_ok") and lakehouse.get("trino_ok"))), + _edge("e-s3", "lakehouse", "s3", "s3-kafka-consumer", "pipeline", bool(s3_flow_ok)), + _edge("e-iceberg", "lakehouse", "s3", "Trino Iceberg", "query", bool(lakehouse.get("trino_ok") and objectscale_ok)), + _edge("e-trino-db", "lakehouse", "db", "federated SQL", "query", bool(lakehouse.get("trino_ok"))), + _edge("e-hdfs", "lakehouse", "hadoop", "parallel layer", "parallel", bool(hdfs_ok)), + _edge("e-monitor-docker", "command", "docker", "monitor", "infra", True), + _edge("e-monitor-gpu", "command", "gpu", "LLM", "infra", bool(gpu.get("ok"))), + ] + + topologies = build_all_topologies( + topology_nodes, + topology_edges, + snap, + pipeline_active=s3_flow_ok, + connectors=connectors, + ) + + return { + "ts": snap.get("ts"), + "zones": zones, + "topology": topologies["architecture"], + "topologies": topologies, + "gpu": { + "level": "ok" if gpu.get("ok") and gpu.get("inference_active") else ("warn" if gpu.get("ok") else "down"), + "model": gpu.get("active_model"), + "inference_active": gpu.get("inference_active"), + "gpu_count": gpu.get("gpu_count", 0), + "avg_util": round( + sum(g.get("util_gpu", 0) for g in gpu.get("gpus", [])) / max(len(gpu.get("gpus", [])), 1), + 1, + ), + "gpus": gpu.get("gpus", []), + }, + "totals": { + "apps_running": sum(z["running"] for z in zones if z["id"] not in ("hadoop",)) + (hadoop.get("live_datanodes") or 0), + "apps_total": sum(z["total"] for z in zones), + "connectors": len(connectors), + "vms": len(topology_nodes), + "pipeline_active": s3_flow_ok, + }, + } diff --git a/caddy/Caddyfile b/caddy/Caddyfile index 89a87c3..519744f 100644 --- a/caddy/Caddyfile +++ b/caddy/Caddyfile @@ -1,4 +1,17 @@ :80 { - reverse_proxy /api/* api:3201 - reverse_proxy /* ui:80 + handle_path /rag/* { + reverse_proxy rag-api:5020 + } + handle_path /jupyter/* { + reverse_proxy jupyter:8888 + } + handle_path /dq/* { + reverse_proxy dq-api:5010 + } + handle /api/* { + reverse_proxy api:3201 + } + handle { + reverse_proxy ui:80 + } } diff --git a/config/command-center/.env.example b/config/command-center/.env.example new file mode 100644 index 0000000..abd5018 --- /dev/null +++ b/config/command-center/.env.example @@ -0,0 +1,27 @@ +# Command Center — VM304 (10.0.21.33) +# Copy to /opt/atc-agents/.env — never commit secrets + +# Postgres (internal) +POSTGRES_USER=atc +POSTGRES_PASSWORD=change-me +POSTGRES_DB=atc_agents + +# GPU / LLM (VM303) +GPU_URL=http://10.0.20.106:9000 +LLM_URL=http://10.0.20.106:8001/v1 +LLM_MODEL=gpt-4o +LLM_API_KEY=sk-local + +# ObjectScale S3 (VM objectscale 10.0.20.111) +S3_ENDPOINT=http://10.0.20.111:9020 +S3_ACCESS_KEY=object_admin1 +S3_SECRET_KEY=REDACTED-use-deploy-yml-or-ecs-admin +S3_REGION=us-east-1 + +# Jupyter +JUPYTER_TOKEN=change-me-jupyter-token + +# Lakehouse / ETL (optional probes) +LAKEHOUSE_HOST=10.0.21.50 +AIRFLOW_URL=http://10.0.21.55:8080 +KAFKA_UI_URL=http://10.0.21.36:9000 diff --git a/config/command-center/Caddyfile b/config/command-center/Caddyfile new file mode 100644 index 0000000..519744f --- /dev/null +++ b/config/command-center/Caddyfile @@ -0,0 +1,17 @@ +:80 { + handle_path /rag/* { + reverse_proxy rag-api:5020 + } + handle_path /jupyter/* { + reverse_proxy jupyter:8888 + } + handle_path /dq/* { + reverse_proxy dq-api:5010 + } + handle /api/* { + reverse_proxy api:3201 + } + handle { + reverse_proxy ui:80 + } +} diff --git a/config/command-center/docker-compose.yml b/config/command-center/docker-compose.yml new file mode 100644 index 0000000..7ba0db3 --- /dev/null +++ b/config/command-center/docker-compose.yml @@ -0,0 +1,150 @@ +services: + redis: + image: redis:7-alpine + restart: unless-stopped + volumes: + - redis_data:/data + + postgres: + image: postgres:16-alpine + restart: unless-stopped + environment: + POSTGRES_USER: atc + POSTGRES_PASSWORD: atc-agents-pg + POSTGRES_DB: atc_agents + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U atc -d atc_agents"] + interval: 5s + timeout: 5s + retries: 10 + + api: + build: ./api + restart: unless-stopped + env_file: + - .env + environment: + REDIS_URL: redis://redis:6379/0 + DOCKHAND_URL: http://10.0.21.45:8082 + DATABASE_URL: postgresql+psycopg2://atc:atc-agents-pg@postgres:5432/atc_agents + SQLITE_FALLBACK_PATH: /data/atc-agents.db + DOCLING_URL: http://docling-serve:5001 + PRESENTATIONS_DIR: /data/presentations + GPU_URL: http://10.0.20.106:9000 + GPU_UI_URL: http://10.0.20.106:9000 + LLM_URL: http://10.0.20.106:8001/v1 + LLM_MODEL: gpt-4o + LLM_API_KEY: sk-local + LAKEHOUSE_HOST: 10.0.21.50 + AIRFLOW_URL: http://10.0.21.55:8080 + KAFKA_UI_URL: http://10.0.21.36:9000 + HDFS_NN_URL: http://10.0.21.61:9870 + S3_ENDPOINT: ${S3_ENDPOINT:-http://10.0.20.111:9020} + S3_ACCESS_KEY: ${S3_ACCESS_KEY:-object_admin1} + S3_SECRET_KEY: ${S3_SECRET_KEY} + S3_REGION: ${S3_REGION:-us-east-1} + volumes: + - api_data:/data + depends_on: + redis: + condition: service_started + postgres: + condition: service_healthy + + ui: + build: ./ui + restart: unless-stopped + depends_on: + - api + + docling-serve: + image: quay.io/docling-project/docling-serve-cpu + restart: unless-stopped + ports: + - "5001:5001" + environment: + DOCLING_SERVE_ENABLE_UI: "1" + DOCLING_SERVE_MAX_SYNC_WAIT: "300" + + chromadb: + image: chromadb/chroma:0.5.23 + restart: unless-stopped + volumes: + - chroma_data:/chroma/chroma + environment: + ANONYMIZED_TELEMETRY: "false" + + rag-api: + build: ../atc-data-quality/rag-api + restart: unless-stopped + environment: + CHROMA_HOST: chromadb + CHROMA_PORT: 8000 + DOCLING_URL: http://docling-serve:5001 + LLM_URL: http://10.0.20.106:8001/v1 + LLM_MODEL: gpt-4o + LLM_API_KEY: sk-local + RAG_DATA_DIR: /data + volumes: + - rag_data:/data + depends_on: + - chromadb + - docling-serve + + dq-api: + build: ../atc-data-quality/dq-api + restart: unless-stopped + environment: + DOCLING_URL: http://docling-serve:5001 + DQ_DATA_DIR: /data + RAG_URL: http://rag-api:5020 + RAG_COLLECTION: default + volumes: + - dq_data:/data + depends_on: + - docling-serve + - rag-api + + jupyter: + image: quay.io/jupyter/scipy-notebook:latest + restart: unless-stopped + environment: + JUPYTER_TOKEN: ${JUPYTER_TOKEN:-atc-jupyter} + AWS_ACCESS_KEY_ID: ${S3_ACCESS_KEY:-object_admin1} + AWS_SECRET_ACCESS_KEY: ${S3_SECRET_KEY} + S3_ENDPOINT: ${S3_ENDPOINT:-http://10.0.20.111:9020} + AWS_DEFAULT_REGION: ${S3_REGION:-us-east-1} + command: > + start-notebook.sh + --NotebookApp.base_url=/jupyter/ + --NotebookApp.token=${JUPYTER_TOKEN:-atc-jupyter} + --NotebookApp.allow_origin=* + volumes: + - jupyter_data:/home/jovyan/work + + caddy: + image: caddy:2-alpine + restart: unless-stopped + ports: + - "80:80" + volumes: + - ./caddy/Caddyfile:/etc/caddy/Caddyfile:ro + depends_on: + - ui + - api + - dq-api + - docling-serve + - rag-api + - chromadb + - jupyter + +volumes: + chroma_data: + rag_data: + dq_data: + redis_data: + postgres_data: + api_data: + jupyter_data: diff --git a/docker-compose.dockhand.yml b/docker-compose.dockhand.yml new file mode 100644 index 0000000..cb4e669 --- /dev/null +++ b/docker-compose.dockhand.yml @@ -0,0 +1,70 @@ +# Dockhand-managed compose — image-only (no build context on Dockhand host). +# Source of truth for builds: /opt/atc-agents on VM304 (10.0.21.33). +services: + redis: + image: redis:7-alpine + restart: unless-stopped + volumes: + - redis_data:/data + + postgres: + image: postgres:16-alpine + restart: unless-stopped + environment: + POSTGRES_USER: atc + POSTGRES_PASSWORD: atc-agents-pg + POSTGRES_DB: atc_agents + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U atc -d atc_agents"] + interval: 5s + timeout: 5s + retries: 10 + + api: + image: atc-agents-api:latest + restart: unless-stopped + environment: + REDIS_URL: redis://redis:6379/0 + DOCKHAND_URL: http://10.0.21.45:8082 + DATABASE_URL: postgresql+psycopg2://atc:atc-agents-pg@postgres:5432/atc_agents + SQLITE_FALLBACK_PATH: /data/atc-agents.db + GPU_URL: http://10.0.20.106:9000 + GPU_UI_URL: http://10.0.20.106:9000 + LLM_URL: http://10.0.20.106:8001/v1 + LLM_MODEL: qwen2.5-32b-gptq + LLM_API_KEY: sk-local + LAKEHOUSE_HOST: 10.0.21.50 + AIRFLOW_URL: http://10.0.21.55:8080 + KAFKA_UI_URL: http://10.0.21.36:9000 + HDFS_NN_URL: http://10.0.21.61:9870 + volumes: + - api_data:/data + depends_on: + redis: + condition: service_started + postgres: + condition: service_healthy + + ui: + image: atc-agents-ui:latest + restart: unless-stopped + depends_on: + - api + + caddy: + image: caddy:2-alpine + restart: unless-stopped + ports: + - "80:80" + volumes: + - ./caddy/Caddyfile:/etc/caddy/Caddyfile:ro + depends_on: + - ui + - api + +volumes: + redis_data: + postgres_data: + api_data: diff --git a/docker-compose.yml b/docker-compose.yml index 1496864..7ba0db3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,17 +5,53 @@ services: volumes: - redis_data:/data + postgres: + image: postgres:16-alpine + restart: unless-stopped + environment: + POSTGRES_USER: atc + POSTGRES_PASSWORD: atc-agents-pg + POSTGRES_DB: atc_agents + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U atc -d atc_agents"] + interval: 5s + timeout: 5s + retries: 10 + api: build: ./api restart: unless-stopped + env_file: + - .env environment: REDIS_URL: redis://redis:6379/0 DOCKHAND_URL: http://10.0.21.45:8082 - DATABASE_URL: sqlite:////data/atc-agents.db + DATABASE_URL: postgresql+psycopg2://atc:atc-agents-pg@postgres:5432/atc_agents + SQLITE_FALLBACK_PATH: /data/atc-agents.db + DOCLING_URL: http://docling-serve:5001 + PRESENTATIONS_DIR: /data/presentations + GPU_URL: http://10.0.20.106:9000 + GPU_UI_URL: http://10.0.20.106:9000 + LLM_URL: http://10.0.20.106:8001/v1 + LLM_MODEL: gpt-4o + LLM_API_KEY: sk-local + LAKEHOUSE_HOST: 10.0.21.50 + AIRFLOW_URL: http://10.0.21.55:8080 + KAFKA_UI_URL: http://10.0.21.36:9000 + HDFS_NN_URL: http://10.0.21.61:9870 + S3_ENDPOINT: ${S3_ENDPOINT:-http://10.0.20.111:9020} + S3_ACCESS_KEY: ${S3_ACCESS_KEY:-object_admin1} + S3_SECRET_KEY: ${S3_SECRET_KEY} + S3_REGION: ${S3_REGION:-us-east-1} volumes: - api_data:/data depends_on: - - redis + redis: + condition: service_started + postgres: + condition: service_healthy ui: build: ./ui @@ -23,6 +59,71 @@ services: depends_on: - api + docling-serve: + image: quay.io/docling-project/docling-serve-cpu + restart: unless-stopped + ports: + - "5001:5001" + environment: + DOCLING_SERVE_ENABLE_UI: "1" + DOCLING_SERVE_MAX_SYNC_WAIT: "300" + + chromadb: + image: chromadb/chroma:0.5.23 + restart: unless-stopped + volumes: + - chroma_data:/chroma/chroma + environment: + ANONYMIZED_TELEMETRY: "false" + + rag-api: + build: ../atc-data-quality/rag-api + restart: unless-stopped + environment: + CHROMA_HOST: chromadb + CHROMA_PORT: 8000 + DOCLING_URL: http://docling-serve:5001 + LLM_URL: http://10.0.20.106:8001/v1 + LLM_MODEL: gpt-4o + LLM_API_KEY: sk-local + RAG_DATA_DIR: /data + volumes: + - rag_data:/data + depends_on: + - chromadb + - docling-serve + + dq-api: + build: ../atc-data-quality/dq-api + restart: unless-stopped + environment: + DOCLING_URL: http://docling-serve:5001 + DQ_DATA_DIR: /data + RAG_URL: http://rag-api:5020 + RAG_COLLECTION: default + volumes: + - dq_data:/data + depends_on: + - docling-serve + - rag-api + + jupyter: + image: quay.io/jupyter/scipy-notebook:latest + restart: unless-stopped + environment: + JUPYTER_TOKEN: ${JUPYTER_TOKEN:-atc-jupyter} + AWS_ACCESS_KEY_ID: ${S3_ACCESS_KEY:-object_admin1} + AWS_SECRET_ACCESS_KEY: ${S3_SECRET_KEY} + S3_ENDPOINT: ${S3_ENDPOINT:-http://10.0.20.111:9020} + AWS_DEFAULT_REGION: ${S3_REGION:-us-east-1} + command: > + start-notebook.sh + --NotebookApp.base_url=/jupyter/ + --NotebookApp.token=${JUPYTER_TOKEN:-atc-jupyter} + --NotebookApp.allow_origin=* + volumes: + - jupyter_data:/home/jovyan/work + caddy: image: caddy:2-alpine restart: unless-stopped @@ -33,7 +134,17 @@ services: depends_on: - ui - api + - dq-api + - docling-serve + - rag-api + - chromadb + - jupyter volumes: + chroma_data: + rag_data: + dq_data: redis_data: + postgres_data: api_data: + jupyter_data: diff --git a/docs/runbook-vm304.md b/docs/runbook-vm304.md new file mode 100644 index 0000000..442bf5d --- /dev/null +++ b/docs/runbook-vm304.md @@ -0,0 +1,46 @@ +# Command Center VM304 + +| Item | Value | +|------|-------| +| VM | MCP · Proxmox VMID **304** | +| IP | `10.0.21.33` | +| SSH | `root@10.0.21.33` | +| Gitea | `http://atc-mgt01.dell-atc.lan:3001/mo/atc-agents` | + +## Stack + +- **ui** — React (Data Platform, Presentation, DQ, Knowledge Chat, S3 Storage, Jupyter link) +- **api** — FastAPI + WebSocket + S3 browser API +- **dq-api / rag-api** — from `mo/atc-data-quality` +- **jupyter** — JupyterLab with S3 credentials +- **chromadb, docling, postgres, redis, caddy** + +## ObjectScale S3 + +| Item | Value | +|------|-------| +| Endpoint | `http://10.0.20.111:9020` | +| Namespace | `ns1` | +| Default user | `object_admin1` (see ECS deploy.yml) | +| UI browse | Command Center → **Object Storage** | + +## Jupyter + +- URL: `http://10.0.21.33/jupyter/` +- Token: `JUPYTER_TOKEN` in `.env` +- Work dir persisted in Docker volume `jupyter_data` +- Preconfigured: `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `S3_ENDPOINT` + +Example in notebook: + +```python +import boto3, os +s3 = boto3.client("s3", endpoint_url=os.environ["S3_ENDPOINT"]) +print(s3.list_buckets()) +``` + +## Gitea sync + +```bash +./scripts/sync-gitea.sh +``` diff --git a/index.html b/index.html new file mode 100644 index 0000000..0b2ddd6 --- /dev/null +++ b/index.html @@ -0,0 +1,15 @@ + + + + + + ATC Command Center + + + + + +
+ + + diff --git a/lab_context.py b/lab_context.py new file mode 100644 index 0000000..b169f98 --- /dev/null +++ b/lab_context.py @@ -0,0 +1,541 @@ +"""Live lab metrics for all ATC domains — fed to vLLM as context.""" + +from __future__ import annotations + +import asyncio +import json +import os +import time +from datetime import datetime, timezone +from typing import Any + +import httpx + +from agent_terminal import TerminalLogFn + +DOCKHAND_URL = os.getenv("DOCKHAND_URL", "http://10.0.21.45:8082") +HDFS_NN_URL = os.getenv("HDFS_NN_URL", "http://10.0.21.61:9870") +LAKEHOUSE_HOST = os.getenv("LAKEHOUSE_HOST", "10.0.21.50") +AIRFLOW_URL = os.getenv("AIRFLOW_URL", "http://10.0.21.55:8080") +KAFKA_UI_URL = os.getenv("KAFKA_UI_URL", "http://10.0.21.36:9000") +KAFKA_CONNECT_URL = os.getenv("KAFKA_CONNECT_URL", f"http://{LAKEHOUSE_HOST}:8083") +TRINO_URL = os.getenv("TRINO_URL", f"http://{LAKEHOUSE_HOST}:8089") +SPARK_UI_URL = os.getenv("SPARK_UI_URL", f"http://{LAKEHOUSE_HOST}:8080") +GPU_URL = os.getenv("GPU_URL", "http://10.0.20.106:9000") + +AGENT_PRIMARY_DOMAIN = { + "infra-sentinel": "docker", + "data-custodian": "databases", + "lakehouse-ops": "lakehouse", + "hadoop-ranger": "hadoop", + "etl-guardian": "etl", +} + + +async def _log(log: TerminalLogFn | None, level: str, phase: str, text: str) -> None: + if log: + await log(level, phase, text) + + +async def _get_json( + client: httpx.AsyncClient, + url: str, + log: TerminalLogFn | None = None, + label: str = "", + timeout: float = 6.0, +) -> Any | None: + name = label or url + t0 = time.monotonic() + await _log(log, "cmd", "fetch", f"$ GET {url}") + try: + r = await client.get(url, timeout=timeout) + ms = int((time.monotonic() - t0) * 1000) + if r.status_code < 400: + await _log(log, "ok", "fetch", f"← {r.status_code} {name} ({ms}ms)") + return r.json() + await _log(log, "warn", "fetch", f"← {r.status_code} {name} ({ms}ms)") + except Exception as exc: + ms = int((time.monotonic() - t0) * 1000) + await _log(log, "err", "fetch", f"✗ {name}: {exc} ({ms}ms)") + return None + + +async def _probe_ok( + client: httpx.AsyncClient, + url: str, + log: TerminalLogFn | None = None, + label: str = "", +) -> bool: + name = label or url + t0 = time.monotonic() + await _log(log, "cmd", "probe", f"$ GET {url}") + try: + r = await client.get(url, timeout=4.0) + ms = int((time.monotonic() - t0) * 1000) + ok = r.status_code < 500 + await _log(log, "ok" if ok else "warn", "probe", f"← {r.status_code} {name} ({'UP' if ok else 'DOWN'}, {ms}ms)") + return ok + except Exception as exc: + ms = int((time.monotonic() - t0) * 1000) + await _log(log, "err", "probe", f"✗ {name}: {exc} ({ms}ms)") + return False + + +def _container_rows(containers: list[dict], host: str = "") -> list[dict[str, Any]]: + rows = [] + for c in containers: + ports = sorted({str(p.get("PublicPort")) for p in c.get("ports", []) if p.get("PublicPort")}) + rows.append({ + "name": c.get("name"), + "state": c.get("state"), + "image": c.get("image"), + "status": c.get("status"), + "ports": ports, + "host": host, + }) + return rows + + +async def dockhand_containers( + client: httpx.AsyncClient, + env_id: int, + log: TerminalLogFn | None = None, +) -> list[dict]: + url = f"{DOCKHAND_URL}/api/containers?env={env_id}" + await _log(log, "cmd", "fetch", f"$ GET {url}") + t0 = time.monotonic() + try: + r = await client.get(f"{DOCKHAND_URL}/api/containers", params={"env": env_id}, timeout=8.0) + ms = int((time.monotonic() - t0) * 1000) + r.raise_for_status() + data = r.json() + await _log(log, "ok", "fetch", f"← Dockhand env {env_id}: {len(data)} containers ({ms}ms)") + return data + except Exception as exc: + ms = int((time.monotonic() - t0) * 1000) + await _log(log, "err", "fetch", f"✗ Dockhand env {env_id}: {exc} ({ms}ms)") + return [] + + +async def collect_hdfs(client: httpx.AsyncClient, log: TerminalLogFn | None = None) -> dict[str, Any]: + ctx: dict[str, Any] = {"reachable": False, "namenode": HDFS_NN_URL} + await _log(log, "info", "fetch", "▸ HDFS NameNode JMX metrics") + try: + fs_url = f"{HDFS_NN_URL}/jmx?qry=Hadoop:service=NameNode,name=FSNamesystem" + nn_url = f"{HDFS_NN_URL}/jmx?qry=Hadoop:service=NameNode,name=NameNodeInfo" + t0 = time.monotonic() + await _log(log, "cmd", "fetch", f"$ GET {fs_url}") + await _log(log, "cmd", "fetch", f"$ GET {nn_url}") + fs_r, nn_r = await asyncio.gather( + client.get(fs_url), + client.get(nn_url), + return_exceptions=True, + ) + ms = int((time.monotonic() - t0) * 1000) + if isinstance(fs_r, httpx.Response) and fs_r.status_code == 200: + beans = fs_r.json().get("beans", []) + if beans: + b = beans[0] + ctx.update({ + "reachable": True, + "hostname": b.get("tag.Hostname"), + "ha_state": b.get("tag.HAState"), + "capacity_total_gb": b.get("CapacityTotalGB"), + "capacity_used_gb": b.get("CapacityUsedGB"), + "capacity_remaining_gb": b.get("CapacityRemainingGB"), + "files_total": b.get("FilesTotal"), + "blocks_total": b.get("BlocksTotal"), + "live_datanodes": b.get("NumLiveDataNodes"), + "dead_datanodes": b.get("NumDeadDataNodes"), + "missing_blocks": b.get("MissingBlocks"), + "under_replicated_blocks": b.get("UnderReplicatedBlocks"), + "corrupt_blocks": b.get("CorruptBlocks"), + "default_replication_factor": 3, + }) + await _log( + log, "ok", "fetch", + f"← HDFS: {b.get('CapacityUsedGB')}GB used, {b.get('FilesTotal')} files, " + f"{b.get('NumLiveDataNodes')} datanodes ({ms}ms)", + ) + else: + await _log(log, "warn", "fetch", f"← FSNamesystem JMX failed ({ms}ms)") + + if isinstance(nn_r, httpx.Response) and nn_r.status_code == 200: + beans = nn_r.json().get("beans", []) + if beans: + b = beans[0] + live = json.loads(b.get("LiveNodes") or "{}") + ctx["hdfs_version"] = b.get("Version") + ctx["safemode"] = b.get("Safemode") or "off" + ctx["percent_used"] = round(float(b.get("PercentUsed", 0)) * 100, 4) + ctx["datanodes"] = [ + { + "host": host.split(":")[0], + "capacity_gb": round(node.get("capacity", 0) / (1024**3), 1), + "used_gb": round(node.get("used", 0) / (1024**3), 4), + "blocks": node.get("numBlocks", 0), + "state": node.get("adminState"), + } + for host, node in live.items() + ] + except Exception as exc: + ctx["error"] = str(exc) + await _log(log, "err", "fetch", f"✗ HDFS: {exc}") + return ctx + + +async def collect_etl(client: httpx.AsyncClient, log: TerminalLogFn | None = None) -> dict[str, Any]: + await _log(log, "info", "fetch", "▸ ETL stack (Airflow, Kafka, Spark)") + health, kafka_ok, spark_ok = await asyncio.gather( + _get_json(client, f"{AIRFLOW_URL}/api/v2/monitor/health", log, "Airflow health"), + _probe_ok(client, KAFKA_UI_URL, log, "Kafka UI"), + _probe_ok(client, SPARK_UI_URL, log, "Spark UI"), + ) + connectors: list[str] = [] + await _log(log, "cmd", "fetch", f"$ GET {KAFKA_CONNECT_URL}/connectors") + t0 = time.monotonic() + try: + r = await client.get(f"{KAFKA_CONNECT_URL}/connectors", timeout=5.0) + ms = int((time.monotonic() - t0) * 1000) + if r.status_code == 200: + connectors = r.json() if isinstance(r.json(), list) else [] + await _log(log, "ok", "fetch", f"← Kafka Connect: {len(connectors)} connectors ({ms}ms)") + for c in connectors: + await _log(log, "info", "fetch", f" · {c}") + else: + await _log(log, "warn", "fetch", f"← Kafka Connect {r.status_code} ({ms}ms)") + except Exception as exc: + await _log(log, "err", "fetch", f"✗ Kafka Connect: {exc}") + + airflow_detail: dict[str, str] = {} + if isinstance(health, dict): + for comp, info in health.items(): + if isinstance(info, dict) and "status" in info: + airflow_detail[comp] = info["status"] + await _log(log, "info", "fetch", f" Airflow {comp}: {info['status']}") + + return { + "airflow_url": AIRFLOW_URL, + "airflow_healthy": airflow_detail.get("scheduler") == "healthy", + "airflow_components": airflow_detail, + "kafka_ui_url": KAFKA_UI_URL, + "kafka_ui_ok": kafka_ok, + "kafka_connect_url": KAFKA_CONNECT_URL, + "connectors": connectors, + "spark_ui_url": SPARK_UI_URL, + "spark_ui_ok": spark_ok, + } + + +async def collect_lakehouse( + client: httpx.AsyncClient, + containers: list[dict], + log: TerminalLogFn | None = None, +) -> dict[str, Any]: + await _log(log, "info", "fetch", "▸ Lakehouse (Trino, Spark, Kafka Connect)") + trino_info = await _get_json(client, f"{TRINO_URL}/v1/info", log, "Trino /v1/info") + running = sum(1 for c in containers if c.get("state") == "running") + for c in containers: + ports = ",".join(str(p.get("PublicPort")) for p in c.get("ports", []) if p.get("PublicPort")) or "internal" + await _log(log, "info", "fetch", f" · {c.get('name')}: {c.get('state')} ports={ports}") + return { + "host": LAKEHOUSE_HOST, + "trino_url": TRINO_URL, + "trino_ok": trino_info is not None, + "trino_version": (trino_info or {}).get("nodeVersion", {}).get("version"), + "trino_uptime": (trino_info or {}).get("uptime"), + "trino_coordinator": (trino_info or {}).get("coordinator"), + "spark_ui_url": SPARK_UI_URL, + "kafka_connect_url": KAFKA_CONNECT_URL, + "containers": _container_rows(containers, LAKEHOUSE_HOST), + "running": running, + "total": len(containers), + } + + +async def collect_databases( + client: httpx.AsyncClient, + containers: list[dict], + log: TerminalLogFn | None = None, +) -> dict[str, Any]: + await _log(log, "info", "fetch", "▸ Database vault (Dockhand env 5)") + running = sum(1 for c in containers if c.get("state") == "running") + rows = _container_rows(containers) + by_engine: dict[str, list[str]] = {} + for r in rows: + img = (r.get("image") or "").lower() + name = (r.get("name") or "").lower() + if "postgres" in img or "postgres" in name: + engine = "PostgreSQL" + elif "mysql" in img or "mysql" in name: + engine = "MySQL" + elif "mongo" in img or "mongo" in name: + engine = "MongoDB" + elif "cassandra" in img or "cassandra" in name: + engine = "Cassandra" + elif "neo4j" in img or "neo4j" in name: + engine = "Neo4j" + else: + engine = "Other" + port_str = ",".join(r["ports"]) or "internal" + by_engine.setdefault(engine, []).append(f"{r['name']} ({r['state']}, ports {port_str})") + await _log(log, "info", "fetch", f" · {r['name']}: {r['state']} [{engine}] ports={port_str}") + + return { + "dockhand_env": 5, + "running": running, + "total": len(containers), + "containers": rows, + "by_engine": by_engine, + } + + +async def collect_docker_rack( + client: httpx.AsyncClient, + containers: list[dict], + log: TerminalLogFn | None = None, +) -> dict[str, Any]: + await _log(log, "info", "fetch", "▸ Docker rack (Dockhand env 1)") + running = sum(1 for c in containers if c.get("state") == "running") + not_running = [c["name"] for c in containers if c.get("state") != "running"] + for c in containers: + ports = ",".join(str(p.get("PublicPort")) for p in c.get("ports", []) if p.get("PublicPort")) or "internal" + lvl = "info" if c.get("state") == "running" else "warn" + await _log(log, lvl, "fetch", f" · {c.get('name')}: {c.get('state')} ports={ports}") + return { + "dockhand_url": DOCKHAND_URL, + "dockhand_env": 1, + "running": running, + "total": len(containers), + "not_running": not_running, + "containers": _container_rows(containers, "10.0.21.45"), + } + + +async def collect_gpu_metrics(client: httpx.AsyncClient, log: TerminalLogFn | None = None) -> dict[str, Any]: + await _log(log, "info", "fetch", "▸ GPU Lab metrics") + base = {"ok": False, "host": GPU_URL, "ui_url": GPU_URL} + try: + metrics_url = f"{GPU_URL}/api/gpu/metrics" + model_url = f"{GPU_URL}/api/active-model" + await _log(log, "cmd", "fetch", f"$ GET {metrics_url}") + await _log(log, "cmd", "fetch", f"$ GET {model_url}") + t0 = time.monotonic() + metrics_r, model_r = await asyncio.gather( + client.get(metrics_url), + client.get(model_url), + return_exceptions=True, + ) + ms = int((time.monotonic() - t0) * 1000) + gpus: list[dict[str, Any]] = [] + if isinstance(metrics_r, httpx.Response) and metrics_r.status_code == 200: + current = metrics_r.json().get("current", {}) + gpus = [ + { + "index": g["index"], + "name": g["name"], + "util_gpu": g.get("util_gpu", 0), + "memory_used_mib": g.get("memory_used_mib", 0), + "memory_total_mib": g.get("memory_total_mib", 0), + "temperature_c": g.get("temperature_c", 0), + "power_w": g.get("power_w", 0), + } + for g in current.get("gpus", []) + ] + await _log(log, "ok", "fetch", f"← GPU metrics: {len(gpus)} devices ({ms}ms)") + for g in gpus: + await _log( + log, "info", "fetch", + f" GPU{g['index']}: util {g['util_gpu']:.0f}% VRAM " + f"{g['memory_used_mib']:.0f}/{g['memory_total_mib']:.0f} MiB", + ) + + active_model = None + inference_active = False + vllm_url = None + if isinstance(model_r, httpx.Response) and model_r.status_code == 200: + model_data = model_r.json() + active_model = model_data.get("name") + inference_active = bool(model_data.get("inference_active")) + vllm_url = model_data.get("base_url") + await _log(log, "ok", "fetch", f"← Active model: {active_model} inference={'ON' if inference_active else 'OFF'}") + + return { + **base, + "ok": len(gpus) > 0 or inference_active, + "inference_active": inference_active, + "active_model": active_model, + "vllm_url": vllm_url, + "gpu_count": len(gpus), + "gpus": gpus, + } + except Exception as exc: + await _log(log, "err", "fetch", f"✗ GPU Lab: {exc}") + return {**base, "error": str(exc)} + + +def _section_docker(d: dict[str, Any]) -> list[str]: + lines = [ + f"Docker rack (Dockhand env 1): {d['running']}/{d['total']} running", + f"Dockhand: {d['dockhand_url']}", + ] + if d.get("not_running"): + lines.append(f"Not running: {', '.join(d['not_running'])}") + for c in d.get("containers", []): + port_str = ",".join(c["ports"]) if c["ports"] else "internal" + lines.append(f" - {c['name']}: {c['state']} | {c['image']} | ports {port_str}") + return lines + + +def _section_databases(d: dict[str, Any]) -> list[str]: + lines = [f"Databases (Dockhand env {d['dockhand_env']}): {d['running']}/{d['total']} running"] + for engine, items in d.get("by_engine", {}).items(): + lines.append(f" {engine}:") + for item in items: + lines.append(f" - {item}") + return lines + + +def _section_lakehouse(d: dict[str, Any]) -> list[str]: + lines = [ + f"Lakehouse host: {d['host']} — {d['running']}/{d['total']} containers running", + f"Trino: {d['trino_url']} — {'UP' if d['trino_ok'] else 'DOWN'}" + + (f" (v{d['trino_version']}, uptime {d.get('trino_uptime')})" if d.get("trino_ok") else ""), + f"Spark UI: {d['spark_ui_url']}", + f"Kafka Connect: {d['kafka_connect_url']}", + ] + for c in d.get("containers", []): + port_str = ",".join(c["ports"]) if c["ports"] else "internal" + lines.append(f" - {c['name']}: {c['state']} | {c['image']} | ports {port_str}") + return lines + + +def _section_etl(d: dict[str, Any]) -> list[str]: + lines = [ + f"Airflow ({d['airflow_url']}): {'HEALTHY' if d['airflow_healthy'] else 'DEGRADED'}", + ] + for comp, st in d.get("airflow_components", {}).items(): + lines.append(f" - {comp}: {st}") + lines.append(f"Kafka UI ({d['kafka_ui_url']}): {'UP' if d['kafka_ui_ok'] else 'DOWN'}") + lines.append(f"Kafka Connect ({d['kafka_connect_url']}): connectors {d.get('connectors') or 'none listed'}") + if d.get("connectors"): + lines.append(" Registered connector names (exact): " + ", ".join(d["connectors"])) + lines.append(f"Spark UI ({d['spark_ui_url']}): {'UP' if d['spark_ui_ok'] else 'DOWN'}") + return lines + + +def _section_hadoop(h: dict[str, Any]) -> list[str]: + lines = ["HDFS / Hadoop:"] + if not h.get("reachable"): + lines.append(f" UNREACHABLE: {h.get('error', 'NameNode probe failed')}") + return lines + lines.extend([ + f" NameNode: {h['namenode']} ({h.get('hostname')}, HA {h.get('ha_state')})", + f" Version: {h.get('hdfs_version')}, safemode: {h.get('safemode')}", + f" Capacity: {h.get('capacity_used_gb')} GB used / {h.get('capacity_total_gb')} GB total " + f"({h.get('capacity_remaining_gb')} GB free, {h.get('percent_used', 0)}% used)", + f" Files: {h.get('files_total')}, Blocks: {h.get('blocks_total')}", + f" DataNodes: {h.get('live_datanodes')} live, {h.get('dead_datanodes')} dead", + f" Replication factor (dfs.replication): {h.get('default_replication_factor')}", + f" Block health: missing={h.get('missing_blocks')}, under-replicated={h.get('under_replicated_blocks')}, corrupt={h.get('corrupt_blocks')}", + ]) + for dn in h.get("datanodes", []): + lines.append( + f" - {dn['host']}: {dn['used_gb']} GB / {dn['capacity_gb']} GB, {dn['blocks']} blocks, {dn['state']}" + ) + if (h.get("capacity_used_gb") or 0) < 0.01 and (h.get("files_total") or 0) > 0: + lines.append(" Note: metadata/small files only — almost no user data stored yet.") + return lines + + +def _section_gpu(g: dict[str, Any]) -> list[str]: + lines = ["GPU Lab / vLLM inference:"] + if not g.get("ok"): + lines.append(f" OFFLINE: {g.get('error', 'unreachable')}") + return lines + lines.extend([ + f" Manager: {g.get('ui_url')}", + f" Model: {g.get('active_model')} (inference {'ON' if g.get('inference_active') else 'OFF'})", + f" vLLM endpoint: {g.get('vllm_url')}", + f" GPUs: {g.get('gpu_count')}x V100", + ]) + for gpu in g.get("gpus", []): + lines.append( + f" GPU{gpu['index']}: util {gpu['util_gpu']:.0f}%, " + f"VRAM {gpu['memory_used_mib']:.0f}/{gpu['memory_total_mib']:.0f} MiB, " + f"{gpu['temperature_c']}°C, {gpu['power_w']:.0f}W" + ) + return lines + + +SECTION_BUILDERS = { + "docker": _section_docker, + "databases": _section_databases, + "lakehouse": _section_lakehouse, + "etl": _section_etl, + "hadoop": _section_hadoop, + "gpu": _section_gpu, +} + +DOMAIN_ORDER = ["docker", "databases", "lakehouse", "etl", "hadoop", "gpu"] + + +async def collect_full_lab_context( + gpu_data: dict[str, Any] | None = None, + log: TerminalLogFn | None = None, +) -> dict[str, Any]: + """Gather all lab domains in parallel with optional live terminal logging.""" + await _log(log, "info", "fetch", "═══ Lab snapshot collection started ═══") + async with httpx.AsyncClient(timeout=10.0, verify=False) as client: + if gpu_data is None: + gpu_data = await collect_gpu_metrics(client, log) + + docker_raw, db_raw, lake_raw, hdfs, etl = await asyncio.gather( + dockhand_containers(client, 1, log), + dockhand_containers(client, 5, log), + dockhand_containers(client, 9, log), + collect_hdfs(client, log), + collect_etl(client, log), + ) + docker = await collect_docker_rack(client, docker_raw, log) + databases = await collect_databases(client, db_raw, log) + lakehouse = await collect_lakehouse(client, lake_raw, log) + + await _log(log, "ok", "fetch", "═══ Lab snapshot complete ═══") + return { + "ts": datetime.now(timezone.utc).isoformat(), + "docker": docker, + "databases": databases, + "lakehouse": lakehouse, + "etl": etl, + "hadoop": hdfs, + "gpu": gpu_data, + } + + +def format_context_for_agent(agent_id: str, snapshot: dict[str, Any]) -> str: + """Format full lab snapshot for LLM; primary domain first.""" + primary = AGENT_PRIMARY_DOMAIN.get(agent_id, "docker") + lines = [ + f"ATC Lab live snapshot — {snapshot.get('ts')}", + f"Your primary domain: {primary.upper()}", + ] + if snapshot.get("domains_summary"): + lines.append(f"Health summary: {json.dumps(snapshot['domains_summary'], default=str)}") + lines.extend(["", f"=== PRIMARY: {primary.upper()} ==="]) + + if primary in snapshot and primary in SECTION_BUILDERS: + lines.extend(SECTION_BUILDERS[primary](snapshot[primary])) + lines.append("") + lines.append("=== FULL LAB (all domains) ===") + + for domain in DOMAIN_ORDER: + if domain == primary: + continue + if domain not in snapshot or domain not in SECTION_BUILDERS: + continue + lines.append("") + lines.append(f"--- {domain.upper()} ---") + lines.extend(SECTION_BUILDERS[domain](snapshot[domain])) + + return "\n".join(lines) diff --git a/main.py b/main.py new file mode 100644 index 0000000..f6d8283 --- /dev/null +++ b/main.py @@ -0,0 +1,662 @@ +"""ATC Command Center API — FastAPI backend.""" + +from __future__ import annotations + +import asyncio +import json +import os +import time +import uuid +from contextlib import asynccontextmanager +from datetime import datetime, timezone +from typing import Any + +import httpx +import redis.asyncio as aioredis +from fastapi import FastAPI, WebSocket, WebSocketDisconnect +from fastapi.middleware.cors import CORSMiddleware +from agent_terminal import ( + get_all_terminals, + get_terminal_lines, + init_terminals, + make_logger, + set_terminal_publisher, + terminal_log, +) +from lab_context import collect_full_lab_context, format_context_for_agent +from workload import build_workload_payload +from pydantic import BaseModel, Field +from sqlalchemy import Column, DateTime, String, Text, create_engine, select +from sqlalchemy.orm import DeclarativeBase, Session, sessionmaker + +REDIS_URL = os.getenv("REDIS_URL", "redis://redis:6379/0") +DOCKHAND_URL = os.getenv("DOCKHAND_URL", "http://10.0.21.45:8082") +DATABASE_URL = os.getenv("DATABASE_URL", "sqlite:////data/atc-agents.db") +GPU_URL = os.getenv("GPU_URL", "http://10.0.20.106:9000") +GPU_UI_URL = os.getenv("GPU_UI_URL", GPU_URL) +LLM_URL = os.getenv("LLM_URL", "http://10.0.20.106:8001/v1") +LLM_MODEL = os.getenv("LLM_MODEL", "qwen2.5-32b-gptq") +LLM_API_KEY = os.getenv("LLM_API_KEY", "sk-local") +LLM_TIMEOUT = float(os.getenv("LLM_TIMEOUT", "120")) + +AGENTS = [ + { + "id": "etl-guardian", + "name": "ETL Guardian", + "color": "#00f0ff", + "zone": "etl", + "role": "Airflow, Kafka, Debezium, S3 pipeline", + "icon": "⚡", + "motto": "Pipelines never sleep", + "capabilities": ["Airflow", "Kafka", "Debezium", "S3", "Connectors"], + "suggested_prompts": [ + "Hoe staat Debezium er voor?", + "Zijn alle Airflow DAGs healthy?", + "Kafka connector status?", + ], + }, + { + "id": "lakehouse-ops", + "name": "Lakehouse Ops", + "color": "#ff00aa", + "zone": "lakehouse", + "role": "Spark, Trino, Iceberg", + "icon": "🏔️", + "motto": "Query the lake, trust the table", + "capabilities": ["Spark", "Trino", "Iceberg", "Delta", "SQL"], + "suggested_prompts": [ + "Lakehouse stack status?", + "Is Trino bereikbaar?", + "Hoeveel lakehouse containers draaien?", + ], + }, + { + "id": "data-custodian", + "name": "Data Custodian", + "color": "#ffaa00", + "zone": "db", + "role": "PostgreSQL, MySQL, Mongo, Cassandra, Neo4j", + "icon": "🛡️", + "motto": "Guardian of every row", + "capabilities": ["PostgreSQL", "MySQL", "MongoDB", "Cassandra", "Neo4j"], + "suggested_prompts": [ + "Database containers status?", + "Welke DB's draaien niet?", + "Postgres health check", + ], + }, + { + "id": "hadoop-ranger", + "name": "Hadoop Ranger", + "color": "#39ff14", + "zone": "hadoop", + "role": "HDFS, YARN cluster", + "icon": "🌲", + "motto": "Patrol the data forest", + "capabilities": ["HDFS", "YARN", "NameNode", "DataNodes"], + "suggested_prompts": [ + "Is HDFS NameNode up?", + "Hadoop cluster status?", + "YARN nodes healthy?", + ], + }, + { + "id": "infra-sentinel", + "name": "Infra Sentinel", + "color": "#b366ff", + "zone": "docker", + "role": "Docker, Proxmox, GPU, monitoring", + "icon": "👁️", + "motto": "See everything, miss nothing", + "capabilities": ["Docker", "Proxmox", "GPU", "vLLM", "Monitoring"], + "suggested_prompts": [ + "Hoe staat de GPU?", + "Welk LLM model draait er?", + "Docker containers overzicht", + ], + }, +] + +ZONES = [ + {"id": "docker", "label": "DOCKER RACK", "x": 8, "color": "#b366ff"}, + {"id": "db", "label": "DB VAULT", "x": 28, "color": "#ffaa00"}, + {"id": "lakehouse", "label": "LAKEHOUSE HUB", "x": 50, "color": "#ff00aa"}, + {"id": "hadoop", "label": "HADOOP CLUSTER", "x": 72, "color": "#39ff14"}, + {"id": "etl", "label": "ETL PIPE", "x": 92, "color": "#00f0ff"}, +] + +INTENT_KEYWORDS: dict[str, list[str]] = { + "data-custodian": ["database", "postgres", "postgresql", "mysql", "mongo", "mongodb", "cassandra", "neo4j", "sql", "db "], + "lakehouse-ops": ["trino", "spark", "lakehouse", "iceberg", "query", "table"], + "hadoop-ranger": [ + "hadoop", "hdfs", "yarn", "datanode", "namenode", "replicatie", "replication", + "rf factor", "opslag", "bestanden", "blocks", "cluster opslag", "data op", + ], + "infra-sentinel": ["docker", "container", "vm", "proxmox", "infra", "grafana", "gpu", "vllm", "llm", "nvidia", "inference", "model"], + "etl-guardian": ["airflow", "dag", "debezium", "kafka", "connector", "etl", "pipeline", "s3"], +} + + +class Base(DeclarativeBase): + pass + + +class FeedEntry(Base): + __tablename__ = "feed" + + id = Column(String, primary_key=True) + ts = Column(DateTime, default=lambda: datetime.now(timezone.utc)) + agent_id = Column(String) + level = Column(String, default="info") + message = Column(Text) + + +class Approval(Base): + __tablename__ = "approvals" + + id = Column(String, primary_key=True) + ts = Column(DateTime, default=lambda: datetime.now(timezone.utc)) + agent_id = Column(String) + action = Column(Text) + reason = Column(Text) + status = Column(String, default="pending") + + +engine = create_engine(DATABASE_URL, connect_args={"check_same_thread": False}) +SessionLocal = sessionmaker(bind=engine) +Base.metadata.create_all(engine) + +redis_client: aioredis.Redis | None = None +ws_clients: set[WebSocket] = set() + + +class PromptRequest(BaseModel): + message: str = Field(min_length=1, max_length=2000) + agent_id: str | None = None + + +class ApprovalDecision(BaseModel): + approved: bool + + +def route_agent(message: str) -> str: + lower = message.lower() + # Storage/data questions default to Hadoop unless clearly about databases + if any(w in lower for w in ("data", "opslag", "gb", "replicatie", "replication", "hdfs", "hadoop")): + if not any(w in lower for w in ("postgres", "mysql", "mongo", "database", "sql", "neo4j", "cassandra")): + return "hadoop-ranger" + scores = {aid: sum(1 for kw in kws if kw in lower) for aid, kws in INTENT_KEYWORDS.items()} + best = max(scores, key=scores.get) + if scores[best] == 0: + return "infra-sentinel" + return best + + +async def gather_agent_context( + agent_id: str, + status: dict[str, Any], + log: Any | None = None, +) -> str: + """Full lab snapshot for vLLM — all domains, agent's primary domain highlighted.""" + snapshot = await collect_full_lab_context(gpu_data=status.get("gpu"), log=log) + snapshot["domains_summary"] = status.get("domains", {}) + ctx = format_context_for_agent(agent_id, snapshot) + if log: + await log("info", "fetch", f"▸ Context assembled: {len(ctx)} chars for LLM") + return ctx + + +async def ask_llm( + agent_id: str, + message: str, + context: str, + log: Any | None = None, +) -> str | None: + agent = next(a for a in AGENTS if a["id"] == agent_id) + system = f"""Je bent {agent['name']}, een autonomous ops agent in het Dell ATC data lab. +Specialisatie: {agent['role']}. +Motto: {agent.get('motto', '')} + +Je antwoordt namens je domein maar hebt zicht op de HELE lab stack: Docker, databases, lakehouse (Trino/Spark/Kafka Connect), ETL (Airflow/Kafka), Hadoop HDFS, en GPU/vLLM. + +Regels: +- Antwoord in dezelfde taal als de gebruiker (Nederlands of Engels). +- Gebruik ALLEEN de live data hieronder — verzin geen hosts, poorten, cijfers of connector namen. +- Gebruik exact de container/connector namen uit de data (bijv. mysql-hr-connector, niet "Debezium"). +- Als iets DOWN of 0 GB is, zeg dat eerlijk. +- Kort en behulpzaam (max ~10 zinnen); bullet lists mogen als het overzicht helpt. + +--- LIVE LAB DATA (primary domain eerst, daarna volledige stack) --- +{context} +""" + if log: + await log("info", "llm", f"▸ Querying vLLM model={LLM_MODEL}") + await log("cmd", "llm", f"$ POST {LLM_URL.rstrip('/')}/chat/completions") + await log("info", "llm", f" user: {message[:160]}{'…' if len(message) > 160 else ''}") + try: + async with httpx.AsyncClient(timeout=LLM_TIMEOUT) as client: + t0 = time.monotonic() + r = await client.post( + f"{LLM_URL.rstrip('/')}/chat/completions", + headers={ + "Authorization": f"Bearer {LLM_API_KEY}", + "Content-Type": "application/json", + }, + json={ + "model": LLM_MODEL, + "messages": [ + {"role": "system", "content": system}, + {"role": "user", "content": message}, + ], + "max_tokens": 800, + "temperature": 0.25, + }, + ) + r.raise_for_status() + content = r.json()["choices"][0]["message"]["content"].strip() + ms = int((time.monotonic() - t0) * 1000) + if content and content.strip("!"): + if log: + await log("ok", "llm", f"← vLLM response {len(content)} chars ({ms}ms)") + preview = content.replace("\n", " ")[:180] + await log("info", "llm", f" » {preview}{'…' if len(content) > 180 else ''}") + return content + if log: + await log("warn", "llm", f"← Empty or invalid LLM output ({ms}ms)") + except Exception as exc: + if log: + await log("err", "llm", f"✗ vLLM error: {exc}") + return None + + +def fallback_answer(agent_id: str, context: str) -> str: + agent_name = next(a["name"] for a in AGENTS if a["id"] == agent_id) + return f"**{agent_name}** (offline LLM — ruwe data):\n\n{context}" + + +async def publish_event(event: dict[str, Any]) -> None: + payload = json.dumps(event, default=str) + if redis_client: + await redis_client.publish("ops", payload) + dead = [] + for ws in ws_clients: + try: + await ws.send_text(payload) + except Exception: + dead.append(ws) + for ws in dead: + ws_clients.discard(ws) + + +def add_feed(agent_id: str, message: str, level: str = "info") -> dict: + entry_id = str(uuid.uuid4())[:8] + with SessionLocal() as db: + row = FeedEntry(id=entry_id, agent_id=agent_id, message=message, level=level) + db.add(row) + db.commit() + return { + "id": entry_id, + "ts": datetime.now(timezone.utc).isoformat(), + "agent_id": agent_id, + "message": message, + "level": level, + } + + +async def dockhand_env_containers(env_id: int) -> list[dict]: + try: + async with httpx.AsyncClient(timeout=8.0) as client: + r = await client.get(f"{DOCKHAND_URL}/api/containers", params={"env": env_id}) + r.raise_for_status() + return r.json() + except Exception: + return [] + + +async def probe_url(url: str) -> bool: + try: + async with httpx.AsyncClient(timeout=4.0, verify=False) as client: + r = await client.get(url) + return r.status_code < 500 + except Exception: + return False + + +async def collect_gpu() -> dict[str, Any]: + host = GPU_URL.replace("http://", "").replace("https://", "").split("/")[0] + base = {"ok": False, "host": host, "ui_url": GPU_UI_URL} + + try: + async with httpx.AsyncClient(timeout=6.0) as client: + metrics_r, model_r, integration_r = await asyncio.gather( + client.get(f"{GPU_URL}/api/gpu/metrics"), + client.get(f"{GPU_URL}/api/active-model"), + client.get(f"{GPU_URL}/api/integration"), + return_exceptions=True, + ) + + gpus: list[dict[str, Any]] = [] + if isinstance(metrics_r, httpx.Response) and metrics_r.status_code == 200: + current = metrics_r.json().get("current", {}) + gpus = [ + { + "index": g["index"], + "name": g["name"], + "util_gpu": g.get("util_gpu", 0), + "memory_used_mib": g.get("memory_used_mib", 0), + "memory_total_mib": g.get("memory_total_mib", 0), + "temperature_c": g.get("temperature_c", 0), + "power_w": g.get("power_w", 0), + } + for g in current.get("gpus", []) + ] + + active_model = None + inference_active = False + vllm_url = None + + if isinstance(model_r, httpx.Response) and model_r.status_code == 200: + model_data = model_r.json() + active_model = model_data.get("name") + inference_active = bool(model_data.get("inference_active")) + vllm_url = model_data.get("base_url") + + if isinstance(integration_r, httpx.Response) and integration_r.status_code == 200: + integ = integration_r.json() + if not active_model: + active_model = integ.get("active_name") + if not inference_active: + inference_active = bool(integ.get("inference_active")) + if not vllm_url: + vllm_url = integ.get("recommended_base_url") + + return { + **base, + "ok": len(gpus) > 0 or inference_active, + "inference_active": inference_active, + "active_model": active_model, + "vllm_url": vllm_url, + "gpu_count": len(gpus), + "gpus": gpus, + } + except Exception as exc: + return {**base, "error": str(exc)} + + +async def collect_status() -> dict[str, Any]: + db_containers = await dockhand_env_containers(5) + db_running = sum(1 for c in db_containers if c.get("state") == "running") + db_total = len(db_containers) or 6 + + lake_containers = await dockhand_env_containers(9) + lake_running = sum(1 for c in lake_containers if c.get("state") == "running") + lake_total = len(lake_containers) or 6 + + docker_containers = await dockhand_env_containers(1) + docker_running = sum(1 for c in docker_containers if c.get("state") == "running") + + hdfs_ok = await probe_url("http://10.0.21.61:9870") + kafka_ok = await probe_url("http://10.0.21.36:9000") + airflow_ok = await probe_url("http://10.0.21.55:8080") + + def level(running: int, total: int) -> str: + if total == 0: + return "unknown" + ratio = running / total + if ratio >= 0.9: + return "ok" + if ratio >= 0.5: + return "warn" + return "down" + + gpu = await collect_gpu() + gpu_level = "ok" if gpu.get("ok") and gpu.get("inference_active") else ("warn" if gpu.get("ok") else "down") + gpu_label = gpu.get("active_model") or (f"{gpu.get('gpu_count', 0)} GPUs" if gpu.get("ok") else "offline") + + return { + "ts": datetime.now(timezone.utc).isoformat(), + "domains": { + "docker": {"level": "ok" if docker_running >= 5 else "warn", "label": f"{docker_running} containers", "running": docker_running}, + "databases": {"level": level(db_running, db_total), "label": f"{db_running}/{db_total} up", "running": db_running, "total": db_total}, + "lakehouse": {"level": level(lake_running, lake_total), "label": f"{lake_running}/{lake_total} up", "running": lake_running, "total": lake_total}, + "hadoop": {"level": "ok" if hdfs_ok else "warn", "label": "NN up" if hdfs_ok else "NN check"}, + "etl": {"level": "ok" if kafka_ok and airflow_ok else "warn", "label": "Kafka+Airflow"}, + "gpu": {"level": gpu_level, "label": gpu_label}, + }, + "gpu": gpu, + "kafka_ok": kafka_ok, + "airflow_ok": airflow_ok, + "hdfs_ok": hdfs_ok, + } + + +async def run_agent_task(agent_id: str, message: str, prompt_id: str) -> str: + zone = next(a["zone"] for a in AGENTS if a["id"] == agent_id) + agent_name = next(a["name"] for a in AGENTS if a["id"] == agent_id) + log = make_logger(agent_id, prompt_id) + + await terminal_log( + agent_id, + f"[{prompt_id}] ▶ Mission accepted: {message}", + level="info", + phase="dispatch", + prompt_id=prompt_id, + ) + await publish_event({"type": "agent_dispatch", "agent_id": agent_id, "zone": zone, "prompt_id": prompt_id}) + await asyncio.sleep(0.4) + await terminal_log(agent_id, f"[{prompt_id}] Walking to zone: {zone}", level="info", phase="dispatch", prompt_id=prompt_id) + await publish_event({"type": "agent_fetch", "agent_id": agent_id, "zone": zone, "prompt_id": prompt_id}) + + await log("info", "fetch", f"[{prompt_id}] Collecting live lab metrics…") + status = await collect_status() + context = await gather_agent_context(agent_id, status, log=log) + + answer = await ask_llm(agent_id, message, context, log=log) + if not answer: + await log("warn", "llm", "LLM fallback — returning raw context") + answer = fallback_answer(agent_id, context) + + await asyncio.sleep(0.3) + await terminal_log(agent_id, f"[{prompt_id}] ✓ Mission complete", level="ok", phase="done", prompt_id=prompt_id) + await publish_event({"type": "agent_return", "agent_id": agent_id, "zone": zone, "prompt_id": prompt_id}) + feed = add_feed(agent_id, f"{agent_name} answered: {message[:60]}", "info") + await publish_event({"type": "feed", "entry": feed}) + await publish_event({"type": "prompt_result", "prompt_id": prompt_id, "agent_id": agent_id, "answer": answer}) + return answer + + +async def heartbeat_loop() -> None: + while True: + try: + status = await collect_status() + workload = await collect_workload() + await publish_event({"type": "status", "data": status}) + await publish_event({"type": "workload", "data": workload}) + for domain, info in status["domains"].items(): + if info["level"] == "down": + agent = "data-custodian" if domain == "databases" else "infra-sentinel" + feed = add_feed(agent, f"Alert: {domain} is DOWN ({info['label']})", "warn") + await publish_event({"type": "feed", "entry": feed}) + except Exception as exc: + await publish_event({"type": "error", "message": str(exc)}) + await asyncio.sleep(60) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + global redis_client + redis_client = aioredis.from_url(REDIS_URL, decode_responses=True) + set_terminal_publisher(publish_event) + init_terminals([a["id"] for a in AGENTS]) + for a in AGENTS: + await terminal_log(a["id"], f"{a['name']} terminal online — awaiting missions", level="info", phase="boot") + task = asyncio.create_task(heartbeat_loop()) + add_feed("infra-sentinel", "ATC Command Center API online", "info") + yield + task.cancel() + if redis_client: + await redis_client.close() + + +app = FastAPI(title="ATC Command Center API", lifespan=lifespan) +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + + +@app.get("/api/health") +async def health(): + llm_ok = False + try: + async with httpx.AsyncClient(timeout=4.0) as client: + r = await client.get(f"{LLM_URL.rstrip('/')}/models", headers={"Authorization": f"Bearer {LLM_API_KEY}"}) + llm_ok = r.status_code == 200 + except Exception: + pass + return {"ok": True, "ts": datetime.now(timezone.utc).isoformat(), "llm_url": LLM_URL, "llm_ok": llm_ok, "llm_model": LLM_MODEL} + + +async def collect_workload() -> dict[str, Any]: + gpu = await collect_gpu() + snap = await collect_full_lab_context(gpu_data=gpu) + return build_workload_payload(snap) + + +@app.get("/api/workload") +async def get_workload(): + return await collect_workload() + + +@app.get("/api/status") +async def get_status(): + return await collect_status() + + +@app.get("/api/gpu") +async def get_gpu(): + return await collect_gpu() + + +def agent_stats() -> dict[str, dict[str, Any]]: + stats: dict[str, dict[str, Any]] = {a["id"]: {"tasks": 0, "last_active": None, "alerts": 0} for a in AGENTS} + with SessionLocal() as db: + rows = db.execute(select(FeedEntry).order_by(FeedEntry.ts.desc()).limit(200)).scalars().all() + for r in rows: + aid = r.agent_id + if aid not in stats: + continue + stats[aid]["tasks"] += 1 + if r.level == "warn": + stats[aid]["alerts"] += 1 + if stats[aid]["last_active"] is None and r.ts: + stats[aid]["last_active"] = r.ts.isoformat() + return stats + + +@app.get("/api/agents") +async def get_agents(): + stats = agent_stats() + enriched = [{**a, "stats": stats.get(a["id"], {})} for a in AGENTS] + return {"agents": enriched, "zones": ZONES} + + +@app.get("/api/terminals") +async def get_terminals(limit: int = 200): + return {"terminals": get_all_terminals(limit)} + + +@app.get("/api/terminals/{agent_id}") +async def get_agent_terminal(agent_id: str, limit: int = 200): + valid = {a["id"] for a in AGENTS} + if agent_id not in valid: + return {"error": "unknown agent"} + return {"agent_id": agent_id, "lines": get_terminal_lines(agent_id, limit)} + + +@app.get("/api/feed") +async def get_feed(limit: int = 50): + with SessionLocal() as db: + rows = db.execute(select(FeedEntry).order_by(FeedEntry.ts.desc()).limit(limit)).scalars().all() + return { + "entries": [ + { + "id": r.id, + "ts": r.ts.isoformat() if r.ts else None, + "agent_id": r.agent_id, + "message": r.message, + "level": r.level, + } + for r in rows + ] + } + + +@app.get("/api/approvals") +async def get_approvals(): + with SessionLocal() as db: + rows = db.execute(select(Approval).where(Approval.status == "pending")).scalars().all() + return { + "approvals": [ + { + "id": r.id, + "ts": r.ts.isoformat() if r.ts else None, + "agent_id": r.agent_id, + "action": r.action, + "reason": r.reason, + "status": r.status, + } + for r in rows + ] + } + + +@app.post("/api/approvals/{approval_id}/decide") +async def decide_approval(approval_id: str, body: ApprovalDecision): + with SessionLocal() as db: + row = db.get(Approval, approval_id) + if not row: + return {"error": "not found"} + row.status = "approved" if body.approved else "denied" + db.commit() + agent_id = row.agent_id + action = row.action + msg = f"Approval {'approved' if body.approved else 'denied'}: {action}" + feed = add_feed(agent_id, msg, "info" if body.approved else "warn") + await publish_event({"type": "feed", "entry": feed}) + await publish_event({"type": "approval_update", "id": approval_id, "status": row.status}) + return {"ok": True, "status": row.status} + + +@app.post("/api/prompt") +async def post_prompt(body: PromptRequest): + prompt_id = str(uuid.uuid4())[:8] + valid_ids = {a["id"] for a in AGENTS} + agent_id = body.agent_id if body.agent_id in valid_ids else route_agent(body.message) + add_feed(agent_id, f"Prompt received: {body.message}", "info") + asyncio.create_task(run_agent_task(agent_id, body.message, prompt_id)) + return {"prompt_id": prompt_id, "agent_id": agent_id, "status": "dispatched"} + + +@app.websocket("/api/ws/ops") +async def ws_ops(websocket: WebSocket): + await websocket.accept() + ws_clients.add(websocket) + try: + status = await collect_status() + workload = await collect_workload() + await websocket.send_text(json.dumps({"type": "status", "data": status}, default=str)) + await websocket.send_text(json.dumps({"type": "workload", "data": workload}, default=str)) + await websocket.send_text(json.dumps({ + "type": "terminal_history", + "terminals": get_all_terminals(150), + }, default=str)) + while True: + await websocket.receive_text() + except WebSocketDisconnect: + pass + finally: + ws_clients.discard(websocket) diff --git a/nginx.conf b/nginx.conf new file mode 100644 index 0000000..3d24bfe --- /dev/null +++ b/nginx.conf @@ -0,0 +1,17 @@ +server { + listen 80; + root /usr/share/nginx/html; + index index.html; + + location /api/ { + proxy_pass http://api:3201/api/; + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host $host; + } + + location / { + try_files $uri $uri/ /index.html; + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..0aa1dcf --- /dev/null +++ b/package.json @@ -0,0 +1,26 @@ +{ + "name": "atc-command-center", + "private": true, + "version": "0.1.0", + "type": "module", + "scripts": { + "dev": "vite", + "build": "tsc -b && vite build", + "preview": "vite preview" + }, + "dependencies": { + "framer-motion": "^11.15.0", + "react": "^18.3.1", + "react-dom": "^18.3.1" + }, + "devDependencies": { + "@types/react": "^18.3.12", + "@types/react-dom": "^18.3.1", + "@vitejs/plugin-react": "^4.3.4", + "autoprefixer": "^10.4.20", + "postcss": "^8.4.49", + "tailwindcss": "^3.4.16", + "typescript": "^5.7.2", + "vite": "^6.0.3" + } +} diff --git a/postcss.config.js b/postcss.config.js new file mode 100644 index 0000000..2e7af2b --- /dev/null +++ b/postcss.config.js @@ -0,0 +1,6 @@ +export default { + plugins: { + tailwindcss: {}, + autoprefixer: {}, + }, +} diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..c506982 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,9 @@ +fastapi==0.115.6 +uvicorn[standard]==0.34.0 +redis==5.2.1 +httpx==0.28.1 +sqlalchemy==2.0.36 +aiosqlite==0.20.0 +pydantic==2.10.4 +python-multipart==0.0.20 +websockets==14.1 diff --git a/scripts/deploy.sh b/scripts/deploy.sh new file mode 100755 index 0000000..a75b0f2 --- /dev/null +++ b/scripts/deploy.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# Deploy Command Center stack on VM304 +set -euo pipefail +cd "$(dirname "$0")/.." +[ -f .env ] || { echo "Create .env from config/command-center/.env.example"; exit 1; } +docker compose --env-file .env up -d --build +echo "OK — http://10.0.21.33/" diff --git a/scripts/sync-gitea.sh b/scripts/sync-gitea.sh new file mode 100755 index 0000000..e21109c --- /dev/null +++ b/scripts/sync-gitea.sh @@ -0,0 +1,80 @@ +#!/bin/bash +# Sync Command Center + Data Quality to Gitea (run on VM304 as root) +set -euo pipefail + +GITEA="http://atc-mgt01.dell-atc.lan:3001" +AUTH="mo:Dell2026!" + +echo "==> Ensure atc-data-quality repo on Gitea" +code=$(curl -s -o /dev/null -w "%{http_code}" -u "$AUTH" "$GITEA/api/v1/repos/mo/atc-data-quality") +if [ "$code" = "404" ]; then + curl -s -u "$AUTH" -H "Content-Type: application/json" \ + -d '{"name":"atc-data-quality","description":"ATC Data Quality + RAG APIs (Docling, ChromaDB, LangChain)","private":false,"auto_init":false}' \ + "$GITEA/api/v1/user/repos" + echo "Created mo/atc-data-quality" +fi + +echo "==> Init/push atc-data-quality" +cd /opt/atc-data-quality +if [ ! -d .git ]; then + git init + git config user.email "mo@dell-atc.lan" + git config user.name "mo" + git remote add origin "$GITEA/mo/atc-data-quality.git" +fi +cat > .gitignore <<'GI' +__pycache__/ +*.pyc +.env +GI +cat > README.md <<'MD' +# ATC Data Quality + RAG + +| Service | Port (internal) | Route | +|---------|-----------------|-------| +| dq-api | 5010 | `/dq/*` | +| rag-api | 5020 | `/rag/*` | + +Deploy with sibling repo `mo/atc-agents` — see `config/data-quality/README.md` there. +MD +git add -A +git commit -m "Add DQ + RAG APIs with Docling, ChromaDB, persistent ingest" || true +git push -u origin main 2>/dev/null || git push -u origin master 2>/dev/null || \ + git branch -M main && git push -u origin main --force + +echo "==> Structure + push atc-agents" +cd /opt/atc-agents +mkdir -p config/command-center config/data-quality config/jupyter docs scripts +cp docker-compose.yml config/command-center/ +cp caddy/Caddyfile config/command-center/ +cp config/command-center/.env.example config/command-center/.env.example 2>/dev/null || true + +cat > config/data-quality/README.md <<'MD' +# Data Quality services + +Built from **`mo/atc-data-quality`** — clone to `/opt/atc-data-quality`. + +Docker compose build contexts: +- `../atc-data-quality/dq-api` +- `../atc-data-quality/rag-api` +MD + +cat > config/jupyter/README.md <<'MD' +# JupyterLab + +Included in root `docker-compose.yml` as service `jupyter`. +Access: http://10.0.21.33/jupyter/ +Token: JUPYTER_TOKEN in `.env` +MD + +cp /tmp/deploy.sh scripts/ 2>/dev/null || cp scripts/deploy.sh scripts/ 2>/dev/null || true +chmod +x scripts/*.sh 2>/dev/null || true + +git add -A +git status -sb +git commit -m "Command Center v2: DQ, RAG, GPU matrix, S3 browser, Jupyter, Gitea config layout" || true +git push origin main + +echo "==> Done. Repos:" +echo " $GITEA/mo/atc-agents" +echo " $GITEA/mo/atc-data-quality" diff --git a/src/App.tsx b/src/App.tsx new file mode 100644 index 0000000..c544812 --- /dev/null +++ b/src/App.tsx @@ -0,0 +1,296 @@ +import { useCallback, useEffect, useMemo, useState } from 'react' +import { ActivityFeed } from './components/ActivityFeed' +import { AgentRoster } from './components/AgentRoster' +import { AgentTerminalGrid } from './components/AgentTerminalGrid' +import { ChatPanel } from './components/ChatPanel' +import { CommandDock } from './components/CommandDock' +import { GpuPanel } from './components/GpuPanel' +import { AmbientBackground } from './components/AmbientBackground' +import { LiveClusterMap } from './components/LiveClusterMap' +import { LiveDomainGrid } from './components/LiveDomainGrid' +import { ThemeToggle } from './components/ThemeToggle' +import type { Agent, AgentAnim, Approval, ChatMessage, FeedEntry, GpuStatus, StatusData, TerminalLine, WorkloadData } from './types' + +const TABS = ['Overview', 'Terminals', 'Activity', 'Approvals', 'GPU'] as const +type Tab = (typeof TABS)[number] + +function wsUrl() { + const proto = window.location.protocol === 'https:' ? 'wss' : 'ws' + return `${proto}://${window.location.host}/api/ws/ops` +} + +function LiveClock() { + const [now, setNow] = useState(new Date()) + useEffect(() => { + const t = setInterval(() => setNow(new Date()), 1000) + return () => clearInterval(t) + }, []) + return ( + + {now.toLocaleTimeString()} + + ) +} + +export default function App() { + const [tab, setTab] = useState('Overview') + const [agents, setAgents] = useState([]) + const [status, setStatus] = useState(null) + const [workload, setWorkload] = useState(null) + const [gpu, setGpu] = useState(null) + const [feed, setFeed] = useState([]) + const [approvals, setApprovals] = useState([]) + const [chat, setChat] = useState([]) + const [anims, setAnims] = useState>({}) + const [selectedId, setSelectedId] = useState(null) + const [busy, setBusy] = useState(false) + const [terminals, setTerminals] = useState>({}) + const [terminalLayout, setTerminalLayout] = useState<'grid' | 'focus'>('grid') + + const appendTerminal = useCallback((line: TerminalLine) => { + setTerminals((prev) => { + const cur = prev[line.agent_id] || [] + return { ...prev, [line.agent_id]: [...cur, line].slice(-300) } + }) + }, []) + + const selectedAgent = useMemo( + () => agents.find((a) => a.id === selectedId) || null, + [agents, selectedId], + ) + + const load = useCallback(async () => { + const [a, s, f, ap, g, t, w] = await Promise.all([ + fetch('/api/agents').then((r) => r.json()), + fetch('/api/status').then((r) => r.json()), + fetch('/api/feed').then((r) => r.json()), + fetch('/api/approvals').then((r) => r.json()), + fetch('/api/gpu').then((r) => r.json()).catch(() => null), + fetch('/api/terminals').then((r) => r.json()).catch(() => ({ terminals: {} })), + fetch('/api/workload').then((r) => r.json()).catch(() => null), + ]) + setAgents(a.agents || []) + setStatus(s) + setGpu(g || s.gpu || null) + setFeed(f.entries || []) + setApprovals(ap.approvals || []) + if (t.terminals) setTerminals(t.terminals) + if (w?.zones) setWorkload(w) + }, []) + + useEffect(() => { + load() + const ws = new WebSocket(wsUrl()) + ws.onmessage = (ev) => { + const msg = JSON.parse(ev.data) + if (msg.type === 'status') { + setStatus(msg.data) + if (msg.data.gpu) setGpu(msg.data.gpu) + } + if (msg.type === 'workload') setWorkload(msg.data) + if (msg.type === 'terminal') appendTerminal(msg.line) + if (msg.type === 'terminal_history' && msg.terminals) setTerminals(msg.terminals) + if (msg.type === 'feed') setFeed((prev) => [msg.entry, ...prev].slice(0, 100)) + if (msg.type === 'agent_dispatch') { + setSelectedId(msg.agent_id) + setAnims((p) => ({ ...p, [msg.agent_id]: { agentId: msg.agent_id, state: 'walk', zone: msg.zone } })) + } + if (msg.type === 'agent_fetch') { + setAnims((p) => ({ ...p, [msg.agent_id]: { agentId: msg.agent_id, state: 'fetch', zone: msg.zone } })) + } + if (msg.type === 'agent_return') { + setAnims((p) => ({ ...p, [msg.agent_id]: { agentId: msg.agent_id, state: 'return', zone: msg.zone } })) + setTimeout(() => { + setAnims((p) => ({ ...p, [msg.agent_id]: { agentId: msg.agent_id, state: 'idle' } })) + }, 1200) + } + if (msg.type === 'prompt_result') { + setChat((c) => [...c, { role: 'agent', text: msg.answer, agent: msg.agent_id, ts: new Date().toISOString() }]) + setBusy(false) + load() + } + } + const iv = setInterval(load, 15000) + return () => { ws.close(); clearInterval(iv) } + }, [load, appendTerminal]) + + const sendPrompt = async (message: string, agentId?: string) => { + setBusy(true) + setChat((c) => [...c, { role: 'user', text: message, ts: new Date().toISOString() }]) + if (agentId) setSelectedId(agentId) + await fetch('/api/prompt', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ message, agent_id: agentId || undefined }), + }) + } + + const decide = async (id: string, approved: boolean) => { + await fetch(`/api/approvals/${id}/decide`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ approved }), + }) + load() + } + + const allOk = status && Object.values(status.domains).every((d) => d.level === 'ok') + const totalTasks = agents.reduce((n, a) => n + (a.stats?.tasks || 0), 0) + + return ( +
+ +
+
+
+
ATC
+
+

+ Command Center +

+

+ {agents.length} agents · {totalTasks} missions · autonomous ops +

+
+
+
+ + {status && ( + + {allOk ? 'All systems operational' : 'Attention required'} + + )} + {gpu?.ok && gpu.active_model && ( + + GPU · {gpu.active_model} + + )} + +
+
+ + setSelectedId(a.id)} + /> + +
+
+
+

Live agent shells

+

Agent Terminals

+

Volg live hoe elke agent data ophaalt — HTTP probes, Dockhand, JMX, vLLM.

+
+
+ + +
+
+ +
+ +
+
+ + + + +
+ {tab === 'Overview' && ( +
+

Infrastructure

+

Live Cluster Workload

+ +
+ )} + {tab === 'Terminals' && ( +
+

Mission trace

+

+ {selectedAgent ? `${selectedAgent.name} — full terminal` : 'Select an agent'} +

+ {selectedAgent ? ( + + ) : ( +

Klik een agent in de roster of stuur een prompt.

+ )} +
+ )} + {tab === 'Activity' && ( +
+
+
+

Event stream

+

Agent Activity

+
+ {selectedAgent && ( + + )} +
+ +
+ )} + {tab === 'Approvals' && ( +
+ {approvals.length === 0 &&

Geen pending approvals.

} + {approvals.map((a) => { + const agent = agents.find((ag) => ag.id === a.agent_id) + return ( +
+ {agent && {agent.icon}} +
+
{a.action}
+
{a.reason}
+
via {agent?.name || a.agent_id}
+
+ + +
+
+
+ ) + })} +
+ )} + {tab === 'GPU' && } +
+
+ +
+ + +
+
+ + +
+ ) +} diff --git a/src/components/ActivityFeed.tsx b/src/components/ActivityFeed.tsx new file mode 100644 index 0000000..9b89a6c --- /dev/null +++ b/src/components/ActivityFeed.tsx @@ -0,0 +1,55 @@ +import { motion } from 'framer-motion' +import type { Agent, FeedEntry } from '../types' + +type Props = { + feed: FeedEntry[] + agents: Agent[] + filterAgentId?: string | null +} + +export function ActivityFeed({ feed, agents, filterAgentId }: Props) { + const entries = filterAgentId ? feed.filter((e) => e.agent_id === filterAgentId) : feed + + if (entries.length === 0) { + return ( +
+ 📡 +

Geen activiteit{filterAgentId ? ' voor deze agent' : ''}.

+
+ ) + } + + return ( +
+ {entries.map((e, i) => { + const agent = agents.find((a) => a.id === e.agent_id) + return ( + +
+
+ {i < entries.length - 1 &&
} +
+
+
+ + {e.ts ? new Date(e.ts).toLocaleString() : ''} + + + {agent?.icon} {agent?.name || e.agent_id} + + {e.level === 'warn' && WARN} +
+

{e.message}

+
+ + ) + })} +
+ ) +} diff --git a/src/components/AgentCard.tsx b/src/components/AgentCard.tsx new file mode 100644 index 0000000..9ad4dbe --- /dev/null +++ b/src/components/AgentCard.tsx @@ -0,0 +1,79 @@ +import { motion } from 'framer-motion' +import type { CSSProperties } from 'react' +import type { Agent, AgentAnim } from '../types' + +const STATE_LABEL: Record = { + idle: 'Standby', + walk: 'En route', + fetch: 'Fetching data', + return: 'Returning', +} + +type Props = { + agent: Agent + anim: AgentAnim + selected: boolean + onSelect: () => void + onDelegate: () => void +} + +export function AgentCard({ agent, anim, selected, onSelect, onDelegate }: Props) { + const busy = anim.state !== 'idle' + const tasks = agent.stats?.tasks ?? 0 + const alerts = agent.stats?.alerts ?? 0 + + return ( + +
+
+
+ {agent.icon || '🤖'} + +
+
+ + {STATE_LABEL[anim.state]} + + {alerts > 0 && ( + + {alerts} alert{alerts > 1 ? 's' : ''} + + )} +
+
+ +
+

{agent.name}

+

"{agent.motto || agent.role}"

+
+ +
+ {(agent.capabilities || []).slice(0, 4).map((cap) => ( + {cap} + ))} +
+ +
+ {tasks} tasks logged + { e.stopPropagation(); onDelegate() }} + onKeyDown={(e) => { if (e.key === 'Enter') { e.stopPropagation(); onDelegate() } }} + className="delegate-btn text-[10px] font-mono font-semibold px-2 py-1 rounded-lg" + style={{ color: agent.color }} + > + Delegate → + +
+
+ + ) +} diff --git a/src/components/AgentRoster.tsx b/src/components/AgentRoster.tsx new file mode 100644 index 0000000..f4d7379 --- /dev/null +++ b/src/components/AgentRoster.tsx @@ -0,0 +1,53 @@ +import { motion } from 'framer-motion' +import type { Agent, AgentAnim } from '../types' +import { AgentCard } from './AgentCard' + +type Props = { + agents: Agent[] + animations: Record + selectedId: string | null + onSelect: (id: string) => void + onDelegate: (agent: Agent) => void +} + +export function AgentRoster({ agents, animations, selectedId, onSelect, onDelegate }: Props) { + return ( +
+
+
+
+

Autonomous workforce

+

+ Agent Roster +

+

+ Selecteer een agent om te delegeren. Elk teamlid bewaakt een zone op de ops floor. +

+
+
+ + {agents.length} agents online +
+
+ +
+ {agents.map((agent, i) => ( + + onSelect(agent.id)} + onDelegate={() => onDelegate(agent)} + /> + + ))} +
+
+ ) +} diff --git a/src/components/AgentSprite.tsx b/src/components/AgentSprite.tsx new file mode 100644 index 0000000..f3dad1e --- /dev/null +++ b/src/components/AgentSprite.tsx @@ -0,0 +1,216 @@ +import { motion } from 'framer-motion' + +type Props = { + agentId: string + color: string + icon?: string + state: 'idle' | 'walk' | 'fetch' | 'return' + label: string +} + +function uid(agentId: string, name: string) { + return `${agentId}-${name}` +} + +function CharacterBody({ agentId, color, state }: { agentId: string; color: string; state: Props['state'] }) { + const g = uid(agentId, 'bodyGrad') + const glow = uid(agentId, 'glow') + const visor = uid(agentId, 'visor') + const walking = state === 'walk' || state === 'return' + const fetching = state === 'fetch' + + return ( + + + + + + + + + + + + + + + + + + + + + {/* Platform + aura */} + + + + {agentId === 'etl-guardian' && ( + + + + + + + + + + + + + + + + + + ⚡ + + )} + + {agentId === 'lakehouse-ops' && ( + + + + + + + + + + + + + + + + + 🏔 + + )} + + {agentId === 'data-custodian' && ( + + + + + + + + + + + + + + + + 🛡 + + )} + + {agentId === 'hadoop-ranger' && ( + + + + + + + + + + + + + + + + 🌲 + {fetching && ( + + )} + + )} + + {agentId === 'infra-sentinel' && ( + + + + + + + + + + + + + + + + + 👁 + + )} + + {/* Default fallback */} + {!['etl-guardian', 'lakehouse-ops', 'data-custodian', 'hadoop-ranger', 'infra-sentinel'].includes(agentId) && ( + + + + 🤖 + + )} + + {fetching && ( + + + + + )} + + ) +} + +export function AgentSprite({ agentId, color, state, label }: Props) { + const bob = state === 'idle' ? { y: [0, -5, 0] } : state === 'walk' || state === 'return' ? { y: [0, -9, 0] } : { y: [0, -2, 0] } + const scale = state === 'fetch' ? 0.94 : 1 + const busy = state !== 'idle' + + return ( + +
+ {state !== 'idle' && ( + + )} +
+ +
+
+ + {label} +
+ + ) +} diff --git a/src/components/AgentTerminal.tsx b/src/components/AgentTerminal.tsx new file mode 100644 index 0000000..c37af1f --- /dev/null +++ b/src/components/AgentTerminal.tsx @@ -0,0 +1,74 @@ +import { useEffect, useRef } from 'react' +import type { CSSProperties } from 'react' +import type { Agent, TerminalLine } from '../types' + +const LEVEL_CLASS: Record = { + info: 'term-info', + ok: 'term-ok', + warn: 'term-warn', + err: 'term-err', + cmd: 'term-cmd', + llm: 'term-llm', +} + +type Props = { + agent: Agent + lines: TerminalLine[] + active: boolean + expanded?: boolean + onFocus?: () => void +} + +export function AgentTerminal({ agent, lines, active, expanded, onFocus }: Props) { + const bottomRef = useRef(null) + const containerRef = useRef(null) + + useEffect(() => { + if (active || expanded) { + bottomRef.current?.scrollIntoView({ behavior: 'smooth' }) + } + }, [lines, active, expanded]) + + return ( +
e.key === 'Enter' && onFocus?.()} + > +
+
+ {agent.icon} + {agent.name} + {active && LIVE} +
+ {lines.length} lines +
+ +
+ {lines.length === 0 && ( +
+ --:--:-- + Waiting for missions… +
+ )} + {lines.map((line) => ( +
+ {line.ts ? new Date(line.ts).toLocaleTimeString() : ''} + {line.phase} + {line.text} +
+ ))} + {active && ( +
+ + █ +
+ )} +
+
+
+ ) +} diff --git a/src/components/AgentTerminalGrid.tsx b/src/components/AgentTerminalGrid.tsx new file mode 100644 index 0000000..6b68ab7 --- /dev/null +++ b/src/components/AgentTerminalGrid.tsx @@ -0,0 +1,60 @@ +import type { Agent, AgentAnim, TerminalLine } from '../types' +import { AgentTerminal } from './AgentTerminal' + +type Props = { + agents: Agent[] + terminals: Record + animations: Record + selectedId: string | null + onSelect: (id: string) => void + layout?: 'grid' | 'focus' +} + +export function AgentTerminalGrid({ agents, terminals, animations, selectedId, onSelect, layout = 'grid' }: Props) { + const focusId = selectedId || agents[0]?.id + + if (layout === 'focus' && focusId) { + const agent = agents.find((a) => a.id === focusId)! + const anim = animations[focusId] || { agentId: focusId, state: 'idle' as const } + return ( +
+ +
+ {agents.map((a) => ( + + ))} +
+
+ ) + } + + return ( +
+ {agents.map((agent) => { + const anim = animations[agent.id] || { agentId: agent.id, state: 'idle' as const } + return ( + onSelect(agent.id)} + /> + ) + })} +
+ ) +} diff --git a/src/components/AmbientBackground.tsx b/src/components/AmbientBackground.tsx new file mode 100644 index 0000000..5fa8ed7 --- /dev/null +++ b/src/components/AmbientBackground.tsx @@ -0,0 +1,18 @@ +export function AmbientBackground() { + return ( +
+ {[...Array(12)].map((_, i) => ( + + ))} +
+ ) +} diff --git a/src/components/ChatPanel.tsx b/src/components/ChatPanel.tsx new file mode 100644 index 0000000..37edbbf --- /dev/null +++ b/src/components/ChatPanel.tsx @@ -0,0 +1,89 @@ +import { useEffect, useRef } from 'react' +import { motion } from 'framer-motion' +import type { Agent, ChatMessage } from '../types' + +type Props = { + messages: ChatMessage[] + agents: Agent[] + selectedAgent: Agent | null + busy: boolean +} + +function agentFor(agents: Agent[], id?: string) { + return agents.find((a) => a.id === id) +} + +export function ChatPanel({ messages, agents, selectedAgent, busy }: Props) { + const bottomRef = useRef(null) + + useEffect(() => { + bottomRef.current?.scrollIntoView({ behavior: 'smooth' }) + }, [messages, busy]) + + return ( +
+
+
+

Mission control

+

Agent Comms

+
+ {selectedAgent && ( +
+ {selectedAgent.icon} + → {selectedAgent.name} +
+ )} +
+ +
+ {messages.length === 0 && ( +
+ 💬 +

+ {selectedAgent + ? `Stuur een opdracht naar ${selectedAgent.name}. Kies een suggestie hieronder of typ je eigen vraag.` + : 'Selecteer een agent of stel een vraag — routing kiest automatisch de specialist.'} +

+
+ )} + + {messages.map((m, i) => { + const agent = m.role === 'agent' ? agentFor(agents, m.agent) : null + return ( + + {m.role === 'agent' && agent && ( +
+ {agent.icon} +
+ )} +
+
+ {m.role === 'user' ? 'You' : agent?.name || m.agent} + {m.ts && {new Date(m.ts).toLocaleTimeString()}} +
+
{m.text}
+
+ + ) + })} + + {busy && ( + +
⋯
+
+
+ +
+
+ + )} +
+
+
+ ) +} diff --git a/src/components/ClusterViz.tsx b/src/components/ClusterViz.tsx new file mode 100644 index 0000000..496a44f --- /dev/null +++ b/src/components/ClusterViz.tsx @@ -0,0 +1,167 @@ +import { motion } from 'framer-motion' +import { appIcon, shortName } from '../lib/appIcons' +import type { WorkloadZone } from '../types' + +const DESK_X = 50 + +type Props = { + zones: WorkloadZone[] + activeZoneId?: string | null +} + +export function DataFlowLayer({ zones, activeZoneId }: Props) { + return ( + + + {zones.map((z) => ( + + + + + + ))} + + {zones.map((z, i) => ( + + + {[0, 1, 2].map((p) => ( + + ))} + + ))} + + ) +} + +type ZoneProps = { + zone: WorkloadZone + active: boolean + agentBusy: boolean +} + +export function ZoneTower({ zone, active, agentBusy }: ZoneProps) { + const pulse = zone.level === 'ok' + const apps = zone.apps.slice(0, 5) + + return ( +
+ +
{zone.label}
+
+ + {zone.running}/{zone.total || zone.apps.length} +
+ {zone.id === 'hadoop' && zone.hdfs_total_gb != null && ( +
{zone.hdfs_used_gb ?? 0}/{zone.hdfs_total_gb} GB
+ )} + {zone.id === 'lakehouse' && ( +
{zone.trino_ok ? 'Trino ●' : 'Trino ○'}
+ )} + + +
+ {apps.map((app, i) => ( + + {appIcon(app.name, app.image)} + {shortName(app.name, 10)} + + ))} +
+
+ ) +} + +export function GpuBeacon({ model, util, count, level, active }: { + model?: string | null + util?: number + count?: number + level: string + active?: boolean +}) { + return ( + +
⚡ GPU LAB
+
{shortName(model || 'offline', 18)}
+
{count ?? 0}× V100 · {Math.round(util ?? 0)}%
+
+ +
+ + ) +} + +export function WorkloadTicker({ zones }: { zones: WorkloadZone[] }) { + const allApps = zones.flatMap((z) => + z.apps.map((a) => ({ ...a, zoneColor: z.color, zoneId: z.id })), + ) + + return ( +
+
LIVE WORKLOAD
+
+ + {[...allApps, ...allApps].map((app, i) => ( + + {appIcon(app.name, app.image)} {app.name} + + ))} + +
+
+ ) +} diff --git a/src/components/CommandDock.tsx b/src/components/CommandDock.tsx new file mode 100644 index 0000000..b32c001 --- /dev/null +++ b/src/components/CommandDock.tsx @@ -0,0 +1,69 @@ +import { FormEvent, useState } from 'react' +import { motion } from 'framer-motion' +import type { Agent } from '../types' + +type Props = { + onSubmit: (message: string, agentId?: string) => void + busy: boolean + selectedAgent: Agent | null +} + +export function CommandDock({ onSubmit, busy, selectedAgent }: Props) { + const [text, setText] = useState('') + const prompts = selectedAgent?.suggested_prompts || [ + 'Lab health overview?', + 'Hoe staat de GPU?', + 'Database status?', + ] + + const handle = (e: FormEvent) => { + e.preventDefault() + if (!text.trim() || busy) return + onSubmit(text.trim(), selectedAgent?.id) + setText('') + } + + const sendQuick = (prompt: string) => { + if (busy) return + onSubmit(prompt, selectedAgent?.id) + } + + return ( +
+
+ {prompts.map((p) => ( + sendQuick(p)} + className="quick-prompt-chip" + style={selectedAgent ? { borderColor: `${selectedAgent.color}55`, color: selectedAgent.color } : undefined} + > + {p} + + ))} +
+ +
+
+ {selectedAgent && ( + {selectedAgent.icon} + )} + setText(e.target.value)} + disabled={busy} + /> +
+ +
+
+ ) +} diff --git a/src/components/GpuPanel.tsx b/src/components/GpuPanel.tsx new file mode 100644 index 0000000..fabc241 --- /dev/null +++ b/src/components/GpuPanel.tsx @@ -0,0 +1,137 @@ +import type { GpuStatus } from '../types' + +type Props = { + gpu: GpuStatus | null + compact?: boolean +} + +function memPct(used: number, total: number) { + if (!total) return 0 + return Math.round((used / total) * 100) +} + +export function GpuPanel({ gpu, compact }: Props) { + if (!gpu) { + return ( +
+
+
+
+ ) + } + + const online = gpu.ok + const gpus = gpu.gpus || [] + const avgUtil = gpus.length ? gpus.reduce((s, g) => s + g.util_gpu, 0) / gpus.length : 0 + const avgVram = gpus.length ? gpus.reduce((s, g) => s + memPct(g.memory_used_mib, g.memory_total_mib), 0) / gpus.length : 0 + + if (compact) { + return ( +
+
+
+
+ ⚡ +

GPU Lab

+
+ + Open ↗ + +
+ {gpu.active_model && ( +
{gpu.active_model}
+ )} +
+ {gpu.gpu_count ?? gpus.length}× V100 + {Math.round(avgUtil)}% util + {Math.round(avgVram)}% VRAM + + {online ? (gpu.inference_active ? 'ON' : 'STBY') : 'OFF'} + +
+
+
+
+
+ ) + } + + return ( +
+
+ +
+
+

Inference cluster

+
+ ⚡ +

GPU Lab

+ + {online ? (gpu.inference_active ? 'INFERENCE ON' : 'STANDBY') : 'OFFLINE'} + +
+

+ atc-gpu-dev · {gpu.host} · {gpu.gpu_count ?? gpus.length}× V100 +

+
+ + Open GPU Manager ↗ + +
+ + {gpu.active_model && ( +
+
Active model
+
{gpu.active_model}
+ {gpu.vllm_url && ( +
{gpu.vllm_url}
+ )} +
+ )} + + {!online && ( +

{gpu.error || 'GPU manager unreachable'}

+ )} + +
+ {gpus.map((g) => { + const pct = memPct(g.memory_used_mib, g.memory_total_mib) + return ( +
+
+ GPU {g.index} + {g.temperature_c}°C · {g.power_w}W +
+
{g.name}
+
+ Util {Math.round(g.util_gpu)}% + VRAM {pct}% +
+
+
+
+
+ ) + })} +
+
+ ) +} diff --git a/src/components/LiveClusterMap.tsx b/src/components/LiveClusterMap.tsx new file mode 100644 index 0000000..249bfd8 --- /dev/null +++ b/src/components/LiveClusterMap.tsx @@ -0,0 +1,154 @@ +import { motion, AnimatePresence } from 'framer-motion' +import type { CSSProperties } from 'react' +import { AgentSprite } from './AgentSprite' +import { DataFlowLayer, GpuBeacon, WorkloadTicker, ZoneTower } from './ClusterViz' +import type { Agent, AgentAnim, WorkloadData } from '../types' + +const ZONE_X: Record = { + docker: 8, + db: 28, + lakehouse: 50, + hadoop: 72, + etl: 92, +} + +const STATE_LABEL: Record = { + idle: '', + walk: '→ zone', + fetch: '⟳ fetch', + return: '← desk', +} + +const DESK_X = 50 + +type Props = { + agents: Agent[] + workload: WorkloadData | null + animations: Record + selectedId: string | null +} + +export function LiveClusterMap({ agents, workload, animations, selectedId }: Props) { + const zones = workload?.zones || [] + const activeCount = agents.filter((a) => (animations[a.id]?.state || 'idle') !== 'idle').length + const busyAgent = agents.find((a) => (animations[a.id]?.state || 'idle') !== 'idle') + const mappedBusyZone = busyAgent?.zone ?? null + + return ( +
+
+
+ +
+
+

Live simulation

+

+ Cluster Ops Floor +

+ {workload && ( +

+ {workload.totals.apps_running} workloads active · {workload.totals.connectors} connectors · GPU {workload.gpu.avg_util ?? 0}% +

+ )} +
+
+ {activeCount > 0 && ( + + {activeCount} agent{activeCount > 1 ? 's' : ''} deployed + + )} + ● LIVE +
+
+ + {workload && ( +
+ +
+ )} + +
+ + {zones.map((z) => ( + + ))} + + + + COMMAND DESK + +
+ + {workload && zones.length > 0 && ( +
+ +
+ )} + +
+
+ {agents.map((agent, i) => { + const anim = animations[agent.id] || { agentId: agent.id, state: 'idle' as const } + const targetX = anim.state === 'idle' ? 10 + i * 18 : ZONE_X[anim.zone || agent.zone] ?? DESK_X + const y = anim.state === 'fetch' ? 10 : anim.state === 'idle' ? 0 : 6 + const selected = selectedId === agent.id + const busy = anim.state !== 'idle' + + return ( + + + {busy && STATE_LABEL[anim.state] && ( + + {STATE_LABEL[anim.state]} + + )} + + +
+ +
+ + ) + })} +
+
+ ) +} diff --git a/src/components/LiveDomainGrid.tsx b/src/components/LiveDomainGrid.tsx new file mode 100644 index 0000000..aff4f1b --- /dev/null +++ b/src/components/LiveDomainGrid.tsx @@ -0,0 +1,110 @@ +import { motion } from 'framer-motion' +import { appIcon, LEVEL_COLOR } from '../lib/appIcons' +import type { WorkloadData } from '../types' + +type Props = { + workload: WorkloadData | null +} + +export function LiveDomainGrid({ workload }: Props) { + if (!workload) { + return
+ } + + return ( +
+
+ {workload.zones.map((zone, zi) => { + const pct = zone.total ? Math.round((zone.running / zone.total) * 100) : 100 + return ( + +
+
+
+ {zone.label} +
+
+ {zone.running}/{zone.total || zone.apps.length} +
+
+ +
+ +
+ +
+ +
+ {zone.apps.slice(0, 6).map((app, ai) => ( + + {appIcon(app.name, app.image)} + {app.name.split('_')[0]} + + ))} +
+ + ) + })} + + +
GPU LAB
+
+ {workload.gpu.gpu_count ?? 0}× V100 +
+
+ {workload.gpu.model || 'offline'} +
+
+ +
+
+ {(workload.gpu.gpus || []).map((g) => ( + + G{g.index} + + ))} +
+ +
+
+ ) +} diff --git a/src/components/OpsFloor.tsx b/src/components/OpsFloor.tsx new file mode 100644 index 0000000..3540308 --- /dev/null +++ b/src/components/OpsFloor.tsx @@ -0,0 +1,134 @@ +import { motion, AnimatePresence } from 'framer-motion' +import type { CSSProperties } from 'react' +import { AgentSprite } from './AgentSprite' +import type { Agent, AgentAnim, Zone } from '../types' + +const ZONE_X: Record = { + docker: 8, + db: 28, + lakehouse: 50, + hadoop: 72, + etl: 92, +} + +const STATE_LABEL: Record = { + idle: '', + walk: '→ zone', + fetch: '⟳ fetch', + return: '← desk', +} + +const DESK_X = 50 + +type Props = { + agents: Agent[] + zones: Zone[] + animations: Record + selectedId: string | null +} + +export function OpsFloor({ agents, zones, animations, selectedId }: Props) { + const activeCount = agents.filter((a) => (animations[a.id]?.state || 'idle') !== 'idle').length + + return ( +
+
+
+
+

Live simulation

+

+ Ops Floor +

+
+
+ {activeCount > 0 && ( + + {activeCount} agent{activeCount > 1 ? 's' : ''} deployed + + )} + ● LIVE +
+
+ +
+ {zones.map((z) => ( +
+ + {z.label} + +
+ ))} + +
+ COMMAND DESK +
+ + + {zones.map((z) => ( + + ))} + +
+ +
+ {agents.map((agent, i) => { + const anim = animations[agent.id] || { agentId: agent.id, state: 'idle' as const } + const targetX = anim.state === 'idle' ? 10 + i * 18 : ZONE_X[anim.zone || agent.zone] ?? DESK_X + const y = anim.state === 'fetch' ? 10 : anim.state === 'idle' ? 0 : 6 + const selected = selectedId === agent.id + const busy = anim.state !== 'idle' + + return ( + + + {busy && STATE_LABEL[anim.state] && ( + + {STATE_LABEL[anim.state]} + + )} + + +
+ +
+ + ) + })} +
+
+ ) +} diff --git a/src/components/PromptBar.tsx b/src/components/PromptBar.tsx new file mode 100644 index 0000000..f507088 --- /dev/null +++ b/src/components/PromptBar.tsx @@ -0,0 +1,33 @@ +import { FormEvent, useState } from 'react' + +type Props = { + onSubmit: (message: string) => void + busy: boolean +} + +export function PromptBar({ onSubmit, busy }: Props) { + const [text, setText] = useState('') + + const handle = (e: FormEvent) => { + e.preventDefault() + if (!text.trim() || busy) return + onSubmit(text.trim()) + setText('') + } + + return ( +
+ 💬 + setText(e.target.value)} + disabled={busy} + /> + +
+ ) +} diff --git a/src/components/ThemeToggle.tsx b/src/components/ThemeToggle.tsx new file mode 100644 index 0000000..2276663 --- /dev/null +++ b/src/components/ThemeToggle.tsx @@ -0,0 +1,23 @@ +import { useTheme } from '../context/ThemeContext' + +export function ThemeToggle() { + const { theme, toggle } = useTheme() + const isDark = theme === 'dark' + + return ( + + ) +} diff --git a/src/context/ThemeContext.tsx b/src/context/ThemeContext.tsx new file mode 100644 index 0000000..d206267 --- /dev/null +++ b/src/context/ThemeContext.tsx @@ -0,0 +1,46 @@ +import { createContext, useContext, useEffect, useState, type ReactNode } from 'react' + +export type Theme = 'light' | 'dark' + +type ThemeContextValue = { + theme: Theme + toggle: () => void + setTheme: (t: Theme) => void +} + +const ThemeContext = createContext(null) +const STORAGE_KEY = 'atc-command-center-theme' + +function readStored(): Theme { + const v = localStorage.getItem(STORAGE_KEY) + return v === 'dark' || v === 'light' ? v : 'light' +} + +export function ThemeProvider({ children }: { children: ReactNode }) { + const [theme, setThemeState] = useState(() => { + if (typeof window === 'undefined') return 'light' + return readStored() + }) + + useEffect(() => { + const root = document.documentElement + root.classList.remove('light', 'dark') + root.classList.add(theme) + localStorage.setItem(STORAGE_KEY, theme) + }, [theme]) + + const setTheme = (t: Theme) => setThemeState(t) + const toggle = () => setThemeState((t) => (t === 'light' ? 'dark' : 'light')) + + return ( + + {children} + + ) +} + +export function useTheme() { + const ctx = useContext(ThemeContext) + if (!ctx) throw new Error('useTheme outside ThemeProvider') + return ctx +} diff --git a/src/index.css b/src/index.css new file mode 100644 index 0000000..341e2fd --- /dev/null +++ b/src/index.css @@ -0,0 +1,629 @@ +@tailwind base; +@tailwind components; +@tailwind utilities; + +:root, +.light { + --bg-1: #e8f4fc; + --bg-2: #f4f7fb; + --bg-3: #f5f0ff; + --grid-color: rgba(8, 145, 178, 0.06); + --surface: rgba(255, 255, 255, 0.72); + --surface-strong: rgba(255, 255, 255, 0.94); + --surface-elevated: #f1f5f9; + --surface-muted: #e2e8f0; + --text: #0f172a; + --text-muted: #64748b; + --text-faint: #94a3b8; + --border: rgba(15, 23, 42, 0.09); + --accent: #0891b2; + --accent-gpu: #65a30d; + --accent-secondary: #7c3aed; + --glow: rgba(8, 145, 178, 0.22); + --status-ok: #16a34a; + --status-ok-bg: rgba(22, 163, 74, 0.1); + --status-warn: #d97706; + --status-warn-bg: rgba(217, 119, 6, 0.1); + --status-down: #dc2626; + --status-down-bg: rgba(220, 38, 38, 0.08); + --shadow: 0 12px 40px rgba(15, 40, 80, 0.09); + --floor-bg: linear-gradient(180deg, #f1f5f9 0%, #ffffff 100%); + --aurora-1: rgba(8, 145, 178, 0.12); + --aurora-2: rgba(124, 58, 237, 0.08); +} + +.dark { + --bg-1: #050810; + --bg-2: #0a0f1a; + --bg-3: #100818; + --grid-color: rgba(34, 211, 238, 0.05); + --surface: rgba(12, 18, 32, 0.78); + --surface-strong: rgba(16, 22, 38, 0.94); + --surface-elevated: #1a2236; + --surface-muted: #243049; + --text: #eef2f9; + --text-muted: #94a3b8; + --text-faint: #64748b; + --border: rgba(34, 211, 238, 0.14); + --accent: #22d3ee; + --accent-gpu: #a3e635; + --accent-secondary: #c084fc; + --glow: rgba(34, 211, 238, 0.28); + --status-ok: #4ade80; + --status-ok-bg: rgba(74, 222, 128, 0.12); + --status-warn: #fbbf24; + --status-warn-bg: rgba(251, 191, 36, 0.12); + --status-down: #f87171; + --status-down-bg: rgba(248, 113, 113, 0.12); + --shadow: 0 12px 48px rgba(0, 0, 0, 0.5); + --floor-bg: linear-gradient(180deg, #121a2e 0%, #0a0e18 100%); + --aurora-1: rgba(34, 211, 238, 0.15); + --aurora-2: rgba(192, 132, 252, 0.1); +} + +body { + margin: 0; + min-height: 100vh; + color: var(--text); + background: linear-gradient(145deg, var(--bg-1) 0%, var(--bg-2) 45%, var(--bg-3) 100%); + background-attachment: fixed; + transition: background 0.4s ease, color 0.4s ease; +} + +body::before { + content: ''; + position: fixed; + inset: 0; + background-image: + radial-gradient(ellipse 80% 50% at 20% 0%, var(--aurora-1), transparent 50%), + radial-gradient(ellipse 60% 40% at 80% 10%, var(--aurora-2), transparent 45%), + linear-gradient(var(--grid-color) 1px, transparent 1px), + linear-gradient(90deg, var(--grid-color) 1px, transparent 1px); + background-size: 100% 100%, 100% 100%, 40px 40px, 40px 40px; + pointer-events: none; + z-index: 0; +} + +#root { position: relative; z-index: 1; } + +.section-eyebrow { + @apply text-[10px] font-mono uppercase tracking-[0.2em] text-[var(--text-faint)] mb-0.5; +} + +.panel { + background: var(--surface-strong); + backdrop-filter: blur(20px); + border: 1px solid var(--border); + box-shadow: var(--shadow); + transition: background 0.35s ease, border-color 0.35s ease, box-shadow 0.35s ease; +} + +.neon-text { text-shadow: 0 0 32px var(--glow); } + +.logo-mark { + @apply w-12 h-12 rounded-2xl flex items-center justify-center text-sm font-bold text-white shrink-0; + background: linear-gradient(135deg, var(--accent), var(--accent-secondary)); + box-shadow: 0 4px 28px var(--glow); +} + +.header-aurora { + position: absolute; + inset: 0; + background: linear-gradient(90deg, transparent, var(--aurora-1), transparent); + opacity: 0.6; + pointer-events: none; +} + +.status-pill { + @apply text-xs font-mono px-3 py-1.5 rounded-full border inline-flex items-center gap-1.5; +} +.status-pill::before { content: '●'; font-size: 8px; } +.status-pill.ok { color: var(--status-ok); border-color: var(--status-ok); background: var(--status-ok-bg); } +.status-pill.warn { color: var(--status-warn); border-color: var(--status-warn); background: var(--status-warn-bg); } +.status-pill.gpu { color: var(--accent-gpu); border-color: var(--accent-gpu); background: color-mix(in srgb, var(--accent-gpu) 10%, transparent); } + +.live-badge { + @apply text-xs font-mono px-2.5 py-1 rounded-full border animate-pulse; + color: var(--status-ok); + border-color: var(--status-ok); + background: var(--status-ok-bg); +} + +/* Agent cards */ +.agent-roster-glow { + background: radial-gradient(ellipse at 50% 0%, var(--aurora-1), transparent 70%); +} + +.agent-card { + @apply rounded-2xl p-[1px] transition-all duration-300 cursor-pointer; + background: var(--border); +} +.agent-card:hover, +.agent-card.selected { + background: linear-gradient(135deg, var(--agent-color, var(--accent)), var(--accent-secondary)); + box-shadow: 0 8px 32px color-mix(in srgb, var(--agent-color, var(--accent)) 25%, transparent); +} +.agent-card-inner { + background: var(--surface-strong); + min-height: 200px; +} +.agent-card.selected .agent-card-inner { + background: color-mix(in srgb, var(--agent-color, var(--accent)) 4%, var(--surface-strong)); +} + +.agent-avatar { + @apply relative w-11 h-11 rounded-xl flex items-center justify-center border-2; +} +.agent-status-dot { + @apply absolute -bottom-0.5 -right-0.5 w-3 h-3 rounded-full border-2; + border-color: var(--surface-strong); +} +.agent-status-dot.active { animation: pulse-dot 1.2s ease infinite; } + +.agent-state-pill { + @apply text-[10px] font-mono px-2 py-0.5 rounded-full bg-[var(--surface-elevated)]; +} +.agent-state-pill.busy { background: color-mix(in srgb, currentColor 12%, transparent); } + +.cap-chip { + @apply text-[9px] font-mono px-1.5 py-0.5 rounded-md; + background: var(--surface-elevated); + color: var(--text-muted); + border: 1px solid var(--border); +} + +.delegate-btn { + background: color-mix(in srgb, currentColor 8%, transparent); + border: 1px solid color-mix(in srgb, currentColor 25%, transparent); + transition: all 0.15s ease; +} +.delegate-btn:hover { + background: color-mix(in srgb, currentColor 18%, transparent); +} + +/* Ops floor */ +.ops-floor-stage { + background: var(--floor-bg); + border-color: var(--border); + overflow: hidden; +} +.ops-floor-scan { + position: absolute; + inset: 0; + background: linear-gradient(180deg, transparent 0%, color-mix(in srgb, var(--accent) 4%, transparent) 50%, transparent 100%); + animation: scan 4s ease-in-out infinite; + pointer-events: none; +} +.zone-node { + background: var(--surface-strong); +} + +.sprite-wrap.selected::after { + content: ''; + position: absolute; + inset: -10px -6px; + border-radius: 50%; + border: 2px dashed var(--sprite-color, var(--accent)); + opacity: 0.65; + animation: spin-slow 10s linear infinite; + pointer-events: none; +} +.sprite-wrap { position: relative; } +.sprite-wrap.busy { + filter: drop-shadow(0 0 16px var(--sprite-color)) drop-shadow(0 4px 8px rgba(0,0,0,0.3)); +} + +.sprite-figure { + position: relative; + display: flex; + align-items: center; + justify-content: center; +} +.sprite-figure-busy .sprite-svg { + filter: drop-shadow(0 0 8px var(--sprite-color, var(--accent))); +} +.sprite-ring-outer { + position: absolute; + inset: -4px 2px 16px 2px; + border-radius: 50%; + border: 1.5px solid; + pointer-events: none; +} +.sprite-holo-shimmer { + position: absolute; + inset: 0; + border-radius: 40%; + pointer-events: none; + animation: holo-shimmer 3s ease-in-out infinite; + opacity: 0.7; +} +.sprite-nameplate { + @apply flex items-center gap-1.5 mt-1.5 px-2.5 py-0.5 rounded-full border; + background: color-mix(in srgb, var(--surface-strong) 85%, transparent); + backdrop-filter: blur(6px); +} +.sprite-nameplate-dot { + @apply w-1.5 h-1.5 rounded-full shrink-0; +} +.sprite-nameplate-text { + @apply text-[10px] font-mono font-bold tracking-wide; +} +.sprite-platform-pulse { + animation: platform-pulse 2s ease-in-out infinite; +} +.sprite-core-pulse { + animation: pulse-dot 1.5s ease infinite; +} +.sprite-svg { overflow: visible; } + +.sprite-ring { + position: absolute; + inset: -6px; + border-radius: 50%; + border: 2px solid; + animation: pulse-ring 1.5s ease infinite; +} + +/* Activity feed */ +.activity-feed { scrollbar-width: thin; } +.activity-item { @apply flex gap-3; } +.activity-rail { @apply flex flex-col items-center w-4 shrink-0; } +.activity-dot { @apply w-2.5 h-2.5 rounded-full shrink-0 mt-1.5; } +.activity-line { @apply w-px flex-1 bg-[var(--border)] min-h-[1rem]; } +.activity-agent-badge { + @apply text-[10px] font-mono px-2 py-0.5 rounded-full border; + background: var(--surface-elevated); +} + +.empty-state { + @apply text-sm text-[var(--text-muted)] text-center py-8; +} + +/* Chat */ +.chat-message { @apply flex gap-2 items-end; } +.chat-message.user { @apply flex-row-reverse; } +.chat-avatar { + @apply w-8 h-8 rounded-xl flex items-center justify-center text-sm shrink-0 border; +} +.chat-bubble { @apply rounded-2xl px-3 py-2 max-w-[90%]; } +.chat-meta { + @apply text-[10px] font-mono text-[var(--text-faint)] flex gap-2 mb-1; +} +.chat-text { @apply text-sm leading-relaxed; } +.chat-bubble-user { + background: color-mix(in srgb, var(--accent) 14%, var(--surface-elevated)); + border: 1px solid color-mix(in srgb, var(--accent) 28%, transparent); +} +.chat-bubble-agent { + background: var(--surface-elevated); + border: 1px solid var(--border); +} + +.thinking-pulse { animation: pulse-dot 1s ease infinite; } +.typing-indicator { @apply flex gap-1 py-1; } +.typing-indicator span { + @apply w-1.5 h-1.5 rounded-full bg-[var(--text-faint)]; + animation: typing 1.2s ease infinite; +} +.typing-indicator span:nth-child(2) { animation-delay: 0.15s; } +.typing-indicator span:nth-child(3) { animation-delay: 0.3s; } + +/* Command dock */ +.command-dock { border-color: color-mix(in srgb, var(--accent) 20%, var(--border)); } +.command-input-wrap { + background: var(--surface-elevated); + border: 1px solid var(--border); + transition: border-color 0.2s, box-shadow 0.2s; +} +.command-input-wrap:focus-within { + border-color: var(--accent); + box-shadow: 0 0 0 3px var(--glow); +} + +.quick-prompt-chip { + @apply text-xs font-mono px-3 py-1.5 rounded-full border transition-all disabled:opacity-40; + background: var(--surface-elevated); + border-color: var(--border); + color: var(--text-muted); +} +.quick-prompt-chip:hover:not(:disabled) { + border-color: var(--accent); + color: var(--accent); + background: color-mix(in srgb, var(--accent) 8%, var(--surface-elevated)); +} + +/* Shared */ +.status-card { + background: var(--surface-strong); + border: 1px solid var(--border); + transition: transform 0.15s ease, box-shadow 0.15s ease; +} +.status-card:hover { transform: translateY(-2px); box-shadow: var(--shadow); } + +.gpu-card { background: var(--surface-elevated); transition: border-color 0.2s ease; } +.gpu-card:hover { border-color: var(--accent-gpu); } + +.btn-primary { + @apply px-5 py-2.5 rounded-xl font-display text-sm font-semibold text-white transition-all hover:brightness-110 disabled:opacity-40; + background: linear-gradient(135deg, var(--accent), var(--accent-secondary)); + box-shadow: 0 4px 24px var(--glow); +} +.btn-secondary { + @apply px-3 py-1.5 rounded-lg font-mono border transition-all; + background: var(--surface-elevated); + border-color: var(--border); + color: var(--accent); +} +.btn-secondary:hover { border-color: var(--accent); box-shadow: 0 0 16px var(--glow); } + +.tab-btn { + @apply px-4 py-2 rounded-xl text-sm font-mono border transition-all inline-flex items-center gap-2; + border-color: var(--border); + color: var(--text-muted); + background: var(--surface); +} +.tab-btn.active { + border-color: var(--accent); + color: var(--accent); + background: var(--surface-strong); + box-shadow: 0 0 24px var(--glow); + font-weight: 600; +} +.tab-badge { + @apply text-[10px] px-1.5 py-0.5 rounded-full font-bold; + background: var(--accent-secondary); + color: white; +} + +.theme-toggle { + @apply flex items-center gap-2 px-2 py-1 rounded-xl border transition-all; + border-color: var(--border); + background: var(--surface-elevated); +} +.theme-toggle:hover { border-color: var(--accent); } +.theme-toggle-track { @apply relative w-11 h-6 rounded-full transition-colors; background: var(--surface-muted); } +.theme-toggle-track.is-dark { background: linear-gradient(90deg, #1e293b, #312e81); } +.theme-toggle-thumb { + @apply absolute top-0.5 left-0.5 w-5 h-5 rounded-full flex items-center justify-center text-xs transition-transform; + background: var(--surface-strong); + box-shadow: 0 2px 6px rgba(0, 0, 0, 0.2); +} +.theme-toggle-track.is-dark .theme-toggle-thumb { transform: translateX(1.25rem); } + +.prompt-input { + @apply flex-1 rounded-xl px-4 py-2.5 outline-none font-mono text-sm transition; + background: var(--surface-elevated); + border: 1px solid var(--border); + color: var(--text); +} +.prompt-input::placeholder { color: var(--text-faint); } +.prompt-input:focus { border-color: var(--accent); box-shadow: 0 0 0 3px var(--glow); } + +.agent-sprite .sprite-body { fill: var(--surface-strong); } +.agent-sprite .sprite-limb { fill: var(--surface-elevated); } +.agent-sprite .sprite-head { fill: var(--surface-strong); } + +/* Agent terminals */ +.agent-terminal { + @apply rounded-xl border overflow-hidden flex flex-col cursor-pointer transition-all; + border-color: var(--border); + background: #0a0e14; + min-height: 200px; + max-height: 240px; +} +.dark .agent-terminal { background: #060a10; } +.light .agent-terminal { background: #0f172a; } + +.agent-terminal.active { + border-color: color-mix(in srgb, var(--term-accent, var(--accent)) 55%, transparent); + box-shadow: 0 0 24px color-mix(in srgb, var(--term-accent, var(--accent)) 15%, transparent); +} +.agent-terminal.expanded { + max-height: 480px; + min-height: 400px; +} +.agent-terminal-header { + @apply flex items-center justify-between gap-2 px-3 py-2 border-b; + border-color: rgba(255,255,255,0.06); + background: rgba(0,0,0,0.25); +} +.agent-terminal-body { + @apply flex-1 overflow-y-auto p-2 font-mono text-[11px] leading-relaxed; + scrollbar-width: thin; +} +.term-line { + @apply flex gap-2 py-0.5; + word-break: break-word; +} +.term-ts { + @apply shrink-0 text-[10px] opacity-50 w-[4.5rem]; + color: #64748b; +} +.term-phase { + @apply shrink-0 text-[9px] uppercase w-10 opacity-40 hidden sm:inline; +} +.term-text { flex: 1; } +.term-info .term-text { color: #94a3b8; } +.term-ok .term-text { color: #4ade80; } +.term-warn .term-text { color: #fbbf24; } +.term-err .term-text { color: #f87171; } +.term-cmd .term-text { color: #67e8f9; } +.term-llm .term-text { color: #c084fc; } +.term-live-badge { + @apply text-[9px] font-mono px-1.5 py-0.5 rounded-full animate-pulse; + color: var(--term-accent, var(--accent)); + border: 1px solid color-mix(in srgb, var(--term-accent, var(--accent)) 40%, transparent); +} +.term-cursor { animation: blink 1s step-end infinite; color: var(--term-accent, var(--accent)); } +.agent-terminal-focus { @apply flex flex-col gap-3; } +.agent-terminal-tabs { @apply flex flex-wrap gap-2; } +.agent-terminal-tab { + @apply text-xs font-mono px-3 py-1.5 rounded-lg border transition-all; + border-color: var(--border); + color: var(--text-muted); + background: var(--surface-elevated); +} +.agent-terminal-tab.active { font-weight: 600; background: var(--surface-strong); } + +@keyframes blink { + 0%, 100% { opacity: 1; } + 50% { opacity: 0; } +} + +@keyframes pulse-dot { + 0%, 100% { opacity: 1; transform: scale(1); } + 50% { opacity: 0.6; transform: scale(1.15); } +} +@keyframes pulse-ring { + 0%, 100% { opacity: 0.8; transform: scale(1); } + 50% { opacity: 0.4; transform: scale(1.08); } +} +@keyframes scan { + 0%, 100% { transform: translateY(-100%); opacity: 0; } + 50% { opacity: 1; } + 100% { transform: translateY(100%); } +} +@keyframes spin-slow { to { transform: rotate(360deg); } } +@keyframes holo-shimmer { + 0%, 100% { opacity: 0.4; transform: translateX(-2px); } + 50% { opacity: 0.85; transform: translateX(2px); } +} +@keyframes platform-pulse { + 0%, 100% { opacity: 0.35; transform: scaleX(1); } + 50% { opacity: 0.7; transform: scaleX(1.08); } +} +@keyframes typing { + 0%, 60%, 100% { transform: translateY(0); opacity: 0.4; } + 30% { transform: translateY(-4px); opacity: 1; } +} + +/* Live cluster map */ +.live-cluster-map { isolation: isolate; } +.cluster-ambient { + position: absolute; + inset: 0; + background: radial-gradient(ellipse 70% 50% at 50% 30%, var(--aurora-1), transparent 60%); + pointer-events: none; +} +.cluster-stage-grid { + position: absolute; + inset: 0; + background-image: linear-gradient(var(--grid-color) 1px, transparent 1px), + linear-gradient(90deg, var(--grid-color) 1px, transparent 1px); + background-size: 24px 24px; + opacity: 0.5; + border-radius: inherit; +} +.cluster-agent-stage { overflow: hidden; } + +.zone-tower { + @apply rounded-xl px-2.5 py-2 text-center min-w-[84px] border-2; + background: color-mix(in srgb, var(--surface-strong) 90%, transparent); + backdrop-filter: blur(8px); +} +.zone-tower-live { animation: tower-breathe 3s ease-in-out infinite; } +.zone-tower-active { transform: scale(1.05); } +.zone-tower-label { @apply text-[8px] font-mono font-bold tracking-wider; } +.zone-tower-stats { @apply text-[10px] font-mono font-semibold mt-0.5 flex items-center justify-center gap-1; } +.zone-tower-extra { @apply text-[8px] font-mono opacity-70 mt-0.5; } +.zone-level-dot { @apply w-1.5 h-1.5 rounded-full; } +.zone-level-dot.level-ok { background: var(--status-ok); box-shadow: 0 0 6px var(--status-ok); } +.zone-level-dot.level-warn { background: var(--status-warn); } +.zone-level-dot.level-down { background: var(--status-down); } + +.zone-app-orbit { + @apply flex flex-col gap-0.5 mt-1 max-w-[90px]; +} +.zone-app-chip { + @apply flex items-center gap-1 text-[8px] font-mono px-1.5 py-0.5 rounded-md border; + background: var(--surface-strong); +} +.zone-app-chip.state-running { opacity: 1; } +.zone-app-chip.state-down, .zone-app-chip.state-created { opacity: 0.45; filter: grayscale(0.5); } + +.command-desk { + @apply relative px-4 py-2 rounded-xl text-[9px] font-mono font-bold tracking-widest border-2; + border-color: var(--accent); + color: var(--accent); + background: var(--surface-strong); +} +.command-desk-ring { + @apply absolute inset-0 rounded-xl border border-[var(--accent)] opacity-30 animate-ping; + pointer-events: none; +} + +.gpu-beacon { + @apply rounded-xl border px-3 py-2 text-left min-w-[140px]; + border-color: var(--accent-gpu); + background: color-mix(in srgb, var(--accent-gpu) 8%, var(--surface-strong)); +} +.gpu-beacon-title { @apply text-[9px] font-mono font-bold text-[var(--accent-gpu)]; } +.gpu-beacon-model { @apply text-xs font-semibold text-[var(--text)] mt-0.5; } +.gpu-beacon-meta { @apply text-[9px] font-mono text-[var(--text-muted)]; } +.gpu-beacon-bar { @apply h-1 rounded-full bg-[var(--surface-muted)] mt-2 overflow-hidden; } +.gpu-beacon-fill { @apply h-full rounded-full bg-[var(--accent-gpu)]; } + +.workload-ticker-wrap { + @apply flex items-center gap-3 overflow-hidden rounded-xl border px-3 py-2; + border-color: var(--border); + background: var(--surface-elevated); +} +.workload-ticker-label { + @apply text-[9px] font-mono font-bold tracking-widest shrink-0 text-[var(--accent)]; +} +.workload-ticker-track { @apply flex-1 overflow-hidden; } +.workload-ticker-inner { @apply flex gap-2 whitespace-nowrap; } +.ticker-chip { + @apply inline-flex items-center gap-1 text-[10px] font-mono px-2 py-0.5 rounded-full border; + background: var(--surface-strong); +} +.ticker-chip.state-running { opacity: 1; } +.ticker-chip.state-down, .ticker-chip.state-created { opacity: 0.4; } + +.live-domain-card { + @apply rounded-xl p-4 border-2 transition-all; + background: var(--surface-strong); +} +.domain-pulse-dot { @apply w-2.5 h-2.5 rounded-full shrink-0; } +.domain-pulse-dot.level-ok { background: var(--status-ok); } +.domain-pulse-dot.level-warn { background: var(--status-warn); } +.domain-pulse-dot.level-down { background: var(--status-down); } +.domain-progress-bar { @apply h-1.5 rounded-full bg-[var(--surface-muted)] overflow-hidden; } +.domain-progress-fill { @apply h-full rounded-full; } +.domain-app-grid { + @apply grid grid-cols-3 gap-1; +} +.domain-app-tile { + @apply flex flex-col items-center text-[8px] font-mono p-1 rounded-md border border-[var(--border)]; + background: var(--surface-elevated); +} +.domain-app-tile.state-down { opacity: 0.35; } +.gpu-mini-tile { + @apply text-[8px] font-mono px-1.5 py-0.5 rounded border border-[var(--accent-gpu)] text-[var(--accent-gpu)]; + background: color-mix(in srgb, var(--accent-gpu) 10%, transparent); +} + +.ambient-bg { pointer-events: none; } +.ambient-orb { + position: absolute; + width: 120px; + height: 120px; + border-radius: 50%; + background: radial-gradient(circle, var(--aurora-1), transparent 70%); + animation: orb-float linear infinite; + opacity: 0.35; +} + +@keyframes tower-breathe { + 0%, 100% { filter: brightness(1); } + 50% { filter: brightness(1.15); } +} +@keyframes orb-float { + 0%, 100% { transform: translate(0, 0) scale(1); opacity: 0.2; } + 33% { transform: translate(20px, -30px) scale(1.1); opacity: 0.4; } + 66% { transform: translate(-15px, 20px) scale(0.9); opacity: 0.25; } +} +@keyframes ticker-scroll { + from { transform: translateX(0); } + to { transform: translateX(-50%); } +} diff --git a/src/lib/appIcons.ts b/src/lib/appIcons.ts new file mode 100644 index 0000000..68b54a2 --- /dev/null +++ b/src/lib/appIcons.ts @@ -0,0 +1,41 @@ +const ICON_MAP: [RegExp, string][] = [ + [/trino/i, '🔷'], + [/spark/i, '⚡'], + [/kafka/i, '📨'], + [/connect/i, '🔗'], + [/postgres/i, '🐘'], + [/mysql/i, '🐬'], + [/mongo/i, '🍃'], + [/cassandra/i, '💿'], + [/neo4j/i, '🔴'], + [/airflow/i, '🌀'], + [/superset/i, '📊'], + [/forgejo|gitea/i, '🦊'], + [/homepage/i, '🏠'], + [/dockhand/i, '🐳'], + [/redis/i, '⚙️'], + [/nginx|caddy/i, '🌐'], + [/hdfs|namenode|datanode/i, '🌲'], + [/gpu|vllm|nvidia/i, '🎮'], + [/lam-|ldap/i, '👤'], + [/cadvisor/i, '📈'], +] + +export function appIcon(name: string, image?: string): string { + const hay = `${name} ${image || ''}` + for (const [re, icon] of ICON_MAP) { + if (re.test(hay)) return icon + } + return '📦' +} + +export function shortName(name: string, max = 14): string { + return name.length > max ? `${name.slice(0, max - 1)}…` : name +} + +export const LEVEL_COLOR: Record = { + ok: 'var(--status-ok)', + warn: 'var(--status-warn)', + down: 'var(--status-down)', + unknown: 'var(--text-faint)', +} diff --git a/src/main.tsx b/src/main.tsx new file mode 100644 index 0000000..227e4e1 --- /dev/null +++ b/src/main.tsx @@ -0,0 +1,13 @@ +import React from 'react' +import ReactDOM from 'react-dom/client' +import App from './App' +import { ThemeProvider } from './context/ThemeContext' +import './index.css' + +ReactDOM.createRoot(document.getElementById('root')!).render( + + + + + , +) diff --git a/src/types.ts b/src/types.ts new file mode 100644 index 0000000..cd6bb0f --- /dev/null +++ b/src/types.ts @@ -0,0 +1,134 @@ +export type AgentStats = { + tasks: number + last_active: string | null + alerts: number +} + +export type Agent = { + id: string + name: string + color: string + zone: string + role: string + icon?: string + motto?: string + capabilities?: string[] + suggested_prompts?: string[] + stats?: AgentStats +} + +export type Zone = { id: string; label: string; x: number; color: string } + +export type FeedEntry = { + id: string + ts: string + agent_id: string + message: string + level: string +} + +export type DomainStatus = { + level: 'ok' | 'warn' | 'down' | 'unknown' + label: string +} + +export type StatusData = { + ts: string + domains: Record + gpu?: GpuStatus +} + +export type GpuDevice = { + index: number + name: string + util_gpu: number + memory_used_mib: number + memory_total_mib: number + temperature_c: number + power_w: number +} + +export type GpuStatus = { + ok: boolean + host: string + ui_url: string + inference_active?: boolean + active_model?: string | null + vllm_url?: string | null + gpu_count?: number + gpus?: GpuDevice[] + error?: string +} + +export type AgentState = 'idle' | 'walk' | 'fetch' | 'return' + +export type AgentAnim = { + agentId: string + state: AgentState + zone?: string +} + +export type Approval = { + id: string + ts: string + agent_id: string + action: string + reason: string + status: string +} + +export type TerminalLine = { + id: string + ts: string + agent_id: string + level: 'info' | 'ok' | 'warn' | 'err' | 'cmd' | 'llm' + phase: string + text: string + prompt_id?: string +} + +export type WorkloadApp = { + name: string + state: string + image: string + ports: string[] +} + +export type WorkloadZone = { + id: string + label: string + x: number + color: string + level: 'ok' | 'warn' | 'down' | 'unknown' + running: number + total: number + apps: WorkloadApp[] + trino_ok?: boolean + hdfs_used_gb?: number + hdfs_total_gb?: number +} + +export type WorkloadData = { + ts: string + zones: WorkloadZone[] + gpu: { + level: string + model?: string | null + inference_active?: boolean + gpu_count?: number + avg_util?: number + gpus?: GpuDevice[] + } + totals: { + apps_running: number + apps_total: number + connectors: number + } +} + +export type ChatMessage = { + role: 'user' | 'agent' + text: string + agent?: string + ts?: string +} diff --git a/tailwind.config.js b/tailwind.config.js new file mode 100644 index 0000000..5c69503 --- /dev/null +++ b/tailwind.config.js @@ -0,0 +1,14 @@ +/** @type {import('tailwindcss').Config} */ +export default { + content: ['./index.html', './src/**/*.{js,ts,jsx,tsx}'], + darkMode: 'class', + theme: { + extend: { + fontFamily: { + display: ['"Space Grotesk"', 'system-ui', 'sans-serif'], + mono: ['"JetBrains Mono"', 'monospace'], + }, + }, + }, + plugins: [], +} diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..42e0521 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "ES2020", + "useDefineForClassFields": true, + "lib": ["ES2020", "DOM", "DOM.Iterable"], + "module": "ESNext", + "skipLibCheck": true, + "moduleResolution": "bundler", + "allowImportingTsExtensions": true, + "resolveJsonModule": true, + "isolatedModules": true, + "noEmit": true, + "jsx": "react-jsx", + "strict": true + }, + "include": ["src"] +} diff --git a/ui/index.html b/ui/index.html index 7c68b66..81fa1ec 100644 --- a/ui/index.html +++ b/ui/index.html @@ -1,14 +1,14 @@ - + - ATC Command Center + ATC Data & AI Command Center - + - +
diff --git a/ui/nginx.conf b/ui/nginx.conf index 3d24bfe..3fc9183 100644 --- a/ui/nginx.conf +++ b/ui/nginx.conf @@ -11,6 +11,17 @@ server { proxy_set_header Host $host; } + location /assets/ { + add_header Cache-Control "public, max-age=31536000, immutable"; + try_files $uri =404; + } + + location = /index.html { + add_header Cache-Control "no-cache, no-store, must-revalidate"; + add_header Pragma "no-cache"; + add_header Expires "0"; + } + location / { try_files $uri $uri/ /index.html; } diff --git a/ui/package.json b/ui/package.json index 0aa1dcf..75298d3 100644 --- a/ui/package.json +++ b/ui/package.json @@ -1,7 +1,7 @@ { "name": "atc-command-center", "private": true, - "version": "0.1.0", + "version": "2.0.0", "type": "module", "scripts": { "dev": "vite", @@ -9,9 +9,13 @@ "preview": "vite preview" }, "dependencies": { - "framer-motion": "^11.15.0", + "@tanstack/react-query": "^5.62.8", + "class-variance-authority": "^0.7.1", + "clsx": "^2.1.1", + "lucide-react": "^0.469.0", "react": "^18.3.1", - "react-dom": "^18.3.1" + "react-dom": "^18.3.1", + "tailwind-merge": "^2.6.0" }, "devDependencies": { "@types/react": "^18.3.12", diff --git a/ui/src/App.tsx b/ui/src/App.tsx index 3c1347a..abbd7ae 100644 --- a/ui/src/App.tsx +++ b/ui/src/App.tsx @@ -1,218 +1,178 @@ -import { useCallback, useEffect, useState } from 'react' -import { OpsFloor } from './components/OpsFloor' -import { PromptBar } from './components/PromptBar' -import type { Agent, AgentAnim, Approval, FeedEntry, StatusData, Zone } from './types' - -const TABS = ['Overview', 'Agents', 'Feed', 'Approvals', 'Audit'] as const -type Tab = (typeof TABS)[number] - -const LEVEL_COLOR = { ok: '#22aa44', warn: '#cc7700', down: '#dd3355', unknown: '#8b9cb3' } -const LEVEL_BG = { ok: '#e8f8ec', warn: '#fff6e6', down: '#ffeef2', unknown: '#f0f3f8' } - -function wsUrl() { - const proto = window.location.protocol === 'https:' ? 'wss' : 'ws' - const host = window.location.host - return `${proto}://${host}/api/ws/ops` -} +import { useRef, useState } from 'react' +import { useClock } from './hooks/useClock' +import { useCommandCenter } from './hooks/useCommandCenter' +import { useLiveMetrics } from './hooks/useLiveMetrics' +import { SideNav } from './components/layout/SideNav' +import { TopBar } from './components/layout/TopBar' +import { AgentFleet } from './components/features/AgentFleet' +import { ApprovalInbox } from './components/features/ApprovalInbox' +import { ChatDrawer } from './components/features/ChatDrawer' +import { GpuMonitor } from './components/features/GpuMonitor' +import { InfraQuickAccess } from './components/features/InfraQuickAccess' +import { InspectorPanel } from './components/features/InspectorPanel' +import { PlatformTopology } from './components/features/PlatformTopology' +import { PresentationView } from './components/features/PresentationView' +import { DataQualityView } from './components/features/DataQualityView' +import { KnowledgeChatView } from './components/features/KnowledgeChatView' +import { StorageView } from './components/features/StorageView' +import { TerminalDock } from './components/features/TerminalDock' +import { resolveInfraNode } from './lib/infraCatalog' +import { cn } from './lib/utils' export default function App() { - const [tab, setTab] = useState('Overview') - const [agents, setAgents] = useState([]) - const [zones, setZones] = useState([]) - const [status, setStatus] = useState(null) - const [feed, setFeed] = useState([]) - const [approvals, setApprovals] = useState([]) - const [chat, setChat] = useState<{ role: 'user' | 'agent'; text: string; agent?: string }[]>([]) - const [anims, setAnims] = useState>({}) - const [busy, setBusy] = useState(false) + const clock = useClock() + const cc = useCommandCenter() + const [gpuChatActive, setGpuChatActive] = useState(false) + const gpuBoost = gpuChatActive || cc.mainView === 'knowledge' + const { agentLoads, gpuLive } = useLiveMetrics(cc.agents, cc.gpu, cc.anims, gpuBoost) + const mainScrollRef = useRef(null) - const load = useCallback(async () => { - const [a, s, f, ap] = await Promise.all([ - fetch('/api/agents').then((r) => r.json()), - fetch('/api/status').then((r) => r.json()), - fetch('/api/feed').then((r) => r.json()), - fetch('/api/approvals').then((r) => r.json()), - ]) - setAgents(a.agents || []) - setZones(a.zones || []) - setStatus(s) - setFeed(f.entries || []) - setApprovals(ap.approvals || []) - }, []) + const isPlatform = cc.mainView === 'platform' - useEffect(() => { - load() - const ws = new WebSocket(wsUrl()) - ws.onmessage = (ev) => { - const msg = JSON.parse(ev.data) - if (msg.type === 'status') setStatus(msg.data) - if (msg.type === 'feed') setFeed((prev) => [msg.entry, ...prev].slice(0, 100)) - if (msg.type === 'agent_dispatch') { - setAnims((p) => ({ ...p, [msg.agent_id]: { agentId: msg.agent_id, state: 'walk', zone: msg.zone } })) - } - if (msg.type === 'agent_fetch') { - setAnims((p) => ({ ...p, [msg.agent_id]: { agentId: msg.agent_id, state: 'fetch', zone: msg.zone } })) - } - if (msg.type === 'agent_return') { - setAnims((p) => ({ ...p, [msg.agent_id]: { agentId: msg.agent_id, state: 'return', zone: msg.zone } })) - setTimeout(() => { - setAnims((p) => ({ ...p, [msg.agent_id]: { agentId: msg.agent_id, state: 'idle' } })) - }, 1200) - } - if (msg.type === 'prompt_result') { - setChat((c) => [...c, { role: 'agent', text: msg.answer, agent: msg.agent_id }]) - setBusy(false) - } - } - const iv = setInterval(load, 30000) - return () => { ws.close(); clearInterval(iv) } - }, [load]) - - const sendPrompt = async (message: string) => { - setBusy(true) - setChat((c) => [...c, { role: 'user', text: message }]) - await fetch('/api/prompt', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ message }), - }) + const openApprovals = () => { + cc.setMainView('approvals') + mainScrollRef.current?.scrollTo({ top: 0, behavior: 'smooth' }) } - const decide = async (id: string, approved: boolean) => { - await fetch(`/api/approvals/${id}/decide`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ approved }), - }) - load() - } - - const allOk = status && Object.values(status.domains).every((d) => d.level === 'ok') + const terminalSubject = cc.terminalSubjectId + const terminalLabel = (() => { + if (cc.selectedAgent) return cc.selectedAgent.name.split(' ·')[0] + const infra = resolveInfraNode(terminalSubject) + if (infra) return infra.label + if (cc.selectedNode) return cc.selectedNode.label + return 'Lab' + })() return ( -
-
-
-
- ATC -
-
-

Command Center

-

Agent ops floor · Dell ATC Lab

-
-
-
- {status && ( - - {allOk ? '● All systems operational' : '● Attention required'} - - )} - {status && ( - - Scan {new Date(status.ts).toLocaleTimeString()} - - )} -
-
+
+ - +
+ - - -
-
- {tab === 'Overview' && status && ( -
- {Object.entries(status.domains).map(([key, d]) => ( -
-
{key}
-
{d.label}
-
-
- {d.level} +
+
+
+ {isPlatform && ( + <> +
+ +
-
- ))} -
- )} - {tab === 'Agents' && ( -
- {agents.map((a) => ( -
-
{a.name}
-
{a.role}
-
zone: {a.zone}
-
- ))} -
- )} - {tab === 'Feed' && ( -
- {feed.map((e) => ( -
- {e.ts ? new Date(e.ts).toLocaleTimeString() : ''} - a.id === e.agent_id)?.color || '#888' }}>{e.agent_id} - {e.message} -
- ))} -
- )} - {tab === 'Approvals' && ( -
- {approvals.length === 0 &&

Geen pending approvals.

} - {approvals.map((a) => ( -
-
{a.action}
-
{a.reason}
-
- - -
-
- ))} -
- )} - {tab === 'Audit' && ( -

Audit log — approvals en agent acties (v1 via Feed tab).

- )} -
+ + + )} - +
- + cc.setChatExpanded(!cc.chatExpanded)} + chat={cc.chat} + feed={cc.feed} + agents={cc.agents} + approvals={cc.approvals} + selectedAgent={cc.selectedAgent} + promptBusy={cc.promptBusy} + approvalHighlight={cc.approvalHighlight} + filterAgentId={cc.selectedAgentId} + onSendPrompt={cc.sendPrompt} + onDecide={cc.decide} + onDismissHighlight={() => cc.setApprovalHighlight(false)} + />
) } diff --git a/ui/src/components/AgentSprite.tsx b/ui/src/components/AgentSprite.tsx deleted file mode 100644 index eca0191..0000000 --- a/ui/src/components/AgentSprite.tsx +++ /dev/null @@ -1,48 +0,0 @@ -import { motion } from 'framer-motion' - -type Props = { - color: string - state: 'idle' | 'walk' | 'fetch' | 'return' - label: string -} - -export function AgentSprite({ color, state, label }: Props) { - const bob = state === 'idle' ? { y: [0, -3, 0] } : state === 'walk' || state === 'return' ? { y: [0, -6, 0] } : { y: 0 } - const scale = state === 'fetch' ? 0.92 : 1 - - return ( - - - - - - - - - - - - - - - - - - - {state === 'fetch' && ( - - )} - - {label} - - ) -} diff --git a/ui/src/components/OpsFloor.tsx b/ui/src/components/OpsFloor.tsx deleted file mode 100644 index 8e16917..0000000 --- a/ui/src/components/OpsFloor.tsx +++ /dev/null @@ -1,79 +0,0 @@ -import { motion } from 'framer-motion' -import { AgentSprite } from './AgentSprite' -import type { Agent, AgentAnim, Zone } from '../types' - -const ZONE_X: Record = { - docker: 8, - db: 28, - lakehouse: 50, - hadoop: 72, - etl: 92, -} - -const DESK_X = 50 - -type Props = { - agents: Agent[] - zones: Zone[] - animations: Record -} - -export function OpsFloor({ agents, zones, animations }: Props) { - return ( -
-
-

OPS FLOOR

- ● LIVE -
- -
- {zones.map((z) => ( -
-
- {z.label} -
-
- ))} - - {zones.map((z) => ( - - ))} - -
- -
- {agents.map((agent, i) => { - const anim = animations[agent.id] || { agentId: agent.id, state: 'idle' as const } - const targetX = anim.state === 'idle' ? 12 + i * 17 : ZONE_X[anim.zone || agent.zone] ?? DESK_X - const y = anim.state === 'fetch' ? 8 : anim.state === 'idle' ? 0 : 4 - - return ( - - - - ) - })} -
-
- ) -} diff --git a/ui/src/components/PromptBar.tsx b/ui/src/components/PromptBar.tsx deleted file mode 100644 index d9b6762..0000000 --- a/ui/src/components/PromptBar.tsx +++ /dev/null @@ -1,41 +0,0 @@ -import { FormEvent, useState } from 'react' - -type Props = { - onSubmit: (message: string) => void - busy: boolean -} - -export function PromptBar({ onSubmit, busy }: Props) { - const [text, setText] = useState('') - - const handle = (e: FormEvent) => { - e.preventDefault() - if (!text.trim() || busy) return - onSubmit(text.trim()) - setText('') - } - - return ( -
- 💬 - setText(e.target.value)} - disabled={busy} - /> - -
- ) -} diff --git a/ui/src/components/features/ActivityStream.tsx b/ui/src/components/features/ActivityStream.tsx new file mode 100644 index 0000000..ed54630 --- /dev/null +++ b/ui/src/components/features/ActivityStream.tsx @@ -0,0 +1,61 @@ +import type { Agent, FeedEntry } from '../../types' +import { getAgentMeta } from '../../lib/agentMeta' +import { cn } from '../../lib/utils' + +type Props = { + feed: FeedEntry[] + agents: Agent[] + filterAgentId?: string | null + opsOnly?: boolean +} + +const LEVEL: Record = { + info: 'text-foreground-muted', + ok: 'text-success', + warn: 'text-warning', + err: 'text-danger', +} + +function isOpsEvent(message: string): boolean { + const lower = message.toLowerCase() + if (message.includes(' answered:')) return false + if (message.startsWith('Prompt received:')) return false + if (lower.includes('completed a response')) return false + return true +} + +export function ActivityStream({ feed, agents, filterAgentId, opsOnly }: Props) { + let items = filterAgentId ? feed.filter((e) => e.agent_id === filterAgentId) : feed + if (opsOnly) items = items.filter((e) => isOpsEvent(e.message)) + + return ( +
+ {items.length === 0 && ( +

+ {opsOnly ? 'Geen operationele events — antwords staan in Chat.' : 'No activity yet — agents are on standby.'} +

+ )} + {items.map((e) => { + const ag = agents.find((a) => a.id === e.agent_id) + const meta = ag ? getAgentMeta(ag.id) : null + const Icon = meta?.icon + return ( +
+ {Icon && ( + + + + )} +
+
+ {ag?.name.split(' ·')[0] || e.agent_id} + {new Date(e.ts).toLocaleTimeString('en-US', { hour12: false })} +
+

{e.message}

+
+
+ ) + })} +
+ ) +} diff --git a/ui/src/components/features/AgentFleet.tsx b/ui/src/components/features/AgentFleet.tsx new file mode 100644 index 0000000..92fd5df --- /dev/null +++ b/ui/src/components/features/AgentFleet.tsx @@ -0,0 +1,121 @@ +import { ShieldCheck } from 'lucide-react' +import type { Agent, AgentAnim } from '../../types' +import type { AgentLoad } from '../../hooks/useLiveMetrics' +import { agentTaskLabel, getAgentMeta } from '../../lib/agentMeta' +import { cn } from '../../lib/utils' + +type Props = { + agents: Agent[] + animations: Record + selectedId: string | null + loads: Record + approvalCount: number + onSelect: (id: string) => void + onOpenApprovals: () => void +} + +function AgentCard({ + agent, + anim, + load, + selected, + onSelect, +}: { + agent: Agent + anim?: AgentAnim + load?: AgentLoad + selected: boolean + onSelect: () => void +}) { + const meta = getAgentMeta(agent.id) + const Icon = meta.icon + const busy = anim && anim.state !== 'idle' + + return ( + + ) +} + +export function AgentFleet({ agents, animations, selectedId, loads, approvalCount, onSelect, onOpenApprovals }: Props) { + const supervisors = agents.filter((a) => a.supervisor) + const operators = agents.filter((a) => !a.supervisor && a.id !== 'mcp-coordinator') + const mcp = agents.find((a) => a.id === 'mcp-coordinator') + + return ( +
+
+
+

Agent Fleet

+

Klik agent → stel vraag in chat · elk agent bewaakt één domein

+
+
+ + {agents.length} agents +
+
+
+
+

Supervisors

+
+ {supervisors.map((a) => ( + onSelect(a.id)} /> + ))} +
+
+ {mcp && ( +
+

MCP Hub

+ onSelect(mcp.id)} /> +
+ )} +
+

Field Operators

+
+ {operators.map((a) => ( + onSelect(a.id)} /> + ))} +
+
+
+
+ ) +} diff --git a/ui/src/components/features/ApprovalCards.tsx b/ui/src/components/features/ApprovalCards.tsx new file mode 100644 index 0000000..ac88f85 --- /dev/null +++ b/ui/src/components/features/ApprovalCards.tsx @@ -0,0 +1,98 @@ +import { useState } from 'react' +import { Check, ShieldAlert, X } from 'lucide-react' +import type { Agent, Approval } from '../../types' +import { getAgentMeta } from '../../lib/agentMeta' +import { Button } from '../ui/Button' +import { cn } from '../../lib/utils' + +const ACTION_LABELS: Record = { + 'docker.restart': 'Container restart', + 'docker.update': 'Image update', + 'generic.mutate': 'Infrastructure change', +} + +type Props = { + approvals: Approval[] + agents: Agent[] + highlighted: boolean + onDecide: (id: string, approved: boolean, decidedBy: string, note: string) => Promise + onDismissHighlight?: () => void +} + +export function ApprovalCards({ approvals, agents, highlighted, onDecide, onDismissHighlight }: Props) { + const [busyId, setBusyId] = useState(null) + const [decider, setDecider] = useState<'mo-commander' | 'bart-commander'>('mo-commander') + + if (!approvals.length) return null + + const agentOf = (id: string) => agents.find((a) => a.id === id) + + const handle = async (id: string, approved: boolean) => { + setBusyId(id) + try { + await onDecide(id, approved, decider, '') + } finally { + setBusyId(null) + } + } + + return ( +
+
+
+ + Pending approvals ({approvals.length}) +
+
+ + {highlighted && onDismissHighlight && ( + + )} +
+
+
+ {approvals.map((a) => { + const ag = agentOf(a.agent_id) + const meta = ag ? getAgentMeta(ag.id) : null + const Icon = meta?.icon + return ( +
+
+ {Icon && ( + + + + )} +
+ {ACTION_LABELS[a.action_type] || a.action_type} +

{ag?.name || a.agent_id}

+
+ #{a.id.slice(0, 6)} +
+

{a.action}

+ {a.target &&

Target: {a.target}

} +
+ + +
+
+ ) + })} +
+
+ ) +} diff --git a/ui/src/components/features/ApprovalInbox.tsx b/ui/src/components/features/ApprovalInbox.tsx new file mode 100644 index 0000000..f56cff0 --- /dev/null +++ b/ui/src/components/features/ApprovalInbox.tsx @@ -0,0 +1,122 @@ +import { useCallback, useEffect, useMemo, useState } from 'react' +import { Check, ShieldCheck, X } from 'lucide-react' +import { fetchApprovalHistory } from '../../lib/api' +import type { Agent, Approval } from '../../types' +import { getAgentMeta } from '../../lib/agentMeta' +import { Badge } from '../ui/Badge' +import { Button } from '../ui/Button' +import { cn } from '../../lib/utils' + +type Filter = 'pending' | 'approved' | 'denied' | 'all' + +type Props = { + agents: Agent[] + livePending: Approval[] + onDecide: (id: string, approved: boolean, decidedBy: string, note: string) => Promise +} + +export function ApprovalInbox({ agents, livePending, onDecide }: Props) { + const [filter, setFilter] = useState('pending') + const [items, setItems] = useState([]) + const [stats, setStats] = useState({ pending: 0, approved: 0, denied: 0, total: 0 }) + const [selectedId, setSelectedId] = useState(null) + const [decider, setDecider] = useState<'mo-commander' | 'bart-commander'>('mo-commander') + const [note, setNote] = useState('') + const [busy, setBusy] = useState(false) + + const load = useCallback(async () => { + const res = await fetchApprovalHistory(filter === 'all' ? 'all' : filter) + setItems(res.approvals) + if (res.stats) setStats(res.stats) + }, [filter]) + + useEffect(() => { load() }, [load, livePending]) + + const selected = useMemo(() => items.find((a) => a.id === selectedId) || items[0] || null, [items, selectedId]) + const agentOf = (id: string) => agents.find((a) => a.id === id) + + const handleDecide = async (approved: boolean) => { + if (!selected || selected.status !== 'pending') return + setBusy(true) + try { + await onDecide(selected.id, approved, decider, note) + setNote('') + await load() + } finally { + setBusy(false) + } + } + + return ( +
+
+
+ +
+

Approval Inbox

+

Mo & Bart review mutating agent actions

+
+
+
+ {stats.pending} pending + {stats.approved} ok + {stats.denied} denied +
+
+ +
+ {(['pending', 'approved', 'denied', 'all'] as Filter[]).map((f) => ( + + ))} +
+ +
+
+ {!items.length &&

No {filter} requests.

} + {items.map((a) => { + const ag = agentOf(a.agent_id) + const meta = ag ? getAgentMeta(ag.id) : null + const Icon = meta?.icon + return ( + + ) + })} +
+ + {selected && ( +
+ {selected.status} +
+
Agent
{agentOf(selected.agent_id)?.name}
+
Type
{selected.action_type}
+
Action
{selected.action}
+
Reason
{selected.reason}
+
+ {selected.status === 'pending' && ( +
+ + setNote(e.target.value)} placeholder="Note (optional)" className="w-full rounded border border-border bg-surface px-2 py-1 text-[10px] text-foreground-muted" /> +
+ + +
+
+ )} +
+ )} +
+
+ ) +} diff --git a/ui/src/components/features/ArchitectureDiagram.tsx b/ui/src/components/features/ArchitectureDiagram.tsx new file mode 100644 index 0000000..9d03168 --- /dev/null +++ b/ui/src/components/features/ArchitectureDiagram.tsx @@ -0,0 +1,212 @@ +import { useEffect, useState } from 'react' +import { cn } from '../../lib/utils' + +type FlowNode = { id: string; label: string; sub?: string; color: string } +type FlowEdge = { from: string; to: string; label?: string } + +const FLOWS: Record = { + 'full-stack': { + nodes: [ + { id: 'user', label: 'User / Customer', sub: 'Browser', color: '#60a5fa' }, + { id: 'caddy', label: 'Caddy :80', sub: 'Reverse proxy', color: '#38bdf8' }, + { id: 'ui', label: 'Command Center', sub: 'React UI', color: '#818cf8' }, + { id: 'api', label: 'Agents API', sub: 'FastAPI :3201', color: '#a78bfa' }, + { id: 'dq', label: 'DQ API', sub: 'Maturity + Docling', color: '#f59e0b' }, + { id: 'rag', label: 'RAG API', sub: 'LangChain', color: '#34d399' }, + { id: 'chroma', label: 'ChromaDB', sub: 'Vectors (persistent)', color: '#22d3ee' }, + { id: 'docling', label: 'Docling', sub: ':5001', color: '#fb923c' }, + { id: 'llm', label: 'vLLM Llama 70B', sub: 'GPU Lab', color: '#4ade80' }, + { id: 'lake', label: 'Lakehouse', sub: 'Kafka · Spark · Trino', color: '#6366f1' }, + ], + edges: [ + { from: 'user', to: 'caddy', label: 'HTTP' }, + { from: 'caddy', to: 'ui' }, + { from: 'ui', to: 'api' }, + { from: 'ui', to: 'dq' }, + { from: 'ui', to: 'rag' }, + { from: 'dq', to: 'docling' }, + { from: 'rag', to: 'docling' }, + { from: 'rag', to: 'chroma' }, + { from: 'rag', to: 'llm' }, + { from: 'api', to: 'llm' }, + { from: 'api', to: 'lake' }, + ], + }, + 'rag-flow': { + nodes: [ + { id: 'upload', label: 'Upload PDF/CSV', sub: 'Once', color: '#60a5fa' }, + { id: 'store', label: 'File Store', sub: '/data/uploads', color: '#64748b' }, + { id: 'docling', label: 'Docling', sub: 'Parse + OCR', color: '#fb923c' }, + { id: 'chunk', label: 'LangChain Splitter', sub: '800 char chunks', color: '#a78bfa' }, + { id: 'embed', label: 'MiniLM Embeddings', sub: '384-d vectors', color: '#818cf8' }, + { id: 'chroma', label: 'ChromaDB', sub: 'Persistent', color: '#22d3ee' }, + { id: 'query', label: 'Your Question', sub: 'Any time', color: '#60a5fa' }, + { id: 'retrieve', label: 'Similarity Search', sub: 'top-k chunks', color: '#34d399' }, + { id: 'llm', label: 'Llama 70B', sub: 'Answer + sources', color: '#4ade80' }, + ], + edges: [ + { from: 'upload', to: 'store', label: 'save' }, + { from: 'upload', to: 'docling' }, + { from: 'docling', to: 'chunk' }, + { from: 'chunk', to: 'embed' }, + { from: 'embed', to: 'chroma', label: 'index' }, + { from: 'query', to: 'retrieve' }, + { from: 'retrieve', to: 'chroma' }, + { from: 'retrieve', to: 'llm' }, + ], + }, + 'dq-flow': { + nodes: [ + { id: 'data', label: 'Customer Data', sub: 'CSV · Excel · PDF', color: '#60a5fa' }, + { id: 'docling', label: 'Docling', sub: 'Structure + images', color: '#fb923c' }, + { id: 'pandas', label: 'Pandas Profiling', sub: 'Column stats', color: '#a78bfa' }, + { id: 'ge', label: 'Great Expectations', sub: 'Expectation checks', color: '#34d399' }, + { id: 'soda', label: 'Soda Core', sub: 'YAML checks', color: '#22d3ee' }, + { id: 'maturity', label: '6 Dimensions', sub: 'Score 0–100', color: '#f59e0b' }, + { id: 'report', label: 'HTML Report', sub: 'Roadmap + actions', color: '#818cf8' }, + ], + edges: [ + { from: 'data', to: 'docling' }, + { from: 'data', to: 'pandas' }, + { from: 'pandas', to: 'ge' }, + { from: 'pandas', to: 'soda' }, + { from: 'ge', to: 'maturity' }, + { from: 'soda', to: 'maturity' }, + { from: 'maturity', to: 'report' }, + ], + }, + 'lakehouse': { + nodes: [ + { id: 'pg', label: 'PostgreSQL', color: '#60a5fa' }, + { id: 'mysql', label: 'MySQL', color: '#60a5fa' }, + { id: 'mongo', label: 'MongoDB', color: '#60a5fa' }, + { id: 'debezium', label: 'Debezium CDC', color: '#f59e0b' }, + { id: 'kafka', label: 'Kafka', color: '#fb923c' }, + { id: 'spark', label: 'Spark', color: '#a78bfa' }, + { id: 'iceberg', label: 'Iceberg', color: '#22d3ee' }, + { id: 'trino', label: 'Trino', color: '#34d399' }, + { id: 'bi', label: 'Superset BI', color: '#818cf8' }, + ], + edges: [ + { from: 'pg', to: 'debezium' }, + { from: 'mysql', to: 'debezium' }, + { from: 'mongo', to: 'debezium' }, + { from: 'debezium', to: 'kafka' }, + { from: 'kafka', to: 'spark' }, + { from: 'spark', to: 'iceberg' }, + { from: 'iceberg', to: 'trino' }, + { from: 'trino', to: 'bi' }, + ], + }, +} + +const POSITIONS: Record> = { + 'full-stack': { + user: { x: 50, y: 8 }, + caddy: { x: 50, y: 22 }, + ui: { x: 50, y: 38 }, + api: { x: 18, y: 58 }, + dq: { x: 50, y: 58 }, + rag: { x: 82, y: 58 }, + docling: { x: 50, y: 78 }, + chroma: { x: 82, y: 78 }, + llm: { x: 82, y: 92 }, + lake: { x: 18, y: 92 }, + }, + 'rag-flow': { + upload: { x: 12, y: 20 }, + store: { x: 12, y: 45 }, + docling: { x: 35, y: 20 }, + chunk: { x: 58, y: 20 }, + embed: { x: 58, y: 45 }, + chroma: { x: 58, y: 70 }, + query: { x: 82, y: 20 }, + retrieve: { x: 82, y: 45 }, + llm: { x: 82, y: 70 }, + }, + 'dq-flow': { + data: { x: 10, y: 50 }, + docling: { x: 28, y: 25 }, + pandas: { x: 28, y: 75 }, + ge: { x: 52, y: 35 }, + soda: { x: 52, y: 65 }, + maturity: { x: 72, y: 50 }, + report: { x: 90, y: 50 }, + }, + 'lakehouse': { + pg: { x: 8, y: 15 }, + mysql: { x: 8, y: 35 }, + mongo: { x: 8, y: 55 }, + debezium: { x: 28, y: 35 }, + kafka: { x: 45, y: 35 }, + spark: { x: 58, y: 35 }, + iceberg: { x: 72, y: 35 }, + trino: { x: 85, y: 35 }, + bi: { x: 92, y: 55 }, + }, +} + +export function ArchitectureDiagram({ animation }: { animation: string }) { + const flow = FLOWS[animation] || FLOWS['full-stack'] + const positions = POSITIONS[animation] || POSITIONS['full-stack'] + const [tick, setTick] = useState(0) + + useEffect(() => { + const t = setInterval(() => setTick((n) => n + 1), 2200) + return () => clearInterval(t) + }, []) + + const activeEdge = tick % flow.edges.length + + return ( +
+ + {flow.edges.map((edge, i) => { + const from = positions[edge.from] + const to = positions[edge.to] + if (!from || !to) return null + const active = i === activeEdge + return ( + + + {active && ( + + + + )} + + ) + })} + + {flow.nodes.map((node) => { + const pos = positions[node.id] + if (!pos) return null + const lit = flow.edges.some((e, i) => i === activeEdge && (e.from === node.id || e.to === node.id)) + return ( +
+

+ {node.label} +

+ {node.sub &&

{node.sub}

} +
+ ) + })} +
+ ) +} diff --git a/ui/src/components/features/ChatDrawer.tsx b/ui/src/components/features/ChatDrawer.tsx new file mode 100644 index 0000000..d0fee32 --- /dev/null +++ b/ui/src/components/features/ChatDrawer.tsx @@ -0,0 +1,119 @@ +import { useState } from 'react' +import { ChevronDown, ChevronUp, MessageSquare, Radio } from 'lucide-react' +import type { Agent, Approval, ChatMessage, FeedEntry } from '../../types' +import { ActivityStream } from './ActivityStream' +import { ApprovalCards } from './ApprovalCards' +import { CommandBar } from './CommandBar' +import { CommsPanel } from './CommsPanel' +import { cn } from '../../lib/utils' + +type Props = { + expanded: boolean + onToggle: () => void + chat: ChatMessage[] + feed: FeedEntry[] + agents: Agent[] + approvals: Approval[] + selectedAgent: Agent | null + promptBusy: boolean + approvalHighlight: boolean + filterAgentId?: string | null + onSendPrompt: (message: string, agentId?: string) => void + onDecide: (id: string, approved: boolean, decidedBy: string, note: string) => Promise + onDismissHighlight: () => void +} + +export function ChatDrawer({ + expanded, + onToggle, + chat, + feed, + agents, + approvals, + selectedAgent, + promptBusy, + approvalHighlight, + filterAgentId, + onSendPrompt, + onDecide, + onDismissHighlight, +}: Props) { + const [tab, setTab] = useState<'chat' | 'activity'>('chat') + const unread = chat.length + + if (!expanded) { + return ( + + ) + } + + return ( +
+
+
+ {(['chat', 'activity'] as const).map((t) => ( + + ))} +
+ +
+ +
+ {tab === 'chat' ? ( + + ) : ( +
+ + +
+ )} +
+ + {tab === 'chat' && } +
+ ) +} diff --git a/ui/src/components/features/CommandBar.tsx b/ui/src/components/features/CommandBar.tsx new file mode 100644 index 0000000..6a07e09 --- /dev/null +++ b/ui/src/components/features/CommandBar.tsx @@ -0,0 +1,58 @@ +import { useState, type FormEvent } from 'react' +import { Send } from 'lucide-react' +import type { Agent } from '../../types' +import { Button } from '../ui/Button' +import { Input } from '../ui/Input' + +type Props = { + busy: boolean + selectedAgent: Agent | null + onSubmit: (message: string, agentId?: string) => void +} + +export function CommandBar({ busy, selectedAgent, onSubmit }: Props) { + const [input, setInput] = useState('') + + const submit = (e: FormEvent) => { + e.preventDefault() + if (!input.trim() || busy) return + onSubmit(input.trim(), selectedAgent?.id) + setInput('') + } + + const suggestions = selectedAgent?.suggested_prompts?.slice(0, 3) || [ + 'Hoeveel data zit er in de databases?', + 'Wat staat er in PostgreSQL?', + 'MongoDB supplychain overzicht', + ] + + return ( +
+
+ setInput(e.target.value)} + placeholder={selectedAgent ? `Command ${selectedAgent.name.split(' ·')[0]}…` : 'Enter command — auto-routed to specialist…'} + disabled={busy} + className="flex-1" + /> + +
+
+ {suggestions.map((s) => ( + + ))} +
+
+ ) +} diff --git a/ui/src/components/features/CommsPanel.tsx b/ui/src/components/features/CommsPanel.tsx new file mode 100644 index 0000000..1d3e4ff --- /dev/null +++ b/ui/src/components/features/CommsPanel.tsx @@ -0,0 +1,68 @@ +import { useEffect, useRef } from 'react' +import { MessageSquare } from 'lucide-react' +import type { Agent, ChatMessage } from '../../types' +import { getAgentMeta } from '../../lib/agentMeta' +import { cn } from '../../lib/utils' + +type Props = { + messages: ChatMessage[] + agents: Agent[] + selectedAgent: Agent | null + busy: boolean +} + +export function CommsPanel({ messages, agents, selectedAgent, busy }: Props) { + const bottomRef = useRef(null) + const meta = selectedAgent ? getAgentMeta(selectedAgent.id) : null + + useEffect(() => { + bottomRef.current?.scrollIntoView({ behavior: 'smooth' }) + }, [messages, busy]) + + return ( +
+
+

+ Comms +

+ {selectedAgent && meta && ( + + {selectedAgent.name.split(' ·')[0]} + + )} +
+
+ {!messages.length && ( +
+ +

+ Send a command below — routing selects the right specialist automatically. +

+
+ )} + {messages.map((m, i) => { + const ag = m.role === 'agent' ? agents.find((a) => a.id === m.agent) : null + const agMeta = ag ? getAgentMeta(ag.id) : null + return ( +
+
+

+ {m.role === 'user' ? 'You' : ag?.name || m.agent} + {m.ts && ` · ${new Date(m.ts).toLocaleTimeString('en-US', { hour12: false })}`} +

+

{m.text}

+
+
+ ) + })} + {busy && ( +
+ + Agent verzamelt cluster-data en vraagt Llama 70B… verwacht ~30–90 sec +
+ )} +
+
+
+ ) +} diff --git a/ui/src/components/features/DataQualityView.tsx b/ui/src/components/features/DataQualityView.tsx new file mode 100644 index 0000000..2ece40d --- /dev/null +++ b/ui/src/components/features/DataQualityView.tsx @@ -0,0 +1,732 @@ +import { useCallback, useEffect, useState, Fragment } from 'react' +import { + AlertTriangle, + CheckCircle2, + FileSearch, + FileText, + Image, + Layers, + Loader2, + RefreshCw, + Table2, + Upload, + XCircle, +} from 'lucide-react' +import { cn } from '../../lib/utils' +import { subTabActive, subTabIdle } from '../../lib/tabActive' + +type Dimension = { + id: string + label: string + description: string + score: number + level: string + findings: string[] + recommended_actions?: string[] +} + +type ColumnProfile = { + name: string + dtype: string + null_pct: number + unique_count: number + quality_flags: string[] + sample_values?: string[] + numeric?: { min: number; max: number; mean: number; outliers: number } + text?: { avg_length: number; empty_strings: number } + top_values?: { value: string; count: number }[] +} + +type GxCheck = { suite: string; expectation: string; success: boolean; result: string; column?: string } +type SodaCheck = { suite: string; name: string; check: string; outcome: string; detail: string } + +type DocStructure = { + pages: number + pictures: number + tables: number + text_blocks: number + headings: number + paragraphs: number + list_items?: number + form_items: number + key_value_pairs: number + label_counts?: Record + table_details?: { index: number; rows: number; cols: number; cells: number; preview?: string }[] + picture_details?: { index: number; label: string; has_image: boolean; captions: number }[] + outline?: { type: string; text: string; level?: number }[] +} + +type AssessResult = { + ok: boolean + report_id: string + overall_score: number + maturity_level: string + maturity_description?: string + rows: number + columns: number + dimensions: Dimension[] + column_profiles: ColumnProfile[] + action_items: { priority: string; dimension: string; score: number; action: string }[] + checks?: { great_expectations: GxCheck[]; soda_core: SodaCheck[] } + checks_summary: { + great_expectations: { total: number; passed: number } + soda_core: { total: number; warnings: number } + } + docling?: { used: boolean; parse_id?: string; document_structure?: DocStructure; stats?: Record; images?: DocImage[] } + rag_ingest?: { ok: boolean; duplicate?: boolean; chunks?: number; message?: string; error?: string } + report_url: string +} + +type DocImage = { + index: number + label: string + available: boolean + url?: string + width?: number + height?: number + mimetype?: string + dpi?: number + bytes?: number + captions?: string[] +} + +type ParseResult = { + ok: boolean + parse_id: string + filename: string + status: string + processing_time_sec?: number + formats_available: string[] + document_structure: DocStructure + images?: DocImage[] + stats: Record + content: { preview_markdown?: string; preview_html?: string; markdown?: string; html?: string } + table_preview?: string[] + errors?: string[] + parse_json_url?: string +} + +type Capabilities = { + maturity_dimensions: { id: string; label: string; description: string }[] + maturity_levels: { min_score: number; label: string; description: string }[] + supported_data_formats: string[] + supported_document_formats: string[] + tools: Record + docling_online: boolean +} + +type ReportSummary = { + id: string + filename: string + ts: string + overall_score: number + maturity_level: string + rows: number + columns: number +} + +type Tab = 'assess' | 'docling' | 'reports' + +const SCORE_COLOR = (s: number) => (s >= 80 ? 'text-success' : s >= 60 ? 'text-warning' : 'text-danger') +const BAR_COLOR = (s: number) => (s >= 80 ? 'bg-success' : s >= 60 ? 'bg-warning' : 'bg-danger') + +export function DataQualityView() { + const [tab, setTab] = useState('assess') + const [caps, setCaps] = useState(null) + const [loading, setLoading] = useState(false) + const [assess, setAssess] = useState(null) + const [parse, setParse] = useState(null) + const [parseFormat, setParseFormat] = useState<'markdown' | 'html'>('markdown') + const [reports, setReports] = useState([]) + const [error, setError] = useState(null) + const [expandedCol, setExpandedCol] = useState(null) + const [showGx, setShowGx] = useState(false) + const [showSoda, setShowSoda] = useState(false) + + const loadMeta = useCallback(async () => { + try { + const [c, r] = await Promise.all([fetch('/dq/capabilities'), fetch('/dq/reports')]) + if (c.ok) setCaps(await c.json()) + if (r.ok) { + const j = await r.json() + setReports(j.reports || []) + } + } catch { + /* ignore */ + } + }, []) + + useEffect(() => { + loadMeta() + }, [loadMeta]) + + const onAssess = async (file: File) => { + setLoading(true) + setError(null) + setAssess(null) + const fd = new FormData() + fd.append('file', file) + try { + const r = await fetch('/dq/assess', { method: 'POST', body: fd }) + const j = await r.json() + if (!r.ok || !j.ok) { + setError(j.error || j.detail || 'Assessment failed') + return + } + setAssess(j as AssessResult) + loadMeta() + } catch { + setError('Connection failed — check DQ API') + } finally { + setLoading(false) + } + } + + const onParse = async (file: File) => { + setLoading(true) + setError(null) + setParse(null) + const fd = new FormData() + fd.append('file', file) + fd.append('to_formats', 'md,html,json') + const ctrl = new AbortController() + const timer = setTimeout(() => ctrl.abort(), 300000) + try { + const r = await fetch('/dq/parse', { method: 'POST', body: fd, signal: ctrl.signal }) + const j = await r.json() + if (!r.ok || !j.ok) { + setError(typeof j.error === 'string' ? j.error : JSON.stringify(j.error || j).slice(0, 200) || 'Docling parse failed') + return + } + setParse(j as ParseResult) + loadMeta() + } catch (e) { + setError(e instanceof Error && e.name === 'AbortError' ? 'Timeout — document too large or Docling overloaded' : 'Docling unavailable') + } finally { + clearTimeout(timer) + setLoading(false) + } + } + + const tabs: { id: Tab; label: string; icon: typeof FileSearch }[] = [ + { id: 'assess', label: 'Maturity Assessment', icon: FileSearch }, + { id: 'docling', label: 'Docling Parser', icon: FileText }, + { id: 'reports', label: 'Reports', icon: CheckCircle2 }, + ] + + return ( +
+
+
+
+

Data Quality & Maturity Platform

+

+ Full data maturity assessment for customer data — Docling, Great Expectations, Soda Core +

+
+
+ + Docling {caps?.docling_online ? '● online' : '○ offline'} + + + + Docling UI ↗ + +
+
+ + {caps && ( +
+ d.label)} icon={Layers} /> + + + +
+ )} +
+ +
+ {tabs.map(({ id, label, icon: Icon }) => ( + + ))} +
+ +
+ {error && ( +
+ + {error} +
+ )} + + {tab === 'assess' && ( +
+ + + {loading && } + + {assess && ( +
+
+ + + + + 0} /> +
+ + {assess.docling?.used && assess.docling.document_structure && ( + <> + + {assess.docling.images && assess.docling.images.length > 0 && assess.docling.parse_id && ( + + )} + + )} + + {assess.rag_ingest && ( +
+

Knowledge Chat sync

+

+ {assess.rag_ingest.ok + ? (assess.rag_ingest.duplicate + ? `Already in Knowledge Chat — ${assess.rag_ingest.message || 'you can chat immediately.'}` + : `Indexed for chat: ${assess.rag_ingest.chunks ?? '?'} text chunks. Open Knowledge Chat to ask questions.`) + : (assess.rag_ingest.error || 'Could not sync to Knowledge Chat')} +

+
+ )} + +
+ + Full HTML report ↗ + + + +
+ + {showGx && assess.checks?.great_expectations && ( + ({ + name: c.column ? `${c.expectation} [${c.column}]` : c.expectation, + status: c.success ? 'pass' : 'fail', + detail: c.result, + }))} /> + )} + {showSoda && assess.checks?.soda_core && ( + ({ + name: c.name, + status: c.outcome, + detail: `${c.check} — ${c.detail}`, + }))} /> + )} + +
+

6 Maturity Dimensions

+
+ {assess.dimensions.map((d) => ( + + ))} +
+
+ + {assess.action_items.length > 0 && ( +
+

+ Remediation Roadmap +

+
+ {assess.action_items.map((a, i) => ( +
+ {a.priority} + {' · '}{a.dimension} ({a.score}): {a.action} +
+ ))} +
+
+ )} + +
+

+ Column profiles ({assess.column_profiles.length}) +

+ +
+
+ )} +
+ )} + + {tab === 'docling' && ( +
+

+ Docling extracts text, tables, images and document structure from PDF, PowerPoint, Word, Excel and images. + Resultaat: Markdown, HTML, JSON met pagina's, plaatjes, tabellen en outline. +

+ + + {loading && } + + {parse && ( +
+
+ 20 ? parse.filename.slice(0, 18) + '…' : parse.filename} sub={parse.status} /> + + + + + +
+ + + + {parse.images && parse.images.filter((i) => i.available).length > 0 && ( + + )} + + {parse.document_structure?.outline && parse.document_structure.outline.length > 0 && ( +
+

Document outline

+
    + {parse.document_structure.outline.map((o, i) => ( +
  • + {o.type} + {o.text} +
    • + ))} +
+
+ )} + +
+ {(['markdown', 'html'] as const).map((f) => ( + + ))} + {parse.parse_json_url && ( + + Full JSON ↗ + + )} +
+ + {parse.table_preview && parse.table_preview.length > 0 && ( +
+

Tables (markdown preview)

+ 
+                      {parse.table_preview.join('\n')}
+
+
+ )} + +
+

Extracted content

+ {parseFormat === 'html' && (parse.content.preview_html || parse.content.html) ? ( +
+
+
+ ) : ( + 
+                      {parse.content.preview_markdown || parse.content.markdown || '(no content)'}
+
+ )} +
+
+ )} +
+ )} + + {tab === 'reports' && ( +
+ {reports.length === 0 ? ( +

No reports yet — upload customer data in Maturity Assessment.

+ ) : ( + reports.map((r) => ( + +
+

{r.filename}

+

{r.ts} · {r.rows?.toLocaleString()} rows · {r.columns} cols

+
+
+

{r.overall_score}

+

{r.maturity_level}

+
+ + )) + )} +
+ )} +
+
+ ) +} + +function ImageGallery({ images, parseId }: { images: DocImage[]; parseId: string }) { + const available = images.filter((i) => i.available) + const [lightbox, setLightbox] = useState(null) + if (!available.length) { + return ( +
+

+ Images gedetecteerd ({images.length}) — no embedded export +

+

Re-upload the document to extract images (embedded mode).

+
+ ) + } + return ( +
+

+ + Images die Docling ziet ({available.length}) +

+
+ {available.map((img) => ( + + ))} +
+ {lightbox !== null && ( +
setLightbox(null)}> +
e.stopPropagation()}> + {`Image + +
+
+ )} +
+ ) +} + +function DocStructurePanel({ structure, title }: { structure: DocStructure; title: string }) { + return ( +
+

{title}

+
+ {[ + { label: 'Pagina\'s', value: structure.pages, icon: Layers }, + { label: 'Images', value: structure.pictures, icon: Image }, + { label: 'Tables', value: structure.tables, icon: Table2 }, + { label: 'Headings', value: structure.headings }, + { label: 'Paragraphs', value: structure.paragraphs }, + { label: 'Text blocks', value: structure.text_blocks }, + ].map(({ label, value, icon: Icon }) => ( +
+ {Icon && } +

{value}

+

{label}

+
+ ))} +
+ {structure.picture_details && structure.picture_details.length > 0 && ( +
+

Images ({structure.picture_details.length})

+
+ {structure.picture_details.map((p) => ( + + #{p.index + 1} {p.label} {p.has_image ? '🖼' : ''} {p.captions > 0 ? `(${p.captions} captions)` : ''} + + ))} +
+
+ )} + {structure.table_details && structure.table_details.length > 0 && ( +
+

Tables ({structure.table_details.length})

+
+ {structure.table_details.map((t) => ( +
+ Table {t.index + 1}: {t.rows}×{t.cols} ({t.cells} cells) — {t.preview || '…'} +
+ ))} +
+
+ )} +
+ ) +} + +function DimensionCard({ dimension: d }: { dimension: Dimension }) { + return ( +
+
+ {d.label} + {d.score} +
+
+
+
+

{d.description}

+
    + {d.findings.map((f) => ( +
  • ▸{f}
    • + ))} +
+ {d.recommended_actions && d.recommended_actions.length > 0 && ( +

→ {d.recommended_actions[0]}

+ )} +
+ ) +} + +function ColumnTable({ profiles, expandedCol, onToggle }: { profiles: ColumnProfile[]; expandedCol: string | null; onToggle: (n: string | null) => void }) { + return ( +
+ + + + + + + + + {profiles.map((c) => ( + + onToggle(expandedCol === c.name ? null : c.name)}> + + + + + + + {expandedCol === c.name && ( + + + + )} + + ))} + +
ColumnTypeNull%UniqueFlags
{c.name}{c.dtype} 10 && 'font-semibold text-warning')}>{c.null_pct}%{c.unique_count.toLocaleString()}{c.quality_flags.join(', ') || '—'}
+ {c.sample_values?.length ?

Samples: {c.sample_values.join(' · ')}

: null} + {c.numeric &&

Range {c.numeric.min} – {c.numeric.max}, μ={c.numeric.mean}, {c.numeric.outliers} outliers

} + {c.text &&

Avg len {c.text.avg_length}, {c.text.empty_strings} empty strings

} + {c.top_values?.map((tv) => {tv.value} ({tv.count}))} +
+
+ ) +} + +function CheckTable({ title, rows }: { title: string; rows: { name: string; status: string; detail: string }[] }) { + return ( +
+

{title}

+ + + {rows.map((r, i) => ( + + + + + ))} + +
+ + {r.status} + + {r.name} + {r.detail}
+
+ ) +} + +function CapCard({ title, items, icon: Icon }: { title: string; items: string[]; icon: typeof Layers }) { + return ( +
+
+ + {title} +
+

{items.join(' · ')}

+
+ ) +} + +function UploadZone({ label, hint, accept, loading, onFile }: { label: string; hint: string; accept: string; loading: boolean; onFile: (f: File) => void }) { + return ( + + ) +} + +function LoadingMsg({ text }: { text: string }) { + return ( +
+ + {text} +
+ ) +} + +function StatCard({ label, value, sub, accent, warn, icon: Icon }: { label: string; value: string; sub?: string; accent?: boolean; warn?: boolean; icon?: typeof Image }) { + return ( +
+
+ {Icon && } +

{label}

+
+

{value}

+ {sub &&

{sub}

} +
+ ) +} diff --git a/ui/src/components/features/GpuMatrixPanel.tsx b/ui/src/components/features/GpuMatrixPanel.tsx new file mode 100644 index 0000000..1026a73 --- /dev/null +++ b/ui/src/components/features/GpuMatrixPanel.tsx @@ -0,0 +1,205 @@ +import { useEffect, useMemo, useState, type ReactNode } from 'react' +import { Activity, Cpu, ExternalLink, Thermometer, Zap } from 'lucide-react' +import { fetchGpu } from '../../lib/api' +import type { GpuDevice, GpuStatus } from '../../types' +import type { GpuLiveMetrics } from '../../hooks/useLiveMetrics' +import { cn } from '../../lib/utils' + +type Props = { + gpu: GpuStatus | null + live: GpuLiveMetrics + boost?: boolean + onSelectGpu?: () => void +} + +function memPct(used: number, total: number) { + if (!total) return 0 + return Math.round((used / total) * 100) +} + +function utilColor(pct: number) { + if (pct >= 75) return 'bg-danger' + if (pct >= 35) return 'bg-warning' + return 'bg-success' +} + +function GpuRow({ device, liveUtil, active }: { device: GpuDevice; liveUtil: number; active: boolean }) { + const vramPct = memPct(device.memory_used_mib, device.memory_total_mib) + const util = liveUtil ?? device.util_gpu + + return ( +
5 && 'border-docker/30 bg-docker/5', + )} + > +
+ GPU {device.index} + {util.toFixed(0)}% · {vramPct}% VRAM +
+
+ + +
+
+ + + {device.temperature_c?.toFixed(0) ?? '—'}°C + + {device.power_w?.toFixed(0) ?? '—'} W +
+
+ ) +} + +function MetricBar({ label, value, colorClass }: { label: string; value: number; colorClass: string }) { + return ( +
+ {label} +
+
+
+
+ ) +} + +export function GpuMatrixPanel({ gpu, live, boost = false, onSelectGpu }: Props) { + const [localGpu, setLocalGpu] = useState(gpu) + const [lastPoll, setLastPoll] = useState(null) + + useEffect(() => { + setLocalGpu(gpu) + }, [gpu]) + + useEffect(() => { + const poll = async () => { + const g = await fetchGpu() + if (g) { + setLocalGpu(g) + setLastPoll(new Date()) + } + } + poll() + const ms = boost ? 1000 : 3000 + const iv = setInterval(poll, ms) + return () => clearInterval(iv) + }, [boost]) + + const g = localGpu + const devices = g?.gpus || [] + const inferenceOn = g?.ok && g.inference_active + const modelLabel = g?.active_model?.replace(/\s*GPTQ$/i, '').replace(/\s*AWQ$/i, '') || 'No model' + + const avgUtil = useMemo(() => { + if (devices.length) { + const sum = devices.reduce((s, d, i) => s + (live.deviceUtils[i] ?? d.util_gpu), 0) + return sum / devices.length + } + return live.avgUtil + }, [devices, live.avgUtil, live.deviceUtils]) + + const avgVram = useMemo(() => { + if (devices.length) { + return devices.reduce((s, d) => s + memPct(d.memory_used_mib, d.memory_total_mib), 0) / devices.length + } + return live.avgVram + }, [devices, live.avgVram]) + + if (!g?.ok) { + return ( +
+

+ GPU Matrix +

+

GPU Lab offline

+
+ ) + } + + return ( +
+ + +
+ + 20 ? 'text-warning' : 'text-foreground'} /> + } + /> +
+ +
+ {devices.map((d, i) => ( + + ))} +
+ +

+ VRAM avg {avgVram.toFixed(0)}% · poll {boost ? '1s' : '3s'} + {lastPoll && ` · ${lastPoll.toLocaleTimeString()}`} +

+
+ ) +} + +function StatChip({ + label, + value, + accent, + icon, +}: { + label: string + value: string + accent?: string + icon?: ReactNode +}) { + return ( +
+

{icon}{label}

+

{value}

+
+ ) +} diff --git a/ui/src/components/features/GpuMonitor.tsx b/ui/src/components/features/GpuMonitor.tsx new file mode 100644 index 0000000..ab8b043 --- /dev/null +++ b/ui/src/components/features/GpuMonitor.tsx @@ -0,0 +1,56 @@ +import type { ReactNode } from 'react' +import { Cpu, ExternalLink, Zap } from 'lucide-react' +import type { GpuStatus } from '../../types' +import type { GpuLiveMetrics } from '../../hooks/useLiveMetrics' +import { Badge } from '../ui/Badge' + +type Props = { + gpu: GpuStatus | null + live: GpuLiveMetrics +} + +export function GpuMonitor({ gpu, live }: Props) { + if (!gpu) { + return ( +
GPU offline
+ ) + } + + const inferenceOn = gpu.inference_active && gpu.ok + const devices = gpu.gpus || [] + + return ( +
+
+
+ + GPU + + {inferenceOn ? 'ON' : 'Standby'} + +
+ + } /> + + + {devices.slice(0, 4).map((d, i) => ( + + ))} + {gpu.ui_url && ( + + {gpu.host} + + )} +
+
+ ) +} + +function Chip({ label, value, icon }: { label: string; value: string; icon?: ReactNode }) { + return ( +
+ {icon}{label} + {value} +
+ ) +} diff --git a/ui/src/components/features/InfraQuickAccess.tsx b/ui/src/components/features/InfraQuickAccess.tsx new file mode 100644 index 0000000..da41d5b --- /dev/null +++ b/ui/src/components/features/InfraQuickAccess.tsx @@ -0,0 +1,129 @@ +import { ExternalLink, RefreshCw, Terminal } from 'lucide-react' +import type { Agent, WorkloadData } from '../../types' +import { getAgentMeta } from '../../lib/agentMeta' +import { copyShellCommand, INFRA_CATALOG, type InfraNode } from '../../lib/infraCatalog' +import { cn } from '../../lib/utils' + +type Props = { + workload: WorkloadData | null + agents: Agent[] + selectedNodeId: string | null + busy: boolean + onSelectNode: (id: string) => void + onSelectAgent: (id: string) => void + onProbe: (nodeId: string) => void + onOpenTerminal: (nodeId: string) => void +} + +function zoneStats(workload: WorkloadData | null, zoneId: string) { + const z = workload?.zones?.find((x) => x.id === zoneId) + if (!z) return null + return `${z.running}/${z.total}` +} + +export function InfraQuickAccess({ + workload, + agents, + selectedNodeId, + busy, + onSelectNode, + onSelectAgent, + onProbe, + onOpenTerminal, +}: Props) { + const handleShell = async (node: InfraNode) => { + await copyShellCommand(node.ssh) + onSelectNode(node.id) + onOpenTerminal(node.id) + onProbe(node.id) + } + + return ( +
+
+
+

Infrastructure & Apps

+

Klik voor inspector · Shell kopieert SSH en opent live terminal · UI opent de applicatie

+
+ {workload && ( + + {workload.totals.apps_running}/{workload.totals.apps_total} containers + + )} +
+
+ {INFRA_CATALOG.map((node) => { + const Icon = node.icon + const agent = agents.find((a) => a.id === node.agentId) + const meta = agent ? getAgentMeta(agent.id) : null + const active = selectedNodeId === node.id || node.topoIds.includes(selectedNodeId || '') + const stats = zoneStats(workload, node.zone) + return ( +
+ + + {agent && meta && ( + + )} + +
+ + + {node.apps.slice(0, 2).map((app) => ( + + {app.label} + + ))} +
+
+ ) + })} +
+
+ ) +} diff --git a/ui/src/components/features/InspectorPanel.tsx b/ui/src/components/features/InspectorPanel.tsx new file mode 100644 index 0000000..f9adbfc --- /dev/null +++ b/ui/src/components/features/InspectorPanel.tsx @@ -0,0 +1,294 @@ +import { useState, type FormEvent } from 'react' +import { ExternalLink, RefreshCw, Terminal, X } from 'lucide-react' +import type { Agent, FeedEntry, GpuStatus, NodeDetail, TerminalLine, TopologyNode, WorkloadData } from '../../types' +import { AGENT_NODE } from '../../lib/constants' +import { getAgentMeta } from '../../lib/agentMeta' +import { copyShellCommand, resolveInfraNode } from '../../lib/infraCatalog' +import { Button } from '../ui/Button' +import { Card, CardDescription, CardTitle } from '../ui/Card' +import { Input } from '../ui/Input' +import { cn } from '../../lib/utils' + +type Tab = 'overview' | 'apps' | 'terminal' + +type Props = { + node: TopologyNode | null + nodeDetail: NodeDetail | null + agent: Agent | null + agents: Agent[] + workload: WorkloadData | null + gpu: GpuStatus | null + feed: FeedEntry[] + lines: TerminalLine[] + busy: boolean + onProbe: () => void + onAsk: (message: string) => void + onSelectAgent: (id: string) => void + onSendPrompt: (message: string, agentId?: string) => void + onClear: () => void + onOpenTerminal: (nodeId: string) => void + onProbeNodeId: (nodeId: string) => void +} + +export function InspectorPanel({ + node, + nodeDetail, + agent, + agents, + workload, + gpu, + feed, + lines, + busy, + onProbe, + onAsk, + onSelectAgent, + onSendPrompt, + onClear, + onOpenTerminal, + onProbeNodeId, +}: Props) { + const [tab, setTab] = useState('overview') + const [input, setInput] = useState('') + const [shellCopied, setShellCopied] = useState(false) + const d = nodeDetail || node + const infra = resolveInfraNode(d?.id || null) + const linkedAgent = d + ? agents.find((a) => a.id === d.id || AGENT_NODE[a.id] === d.id || a.id === infra?.agentId || a.zone === d.id) + : agent + + const submit = (e: FormEvent) => { + e.preventDefault() + if (!input.trim() || busy) return + onAsk(input.trim()) + setInput('') + setTab('terminal') + if (d?.id) onOpenTerminal(d.id) + } + + const runShell = async () => { + if (!infra) return + await copyShellCommand(infra.ssh) + setShellCopied(true) + setTimeout(() => setShellCopied(false), 2000) + onOpenTerminal(infra.id) + onProbeNodeId(infra.id) + setTab('terminal') + } + + return ( + + ) +} + +function Stat({ label, value, ok }: { label: string; value: string; ok?: boolean }) { + return ( +
+

{label}

+

{value}

+
+ ) +} diff --git a/ui/src/components/features/KnowledgeChatView.tsx b/ui/src/components/features/KnowledgeChatView.tsx new file mode 100644 index 0000000..6b8d82d --- /dev/null +++ b/ui/src/components/features/KnowledgeChatView.tsx @@ -0,0 +1,358 @@ +import { useCallback, useEffect, useRef, useState } from 'react' +import { BookOpen, FileText, Loader2, MessageSquare, RefreshCw, RotateCcw, Send, Upload } from 'lucide-react' +import { cn } from '../../lib/utils' +import { subTabActive, subTabIdle } from '../../lib/tabActive' + +type Collection = { name: string; documents: number; files?: number; filenames?: string[] } +type StoredDoc = { + id: string + filename: string + collection: string + chunks: number + characters?: number + ingested_at: string + bytes?: number +} +type Source = { source?: string; chunk?: number; preview?: string } +type ChatMsg = { role: 'user' | 'assistant'; content: string; sources?: Source[] } + +type Health = { ok: boolean; chroma: boolean; docling: boolean; llm: boolean; embed_model?: string } + +type Props = { onGpuActivity?: (active: boolean) => void } + +export function KnowledgeChatView({ onGpuActivity }: Props = {}) { + const [health, setHealth] = useState(null) + const [collections, setCollections] = useState([]) + const [collection, setCollection] = useState('default') + const [newCol, setNewCol] = useState('') + const [messages, setMessages] = useState([]) + const [input, setInput] = useState('') + const [loading, setLoading] = useState(false) + const [ingesting, setIngesting] = useState(false) + const [error, setError] = useState(null) + const [storedDocs, setStoredDocs] = useState([]) + const [selectedDocId, setSelectedDocId] = useState(null) + const [summarizing, setSummarizing] = useState(false) + const [reindexing, setReindexing] = useState(null) + const bottomRef = useRef(null) + + const loadMeta = useCallback(async () => { + try { + const [h, c, d] = await Promise.all([ + fetch('/rag/health'), + fetch('/rag/collections'), + fetch('/rag/documents'), + ]) + if (h.ok) setHealth(await h.json()) + if (c.ok) { + const j = await c.json() + setCollections(j.collections || []) + } + if (d.ok) { + const j = await d.json() + setStoredDocs(j.documents || []) + } + } catch { + setHealth(null) + } + }, []) + + useEffect(() => { + loadMeta() + }, [loadMeta]) + + useEffect(() => { + bottomRef.current?.scrollIntoView({ behavior: 'smooth' }) + }, [messages, loading]) + + useEffect(() => { + onGpuActivity?.(loading || ingesting || summarizing || reindexing !== null) + }, [loading, ingesting, summarizing, reindexing, onGpuActivity]) + + const onIngest = async (file: File) => { + setIngesting(true) + setError(null) + const fd = new FormData() + fd.append('file', file) + fd.append('collection', collection) + try { + const r = await fetch('/rag/ingest', { method: 'POST', body: fd }) + const j = await r.json() + if (!r.ok || !j.ok) { + setError(j.error || 'Ingest failed') + return + } + setMessages((m) => [...m, { + role: 'assistant', + content: j.duplicate + ? `Already indexed: ${j.filename} (${j.chunks} chunks). You can chat immediately — no re-upload needed.` + : `Indexed ${j.filename} → collection "${j.collection}" — ${j.chunks} chunks (${j.characters?.toLocaleString()} chars). Stored permanently.`, + }]) + loadMeta() + } catch { + setError('RAG API unavailable') + } finally { + setIngesting(false) + } + } + + const onSummarize = async (doc: StoredDoc) => { + setSummarizing(true) + setError(null) + setSelectedDocId(doc.id) + setCollection(doc.collection) + setMessages((m) => [...m, { role: 'user', content: `Summarize: ${doc.filename}` }]) + try { + const r = await fetch('/rag/summarize', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ collection: doc.collection, doc_id: doc.id }), + }) + const j = await r.json() + if (!r.ok || !j.ok) { + setError(j.error || 'Summarize failed') + return + } + setMessages((m) => [...m, { + role: 'assistant', + content: `Summary of ${j.filename} (${j.characters?.toLocaleString()} chars):\n\n${j.summary}`, + }]) + } catch { + setError('Summarize request failed') + } finally { + setSummarizing(false) + } + } + + const onReindex = async (doc: StoredDoc) => { + setReindexing(doc.id) + setError(null) + try { + const r = await fetch(`/rag/documents/${doc.id}/reindex`, { method: 'POST' }) + const j = await r.json() + if (!r.ok || !j.ok) { + setError(j.error || 'Re-index failed') + return + } + setMessages((m) => [...m, { + role: 'assistant', + content: `Re-indexed ${j.filename}: ${j.chunks} clean text chunks (${j.characters?.toLocaleString()} chars). You can now chat and summarize.`, + }]) + loadMeta() + } catch { + setError('Re-index request failed') + } finally { + setReindexing(null) + } + } + + const onSend = async () => { + const msg = input.trim() + if (!msg || loading) return + setInput('') + setError(null) + setMessages((m) => [...m, { role: 'user', content: msg }]) + setLoading(true) + try { + const r = await fetch('/rag/chat', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ message: msg, collection, top_k: 5 }), + }) + const j = await r.json() + if (!r.ok || !j.ok) { + setError(j.error || 'Chat failed') + setLoading(false) + return + } + setMessages((m) => [...m, { role: 'assistant', content: j.answer, sources: j.sources }]) + } catch { + setError('Failed to reach RAG / LLM service') + } finally { + setLoading(false) + } + } + + const createCollection = async () => { + if (!newCol.trim()) return + const fd = new FormData() + fd.append('name', newCol.trim()) + await fetch('/rag/collections', { method: 'POST', body: fd }) + setCollection(newCol.trim()) + setNewCol('') + loadMeta() + } + + const doclingUiUrl = `${window.location.protocol}//${window.location.hostname}:5001/ui/` + + return ( +
+
+
+
+

Knowledge Chat (RAG)

+

+ LangChain + ChromaDB — chat with your ingested documents via Llama 70B +

+
+
+ {health && ( + <> + + + + + )} + +
+
+
+ +
+ + +
+
+ {messages.length === 0 && ( +
+ +

Upload once — documents stay in ChromaDB. Ask anytime without re-uploading.

+

Example: "What maturity gaps exist in the customer dataset?"

+
+ )} + {messages.map((m, i) => ( +
+

{m.content}

+ {m.sources && m.sources.length > 0 && ( +
+

Sources

+ {m.sources.map((s, j) => ( +

+ {s.source} · chunk {s.chunk}: {s.preview?.slice(0, 120)}… +

+ ))} +
+ )} +
+ ))} + {loading && ( +
+ Retrieving context & generating answer… +
+ )} + {error &&

{error}

} +
+
+ +
+ setInput(e.target.value)} + onKeyDown={(e) => e.key === 'Enter' && !e.shiftKey && (e.preventDefault(), onSend())} + placeholder="Ask a question about your ingested data…" + className="min-w-0 flex-1 rounded-lg border border-border bg-surface-overlay px-3 py-2 text-[12px]" + disabled={loading} + /> + +
+
+
+
+ ) +} + +function StatusPill({ ok, label }: { ok: boolean; label: string }) { + return ( + + {label} {ok ? '●' : '○'} + + ) +} diff --git a/ui/src/components/features/PlatformTopology.tsx b/ui/src/components/features/PlatformTopology.tsx new file mode 100644 index 0000000..055b1c8 --- /dev/null +++ b/ui/src/components/features/PlatformTopology.tsx @@ -0,0 +1,393 @@ +import { useEffect, useMemo, useState } from 'react' +import { Box } from 'lucide-react' +import type { AgentAnim, WorkloadData } from '../../types' +import { Badge } from '../ui/Badge' +import { cn } from '../../lib/utils' + +/* ── Pipeline model ─────────────────────────────────────────────── */ + +type TopoNode = { id: string; label: string; sub: string; metricKey: string } + +type TopoStage = { + id: string + num: number + title: string + subtitle: string + accent: string + nodes: TopoNode[] +} + +type FlowKind = 'orchestration' | 'cdc' | 'stream' | 'etl' | 'query' | 'serve' + +type FlowEdge = { + from: string + to: string + kind: FlowKind + label: string +} + +const STAGES: TopoStage[] = [ + { + id: 'sources', num: 1, title: 'SOURCES', subtitle: 'Operational databases', accent: 'topo-stage-col--sources', + nodes: [ + { id: 'postgresql', label: 'PostgreSQL', sub: 'OLTP · primary', metricKey: 'postgresql' }, + { id: 'mysql', label: 'MySQL', sub: 'Replica set', metricKey: 'mysql' }, + { id: 'mongodb', label: 'MongoDB', sub: 'Document store', metricKey: 'mongodb' }, + { id: 'cassandra', label: 'Cassandra', sub: 'Wide-column', metricKey: 'cassandra' }, + ], + }, + { + id: 'ingestion', num: 2, title: 'INGESTION & STREAMING', subtitle: 'CDC · event bus · orchestration', accent: 'topo-stage-col--ingestion', + nodes: [ + { id: 'debezium', label: 'Debezium', sub: 'CDC connectors', metricKey: 'debezium' }, + { id: 'kafka', label: 'Apache Kafka', sub: 'Event bus', metricKey: 'kafka' }, + { id: 'airflow', label: 'Apache Airflow', sub: 'Daily Python DAGs · source sync', metricKey: 'airflow' }, + ], + }, + { + id: 'compute', num: 3, title: 'COMPUTE', subtitle: 'Processing & query', accent: 'topo-stage-col--compute', + nodes: [ + { id: 'spark', label: 'Apache Spark', sub: 'Batch / micro-batch', metricKey: 'spark' }, + { id: 'trino', label: 'Trino', sub: 'Distributed SQL', metricKey: 'trino' }, + ], + }, + { + id: 'storage', num: 4, title: 'STORAGE', subtitle: 'Lakehouse layer', accent: 'topo-stage-col--storage', + nodes: [ + { id: 'iceberg', label: 'Iceberg Tables', sub: 'Open table format', metricKey: 'iceberg' }, + { id: 's3', label: 'Dell ECS S3', sub: 'Object scale', metricKey: 's3' }, + ], + }, + { + id: 'consumers', num: 5, title: 'CONSUMERS', subtitle: 'Analytics & AI', accent: 'topo-stage-col--consumers', + nodes: [ + { id: 'bi', label: 'BI / Reporting', sub: 'Dashboards', metricKey: 'bi' }, + { id: 'jupyter', label: 'Jupyter Notebooks', sub: 'Data science', metricKey: 'jupyter' }, + { id: 'llm', label: 'GenAI LLM', sub: 'vLLM inference', metricKey: 'llm' }, + ], + }, +] + +/** Full data-foundation flows — Airflow daily Python generation + CDC stream + lakehouse */ +const FLOW_EDGES: FlowEdge[] = [ + // Airflow orchestrates daily Python jobs on every source + { from: 'airflow', to: 'postgresql', kind: 'orchestration', label: 'Daily Python gen' }, + { from: 'airflow', to: 'mysql', kind: 'orchestration', label: 'Daily Python gen' }, + { from: 'airflow', to: 'mongodb', kind: 'orchestration', label: 'Daily Python gen' }, + { from: 'airflow', to: 'cassandra', kind: 'orchestration', label: 'Daily Python gen' }, + // CDC capture from sources + { from: 'postgresql', to: 'debezium', kind: 'cdc', label: 'CDC' }, + { from: 'mysql', to: 'debezium', kind: 'cdc', label: 'CDC' }, + { from: 'mongodb', to: 'debezium', kind: 'cdc', label: 'CDC' }, + { from: 'cassandra', to: 'debezium', kind: 'cdc', label: 'CDC' }, + // Streaming bus + { from: 'debezium', to: 'kafka', kind: 'stream', label: 'Events' }, + { from: 'airflow', to: 'kafka', kind: 'orchestration', label: 'DAG trigger' }, + // ETL compute + { from: 'kafka', to: 'spark', kind: 'etl', label: 'Micro-batch' }, + { from: 'airflow', to: 'spark', kind: 'orchestration', label: 'Pipeline DAG' }, + { from: 'spark', to: 'iceberg', kind: 'etl', label: 'Lake write' }, + { from: 'spark', to: 's3', kind: 'etl', label: 'Object export' }, + // Query & serve + { from: 'iceberg', to: 'trino', kind: 'query', label: 'SQL' }, + { from: 'trino', to: 'bi', kind: 'serve', label: 'Reports' }, + { from: 'iceberg', to: 'jupyter', kind: 'serve', label: 'Notebooks' }, + { from: 's3', to: 'jupyter', kind: 'serve', label: 'Datasets' }, + { from: 'trino', to: 'llm', kind: 'serve', label: 'RAG context' }, + { from: 's3', to: 'llm', kind: 'serve', label: 'Model artifacts' }, +] + +const STAGE_BADGE: Record = { + sources: 'border-emerald-400/50 bg-emerald-500/20 text-emerald-300', + ingestion: 'border-cyan-400/50 bg-cyan-500/20 text-cyan-300', + compute: 'border-violet-400/50 bg-violet-500/20 text-violet-300', + storage: 'border-blue-400/50 bg-blue-500/20 text-blue-300', + consumers: 'border-amber-400/50 bg-amber-500/20 text-amber-300', +} + +const FLOW_LEGEND: { kind: FlowKind; label: string; color: string }[] = [ + { kind: 'orchestration', label: 'Airflow orchestration', color: '#f59e0b' }, + { kind: 'cdc', label: 'CDC capture', color: '#22d3ee' }, + { kind: 'stream', label: 'Event stream', color: '#38bdf8' }, + { kind: 'etl', label: 'ETL / compute', color: '#a78bfa' }, + { kind: 'query', label: 'SQL query', color: '#818cf8' }, + { kind: 'serve', label: 'Consumption', color: '#34d399' }, +] + +const EDGE_CLASS: Record = { + orchestration: 'topo-edge-orchestration', + cdc: 'topo-edge-cdc', + stream: 'topo-edge-stream', + etl: 'topo-edge-etl', + query: 'topo-edge-query', + serve: 'topo-edge-serve', +} + +const PARTICLE_FILL: Record = { + orchestration: '#fbbf24', + cdc: '#22d3ee', + stream: '#38bdf8', + etl: '#c4b5fd', + query: '#818cf8', + serve: '#34d399', +} + +const NODE_CLICK_MAP: Record = { + postgresql: 'src-postgres', mysql: 'src-mysql', mongodb: 'src-mongo', cassandra: 'src-cassandra', + debezium: 'cdc-postgres', kafka: 'stream-kafka', airflow: 'src-airflow', spark: 'stream-spark', + trino: 'query-trino', iceberg: 'lake-iceberg', s3: 'lake-s3', bi: 'cons-bi', + jupyter: 'cons-notebooks', llm: 'cons-ml', +} + +const NODE_POS: Record = {} +STAGES.forEach((stage, col) => { + stage.nodes.forEach((node, row) => { + NODE_POS[node.id] = { col, row, rows: stage.nodes.length } + }) +}) + +function nodeCoords(col: number, row: number, rows: number) { + const colW = 100 / 5 + const yPad = 8 + const ySpan = 84 + const y = yPad + ((row + 0.5) / rows) * ySpan + return { + inX: col * colW + colW * 0.08, + outX: col * colW + colW * 0.92, + y, + } +} + +/** Curved path — arcs upward for backward (orchestration) flows */ +function flowPath(x1: number, y1: number, x2: number, y2: number, backward = false) { + if (backward || x2 < x1 - 2) { + const arcY = Math.min(y1, y2) - 14 + return `M ${x1} ${y1} C ${x1} ${arcY}, ${x2} ${arcY}, ${x2} ${y2}` + } + const mx = (x1 + x2) / 2 + return `M ${x1} ${y1} C ${mx} ${y1}, ${mx} ${y2}, ${x2} ${y2}` +} + +type MetricState = Record + +function seedMetrics(): MetricState { + return { + postgresql: '12.4k rows/s', mysql: '8.1k rows/s', mongodb: '2.3k docs/s', cassandra: '5.6k ops/s', + debezium: '4 connectors active', kafka: '142 MB/s', airflow: '18 DAGs · daily 02:00 UTC', + spark: '6 executors live', trino: '3 queries active', iceberg: '847 tables · 2.1 TB', s3: '14.2 TB stored', + bi: '26 dashboards', jupyter: '12 kernels active', llm: 'Checking…', + } +} + +function formatLlmLabel(model?: string | null): string { + if (!model) return 'GenAI LLM' + return model.replace(/\s*GPTQ$/i, '').replace(/\s*AWQ$/i, '').trim() +} + +function formatLlmMetric(workload: WorkloadData | null): string { + const gpu = workload?.gpu + if (!gpu?.model) return 'Connecting…' + if (!gpu.inference_active) return 'Offline' + const gpus = gpu.gpus || [] + const util = gpu.avg_util ?? (gpus.length ? gpus.reduce((s, g) => s + g.util_gpu, 0) / gpus.length : 0) + const vram = gpus.length + ? gpus.reduce((s, g) => s + (g.memory_used_mib / Math.max(g.memory_total_mib, 1)) * 100, 0) / gpus.length + : 0 + if (util >= 1) return `${util.toFixed(0)}% GPU · live` + if (vram >= 50) return `Loaded · ${vram.toFixed(0)}% VRAM` + return 'Inference active' +} + +function jitterMetric(key: string, current: string, workload: WorkloadData | null): string { + if (key === 'llm') return formatLlmMetric(workload) + const n = () => (Math.random() - 0.5) * 2 + const fns: Record string> = { + postgresql: () => `${(12.4 + n() * 0.8).toFixed(1)}k rows/s`, + mysql: () => `${(8.1 + n() * 0.6).toFixed(1)}k rows/s`, + mongodb: () => `${(2.3 + n() * 0.3).toFixed(1)}k docs/s`, + cassandra: () => `${(5.6 + n() * 0.5).toFixed(1)}k ops/s`, + debezium: () => `${Math.max(3, Math.round(4 + n()))} connectors active`, + kafka: () => `${Math.max(80, Math.round(142 + n() * 18))} MB/s`, + airflow: () => `${Math.max(12, Math.round(18 + n() * 2))} DAGs · daily 02:00 UTC`, + spark: () => `${Math.max(4, Math.round(6 + n()))} executors live`, + trino: () => `${Math.max(1, Math.round(3 + n()))} queries active`, + iceberg: () => `${Math.round(847 + n() * 5)} tables · ${(2.1 + n() * 0.05).toFixed(1)} TB`, + s3: () => `${(14.2 + n() * 0.08).toFixed(1)} TB stored`, + bi: () => `${Math.max(20, Math.round(26 + n() * 2))} dashboards`, + jupyter: () => `${Math.max(8, Math.round(12 + n() * 2))} kernels active`, + } + return fns[key]?.() ?? current +} + +type Props = { + workload: WorkloadData | null + animations: Record + selectedNodeId: string | null + onNodeClick: (nodeId: string) => void +} + +export function PlatformTopology({ workload, animations, selectedNodeId, onNodeClick }: Props) { + const [metrics, setMetrics] = useState(seedMetrics) + + const llmLabel = formatLlmLabel(workload?.gpu?.model) + + const pipelineActive = workload?.totals?.pipeline_active ?? true + const anyBusy = useMemo( + () => Object.values(animations).some((a) => a.state !== 'idle'), + [animations], + ) + + const edgesLive = pipelineActive || anyBusy + + useEffect(() => { + setMetrics((prev) => ({ ...prev, llm: formatLlmMetric(workload) })) + }, [workload?.gpu?.model, workload?.gpu?.inference_active, workload?.gpu?.avg_util, workload?.gpu?.gpus]) + + useEffect(() => { + const iv = setInterval(() => { + setMetrics((prev) => { + const next = { ...prev } + for (const k of Object.keys(next)) next[k] = jitterMetric(k, prev[k], workload) + return next + }) + }, 2200) + return () => clearInterval(iv) + }, [workload]) + + const resolvedSel = selectedNodeId + ? Object.entries(NODE_CLICK_MAP).find(([, v]) => v === selectedNodeId)?.[0] ?? null + : null + + return ( +
+
+
+
+ + + +
+

Data Platform Topology

+

+ Airflow daily Python → CDC → stream → lakehouse → consumers +

+
+
+
+ + {pipelineActive ? 'Pipeline active' : 'Degraded'} + + {workload?.totals?.connectors ?? 4} CDC + {FLOW_EDGES.length} flows +
+
+
+ {FLOW_LEGEND.map((item) => ( + + + {item.label} + + ))} +
+
+ +
+ + + + + + + + + {FLOW_EDGES.map((edge, i) => { + const pa = NODE_POS[edge.from] + const pb = NODE_POS[edge.to] + if (!pa || !pb) return null + const a = nodeCoords(pa.col, pa.row, pa.rows) + const b = nodeCoords(pb.col, pb.row, pb.rows) + const backward = edge.kind === 'orchestration' && pb.col < pa.col + const fromX = backward ? a.inX + (a.outX - a.inX) * 0.15 : a.outX + const toX = backward ? b.outX - (b.outX - b.inX) * 0.15 : b.inX + const d = flowPath(fromX, a.y, toX, b.y, backward) + const live = edgesLive + const dur = 1.8 + (i % 5) * 0.35 + return ( + + + + {live && ( + <> + + + + + + + + )} + + ) + })} + + +
+ {STAGES.map((stage) => ( +
+
+
+ + 0{stage.num} + +
+

{stage.title}

+

{stage.subtitle}

+
+
+
+
+ {stage.nodes.map((node) => { + const label = node.id === 'llm' ? llmLabel : node.label + const sub = node.id === 'llm' + ? (workload?.gpu?.inference_active ? 'vLLM · live' : 'vLLM inference') + : node.sub + return ( + + ) + })} +
+
+ ))} +
+
+
+ ) +} diff --git a/ui/src/components/features/PresentationView.tsx b/ui/src/components/features/PresentationView.tsx new file mode 100644 index 0000000..2b058b9 --- /dev/null +++ b/ui/src/components/features/PresentationView.tsx @@ -0,0 +1,218 @@ +import { useCallback, useEffect, useState } from 'react' +import { ExternalLink, FileUp, Monitor, Upload } from 'lucide-react' +import { ArchitectureDiagram } from './ArchitectureDiagram' +import type { PresentationData, PresentationSlide } from '../../types' +import { cn } from '../../lib/utils' +import { subTabActive, subTabIdle } from '../../lib/tabActive' + +type DeckSource = 'live' | 'data-maturity' | 'atc-platform' | string + +const KIND_STYLES: Record = { + hero: 'from-blue-600/25 via-violet-600/20 to-emerald-600/15', + narrative: 'from-slate-600/15 to-blue-600/15', + topology: 'from-cyan-600/20 to-blue-800/15', + zone: 'from-amber-600/15 to-orange-600/10', + gpu: 'from-emerald-600/20 to-green-800/15', + agents: 'from-fuchsia-600/15 to-pink-600/10', + cta: 'from-blue-600/15 to-violet-600/20', + upload: 'from-indigo-600/15 to-purple-600/10', + command: 'from-sky-600/15 to-blue-600/10', + architecture: 'from-teal-600/15 to-cyan-600/10', +} + +async function fetchDeck(id: DeckSource): Promise { + const ctrl = new AbortController() + const timeout = id === 'live' ? 45000 : 10000 + const timer = setTimeout(() => ctrl.abort(), timeout) + try { + const url = id === 'live' ? '/api/presentation' : `/api/presentation/decks/${id}` + const r = await fetch(url, { signal: ctrl.signal }) + if (!r.ok) return null + return (await r.json()) as PresentationData + } catch { + return null + } finally { + clearTimeout(timer) + } +} + +export function PresentationView() { + const [source, setSource] = useState('live') + const [data, setData] = useState(null) + const [slideIdx, setSlideIdx] = useState(0) + const [loading, setLoading] = useState(true) + const [error, setError] = useState(null) + const [uploading, setUploading] = useState(false) + const [uploadMsg, setUploadMsg] = useState(null) + const [customDecks, setCustomDecks] = useState<{ id: string; title: string }[]>([]) + + const load = useCallback(async (deckId: DeckSource) => { + setLoading(true) + setError(null) + const d = await fetchDeck(deckId) + if (d && d.slides?.length) { + setData(d) + setSlideIdx(0) + setLoading(false) + return + } + if (deckId === 'live') { + const fallback = await fetchDeck('data-maturity') + if (fallback?.slides?.length) { + setData(fallback) + setSlideIdx(0) + setError('Live deck timeout — showing Data Maturity template. Click Refresh for live cluster data.') + setLoading(false) + return + } + } + setData(null) + setError('Could not load presentation.') + setLoading(false) + }, []) + + useEffect(() => { + load(source) + fetch('/api/presentation/decks') + .then((r) => r.json()) + .then((j) => { + const uploaded = (j.uploaded || []).map((d: { id: string; title: string }) => ({ id: d.id, title: d.title })) + setCustomDecks(uploaded) + }) + .catch(() => {}) + }, [source, load]) + + useEffect(() => { + const onKey = (e: KeyboardEvent) => { + const n = data?.slides.length || 1 + if (e.key === 'ArrowRight' || e.key === ' ') { e.preventDefault(); setSlideIdx((i) => Math.min(n - 1, i + 1)) } + if (e.key === 'ArrowLeft') setSlideIdx((i) => Math.max(0, i - 1)) + if (e.key === 'f' || e.key === 'F') document.documentElement.requestFullscreen?.() + } + window.addEventListener('keydown', onKey) + return () => window.removeEventListener('keydown', onKey) + }, [data?.slides.length]) + + const slides = data?.slides || [] + const slide: PresentationSlide | undefined = slides[slideIdx] + + const exportHtml = () => { + const id = source === 'live' ? 'live' : source + window.open(`/api/presentation/decks/${id}/html`, '_blank') + } + + const onUpload = async (file: File) => { + setUploading(true) + setUploadMsg(null) + const fd = new FormData() + fd.append('file', file) + try { + const r = await fetch('/api/presentation/upload', { method: 'POST', body: fd }) + const j = await r.json() + if (j.ok && j.deck) { + setCustomDecks((prev) => [{ id: j.deck.id, title: j.deck.title }, ...prev]) + setSource(j.deck.id) + setUploadMsg(`✓ ${j.deck.slide_count} slides loaded from ${file.name}`) + } else { + setUploadMsg(j.error || 'Upload failed') + } + } catch { + setUploadMsg('Upload failed — check connection') + } finally { + setUploading(false) + } + } + + const tabs: { id: DeckSource; label: string }[] = [ + { id: 'live', label: 'Live Cluster' }, + { id: 'stack-architecture', label: 'Stack Architecture' }, + { id: 'data-maturity', label: 'Data Maturity' }, + { id: 'atc-platform', label: 'ATC Platform' }, + ...customDecks.map((d) => ({ id: d.id, label: d.title.slice(0, 18) })), + ] + + return ( +
+
+
+

Presentation

+

+ Live cluster · HTML templates · PPT upload (converts via python-pptx + Docling) +

+
+
+ + DQ Portal + + + Docling + + + +
+
+ +
+ {tabs.map((t) => ( + + ))} + +
+ + {uploadMsg &&

{uploadMsg}

} + {error &&

{error}

} + + {loading ? ( +
+ +

Loading presentation{source === 'live' ? ' (live cluster snapshot, ~15 sec)' : '…'}

+
+ ) : !slide ? ( +
+ +
+ ) : ( + <> +
+
+

{slide.kind || 'slide'} · {slideIdx + 1}/{slides.length}

+

{slide.title}

+ {slide.subtitle &&

{slide.subtitle}

} + {'animation' in slide && slide.animation && ( + + )} +
    + {(slide.bullets || []).map((b: string) => ( +
  • ▸{b}
    • + ))} +
+
+
+
+ +
+ {slides.map((_: PresentationSlide, i: number) => ( +
+ +
+ + )} +
+ ) +} diff --git a/ui/src/components/features/StorageView.tsx b/ui/src/components/features/StorageView.tsx new file mode 100644 index 0000000..a7f0279 --- /dev/null +++ b/ui/src/components/features/StorageView.tsx @@ -0,0 +1,197 @@ +import { useCallback, useEffect, useState } from 'react' +import { ChevronRight, Database, Download, ExternalLink, Folder, HardDrive, Loader2, RefreshCw } from 'lucide-react' +import { cn } from '../../lib/utils' +import { subTabActive, subTabIdle } from '../../lib/tabActive' + +type Bucket = { name: string; created?: string; has_objects?: boolean } +type S3Item = { type: string; name?: string; prefix?: string; key?: string; size_human?: string; modified?: string } + +export function StorageView() { + const [health, setHealth] = useState<{ ok: boolean; endpoint?: string; bucket_names?: string[]; error?: string } | null>(null) + const [buckets, setBuckets] = useState([]) + const [bucket, setBucket] = useState(null) + const [prefix, setPrefix] = useState('') + const [folders, setFolders] = useState([]) + const [objects, setObjects] = useState([]) + const [loading, setLoading] = useState(false) + const [error, setError] = useState(null) + + const loadBuckets = useCallback(async () => { + setLoading(true) + setError(null) + try { + const [h, b] = await Promise.all([ + fetch('/api/storage/s3/health'), + fetch('/api/storage/s3/buckets'), + ]) + if (h.ok) setHealth(await h.json()) + if (b.ok) { + const j = await b.json() + setBuckets(j.buckets || []) + if (!bucket && j.buckets?.length) setBucket(j.buckets[0].name) + } else { + setError('Failed to load buckets') + } + } catch { + setError('S3 API unavailable') + } finally { + setLoading(false) + } + }, [bucket]) + + const loadObjects = useCallback(async (b: string, p: string) => { + setLoading(true) + setError(null) + try { + const r = await fetch(`/api/storage/s3/buckets/${encodeURIComponent(b)}/objects?prefix=${encodeURIComponent(p)}`) + const j = await r.json() + if (!r.ok || !j.ok) { + setError(j.error || 'List failed') + return + } + setFolders(j.folders || []) + setObjects(j.objects || []) + } catch { + setError('Failed to list objects') + } finally { + setLoading(false) + } + }, []) + + useEffect(() => { + loadBuckets() + }, [loadBuckets]) + + useEffect(() => { + if (bucket) loadObjects(bucket, prefix) + }, [bucket, prefix, loadObjects]) + + const crumbs = prefix ? prefix.split('/').filter(Boolean) : [] + + return ( +
+
+
+

+ + ObjectScale S3 Storage +

+

+ Dell ECS · {health?.endpoint || '10.0.20.111:9020'} · live bucket browser +

+
+
+ + Open Jupyter + + +
+
+ +
+ + +
+ {bucket && ( + + )} + + {loading && ( +

+ Loading… +

+ )} + {error &&

{error}

} + +
+ + + + + + + + + + {folders.map((f) => ( + + + + + + ))} + {objects.map((o) => ( + + + + + + + ))} + +
NameSizeModified +
+ + —— +
{o.name || o.key}{o.size_human}{o.modified?.slice(0, 19) || '—'} + {o.key && bucket && ( + + + + )} +
+ {!loading && folders.length === 0 && objects.length === 0 && bucket && ( +

This prefix is empty.

+ )} +
+
+
+
+ ) +} diff --git a/ui/src/components/features/TerminalDock.tsx b/ui/src/components/features/TerminalDock.tsx new file mode 100644 index 0000000..7c41461 --- /dev/null +++ b/ui/src/components/features/TerminalDock.tsx @@ -0,0 +1,72 @@ +import { useEffect, useRef } from 'react' +import { Terminal } from 'lucide-react' +import type { TerminalLine } from '../../types' +import { resolveInfraNode } from '../../lib/infraCatalog' +import { cn } from '../../lib/utils' + +type Props = { + subjectId: string | null + subjectLabel: string + lines: TerminalLine[] + busy: boolean + expanded: boolean + onToggle: () => void +} + +const LEVEL: Record = { + info: 'text-foreground-muted', + ok: 'text-success', + warn: 'text-warning', + err: 'text-danger', + cmd: 'text-docker', + llm: 'text-violet-400', +} + +export function TerminalDock({ subjectId, subjectLabel, lines, busy, expanded, onToggle }: Props) { + const bottomRef = useRef(null) + const infra = resolveInfraNode(subjectId) + + useEffect(() => { + if (expanded) bottomRef.current?.scrollIntoView({ behavior: 'smooth' }) + }, [lines, busy, expanded]) + + return ( +
+ + + {expanded && ( +
+ {infra && ( +

+ $ {infra.ssh} (gekopieerd bij Shell-knop) +

+ )} + {lines.length === 0 && ( +

Selecteer een node of agent · klik Shell of Probe om output te zien

+ )} + {lines.map((line) => ( +
+ + {line.ts ? new Date(line.ts).toLocaleTimeString('en-US', { hour12: false }) : ''} + {' '} + [{line.phase}] {line.text} +
+ ))} + {busy && █} +
+
+ )} +
+ ) +} diff --git a/ui/src/components/layout/SideNav.tsx b/ui/src/components/layout/SideNav.tsx new file mode 100644 index 0000000..85c710f --- /dev/null +++ b/ui/src/components/layout/SideNav.tsx @@ -0,0 +1,162 @@ +import { DatabaseZap, HardDrive, LayoutDashboard, MessageSquare, Presentation, ShieldCheck } from 'lucide-react' +import type { Agent, AgentAnim, GpuStatus } from '../../types' +import type { GpuLiveMetrics } from '../../hooks/useLiveMetrics' +import { getAgentMeta } from '../../lib/agentMeta' +import { cn } from '../../lib/utils' +import { viewTabActive, viewTabIdle } from '../../lib/tabActive' +import { GpuMatrixPanel } from '../features/GpuMatrixPanel' + +type MainView = 'platform' | 'presentation' | 'dataquality' | 'knowledge' | 'storage' | 'approvals' + +type Props = { + agents: Agent[] + animations: Record + gpu: GpuStatus | null + gpuLive: GpuLiveMetrics + gpuBoost?: boolean + selectedAgentId: string | null + selectedNodeId: string | null + mainView: MainView + approvalCount: number + agentsLoading: boolean + onSetMainView: (view: MainView) => void + onOpenApprovals: () => void + onSelectAgent: (id: string) => void + onSelectZone: (id: string) => void +} + +const VIEWS: { id: MainView; label: string; icon: typeof LayoutDashboard }[] = [ + { id: 'platform', label: 'Data Platform', icon: LayoutDashboard }, + { id: 'presentation', label: 'Presentation', icon: Presentation }, + { id: 'dataquality', label: 'Data Quality', icon: DatabaseZap }, + { id: 'knowledge', label: 'Knowledge Chat', icon: MessageSquare }, + { id: 'storage', label: 'Object Storage', icon: HardDrive }, +] + +export function SideNav({ + agents, + animations, + gpu, + gpuLive, + gpuBoost = false, + selectedAgentId, + selectedNodeId, + mainView, + approvalCount, + agentsLoading, + onSetMainView, + onOpenApprovals, + onSelectAgent, + onSelectZone, +}: Props) { + const supervisors = agents.filter((a) => a.supervisor) + const operators = agents.filter((a) => !a.supervisor) + const matrixBoost = gpuBoost || mainView === 'knowledge' + + return ( + + ) +} + +function AgentRow({ + agent, + animations, + selected, + onSelect, +}: { + agent: Agent + animations: Record + selected: boolean + onSelect: (id: string) => void +}) { + const meta = getAgentMeta(agent.id) + const Icon = meta.icon + const busy = (animations[agent.id]?.state || 'idle') !== 'idle' + return ( + + ) +} diff --git a/ui/src/components/layout/ThemeToggle.tsx b/ui/src/components/layout/ThemeToggle.tsx new file mode 100644 index 0000000..d1588c9 --- /dev/null +++ b/ui/src/components/layout/ThemeToggle.tsx @@ -0,0 +1,25 @@ +import { Moon, Sun } from 'lucide-react' +import { useTheme } from '../../context/ThemeContext' +import { cn } from '../../lib/utils' + +export function ThemeToggle() { + const { theme, toggle } = useTheme() + const isDark = theme === 'dark' + + return ( + + ) +} diff --git a/ui/src/components/layout/TopBar.tsx b/ui/src/components/layout/TopBar.tsx new file mode 100644 index 0000000..99f7cb7 --- /dev/null +++ b/ui/src/components/layout/TopBar.tsx @@ -0,0 +1,65 @@ +import { Activity, Bot, Box, Clock, ShieldAlert } from 'lucide-react' +import type { Agent, Approval, StatusData, WorkloadData } from '../../types' +import { Badge } from '../ui/Badge' +import { ThemeToggle } from './ThemeToggle' +import { cn } from '../../lib/utils' + +type Props = { + clock: string + status: StatusData | null + workload: WorkloadData | null + agents: Agent[] + approvals: Approval[] + onApprovalsClick: () => void +} + +export function TopBar({ clock, workload, agents, approvals, onApprovalsClick }: Props) { + const pipelineOk = workload?.totals?.pipeline_active ?? false + const running = workload?.totals?.apps_running ?? 0 + const total = workload?.totals?.apps_total ?? 0 + const activeAgents = agents.filter((a) => (a.stats?.tasks ?? 0) > 0).length + + return ( +
+
+
+ +
+
+

Data & AI Command Center

+

ATC Lab · Enterprise Operations

+
+
+ +
+ + + Pipeline {pipelineOk ? 'active' : 'degraded'} + + {running}/{total} containers + + + {activeAgents} agents + + +
+ +
+ +
+ + {clock} + ● +
+
+
+ ) +} diff --git a/ui/src/components/ui/Badge.tsx b/ui/src/components/ui/Badge.tsx new file mode 100644 index 0000000..9d91381 --- /dev/null +++ b/ui/src/components/ui/Badge.tsx @@ -0,0 +1,25 @@ +import { cva, type VariantProps } from 'class-variance-authority' +import type { HTMLAttributes } from 'react' +import { cn } from '../../lib/utils' + +const badgeVariants = cva( + 'inline-flex items-center gap-1 rounded-md border px-1.5 py-0.5 font-mono text-[10px] font-medium', + { + variants: { + variant: { + default: 'border-border bg-surface-overlay text-foreground-muted dark:bg-surface-overlay dark:text-foreground-muted', + accent: 'border-docker/30 bg-docker-light text-docker dark:border-blue-400/30 dark:bg-blue-500/15 dark:text-blue-200', + success: 'border-success/30 bg-green-50 text-green-700 dark:border-green-500/30 dark:bg-green-500/15 dark:text-green-300', + warning: 'border-warning/30 bg-amber-50 text-amber-700 dark:border-amber-500/30 dark:bg-amber-500/15 dark:text-amber-300', + danger: 'border-danger/30 bg-red-50 text-red-700 dark:border-red-500/30 dark:bg-red-500/15 dark:text-red-300', + }, + }, + defaultVariants: { variant: 'default' }, + }, +) + +type Props = HTMLAttributes & VariantProps + +export function Badge({ className, variant, ...props }: Props) { + return +} diff --git a/ui/src/components/ui/Button.tsx b/ui/src/components/ui/Button.tsx new file mode 100644 index 0000000..0bd345d --- /dev/null +++ b/ui/src/components/ui/Button.tsx @@ -0,0 +1,30 @@ +import { cva, type VariantProps } from 'class-variance-authority' +import type { ButtonHTMLAttributes } from 'react' +import { cn } from '../../lib/utils' + +const buttonVariants = cva( + 'inline-flex items-center justify-center gap-1.5 rounded-md border font-medium transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-docker/40 disabled:pointer-events-none disabled:opacity-50', + { + variants: { + variant: { + default: 'border-docker/30 bg-docker text-foreground hover:bg-docker-dark', + ghost: 'border-transparent text-foreground-muted hover:bg-surface-overlay hover:text-foreground', + outline: 'border-border bg-surface-raised text-foreground hover:bg-surface-overlay', + success: 'border-success/30 bg-green-600 text-foreground hover:bg-green-700', + danger: 'border-danger/30 bg-red-600 text-foreground hover:bg-red-700', + }, + size: { + sm: 'h-7 px-2.5 text-xs', + md: 'h-8 px-3 text-sm', + icon: 'h-8 w-8', + }, + }, + defaultVariants: { variant: 'default', size: 'md' }, + }, +) + +type Props = ButtonHTMLAttributes & VariantProps + +export function Button({ className, variant, size, ...props }: Props) { + return