Add Command Center v2: DQ/RAG integration, S3 browser, Jupyter, GPU matrix.
Mirror mo/atc-GPU layout with config/, docs/, scripts/ for Gitea deploy.
This commit is contained in:
mo
@@ -0,0 +1,292 @@
|
||||
"""Build live presentation deck from cluster snapshot + registry."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from node_registry import NODE_REGISTRY
|
||||
from topology_views import build_all_topologies
|
||||
from workload import build_workload_payload
|
||||
|
||||
|
||||
def _status_badge(level: str) -> str:
|
||||
return {"ok": "● Online", "warn": "◐ Degraded", "down": "○ Offline", "unknown": "? Unknown"}.get(level, level)
|
||||
|
||||
|
||||
def _slide(slide_id: str, title: str, subtitle: str, bullets: list[str], **extra: Any) -> dict[str, Any]:
|
||||
return {"id": slide_id, "title": title, "subtitle": subtitle, "bullets": bullets, **extra}
|
||||
|
||||
|
||||
def build_presentation_payload(snap: dict[str, Any]) -> dict[str, Any]:
|
||||
workload = build_workload_payload(snap)
|
||||
topologies = workload.get("topologies") or build_all_topologies(snap)
|
||||
totals = workload.get("totals", {})
|
||||
zones = workload.get("zones", [])
|
||||
gpu = workload.get("gpu", {})
|
||||
etl = snap.get("etl", {})
|
||||
hadoop = snap.get("hadoop", {})
|
||||
objectscale = snap.get("objectscale", {})
|
||||
command = snap.get("command_center", {})
|
||||
|
||||
slides: list[dict[str, Any]] = []
|
||||
|
||||
slides.append(_slide(
|
||||
"title",
|
||||
"Dell ATC Data Lab",
|
||||
"Live demo & presentation — Command Center",
|
||||
[
|
||||
f"Snapshot: {snap.get('ts', 'now')}",
|
||||
f"Pipeline: {'ACTIVE' if totals.get('pipeline_active') else 'INACTIVE'}",
|
||||
f"Apps running: {totals.get('apps_running', 0)}/{totals.get('apps_total', 0)}",
|
||||
f"CDC connectors: {totals.get('connectors', 0)}",
|
||||
f"LLM: {gpu.get('model') or 'offline'} ({gpu.get('gpu_count', 0)}× V100)",
|
||||
"Command Center → http://10.0.21.33/",
|
||||
],
|
||||
kind="hero",
|
||||
))
|
||||
|
||||
slides.append(_slide(
|
||||
"mission",
|
||||
"Mission",
|
||||
"End-to-end modern data platform on Dell infrastructure",
|
||||
[
|
||||
"Ingest change data from operational databases (PostgreSQL, MySQL, MongoDB, Cassandra)",
|
||||
"Stream via Kafka & Debezium into the lakehouse (Spark, Trino, Iceberg)",
|
||||
"Land curated data on ObjectScale S3 — query with Trino & visualize in Superset",
|
||||
"Parallel HDFS cluster for batch / legacy workloads",
|
||||
"GPU lab powers autonomous ops agents with local vLLM inference",
|
||||
"This dashboard orchestrates agents, approvals, and live cluster visibility",
|
||||
],
|
||||
kind="narrative",
|
||||
))
|
||||
|
||||
arch = topologies.get("architecture") or workload.get("topology") or {}
|
||||
arch_nodes = arch.get("nodes", [])
|
||||
slides.append(_slide(
|
||||
"architecture",
|
||||
"Data Platform Architecture",
|
||||
arch.get("subtitle", "Sources → Ingestion → Compute → Storage → Consumers"),
|
||||
[f"{n.get('label', n.get('id'))}: {n.get('subtitle', n.get('role', ''))}" for n in arch_nodes[:14]],
|
||||
kind="topology",
|
||||
topology=arch,
|
||||
))
|
||||
|
||||
pipeline = topologies.get("pipeline", {})
|
||||
connector_lines = [
|
||||
f" · {cs['name']}: {cs.get('state', '?')}"
|
||||
for cs in (etl.get("connector_status") or [])[:6]
|
||||
]
|
||||
slides.append(_slide(
|
||||
"pipeline",
|
||||
"CDC Pipeline",
|
||||
pipeline.get("subtitle", "Airflow → DB → Debezium → Kafka → Lakehouse → S3"),
|
||||
[
|
||||
f"Airflow: {'healthy' if etl.get('airflow_healthy') else 'degraded'} ({etl.get('airflow_url', '')})",
|
||||
f"Kafka UI: {'UP' if etl.get('kafka_ui_ok') else 'DOWN'}",
|
||||
f"Connectors: {', '.join(etl.get('connectors') or []) or 'none'}",
|
||||
*connector_lines,
|
||||
f"Spark UI: {'UP' if etl.get('spark_ui_ok') else 'DOWN'}",
|
||||
f"ObjectScale: {'reachable' if objectscale.get('reachable') else 'down'} bucket={objectscale.get('bucket', 'data')}",
|
||||
],
|
||||
kind="topology",
|
||||
topology=pipeline,
|
||||
))
|
||||
|
||||
for zone in zones:
|
||||
apps = zone.get("apps") or []
|
||||
app_lines = [
|
||||
f"{a['name']}: {a['state']}" + (f" ({a.get('host', '')})" if a.get("host") else "")
|
||||
for a in apps[:10]
|
||||
]
|
||||
slides.append(_slide(
|
||||
f"zone-{zone['id']}",
|
||||
zone["label"],
|
||||
f"{zone.get('vm', '')} · {zone.get('ip', '')} · {_status_badge(zone.get('level', 'unknown'))}",
|
||||
[
|
||||
f"Containers: {zone.get('running', 0)}/{zone.get('total', 0)} running",
|
||||
*app_lines,
|
||||
],
|
||||
kind="zone",
|
||||
zone=zone,
|
||||
))
|
||||
|
||||
infra_nodes = [
|
||||
nid for nid in NODE_REGISTRY
|
||||
if nid not in ("mo-commander", "bart-commander", "network-watcher", "mcp-coordinator")
|
||||
]
|
||||
slides.append(_slide(
|
||||
"infrastructure",
|
||||
"Infrastructure Map",
|
||||
"Proxmox VMs & services across VLAN 20/21",
|
||||
[
|
||||
f"{NODE_REGISTRY[nid]['label']} — {NODE_REGISTRY[nid].get('vm')} "
|
||||
f"(VMID {NODE_REGISTRY[nid].get('vmid', '?')}) @ {NODE_REGISTRY[nid].get('ip')}"
|
||||
for nid in infra_nodes
|
||||
],
|
||||
kind="registry",
|
||||
))
|
||||
|
||||
dn_lines = [
|
||||
f" · {dn['host']}: {dn.get('used_gb', 0)} GB — {dn.get('state', '')}"
|
||||
for dn in (hadoop.get("datanodes") or [])[:5]
|
||||
]
|
||||
slides.append(_slide(
|
||||
"hadoop",
|
||||
"Hadoop HDFS",
|
||||
"9-node parallel storage cluster",
|
||||
[
|
||||
f"NameNode: {'UP' if hadoop.get('reachable') else 'DOWN'} — {hadoop.get('namenode', '')}",
|
||||
f"Capacity: {hadoop.get('capacity_used_gb', '?')} / {hadoop.get('capacity_total_gb', '?')} GB",
|
||||
f"DataNodes: {hadoop.get('live_datanodes', 0)} live, {hadoop.get('dead_datanodes', 0)} dead",
|
||||
f"Files: {hadoop.get('files_total', 0)}, Blocks: {hadoop.get('blocks_total', 0)}",
|
||||
*dn_lines,
|
||||
],
|
||||
kind="data",
|
||||
))
|
||||
|
||||
gpus = gpu.get("gpus") or snap.get("gpu", {}).get("gpus") or []
|
||||
gpu_lines = [
|
||||
f"GPU{g['index']}: {g.get('util_gpu', 0):.0f}% util, "
|
||||
f"{g.get('memory_used_mib', 0):.0f}/{g.get('memory_total_mib', 0):.0f} MiB"
|
||||
for g in gpus[:4]
|
||||
]
|
||||
slides.append(_slide(
|
||||
"gpu",
|
||||
"GPU Lab & GenAI",
|
||||
f"{gpu.get('model') or 'vLLM'} on atc-gpu-dev (VM 303)",
|
||||
[
|
||||
f"Inference: {'ON' if gpu.get('inference_active') else 'OFF'}",
|
||||
f"API: {snap.get('gpu', {}).get('vllm_url') or 'http://10.0.20.106:8001/v1'}",
|
||||
"Manager: http://10.0.20.106:9000",
|
||||
*gpu_lines,
|
||||
],
|
||||
kind="gpu",
|
||||
))
|
||||
|
||||
slides.append(_slide(
|
||||
"agents",
|
||||
"Autonomous Agents",
|
||||
"Mo & Bart supervise 5 domain operators + MCP hub",
|
||||
[
|
||||
"ETL Guardian — Airflow, Kafka, Debezium, connectors",
|
||||
"Data Custodian — PostgreSQL, MySQL, MongoDB, Cassandra, Neo4j",
|
||||
"Lakehouse Ops — Spark, Trino, Iceberg, ObjectScale S3",
|
||||
"Hadoop Ranger — HDFS NameNode, DataNodes, block health",
|
||||
"Infra Sentinel — Docker rack, GPU lab, Command Center",
|
||||
"All agents receive LIVE cluster snapshot in every LLM prompt",
|
||||
],
|
||||
kind="agents",
|
||||
))
|
||||
|
||||
cc_apps = [f"{c['name']}: {c['state']}" for c in (command.get("containers") or [])]
|
||||
slides.append(_slide(
|
||||
"command",
|
||||
"Command Center",
|
||||
"VM 304 — this presentation runs here",
|
||||
[
|
||||
f"Host: {command.get('host', '10.0.21.33')} (VMID {command.get('vmid', 304)})",
|
||||
f"Stack: {command.get('running', 0)}/{command.get('total', 0)} containers",
|
||||
*cc_apps,
|
||||
"WebSocket ops feed · Approval inbox · Agent terminals",
|
||||
],
|
||||
kind="command",
|
||||
))
|
||||
|
||||
slides.append(_slide(
|
||||
"demo",
|
||||
"Live Demo Tips",
|
||||
"Use this deck during customer presentations",
|
||||
[
|
||||
"Press ← → or click dots to navigate slides",
|
||||
"F = fullscreen presentation mode",
|
||||
"Export HTML opens a standalone deck for projectors / offline",
|
||||
"Ask agents in the Command Bar — they see full cluster context",
|
||||
"Switch to Data Platform tab for interactive topology",
|
||||
"GPU Lab chat: http://10.0.20.106:9000/chat",
|
||||
],
|
||||
kind="cta",
|
||||
))
|
||||
|
||||
return {
|
||||
"ts": snap.get("ts"),
|
||||
"title": "Dell ATC Data Lab",
|
||||
"subtitle": "Live Infrastructure Presentation",
|
||||
"totals": totals,
|
||||
"pipeline_active": totals.get("pipeline_active"),
|
||||
"slides": slides,
|
||||
"slide_count": len(slides),
|
||||
"workload": workload,
|
||||
"topologies": topologies,
|
||||
}
|
||||
|
||||
|
||||
def render_presentation_html(payload: dict[str, Any]) -> str:
|
||||
slides_json = json.dumps(payload.get("slides", []), default=str)
|
||||
title = payload.get("title", "ATC Lab")
|
||||
ts = payload.get("ts", "")
|
||||
return f"""<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<title>{title} — Presentation</title>
|
||||
<style>
|
||||
*{{box-sizing:border-box;margin:0;padding:0}}
|
||||
body{{font-family:system-ui,-apple-system,Segoe UI,sans-serif;background:#0c1929;color:#e8f1ff;height:100vh;overflow:hidden}}
|
||||
.deck{{height:100vh;display:flex;flex-direction:column}}
|
||||
header{{padding:1rem 2rem;display:flex;justify-content:space-between;align-items:center;border-bottom:1px solid rgba(96,165,250,.2);background:rgba(15,27,46,.9)}}
|
||||
header h1{{font-size:1.1rem;font-weight:600}}
|
||||
header .meta{{font-size:.75rem;opacity:.7}}
|
||||
.slide{{flex:1;display:none;padding:3rem 4rem;overflow:auto}}
|
||||
.slide.active{{display:flex;flex-direction:column;justify-content:center}}
|
||||
.slide h2{{font-size:2.4rem;margin-bottom:.5rem;background:linear-gradient(90deg,#60a5fa,#a78bfa);-webkit-background-clip:text;-webkit-text-fill-color:transparent}}
|
||||
.slide h3{{font-size:1rem;opacity:.75;margin-bottom:2rem;font-weight:400}}
|
||||
.slide ul{{list-style:none;font-size:1.15rem;line-height:1.9}}
|
||||
.slide li::before{{content:"▸ ";color:#60a5fa}}
|
||||
.slide.hero h2{{font-size:3.2rem}}
|
||||
nav{{display:flex;gap:.5rem;padding:1rem 2rem;border-top:1px solid rgba(96,165,250,.2);align-items:center}}
|
||||
nav button{{background:#1e3a5f;border:1px solid rgba(96,165,250,.3);color:#e8f1ff;padding:.5rem 1rem;border-radius:6px;cursor:pointer}}
|
||||
nav button:hover{{background:#234876}}
|
||||
.dots{{display:flex;gap:6px;flex:1;justify-content:center;flex-wrap:wrap}}
|
||||
.dot{{width:8px;height:8px;border-radius:50%;background:rgba(96,165,250,.3);cursor:pointer;border:none}}
|
||||
.dot.active{{background:#60a5fa;transform:scale(1.3)}}
|
||||
.counter{{font-size:.8rem;opacity:.6;min-width:4rem;text-align:right}}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="deck">
|
||||
<header><h1>{title}</h1><div class="meta">Dell ATC · Live snapshot {ts}</div></header>
|
||||
<div id="slides"></div>
|
||||
<nav>
|
||||
<button id="prev">← Prev</button>
|
||||
<div class="dots" id="dots"></div>
|
||||
<button id="next">Next →</button>
|
||||
<span class="counter" id="counter"></span>
|
||||
</nav>
|
||||
</div>
|
||||
<script>
|
||||
const slides={slides_json};
|
||||
let i=0;
|
||||
const container=document.getElementById("slides");
|
||||
const dots=document.getElementById("dots");
|
||||
const counter=document.getElementById("counter");
|
||||
slides.forEach((s,idx)=>{{
|
||||
const el=document.createElement("section");
|
||||
el.className="slide"+(s.kind==="hero"?" hero":"")+(idx===0?" active":"");
|
||||
const bullets=(s.bullets||[]).map(b=>"<li>"+b+"</li>").join("");
|
||||
el.innerHTML="<h2>"+s.title+"</h2><h3>"+(s.subtitle||"")+"</h3><ul>"+bullets+"</ul>";
|
||||
container.appendChild(el);
|
||||
const d=document.createElement("button");
|
||||
d.className="dot"+(idx===0?" active":"");
|
||||
d.onclick=()=>go(idx);
|
||||
dots.appendChild(d);
|
||||
}});
|
||||
function go(n){{i=Math.max(0,Math.min(slides.length-1,n));document.querySelectorAll(".slide").forEach((e,j)=>e.classList.toggle("active",j===i));document.querySelectorAll(".dot").forEach((e,j)=>e.classList.toggle("active",j===i));counter.textContent=(i+1)+"/"+slides.length;}}
|
||||
document.getElementById("prev").onclick=()=>go(i-1);
|
||||
document.getElementById("next").onclick=()=>go(i+1);
|
||||
document.onkeydown=e=>{{if(e.key==="ArrowRight"||e.key===" ")go(i+1);if(e.key==="ArrowLeft")go(i-1);if(e.key==="f"||e.key==="F")document.documentElement.requestFullscreen?.();}};
|
||||
go(0);
|
||||
</script>
|
||||
</body>
|
||||
</html>"""
|
||||
Reference in New Issue
Block a user