293 lines
12 KiB
Python
293 lines
12 KiB
Python
|
"""Build live presentation deck from cluster snapshot + registry."""
|
|||
|
|
|
|||
|
|
from __future__ import annotations
|
|||
|
|
|
|||
|
|
import json
|
|||
|
|
from typing import Any
|
|||
|
|
|
|||
|
|
from node_registry import NODE_REGISTRY
|
|||
|
|
from topology_views import build_all_topologies
|
|||
|
|
from workload import build_workload_payload
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _status_badge(level: str) -> str:
|
|||
|
|
return {"ok": "● Online", "warn": "◐ Degraded", "down": "○ Offline", "unknown": "? Unknown"}.get(level, level)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _slide(slide_id: str, title: str, subtitle: str, bullets: list[str], **extra: Any) -> dict[str, Any]:
|
|||
|
|
return {"id": slide_id, "title": title, "subtitle": subtitle, "bullets": bullets, **extra}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def build_presentation_payload(snap: dict[str, Any]) -> dict[str, Any]:
|
|||
|
|
workload = build_workload_payload(snap)
|
|||
|
|
topologies = workload.get("topologies") or build_all_topologies(snap)
|
|||
|
|
totals = workload.get("totals", {})
|
|||
|
|
zones = workload.get("zones", [])
|
|||
|
|
gpu = workload.get("gpu", {})
|
|||
|
|
etl = snap.get("etl", {})
|
|||
|
|
hadoop = snap.get("hadoop", {})
|
|||
|
|
objectscale = snap.get("objectscale", {})
|
|||
|
|
command = snap.get("command_center", {})
|
|||
|
|
|
|||
|
|
slides: list[dict[str, Any]] = []
|
|||
|
|
|
|||
|
|
slides.append(_slide(
|
|||
|
|
"title",
|
|||
|
|
"Dell ATC Data Lab",
|
|||
|
|
"Live demo & presentation — Command Center",
|
|||
|
|
[
|
|||
|
|
f"Snapshot: {snap.get('ts', 'now')}",
|
|||
|
|
f"Pipeline: {'ACTIVE' if totals.get('pipeline_active') else 'INACTIVE'}",
|
|||
|
|
f"Apps running: {totals.get('apps_running', 0)}/{totals.get('apps_total', 0)}",
|
|||
|
|
f"CDC connectors: {totals.get('connectors', 0)}",
|
|||
|
|
f"LLM: {gpu.get('model') or 'offline'} ({gpu.get('gpu_count', 0)}× V100)",
|
|||
|
|
"Command Center → http://10.0.21.33/",
|
|||
|
|
],
|
|||
|
|
kind="hero",
|
|||
|
|
))
|
|||
|
|
|
|||
|
|
slides.append(_slide(
|
|||
|
|
"mission",
|
|||
|
|
"Mission",
|
|||
|
|
"End-to-end modern data platform on Dell infrastructure",
|
|||
|
|
[
|
|||
|
|
"Ingest change data from operational databases (PostgreSQL, MySQL, MongoDB, Cassandra)",
|
|||
|
|
"Stream via Kafka & Debezium into the lakehouse (Spark, Trino, Iceberg)",
|
|||
|
|
"Land curated data on ObjectScale S3 — query with Trino & visualize in Superset",
|
|||
|
|
"Parallel HDFS cluster for batch / legacy workloads",
|
|||
|
|
"GPU lab powers autonomous ops agents with local vLLM inference",
|
|||
|
|
"This dashboard orchestrates agents, approvals, and live cluster visibility",
|
|||
|
|
],
|
|||
|
|
kind="narrative",
|
|||
|
|
))
|
|||
|
|
|
|||
|
|
arch = topologies.get("architecture") or workload.get("topology") or {}
|
|||
|
|
arch_nodes = arch.get("nodes", [])
|
|||
|
|
slides.append(_slide(
|
|||
|
|
"architecture",
|
|||
|
|
"Data Platform Architecture",
|
|||
|
|
arch.get("subtitle", "Sources → Ingestion → Compute → Storage → Consumers"),
|
|||
|
|
[f"{n.get('label', n.get('id'))}: {n.get('subtitle', n.get('role', ''))}" for n in arch_nodes[:14]],
|
|||
|
|
kind="topology",
|
|||
|
|
topology=arch,
|
|||
|
|
))
|
|||
|
|
|
|||
|
|
pipeline = topologies.get("pipeline", {})
|
|||
|
|
connector_lines = [
|
|||
|
|
f" · {cs['name']}: {cs.get('state', '?')}"
|
|||
|
|
for cs in (etl.get("connector_status") or [])[:6]
|
|||
|
|
]
|
|||
|
|
slides.append(_slide(
|
|||
|
|
"pipeline",
|
|||
|
|
"CDC Pipeline",
|
|||
|
|
pipeline.get("subtitle", "Airflow → DB → Debezium → Kafka → Lakehouse → S3"),
|
|||
|
|
[
|
|||
|
|
f"Airflow: {'healthy' if etl.get('airflow_healthy') else 'degraded'} ({etl.get('airflow_url', '')})",
|
|||
|
|
f"Kafka UI: {'UP' if etl.get('kafka_ui_ok') else 'DOWN'}",
|
|||
|
|
f"Connectors: {', '.join(etl.get('connectors') or []) or 'none'}",
|
|||
|
|
*connector_lines,
|
|||
|
|
f"Spark UI: {'UP' if etl.get('spark_ui_ok') else 'DOWN'}",
|
|||
|
|
f"ObjectScale: {'reachable' if objectscale.get('reachable') else 'down'} bucket={objectscale.get('bucket', 'data')}",
|
|||
|
|
],
|
|||
|
|
kind="topology",
|
|||
|
|
topology=pipeline,
|
|||
|
|
))
|
|||
|
|
|
|||
|
|
for zone in zones:
|
|||
|
|
apps = zone.get("apps") or []
|
|||
|
|
app_lines = [
|
|||
|
|
f"{a['name']}: {a['state']}" + (f" ({a.get('host', '')})" if a.get("host") else "")
|
|||
|
|
for a in apps[:10]
|
|||
|
|
]
|
|||
|
|
slides.append(_slide(
|
|||
|
|
f"zone-{zone['id']}",
|
|||
|
|
zone["label"],
|
|||
|
|
f"{zone.get('vm', '')} · {zone.get('ip', '')} · {_status_badge(zone.get('level', 'unknown'))}",
|
|||
|
|
[
|
|||
|
|
f"Containers: {zone.get('running', 0)}/{zone.get('total', 0)} running",
|
|||
|
|
*app_lines,
|
|||
|
|
],
|
|||
|
|
kind="zone",
|
|||
|
|
zone=zone,
|
|||
|
|
))
|
|||
|
|
|
|||
|
|
infra_nodes = [
|
|||
|
|
nid for nid in NODE_REGISTRY
|
|||
|
|
if nid not in ("mo-commander", "bart-commander", "network-watcher", "mcp-coordinator")
|
|||
|
|
]
|
|||
|
|
slides.append(_slide(
|
|||
|
|
"infrastructure",
|
|||
|
|
"Infrastructure Map",
|
|||
|
|
"Proxmox VMs & services across VLAN 20/21",
|
|||
|
|
[
|
|||
|
|
f"{NODE_REGISTRY[nid]['label']} — {NODE_REGISTRY[nid].get('vm')} "
|
|||
|
|
f"(VMID {NODE_REGISTRY[nid].get('vmid', '?')}) @ {NODE_REGISTRY[nid].get('ip')}"
|
|||
|
|
for nid in infra_nodes
|
|||
|
|
],
|
|||
|
|
kind="registry",
|
|||
|
|
))
|
|||
|
|
|
|||
|
|
dn_lines = [
|
|||
|
|
f" · {dn['host']}: {dn.get('used_gb', 0)} GB — {dn.get('state', '')}"
|
|||
|
|
for dn in (hadoop.get("datanodes") or [])[:5]
|
|||
|
|
]
|
|||
|
|
slides.append(_slide(
|
|||
|
|
"hadoop",
|
|||
|
|
"Hadoop HDFS",
|
|||
|
|
"9-node parallel storage cluster",
|
|||
|
|
[
|
|||
|
|
f"NameNode: {'UP' if hadoop.get('reachable') else 'DOWN'} — {hadoop.get('namenode', '')}",
|
|||
|
|
f"Capacity: {hadoop.get('capacity_used_gb', '?')} / {hadoop.get('capacity_total_gb', '?')} GB",
|
|||
|
|
f"DataNodes: {hadoop.get('live_datanodes', 0)} live, {hadoop.get('dead_datanodes', 0)} dead",
|
|||
|
|
f"Files: {hadoop.get('files_total', 0)}, Blocks: {hadoop.get('blocks_total', 0)}",
|
|||
|
|
*dn_lines,
|
|||
|
|
],
|
|||
|
|
kind="data",
|
|||
|
|
))
|
|||
|
|
|
|||
|
|
gpus = gpu.get("gpus") or snap.get("gpu", {}).get("gpus") or []
|
|||
|
|
gpu_lines = [
|
|||
|
|
f"GPU{g['index']}: {g.get('util_gpu', 0):.0f}% util, "
|
|||
|
|
f"{g.get('memory_used_mib', 0):.0f}/{g.get('memory_total_mib', 0):.0f} MiB"
|
|||
|
|
for g in gpus[:4]
|
|||
|
|
]
|
|||
|
|
slides.append(_slide(
|
|||
|
|
"gpu",
|
|||
|
|
"GPU Lab & GenAI",
|
|||
|
|
f"{gpu.get('model') or 'vLLM'} on atc-gpu-dev (VM 303)",
|
|||
|
|
[
|
|||
|
|
f"Inference: {'ON' if gpu.get('inference_active') else 'OFF'}",
|
|||
|
|
f"API: {snap.get('gpu', {}).get('vllm_url') or 'http://10.0.20.106:8001/v1'}",
|
|||
|
|
"Manager: http://10.0.20.106:9000",
|
|||
|
|
*gpu_lines,
|
|||
|
|
],
|
|||
|
|
kind="gpu",
|
|||
|
|
))
|
|||
|
|
|
|||
|
|
slides.append(_slide(
|
|||
|
|
"agents",
|
|||
|
|
"Autonomous Agents",
|
|||
|
|
"Mo & Bart supervise 5 domain operators + MCP hub",
|
|||
|
|
[
|
|||
|
|
"ETL Guardian — Airflow, Kafka, Debezium, connectors",
|
|||
|
|
"Data Custodian — PostgreSQL, MySQL, MongoDB, Cassandra, Neo4j",
|
|||
|
|
"Lakehouse Ops — Spark, Trino, Iceberg, ObjectScale S3",
|
|||
|
|
"Hadoop Ranger — HDFS NameNode, DataNodes, block health",
|
|||
|
|
"Infra Sentinel — Docker rack, GPU lab, Command Center",
|
|||
|
|
"All agents receive LIVE cluster snapshot in every LLM prompt",
|
|||
|
|
],
|
|||
|
|
kind="agents",
|
|||
|
|
))
|
|||
|
|
|
|||
|
|
cc_apps = [f"{c['name']}: {c['state']}" for c in (command.get("containers") or [])]
|
|||
|
|
slides.append(_slide(
|
|||
|
|
"command",
|
|||
|
|
"Command Center",
|
|||
|
|
"VM 304 — this presentation runs here",
|
|||
|
|
[
|
|||
|
|
f"Host: {command.get('host', '10.0.21.33')} (VMID {command.get('vmid', 304)})",
|
|||
|
|
f"Stack: {command.get('running', 0)}/{command.get('total', 0)} containers",
|
|||
|
|
*cc_apps,
|
|||
|
|
"WebSocket ops feed · Approval inbox · Agent terminals",
|
|||
|
|
],
|
|||
|
|
kind="command",
|
|||
|
|
))
|
|||
|
|
|
|||
|
|
slides.append(_slide(
|
|||
|
|
"demo",
|
|||
|
|
"Live Demo Tips",
|
|||
|
|
"Use this deck during customer presentations",
|
|||
|
|
[
|
|||
|
|
"Press ← → or click dots to navigate slides",
|
|||
|
|
"F = fullscreen presentation mode",
|
|||
|
|
"Export HTML opens a standalone deck for projectors / offline",
|
|||
|
|
"Ask agents in the Command Bar — they see full cluster context",
|
|||
|
|
"Switch to Data Platform tab for interactive topology",
|
|||
|
|
"GPU Lab chat: http://10.0.20.106:9000/chat",
|
|||
|
|
],
|
|||
|
|
kind="cta",
|
|||
|
|
))
|
|||
|
|
|
|||
|
|
return {
|
|||
|
|
"ts": snap.get("ts"),
|
|||
|
|
"title": "Dell ATC Data Lab",
|
|||
|
|
"subtitle": "Live Infrastructure Presentation",
|
|||
|
|
"totals": totals,
|
|||
|
|
"pipeline_active": totals.get("pipeline_active"),
|
|||
|
|
"slides": slides,
|
|||
|
|
"slide_count": len(slides),
|
|||
|
|
"workload": workload,
|
|||
|
|
"topologies": topologies,
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def render_presentation_html(payload: dict[str, Any]) -> str:
|
|||
|
|
slides_json = json.dumps(payload.get("slides", []), default=str)
|
|||
|
|
title = payload.get("title", "ATC Lab")
|
|||
|
|
ts = payload.get("ts", "")
|
|||
|
|
return f"""<!DOCTYPE html>
|
|||
|
|
<html lang="en">
|
|||
|
|
<head>
|
|||
|
|
<meta charset="utf-8"/>
|
|||
|
|
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
|||
|
|
<title>{title} — Presentation</title>
|
|||
|
|
<style>
|
|||
|
|
*{{box-sizing:border-box;margin:0;padding:0}}
|
|||
|
|
body{{font-family:system-ui,-apple-system,Segoe UI,sans-serif;background:#0c1929;color:#e8f1ff;height:100vh;overflow:hidden}}
|
|||
|
|
.deck{{height:100vh;display:flex;flex-direction:column}}
|
|||
|
|
header{{padding:1rem 2rem;display:flex;justify-content:space-between;align-items:center;border-bottom:1px solid rgba(96,165,250,.2);background:rgba(15,27,46,.9)}}
|
|||
|
|
header h1{{font-size:1.1rem;font-weight:600}}
|
|||
|
|
header .meta{{font-size:.75rem;opacity:.7}}
|
|||
|
|
.slide{{flex:1;display:none;padding:3rem 4rem;overflow:auto}}
|
|||
|
|
.slide.active{{display:flex;flex-direction:column;justify-content:center}}
|
|||
|
|
.slide h2{{font-size:2.4rem;margin-bottom:.5rem;background:linear-gradient(90deg,#60a5fa,#a78bfa);-webkit-background-clip:text;-webkit-text-fill-color:transparent}}
|
|||
|
|
.slide h3{{font-size:1rem;opacity:.75;margin-bottom:2rem;font-weight:400}}
|
|||
|
|
.slide ul{{list-style:none;font-size:1.15rem;line-height:1.9}}
|
|||
|
|
.slide li::before{{content:"▸ ";color:#60a5fa}}
|
|||
|
|
.slide.hero h2{{font-size:3.2rem}}
|
|||
|
|
nav{{display:flex;gap:.5rem;padding:1rem 2rem;border-top:1px solid rgba(96,165,250,.2);align-items:center}}
|
|||
|
|
nav button{{background:#1e3a5f;border:1px solid rgba(96,165,250,.3);color:#e8f1ff;padding:.5rem 1rem;border-radius:6px;cursor:pointer}}
|
|||
|
|
nav button:hover{{background:#234876}}
|
|||
|
|
.dots{{display:flex;gap:6px;flex:1;justify-content:center;flex-wrap:wrap}}
|
|||
|
|
.dot{{width:8px;height:8px;border-radius:50%;background:rgba(96,165,250,.3);cursor:pointer;border:none}}
|
|||
|
|
.dot.active{{background:#60a5fa;transform:scale(1.3)}}
|
|||
|
|
.counter{{font-size:.8rem;opacity:.6;min-width:4rem;text-align:right}}
|
|||
|
|
</style>
|
|||
|
|
</head>
|
|||
|
|
<body>
|
|||
|
|
<div class="deck">
|
|||
|
|
<header><h1>{title}</h1><div class="meta">Dell ATC · Live snapshot {ts}</div></header>
|
|||
|
|
<div id="slides"></div>
|
|||
|
|
<nav>
|
|||
|
|
<button id="prev">← Prev</button>
|
|||
|
|
<div class="dots" id="dots"></div>
|
|||
|
|
<button id="next">Next →</button>
|
|||
|
|
<span class="counter" id="counter"></span>
|
|||
|
|
</nav>
|
|||
|
|
</div>
|
|||
|
|
<script>
|
|||
|
|
const slides={slides_json};
|
|||
|
|
let i=0;
|
|||
|
|
const container=document.getElementById("slides");
|
|||
|
|
const dots=document.getElementById("dots");
|
|||
|
|
const counter=document.getElementById("counter");
|
|||
|
|
slides.forEach((s,idx)=>{{
|
|||
|
|
const el=document.createElement("section");
|
|||
|
|
el.className="slide"+(s.kind==="hero"?" hero":"")+(idx===0?" active":"");
|
|||
|
|
const bullets=(s.bullets||[]).map(b=>"<li>"+b+"</li>").join("");
|
|||
|
|
el.innerHTML="<h2>"+s.title+"</h2><h3>"+(s.subtitle||"")+"</h3><ul>"+bullets+"</ul>";
|
|||
|
|
container.appendChild(el);
|
|||
|
|
const d=document.createElement("button");
|
|||
|
|
d.className="dot"+(idx===0?" active":"");
|
|||
|
|
d.onclick=()=>go(idx);
|
|||
|
|
dots.appendChild(d);
|
|||
|
|
}});
|
|||
|
|
function go(n){{i=Math.max(0,Math.min(slides.length-1,n));document.querySelectorAll(".slide").forEach((e,j)=>e.classList.toggle("active",j===i));document.querySelectorAll(".dot").forEach((e,j)=>e.classList.toggle("active",j===i));counter.textContent=(i+1)+"/"+slides.length;}}
|
|||
|
|
document.getElementById("prev").onclick=()=>go(i-1);
|
|||
|
|
document.getElementById("next").onclick=()=>go(i+1);
|
|||
|
|
document.onkeydown=e=>{{if(e.key==="ArrowRight"||e.key===" ")go(i+1);if(e.key==="ArrowLeft")go(i-1);if(e.key==="f"||e.key==="F")document.documentElement.requestFullscreen?.();}};
|
|||
|
|
go(0);
|
|||
|
|
</script>
|
|||
|
|
</body>
|
|||
|
|
</html>"""
|