mo
607 lines
28 KiB
Python
607 lines
28 KiB
Python
"""Five animated topology views from data architecture perspectives."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
|
|
|
|
def _edge(eid: str, src: str, dst: str, label: str, kind: str, active: bool = True) -> dict[str, Any]:
|
|
return {"id": eid, "from": src, "to": dst, "label": label, "kind": kind, "active": active}
|
|
|
|
|
|
def _clone_node(n: dict[str, Any], x: float, y: float, layer: str | None = None) -> dict[str, Any]:
|
|
out = {**n, "x": x, "y": y}
|
|
if layer:
|
|
out["layer"] = layer
|
|
return out
|
|
|
|
|
|
def build_all_topologies(
|
|
base_nodes: list[dict[str, Any]],
|
|
base_edges: list[dict[str, Any]],
|
|
snap: dict[str, Any],
|
|
*,
|
|
pipeline_active: bool,
|
|
connectors: list[str],
|
|
) -> dict[str, dict[str, Any]]:
|
|
by_id = {n["id"]: n for n in base_nodes}
|
|
etl = snap.get("etl", {})
|
|
lake = snap.get("lakehouse", {})
|
|
docker = snap.get("docker", {})
|
|
gpu = snap.get("gpu", {})
|
|
|
|
# ── 1. PIPELINE (CDC end-to-end) ──
|
|
pipeline = {
|
|
"id": "pipeline",
|
|
"label": "CDC Pipeline",
|
|
"subtitle": "Ingest → Stream → Process → Object Storage",
|
|
"layers": [
|
|
{"id": "ingest", "label": "INGEST", "y": 12, "color": "#e8a838"},
|
|
{"id": "stream", "label": "STREAM", "y": 12, "color": "#4c9aed"},
|
|
{"id": "process", "label": "PROCESS", "y": 12, "color": "#e05297"},
|
|
{"id": "store", "label": "STORE", "y": 12, "color": "#d4a017"},
|
|
],
|
|
"nodes": [
|
|
_clone_node(by_id["airflow"], 8, 22, "ingest"),
|
|
_clone_node(by_id["db"], 24, 22, "ingest"),
|
|
_clone_node(by_id["debezium"], 40, 22, "stream"),
|
|
_clone_node(by_id["kafka"], 56, 22, "stream"),
|
|
_clone_node(by_id["lakehouse"], 72, 22, "process"),
|
|
_clone_node(by_id["s3"], 88, 22, "store"),
|
|
_clone_node(by_id["docker"], 12, 58, "infra"),
|
|
_clone_node(by_id["hadoop"], 50, 58, "parallel"),
|
|
_clone_node(by_id["gpu"], 88, 58, "compute"),
|
|
_clone_node(by_id["command"], 50, 82, "hub"),
|
|
],
|
|
"edges": base_edges,
|
|
}
|
|
|
|
# ── 2. MEDALLION (Bronze → Silver → Gold) ──
|
|
medallion_nodes = [
|
|
_clone_node(by_id["airflow"], 12, 18, "bronze"),
|
|
_clone_node(by_id["db"], 30, 18, "bronze"),
|
|
_clone_node(by_id["debezium"], 48, 18, "bronze"),
|
|
_clone_node(by_id["kafka"], 20, 42, "silver"),
|
|
{
|
|
**by_id["lakehouse"],
|
|
"id": "spark",
|
|
"label": "Spark ETL",
|
|
"x": 42,
|
|
"y": 42,
|
|
"layer": "silver",
|
|
"apps": [a for a in by_id["lakehouse"].get("apps", []) if "spark" in a.get("name", "").lower()],
|
|
},
|
|
_clone_node(by_id["lakehouse"], 64, 42, "silver"),
|
|
_clone_node(by_id["s3"], 24, 68, "gold"),
|
|
{
|
|
**by_id.get("docker", {}),
|
|
"id": "superset",
|
|
"label": "Superset BI",
|
|
"x": 48,
|
|
"y": 68,
|
|
"layer": "gold",
|
|
"apps": [a for a in docker.get("containers", []) if "superset" in f"{a.get('name','')} {a.get('image','')}".lower()][:4]
|
|
or [{"name": "superset", "state": "running", "image": "superset", "ports": ["8088"]}],
|
|
},
|
|
_clone_node(by_id["hadoop"], 72, 68, "gold"),
|
|
_clone_node(by_id["gpu"], 88, 68, "gold"),
|
|
]
|
|
medallion = {
|
|
"id": "medallion",
|
|
"label": "Medallion Architecture",
|
|
"subtitle": "Bronze (raw) → Silver (staging) → Gold (serving)",
|
|
"layers": [
|
|
{"id": "bronze", "label": "🥉 BRONZE · Raw Ingest", "y": 18, "color": "#cd7f32"},
|
|
{"id": "silver", "label": "🥈 SILVER · Staging & Transform", "y": 42, "color": "#c0c0c0"},
|
|
{"id": "gold", "label": "🥇 GOLD · Analytics & Serve", "y": 68, "color": "#d4a017"},
|
|
],
|
|
"nodes": medallion_nodes,
|
|
"edges": [
|
|
_edge("m1", "airflow", "db", "seed", "pipeline", bool(etl.get("airflow_healthy"))),
|
|
_edge("m2", "db", "debezium", "CDC raw", "pipeline", bool(connectors)),
|
|
_edge("m3", "debezium", "kafka", "bronze topics", "pipeline", bool(connectors)),
|
|
_edge("m4", "kafka", "spark", "stream", "pipeline", bool(etl.get("kafka_ui_ok"))),
|
|
_edge("m5", "spark", "lakehouse", "transform", "pipeline", lake.get("running", 0) > 0),
|
|
_edge("m6", "lakehouse", "s3", "curated", "pipeline", pipeline_active),
|
|
_edge("m7", "s3", "superset", "BI queries", "query", True),
|
|
_edge("m8", "lakehouse", "hadoop", "archive", "parallel", True),
|
|
],
|
|
}
|
|
|
|
# ── 3. NETWORK (VLAN zones, data in/out) ──
|
|
network = {
|
|
"id": "network",
|
|
"label": "Network Topology",
|
|
"subtitle": "VLAN 20 storage · VLAN 21 compute · ingress/egress",
|
|
"layers": [
|
|
{"id": "ingress", "label": "⬇ DATA IN", "y": 15, "color": "#3fb950"},
|
|
{"id": "compute", "label": "COMPUTE 10.0.21.x", "y": 42, "color": "#4c9aed"},
|
|
{"id": "storage", "label": "STORAGE 10.0.20.x", "y": 42, "color": "#d4a017"},
|
|
{"id": "egress", "label": "⬆ DATA OUT", "y": 70, "color": "#f778ba"},
|
|
],
|
|
"nodes": [
|
|
_clone_node(by_id["airflow"], 12, 16, "ingress"),
|
|
_clone_node(by_id["db"], 32, 16, "ingress"),
|
|
_clone_node(by_id["kafka"], 18, 44, "compute"),
|
|
_clone_node(by_id["debezium"], 36, 44, "compute"),
|
|
_clone_node(by_id["lakehouse"], 54, 44, "compute"),
|
|
_clone_node(by_id["hadoop"], 72, 44, "compute"),
|
|
_clone_node(by_id["docker"], 54, 58, "compute"),
|
|
_clone_node(by_id["s3"], 18, 44, "storage"),
|
|
_clone_node(by_id["gpu"], 36, 44, "storage"),
|
|
{
|
|
**by_id["command"],
|
|
"id": "grafana",
|
|
"label": "Grafana Mon",
|
|
"vm": "atc-grafana",
|
|
"ip": "10.0.20.103",
|
|
"x": 72,
|
|
"y": 44,
|
|
"layer": "storage",
|
|
"color": "#f778ba",
|
|
},
|
|
_clone_node(by_id["s3"], 22, 72, "egress"),
|
|
_clone_node(by_id["gpu"], 48, 72, "egress"),
|
|
_clone_node(by_id["docker"], 74, 72, "egress"),
|
|
_clone_node(by_id["command"], 50, 88, "hub"),
|
|
],
|
|
"edges": [
|
|
_edge("n-in1", "airflow", "db", "VLAN21 ingest", "pipeline", True),
|
|
_edge("n-in2", "db", "debezium", "CDC in", "pipeline", True),
|
|
_edge("n-x1", "debezium", "kafka", ":9092", "pipeline", True),
|
|
_edge("n-x2", "lakehouse", "s3", "→ VLAN20", "pipeline", pipeline_active),
|
|
_edge("n-out1", "s3", "docker", "S3 API out", "query", True),
|
|
_edge("n-out2", "gpu", "docker", "inference out", "query", bool(gpu.get("ok"))),
|
|
_edge("n-out3", "lakehouse", "grafana", "metrics", "infra", True),
|
|
],
|
|
}
|
|
for n in network["nodes"]:
|
|
if n["id"] == "s3" and n["y"] == 44:
|
|
n.update({"x": 18, "y": 44})
|
|
if n["id"] == "gpu" and n.get("layer") == "storage":
|
|
n.update({"x": 36, "y": 44})
|
|
|
|
# ── 4. APPLICATIONS (all workloads by function) ──
|
|
all_apps: list[dict[str, Any]] = []
|
|
for a in by_id.get("docker", {}).get("apps", []):
|
|
all_apps.append(a)
|
|
for a in by_id.get("db", {}).get("apps", []):
|
|
all_apps.append(a)
|
|
for a in by_id.get("lakehouse", {}).get("apps", []):
|
|
all_apps.append(a)
|
|
all_apps.append({"name": "Airflow", "state": "running" if etl.get("airflow_healthy") else "down", "image": "airflow", "ports": ["8080"]})
|
|
all_apps.append({"name": "Kafka", "state": "running" if etl.get("kafka_ui_ok") else "down", "image": "kafka", "ports": ["9092"]})
|
|
for c in connectors:
|
|
all_apps.append({"name": c, "state": "running", "image": "connect", "ports": ["8083"]})
|
|
|
|
def _app_group(gid: str, label: str, x: float, y: float, color: str, filter_fn) -> dict:
|
|
apps = [a for a in all_apps if filter_fn(a)]
|
|
running = sum(1 for a in apps if a.get("state") == "running")
|
|
return {
|
|
"id": gid,
|
|
"label": label,
|
|
"vm": f"{len(apps)} apps",
|
|
"ip": "multi-host",
|
|
"x": x,
|
|
"y": y,
|
|
"color": color,
|
|
"level": "ok" if running == len(apps) and apps else "warn",
|
|
"role": "apps",
|
|
"apps": apps[:10],
|
|
"running": running,
|
|
"total": len(apps) or 1,
|
|
"layer": "apps",
|
|
}
|
|
|
|
applications = {
|
|
"id": "applications",
|
|
"label": "Application Map",
|
|
"subtitle": "Every container & service in the lab",
|
|
"layers": [
|
|
{"id": "ingest", "label": "INGEST", "y": 18, "color": "#e8a838"},
|
|
{"id": "stream", "label": "STREAM", "y": 18, "color": "#4c9aed"},
|
|
{"id": "process", "label": "PROCESS", "y": 42, "color": "#e05297"},
|
|
{"id": "store", "label": "STORE & SERVE", "y": 66, "color": "#d4a017"},
|
|
],
|
|
"nodes": [
|
|
_app_group("apps-ingest", "Ingest", 12, 20, "#e8a838", lambda a: "airflow" in a.get("name", "").lower() or "airflow" in a.get("image", "").lower()),
|
|
_app_group("apps-sources", "Source DBs", 30, 20, "#ffaa00", lambda a: any(k in f"{a.get('name','')} {a.get('image','')}".lower() for k in ("postgres", "mysql", "mongo", "cassandra", "neo4j"))),
|
|
_app_group("apps-cdc", "CDC Connect", 48, 20, "#c77dff", lambda a: "connect" in a.get("image", "").lower() or "connector" in a.get("name", "").lower()),
|
|
_app_group("apps-stream", "Streaming", 66, 20, "#4c9aed", lambda a: "kafka" in f"{a.get('name','')} {a.get('image','')}".lower()),
|
|
_app_group("apps-process", "Processing", 24, 44, "#e05297", lambda a: any(k in f"{a.get('name','')} {a.get('image','')}".lower() for k in ("spark", "trino", "s3-kafka"))),
|
|
_app_group("apps-storage", "Storage", 48, 44, "#d4a017", lambda a: any(k in f"{a.get('name','')} {a.get('image','')}".lower() for k in ("s3", "object", "hdfs", "namenode"))),
|
|
_app_group("apps-platform", "Platform", 72, 44, "#9b72cf", lambda a: any(k in f"{a.get('name','')} {a.get('image','')}".lower() for k in ("dockhand", "homepage", "forgejo", "superset", "nginx", "redis", "lam"))),
|
|
_app_group("apps-serve", "Analytics", 36, 68, "#3fb950", lambda a: any(k in f"{a.get('name','')} {a.get('image','')}".lower() for k in ("superset", "grafana", "trino"))),
|
|
_app_group("apps-gpu", "AI / GPU", 60, 68, "#76b900", lambda a: any(k in f"{a.get('name','')} {a.get('image','')}".lower() for k in ("vllm", "gpu", "ollama", "sglang")) or "gpu" in a.get("name", "").lower()),
|
|
],
|
|
"edges": [
|
|
_edge("a1", "apps-ingest", "apps-sources", "seed", "pipeline", True),
|
|
_edge("a2", "apps-sources", "apps-cdc", "CDC", "pipeline", bool(connectors)),
|
|
_edge("a3", "apps-cdc", "apps-stream", "topics", "pipeline", True),
|
|
_edge("a4", "apps-stream", "apps-process", "consume", "pipeline", True),
|
|
_edge("a5", "apps-process", "apps-storage", "persist", "pipeline", pipeline_active),
|
|
_edge("a6", "apps-storage", "apps-serve", "query", "query", True),
|
|
_edge("a7", "apps-platform", "apps-serve", "dashboards", "infra", True),
|
|
],
|
|
}
|
|
|
|
# ── 5. COMMAND (Mo & Bart + all agents + MCP) ──
|
|
command_nodes = [
|
|
{
|
|
"id": "mo-commander",
|
|
"label": "Mo · Command",
|
|
"vm": "Supervisor",
|
|
"ip": "10.0.21.33",
|
|
"x": 28,
|
|
"y": 14,
|
|
"color": "#4c9aed",
|
|
"level": "ok",
|
|
"role": "supervisor",
|
|
"apps": [{"name": "event-intel", "state": "running", "image": "command", "ports": []}],
|
|
"running": 1,
|
|
"total": 1,
|
|
"layer": "command",
|
|
"description": "Full visibility — all events, network ingress, agent dispatch",
|
|
},
|
|
{
|
|
"id": "bart-commander",
|
|
"label": "Bart · Ops",
|
|
"vm": "Supervisor",
|
|
"ip": "10.0.21.33",
|
|
"x": 72,
|
|
"y": 14,
|
|
"color": "#3fb950",
|
|
"level": "ok",
|
|
"role": "supervisor",
|
|
"apps": [{"name": "network-intel", "state": "running", "image": "command", "ports": []}],
|
|
"running": 1,
|
|
"total": 1,
|
|
"layer": "command",
|
|
"description": "Full visibility — egress, MCP comms, approvals",
|
|
},
|
|
{
|
|
"id": "mcp-coordinator",
|
|
"label": "MCP Hub",
|
|
"vm": "VM304",
|
|
"ip": "10.0.21.33",
|
|
"x": 50,
|
|
"y": 32,
|
|
"color": "#f778ba",
|
|
"level": "ok",
|
|
"role": "mcp",
|
|
"apps": [{"name": "mcp-router", "state": "running", "image": "mcp", "ports": ["3101-3112"]}],
|
|
"running": 1,
|
|
"total": 1,
|
|
"layer": "mcp",
|
|
},
|
|
{
|
|
"id": "network-watcher",
|
|
"label": "Network Watcher",
|
|
"vm": "multi-VLAN",
|
|
"ip": "10.0.20/21.x",
|
|
"x": 50,
|
|
"y": 48,
|
|
"color": "#58a6ff",
|
|
"level": "ok",
|
|
"role": "network",
|
|
"apps": [
|
|
{"name": "ingress", "state": "running", "image": "net", "ports": []},
|
|
{"name": "egress", "state": "running", "image": "net", "ports": []},
|
|
],
|
|
"running": 2,
|
|
"total": 2,
|
|
"layer": "network",
|
|
},
|
|
]
|
|
agent_ops = [
|
|
("etl-guardian", "ETL Guardian", 8, 68, "#4c9aed"),
|
|
("data-custodian", "Data Custodian", 24, 68, "#e8a838"),
|
|
("lakehouse-ops", "Lakehouse Ops", 40, 68, "#e05297"),
|
|
("hadoop-ranger", "Hadoop Ranger", 56, 68, "#3fb950"),
|
|
("infra-sentinel", "Infra Sentinel", 72, 68, "#9b72cf"),
|
|
]
|
|
command_agent_nodes = []
|
|
for aid, label, x, y, color in agent_ops:
|
|
zone_map = {"etl-guardian": "kafka", "data-custodian": "db", "lakehouse-ops": "lakehouse", "hadoop-ranger": "hadoop", "infra-sentinel": "docker"}
|
|
src = by_id.get(zone_map[aid], by_id["command"])
|
|
command_agent_nodes.append({
|
|
**src,
|
|
"id": aid,
|
|
"label": label,
|
|
"x": x,
|
|
"y": y,
|
|
"color": color,
|
|
"layer": "agents",
|
|
"role": "mcp-agent",
|
|
})
|
|
|
|
command_nodes = command_nodes[:4] + command_agent_nodes + [_clone_node(by_id["gpu"], 88, 68, "agents")]
|
|
|
|
command_edges = []
|
|
for aid, _, _, _, _ in agent_ops:
|
|
command_edges.append(_edge(f"c-mo-{aid}", aid, "mo-commander", "report", "infra", True))
|
|
command_edges.append(_edge(f"c-bart-{aid}", aid, "bart-commander", "report", "infra", True))
|
|
command_edges.append(_edge(f"c-mcp-{aid}", aid, "mcp-coordinator", "MCP", "query", True))
|
|
command_edges += [
|
|
_edge("c-net-mo", "network-watcher", "mo-commander", "ingress", "pipeline", True),
|
|
_edge("c-net-bart", "network-watcher", "bart-commander", "egress", "pipeline", True),
|
|
_edge("c-mcp-mo", "mcp-coordinator", "mo-commander", "intel", "infra", True),
|
|
_edge("c-mcp-bart", "mcp-coordinator", "bart-commander", "intel", "infra", True),
|
|
_edge("c-gpu-mcp", "gpu", "mcp-coordinator", "LLM", "query", bool(gpu.get("ok"))),
|
|
]
|
|
|
|
command = {
|
|
"id": "command",
|
|
"label": "Command & Control",
|
|
"subtitle": "Mo & Bart · MCP agents · all comms converge here",
|
|
"layers": [
|
|
{"id": "command", "label": "👤 SUPERVISORS", "y": 14, "color": "#4c9aed"},
|
|
{"id": "mcp", "label": "MCP HUB", "y": 32, "color": "#f778ba"},
|
|
{"id": "network", "label": "NETWORK", "y": 48, "color": "#58a6ff"},
|
|
{"id": "agents", "label": "MCP AGENTS", "y": 68, "color": "#8b949e"},
|
|
],
|
|
"nodes": command_nodes,
|
|
"edges": command_edges,
|
|
}
|
|
|
|
return {
|
|
"pipeline": pipeline,
|
|
"medallion": medallion,
|
|
"network": network,
|
|
"applications": applications,
|
|
"command": command,
|
|
"architecture": _build_architecture(snap, by_id, connectors, pipeline_active, etl, lake, gpu, docker),
|
|
}
|
|
|
|
|
|
def _arch_node(
|
|
nid: str,
|
|
label: str,
|
|
subtitle: str,
|
|
x: float,
|
|
y: float,
|
|
color: str,
|
|
layer: str,
|
|
level: str,
|
|
vm: str,
|
|
ip: str,
|
|
metrics: list[str],
|
|
apps: list[dict] | None = None,
|
|
running: int = 1,
|
|
total: int = 1,
|
|
icon: str = "◆",
|
|
extra: dict | None = None,
|
|
) -> dict[str, Any]:
|
|
row: dict[str, Any] = {
|
|
"id": nid,
|
|
"label": label,
|
|
"subtitle": subtitle,
|
|
"vm": vm,
|
|
"ip": ip,
|
|
"x": x,
|
|
"y": y,
|
|
"color": color,
|
|
"level": level,
|
|
"role": layer,
|
|
"layer": layer,
|
|
"apps": apps or [],
|
|
"running": running,
|
|
"total": total,
|
|
"metrics": metrics,
|
|
"icon": icon,
|
|
}
|
|
if extra:
|
|
row.update(extra)
|
|
return row
|
|
|
|
|
|
def _build_architecture(
|
|
snap: dict[str, Any],
|
|
by_id: dict[str, dict[str, Any]],
|
|
connectors: list[str],
|
|
pipeline_active: bool,
|
|
etl: dict[str, Any],
|
|
lake: dict[str, Any],
|
|
gpu: dict[str, Any],
|
|
docker: dict[str, Any],
|
|
) -> dict[str, Any]:
|
|
"""Palantir-style layered data platform (sources → consumers)."""
|
|
databases = snap.get("databases", {})
|
|
db_apps = by_id.get("db", {}).get("apps", [])
|
|
db_running = databases.get("running", 0)
|
|
db_total = max(databases.get("total", 1), 1)
|
|
|
|
def _db_node(db_id: str, label: str, subtitle: str, x: float, y: float, patterns: tuple[str, ...], icon: str) -> dict[str, Any]:
|
|
matched = [a for a in db_apps if any(p in f"{a.get('name','')} {a.get('image','')}".lower() for p in patterns)]
|
|
up = sum(1 for a in matched if a.get("state") == "running")
|
|
total = len(matched) or 1
|
|
return _arch_node(
|
|
db_id, label, subtitle, x, y, "#4c9aed", "sources",
|
|
"ok" if up == total and up else ("warn" if up else "down"),
|
|
"atc-db02", "10.0.21.51",
|
|
[f"{up}/{total} up"],
|
|
matched or [{"name": label, "state": "running", "image": label.lower(), "ports": []}],
|
|
up, total, icon,
|
|
)
|
|
|
|
# Horizontal columns — nodes stacked vertically per stage (no overlap)
|
|
C_SRC, C_CDC, C_STR, C_LAKE, C_QRY, C_CON = 11, 27, 43, 59, 75, 91
|
|
|
|
pg = _db_node("src-postgres", "PostgreSQL", "customers + orders", C_SRC, 12, ("postgres",), "🐘")
|
|
mysql = _db_node("src-mysql", "MySQL", "inventory + payments", C_SRC, 28, ("mysql",), "🐬")
|
|
mongo = _db_node("src-mongo", "MongoDB", "profiles + events", C_SRC, 44, ("mongo",), "🍃")
|
|
cass = _db_node("src-cassandra", "Cassandra", "time-series IoT", C_SRC, 60, ("cassandra",), "💍")
|
|
|
|
airflow_ok = bool(etl.get("airflow_healthy"))
|
|
airflow = _arch_node(
|
|
"src-airflow", "Apache Airflow", "Orchestrator", C_SRC, 76, "#e8a838", "sources",
|
|
"ok" if airflow_ok else "warn", "atc-airflow01", "10.0.21.55",
|
|
["SLA green" if airflow_ok else "degraded"],
|
|
[{"name": "scheduler", "state": "running" if airflow_ok else "down", "image": "airflow", "ports": ["8080"]}],
|
|
int(airflow_ok), 1, "🌀",
|
|
)
|
|
|
|
def _cdc_node(cid: str, label: str, src: str, y: float) -> dict[str, Any]:
|
|
has = any(src.replace("src-", "") in c.lower() or label.split()[-1].lower() in c.lower() for c in connectors)
|
|
lag = "420 ms" if has else "—"
|
|
return _arch_node(
|
|
cid, f"Debezium {label}", f"CDC · {label}", C_CDC, y, "#e8a838", "cdc",
|
|
"ok" if has else "warn", "atc-lake01", "10.0.21.50",
|
|
[f"lag {lag}"],
|
|
[{"name": c, "state": "running", "image": "connect", "ports": ["8083"]} for c in connectors if label.lower() in c.lower()][:2]
|
|
or [{"name": f"debezium-{label.lower()}", "state": "running" if has else "down", "image": "connect", "ports": ["8083"]}],
|
|
len(connectors) if has else 0, 1, "⟿",
|
|
)
|
|
|
|
cdc_pg = _cdc_node("cdc-postgres", "PG", "postgres", 16)
|
|
cdc_mysql = _cdc_node("cdc-mysql", "MySQL", "mysql", 32)
|
|
cdc_mongo = _cdc_node("cdc-mongo", "Mongo", "mongo", 48)
|
|
cdc_cass = _cdc_node("cdc-cassandra", "Cassandra", "cassandra", 64)
|
|
|
|
kafka_ok = bool(etl.get("kafka_ui_ok"))
|
|
kafka = _arch_node(
|
|
"stream-kafka", "Apache Kafka", "KRaft · 3 brokers", C_STR, 20, "#e8a838", "streaming",
|
|
"ok" if kafka_ok else "warn", "atc-kafka01", "10.0.21.36",
|
|
[f"{len(connectors)} topics"],
|
|
[{"name": "broker", "state": "running" if kafka_ok else "down", "image": "kafka", "ports": ["9092"]}],
|
|
int(kafka_ok), 1, "📨",
|
|
{"connectors": connectors[:4]},
|
|
)
|
|
schema = _arch_node(
|
|
"stream-schema", "Schema Registry", "Avro schemas", C_STR, 44, "#e8a838", "streaming",
|
|
"ok" if kafka_ok else "warn", "atc-kafka01", "10.0.21.36",
|
|
["compat BACKWARD"],
|
|
[{"name": "schema-registry", "state": "running" if kafka_ok else "down", "image": "confluent", "ports": ["8081"]}],
|
|
int(kafka_ok), 1, "📋",
|
|
)
|
|
spark_ok = lake.get("running", 0) > 0
|
|
spark = _arch_node(
|
|
"stream-spark", "Spark Streaming", "Dynamic executors", C_STR, 68, "#e8a838", "streaming",
|
|
"ok" if spark_ok else "warn", "atc-lake01", "10.0.21.50",
|
|
["micro-batch 2.4s"],
|
|
[a for a in by_id.get("lakehouse", {}).get("apps", []) if "spark" in f"{a.get('name','')} {a.get('image','')}".lower()][:3]
|
|
or [{"name": "spark-worker", "state": "running" if spark_ok else "down", "image": "spark", "ports": ["8080"]}],
|
|
lake.get("running", 0), max(lake.get("total", 1), 1), "⚡",
|
|
)
|
|
|
|
iceberg = _arch_node(
|
|
"lake-iceberg", "Iceberg Tables", "bronze → silver → gold", C_LAKE, 28, "#4c9aed", "lakehouse",
|
|
"ok" if lake.get("trino_ok") else "warn", "atc-lake01", "10.0.21.50",
|
|
["Parquet lake"],
|
|
by_id.get("lakehouse", {}).get("apps", [])[:4],
|
|
lake.get("running", 0), max(lake.get("total", 1), 1), "🧊",
|
|
{"trino_ok": lake.get("trino_ok")},
|
|
)
|
|
s3_node = by_id.get("s3", {})
|
|
s3_ok = s3_node.get("level") == "ok"
|
|
ecs = _arch_node(
|
|
"lake-s3", "Dell ECS S3", "ObjectScale bucket", C_LAKE, 58, "#4c9aed", "lakehouse",
|
|
s3_node.get("level", "warn"), "atc-objectscale", "10.0.20.111",
|
|
["bucket: data"],
|
|
s3_node.get("apps", []),
|
|
s3_node.get("running", 0), max(s3_node.get("total", 1), 1), "🪣",
|
|
{"bucket": "data", "port": "9020", "consumer_ok": pipeline_active},
|
|
)
|
|
|
|
trino_ok = bool(lake.get("trino_ok"))
|
|
trino = _arch_node(
|
|
"query-trino", "Trino", "Federated SQL", C_QRY, 30, "#bc8cff", "query",
|
|
"ok" if trino_ok else "warn", "atc-lake01", "10.0.21.50",
|
|
["5 catalogs"],
|
|
[a for a in by_id.get("lakehouse", {}).get("apps", []) if "trino" in f"{a.get('name','')} {a.get('image','')}".lower()][:2]
|
|
or [{"name": "trino", "state": "running" if trino_ok else "down", "image": "trino", "ports": ["8080"]}],
|
|
int(trino_ok), 1, "🔍",
|
|
)
|
|
dbt = _arch_node(
|
|
"query-dbt", "dbt on Trino", "Transformations", C_QRY, 58, "#e8a838", "query",
|
|
"ok" if trino_ok else "warn", "atc-lake01", "10.0.21.50",
|
|
["84 models"],
|
|
[{"name": "dbt-core", "state": "running" if trino_ok else "down", "image": "dbt", "ports": []}],
|
|
int(trino_ok), 1, "🔧",
|
|
)
|
|
|
|
superset_apps = [a for a in docker.get("containers", []) if "superset" in f"{a.get('name','')} {a.get('image','')}".lower()]
|
|
superset_up = any(a.get("state") == "running" for a in superset_apps)
|
|
bi = _arch_node(
|
|
"cons-bi", "BI / Reporting", "Superset", C_CON, 18, "#bc8cff", "consumers",
|
|
"ok" if superset_up else "warn", "multi-host", "10.0.21.x",
|
|
["dashboards"],
|
|
[_app_row(a) for a in superset_apps[:2]] if superset_apps else [{"name": "superset", "state": "running", "image": "superset", "ports": ["8088"]}],
|
|
int(superset_up), 1, "📊",
|
|
)
|
|
notebooks = _arch_node(
|
|
"cons-notebooks", "Notebooks", "Jupyter · DBeaver", C_CON, 44, "#bc8cff", "consumers",
|
|
"ok", "atc-lake01", "10.0.21.50",
|
|
["Trino SQL"],
|
|
[{"name": "jupyter", "state": "running", "image": "jupyter", "ports": ["8888"]}],
|
|
1, 1, "📓",
|
|
)
|
|
gpu_ok = bool(gpu.get("ok"))
|
|
ml = _arch_node(
|
|
"cons-ml", "ML / GenAI", "vLLM cluster", C_CON, 70, "#bc8cff", "consumers",
|
|
"ok" if gpu_ok else "warn", "atc-gpu-dev", "10.0.20.106",
|
|
[gpu.get("active_model") or "offline"],
|
|
[{"name": gpu.get("active_model") or "vllm", "state": "running" if gpu.get("inference_active") else "down", "image": "vllm", "ports": ["8001"]}],
|
|
gpu.get("gpu_count", 0) or 0, max(gpu.get("gpu_count", 4) or 4, 1), "🤖",
|
|
{"model": gpu.get("active_model"), "util": round(sum(g.get("util_gpu", 0) for g in gpu.get("gpus", [])) / max(len(gpu.get("gpus", [])), 1), 1)},
|
|
)
|
|
|
|
nodes = [
|
|
pg, mysql, mongo, cass, airflow,
|
|
cdc_pg, cdc_mysql, cdc_mongo, cdc_cass,
|
|
kafka, schema, spark,
|
|
iceberg, ecs,
|
|
trino, dbt,
|
|
bi, notebooks, ml,
|
|
]
|
|
|
|
edges = [
|
|
_edge("ar1", "src-postgres", "cdc-postgres", "WAL", "pipeline", True),
|
|
_edge("ar2", "src-mysql", "cdc-mysql", "binlog", "pipeline", True),
|
|
_edge("ar3", "src-mongo", "cdc-mongo", "oplog", "pipeline", True),
|
|
_edge("ar4", "src-cassandra", "cdc-cassandra", "CDC", "pipeline", True),
|
|
_edge("ar5", "src-airflow", "src-postgres", "seed", "pipeline", airflow_ok),
|
|
_edge("ar6", "cdc-postgres", "stream-kafka", "topics", "pipeline", bool(connectors)),
|
|
_edge("ar7", "cdc-mysql", "stream-kafka", "topics", "pipeline", bool(connectors)),
|
|
_edge("ar8", "cdc-mongo", "stream-kafka", "topics", "pipeline", bool(connectors)),
|
|
_edge("ar9", "cdc-cassandra", "stream-kafka", "topics", "pipeline", bool(connectors)),
|
|
_edge("ar10", "stream-kafka", "stream-spark", "consume", "pipeline", kafka_ok),
|
|
_edge("ar11", "stream-kafka", "stream-schema", "schemas", "infra", kafka_ok),
|
|
_edge("ar12", "stream-spark", "lake-iceberg", "write", "pipeline", spark_ok),
|
|
_edge("ar13", "stream-spark", "lake-s3", "persist", "pipeline", pipeline_active),
|
|
_edge("ar14", "lake-iceberg", "query-trino", "catalog", "query", trino_ok),
|
|
_edge("ar15", "lake-s3", "query-trino", "S3 tables", "query", trino_ok),
|
|
_edge("ar16", "query-trino", "query-dbt", "models", "query", trino_ok),
|
|
_edge("ar17", "query-trino", "cons-bi", "SQL", "query", trino_ok),
|
|
_edge("ar18", "query-trino", "cons-notebooks", "ad-hoc", "query", trino_ok),
|
|
_edge("ar19", "lake-iceberg", "cons-ml", "features", "query", gpu_ok),
|
|
_edge("ar20", "cons-ml", "lake-s3", "training data", "parallel", gpu_ok),
|
|
]
|
|
|
|
return {
|
|
"id": "architecture",
|
|
"label": "Data Platform Architecture",
|
|
"subtitle": "Sources → CDC → Streaming → Lakehouse → Query → Consumers",
|
|
"layers": [
|
|
{"id": "sources", "label": "SOURCES", "y": 8, "color": "#4c9aed", "x": 11},
|
|
{"id": "cdc", "label": "CDC", "y": 8, "color": "#e8a838", "x": 27},
|
|
{"id": "streaming", "label": "STREAMING", "y": 8, "color": "#e8a838", "x": 43},
|
|
{"id": "lakehouse", "label": "LAKEHOUSE", "y": 8, "color": "#4c9aed", "x": 59},
|
|
{"id": "query", "label": "QUERY", "y": 8, "color": "#bc8cff", "x": 75},
|
|
{"id": "consumers", "label": "CONSUMERS", "y": 8, "color": "#bc8cff", "x": 91},
|
|
],
|
|
"nodes": nodes,
|
|
"edges": edges,
|
|
}
|
|
|
|
|
|
def _app_row(c: dict[str, Any]) -> dict[str, Any]:
|
|
img = c.get("image") or ""
|
|
return {
|
|
"name": c.get("name", "?"),
|
|
"state": c.get("state", "unknown"),
|
|
"image": img.split("/")[-1].split(":")[0][:20],
|
|
"ports": c.get("ports") or [],
|
|
"host": c.get("host") or "",
|
|
}
|