"""Five animated topology views from data architecture perspectives.""" from __future__ import annotations from typing import Any def _edge(eid: str, src: str, dst: str, label: str, kind: str, active: bool = True) -> dict[str, Any]: return {"id": eid, "from": src, "to": dst, "label": label, "kind": kind, "active": active} def _clone_node(n: dict[str, Any], x: float, y: float, layer: str | None = None) -> dict[str, Any]: out = {**n, "x": x, "y": y} if layer: out["layer"] = layer return out def build_all_topologies( base_nodes: list[dict[str, Any]], base_edges: list[dict[str, Any]], snap: dict[str, Any], *, pipeline_active: bool, connectors: list[str], ) -> dict[str, dict[str, Any]]: by_id = {n["id"]: n for n in base_nodes} etl = snap.get("etl", {}) lake = snap.get("lakehouse", {}) docker = snap.get("docker", {}) gpu = snap.get("gpu", {}) # ── 1. PIPELINE (CDC end-to-end) ── pipeline = { "id": "pipeline", "label": "CDC Pipeline", "subtitle": "Ingest → Stream → Process → Object Storage", "layers": [ {"id": "ingest", "label": "INGEST", "y": 12, "color": "#e8a838"}, {"id": "stream", "label": "STREAM", "y": 12, "color": "#4c9aed"}, {"id": "process", "label": "PROCESS", "y": 12, "color": "#e05297"}, {"id": "store", "label": "STORE", "y": 12, "color": "#d4a017"}, ], "nodes": [ _clone_node(by_id["airflow"], 8, 22, "ingest"), _clone_node(by_id["db"], 24, 22, "ingest"), _clone_node(by_id["debezium"], 40, 22, "stream"), _clone_node(by_id["kafka"], 56, 22, "stream"), _clone_node(by_id["lakehouse"], 72, 22, "process"), _clone_node(by_id["s3"], 88, 22, "store"), _clone_node(by_id["docker"], 12, 58, "infra"), _clone_node(by_id["hadoop"], 50, 58, "parallel"), _clone_node(by_id["gpu"], 88, 58, "compute"), _clone_node(by_id["command"], 50, 82, "hub"), ], "edges": base_edges, } # ── 2. MEDALLION (Bronze → Silver → Gold) ── medallion_nodes = [ _clone_node(by_id["airflow"], 12, 18, "bronze"), _clone_node(by_id["db"], 30, 18, "bronze"), _clone_node(by_id["debezium"], 48, 18, "bronze"), _clone_node(by_id["kafka"], 20, 42, "silver"), { **by_id["lakehouse"], "id": "spark", "label": "Spark ETL", "x": 42, "y": 42, "layer": "silver", "apps": [a for a in by_id["lakehouse"].get("apps", []) if "spark" in a.get("name", "").lower()], }, _clone_node(by_id["lakehouse"], 64, 42, "silver"), _clone_node(by_id["s3"], 24, 68, "gold"), { **by_id.get("docker", {}), "id": "superset", "label": "Superset BI", "x": 48, "y": 68, "layer": "gold", "apps": [a for a in docker.get("containers", []) if "superset" in f"{a.get('name','')} {a.get('image','')}".lower()][:4] or [{"name": "superset", "state": "running", "image": "superset", "ports": ["8088"]}], }, _clone_node(by_id["hadoop"], 72, 68, "gold"), _clone_node(by_id["gpu"], 88, 68, "gold"), ] medallion = { "id": "medallion", "label": "Medallion Architecture", "subtitle": "Bronze (raw) → Silver (staging) → Gold (serving)", "layers": [ {"id": "bronze", "label": "🥉 BRONZE · Raw Ingest", "y": 18, "color": "#cd7f32"}, {"id": "silver", "label": "🥈 SILVER · Staging & Transform", "y": 42, "color": "#c0c0c0"}, {"id": "gold", "label": "🥇 GOLD · Analytics & Serve", "y": 68, "color": "#d4a017"}, ], "nodes": medallion_nodes, "edges": [ _edge("m1", "airflow", "db", "seed", "pipeline", bool(etl.get("airflow_healthy"))), _edge("m2", "db", "debezium", "CDC raw", "pipeline", bool(connectors)), _edge("m3", "debezium", "kafka", "bronze topics", "pipeline", bool(connectors)), _edge("m4", "kafka", "spark", "stream", "pipeline", bool(etl.get("kafka_ui_ok"))), _edge("m5", "spark", "lakehouse", "transform", "pipeline", lake.get("running", 0) > 0), _edge("m6", "lakehouse", "s3", "curated", "pipeline", pipeline_active), _edge("m7", "s3", "superset", "BI queries", "query", True), _edge("m8", "lakehouse", "hadoop", "archive", "parallel", True), ], } # ── 3. NETWORK (VLAN zones, data in/out) ── network = { "id": "network", "label": "Network Topology", "subtitle": "VLAN 20 storage · VLAN 21 compute · ingress/egress", "layers": [ {"id": "ingress", "label": "⬇ DATA IN", "y": 15, "color": "#3fb950"}, {"id": "compute", "label": "COMPUTE 10.0.21.x", "y": 42, "color": "#4c9aed"}, {"id": "storage", "label": "STORAGE 10.0.20.x", "y": 42, "color": "#d4a017"}, {"id": "egress", "label": "⬆ DATA OUT", "y": 70, "color": "#f778ba"}, ], "nodes": [ _clone_node(by_id["airflow"], 12, 16, "ingress"), _clone_node(by_id["db"], 32, 16, "ingress"), _clone_node(by_id["kafka"], 18, 44, "compute"), _clone_node(by_id["debezium"], 36, 44, "compute"), _clone_node(by_id["lakehouse"], 54, 44, "compute"), _clone_node(by_id["hadoop"], 72, 44, "compute"), _clone_node(by_id["docker"], 54, 58, "compute"), _clone_node(by_id["s3"], 18, 44, "storage"), _clone_node(by_id["gpu"], 36, 44, "storage"), { **by_id["command"], "id": "grafana", "label": "Grafana Mon", "vm": "atc-grafana", "ip": "10.0.20.103", "x": 72, "y": 44, "layer": "storage", "color": "#f778ba", }, _clone_node(by_id["s3"], 22, 72, "egress"), _clone_node(by_id["gpu"], 48, 72, "egress"), _clone_node(by_id["docker"], 74, 72, "egress"), _clone_node(by_id["command"], 50, 88, "hub"), ], "edges": [ _edge("n-in1", "airflow", "db", "VLAN21 ingest", "pipeline", True), _edge("n-in2", "db", "debezium", "CDC in", "pipeline", True), _edge("n-x1", "debezium", "kafka", ":9092", "pipeline", True), _edge("n-x2", "lakehouse", "s3", "→ VLAN20", "pipeline", pipeline_active), _edge("n-out1", "s3", "docker", "S3 API out", "query", True), _edge("n-out2", "gpu", "docker", "inference out", "query", bool(gpu.get("ok"))), _edge("n-out3", "lakehouse", "grafana", "metrics", "infra", True), ], } for n in network["nodes"]: if n["id"] == "s3" and n["y"] == 44: n.update({"x": 18, "y": 44}) if n["id"] == "gpu" and n.get("layer") == "storage": n.update({"x": 36, "y": 44}) # ── 4. APPLICATIONS (all workloads by function) ── all_apps: list[dict[str, Any]] = [] for a in by_id.get("docker", {}).get("apps", []): all_apps.append(a) for a in by_id.get("db", {}).get("apps", []): all_apps.append(a) for a in by_id.get("lakehouse", {}).get("apps", []): all_apps.append(a) all_apps.append({"name": "Airflow", "state": "running" if etl.get("airflow_healthy") else "down", "image": "airflow", "ports": ["8080"]}) all_apps.append({"name": "Kafka", "state": "running" if etl.get("kafka_ui_ok") else "down", "image": "kafka", "ports": ["9092"]}) for c in connectors: all_apps.append({"name": c, "state": "running", "image": "connect", "ports": ["8083"]}) def _app_group(gid: str, label: str, x: float, y: float, color: str, filter_fn) -> dict: apps = [a for a in all_apps if filter_fn(a)] running = sum(1 for a in apps if a.get("state") == "running") return { "id": gid, "label": label, "vm": f"{len(apps)} apps", "ip": "multi-host", "x": x, "y": y, "color": color, "level": "ok" if running == len(apps) and apps else "warn", "role": "apps", "apps": apps[:10], "running": running, "total": len(apps) or 1, "layer": "apps", } applications = { "id": "applications", "label": "Application Map", "subtitle": "Every container & service in the lab", "layers": [ {"id": "ingest", "label": "INGEST", "y": 18, "color": "#e8a838"}, {"id": "stream", "label": "STREAM", "y": 18, "color": "#4c9aed"}, {"id": "process", "label": "PROCESS", "y": 42, "color": "#e05297"}, {"id": "store", "label": "STORE & SERVE", "y": 66, "color": "#d4a017"}, ], "nodes": [ _app_group("apps-ingest", "Ingest", 12, 20, "#e8a838", lambda a: "airflow" in a.get("name", "").lower() or "airflow" in a.get("image", "").lower()), _app_group("apps-sources", "Source DBs", 30, 20, "#ffaa00", lambda a: any(k in f"{a.get('name','')} {a.get('image','')}".lower() for k in ("postgres", "mysql", "mongo", "cassandra", "neo4j"))), _app_group("apps-cdc", "CDC Connect", 48, 20, "#c77dff", lambda a: "connect" in a.get("image", "").lower() or "connector" in a.get("name", "").lower()), _app_group("apps-stream", "Streaming", 66, 20, "#4c9aed", lambda a: "kafka" in f"{a.get('name','')} {a.get('image','')}".lower()), _app_group("apps-process", "Processing", 24, 44, "#e05297", lambda a: any(k in f"{a.get('name','')} {a.get('image','')}".lower() for k in ("spark", "trino", "s3-kafka"))), _app_group("apps-storage", "Storage", 48, 44, "#d4a017", lambda a: any(k in f"{a.get('name','')} {a.get('image','')}".lower() for k in ("s3", "object", "hdfs", "namenode"))), _app_group("apps-platform", "Platform", 72, 44, "#9b72cf", lambda a: any(k in f"{a.get('name','')} {a.get('image','')}".lower() for k in ("dockhand", "homepage", "forgejo", "superset", "nginx", "redis", "lam"))), _app_group("apps-serve", "Analytics", 36, 68, "#3fb950", lambda a: any(k in f"{a.get('name','')} {a.get('image','')}".lower() for k in ("superset", "grafana", "trino"))), _app_group("apps-gpu", "AI / GPU", 60, 68, "#76b900", lambda a: any(k in f"{a.get('name','')} {a.get('image','')}".lower() for k in ("vllm", "gpu", "ollama", "sglang")) or "gpu" in a.get("name", "").lower()), ], "edges": [ _edge("a1", "apps-ingest", "apps-sources", "seed", "pipeline", True), _edge("a2", "apps-sources", "apps-cdc", "CDC", "pipeline", bool(connectors)), _edge("a3", "apps-cdc", "apps-stream", "topics", "pipeline", True), _edge("a4", "apps-stream", "apps-process", "consume", "pipeline", True), _edge("a5", "apps-process", "apps-storage", "persist", "pipeline", pipeline_active), _edge("a6", "apps-storage", "apps-serve", "query", "query", True), _edge("a7", "apps-platform", "apps-serve", "dashboards", "infra", True), ], } # ── 5. COMMAND (Mo & Bart + all agents + MCP) ── command_nodes = [ { "id": "mo-commander", "label": "Mo · Command", "vm": "Supervisor", "ip": "10.0.21.33", "x": 28, "y": 14, "color": "#4c9aed", "level": "ok", "role": "supervisor", "apps": [{"name": "event-intel", "state": "running", "image": "command", "ports": []}], "running": 1, "total": 1, "layer": "command", "description": "Full visibility — all events, network ingress, agent dispatch", }, { "id": "bart-commander", "label": "Bart · Ops", "vm": "Supervisor", "ip": "10.0.21.33", "x": 72, "y": 14, "color": "#3fb950", "level": "ok", "role": "supervisor", "apps": [{"name": "network-intel", "state": "running", "image": "command", "ports": []}], "running": 1, "total": 1, "layer": "command", "description": "Full visibility — egress, MCP comms, approvals", }, { "id": "mcp-coordinator", "label": "MCP Hub", "vm": "VM304", "ip": "10.0.21.33", "x": 50, "y": 32, "color": "#f778ba", "level": "ok", "role": "mcp", "apps": [{"name": "mcp-router", "state": "running", "image": "mcp", "ports": ["3101-3112"]}], "running": 1, "total": 1, "layer": "mcp", }, { "id": "network-watcher", "label": "Network Watcher", "vm": "multi-VLAN", "ip": "10.0.20/21.x", "x": 50, "y": 48, "color": "#58a6ff", "level": "ok", "role": "network", "apps": [ {"name": "ingress", "state": "running", "image": "net", "ports": []}, {"name": "egress", "state": "running", "image": "net", "ports": []}, ], "running": 2, "total": 2, "layer": "network", }, ] agent_ops = [ ("etl-guardian", "ETL Guardian", 8, 68, "#4c9aed"), ("data-custodian", "Data Custodian", 24, 68, "#e8a838"), ("lakehouse-ops", "Lakehouse Ops", 40, 68, "#e05297"), ("hadoop-ranger", "Hadoop Ranger", 56, 68, "#3fb950"), ("infra-sentinel", "Infra Sentinel", 72, 68, "#9b72cf"), ] command_agent_nodes = [] for aid, label, x, y, color in agent_ops: zone_map = {"etl-guardian": "kafka", "data-custodian": "db", "lakehouse-ops": "lakehouse", "hadoop-ranger": "hadoop", "infra-sentinel": "docker"} src = by_id.get(zone_map[aid], by_id["command"]) command_agent_nodes.append({ **src, "id": aid, "label": label, "x": x, "y": y, "color": color, "layer": "agents", "role": "mcp-agent", }) command_nodes = command_nodes[:4] + command_agent_nodes + [_clone_node(by_id["gpu"], 88, 68, "agents")] command_edges = [] for aid, _, _, _, _ in agent_ops: command_edges.append(_edge(f"c-mo-{aid}", aid, "mo-commander", "report", "infra", True)) command_edges.append(_edge(f"c-bart-{aid}", aid, "bart-commander", "report", "infra", True)) command_edges.append(_edge(f"c-mcp-{aid}", aid, "mcp-coordinator", "MCP", "query", True)) command_edges += [ _edge("c-net-mo", "network-watcher", "mo-commander", "ingress", "pipeline", True), _edge("c-net-bart", "network-watcher", "bart-commander", "egress", "pipeline", True), _edge("c-mcp-mo", "mcp-coordinator", "mo-commander", "intel", "infra", True), _edge("c-mcp-bart", "mcp-coordinator", "bart-commander", "intel", "infra", True), _edge("c-gpu-mcp", "gpu", "mcp-coordinator", "LLM", "query", bool(gpu.get("ok"))), ] command = { "id": "command", "label": "Command & Control", "subtitle": "Mo & Bart · MCP agents · all comms converge here", "layers": [ {"id": "command", "label": "👤 SUPERVISORS", "y": 14, "color": "#4c9aed"}, {"id": "mcp", "label": "MCP HUB", "y": 32, "color": "#f778ba"}, {"id": "network", "label": "NETWORK", "y": 48, "color": "#58a6ff"}, {"id": "agents", "label": "MCP AGENTS", "y": 68, "color": "#8b949e"}, ], "nodes": command_nodes, "edges": command_edges, } return { "pipeline": pipeline, "medallion": medallion, "network": network, "applications": applications, "command": command, "architecture": _build_architecture(snap, by_id, connectors, pipeline_active, etl, lake, gpu, docker), } def _arch_node( nid: str, label: str, subtitle: str, x: float, y: float, color: str, layer: str, level: str, vm: str, ip: str, metrics: list[str], apps: list[dict] | None = None, running: int = 1, total: int = 1, icon: str = "◆", extra: dict | None = None, ) -> dict[str, Any]: row: dict[str, Any] = { "id": nid, "label": label, "subtitle": subtitle, "vm": vm, "ip": ip, "x": x, "y": y, "color": color, "level": level, "role": layer, "layer": layer, "apps": apps or [], "running": running, "total": total, "metrics": metrics, "icon": icon, } if extra: row.update(extra) return row def _build_architecture( snap: dict[str, Any], by_id: dict[str, dict[str, Any]], connectors: list[str], pipeline_active: bool, etl: dict[str, Any], lake: dict[str, Any], gpu: dict[str, Any], docker: dict[str, Any], ) -> dict[str, Any]: """Palantir-style layered data platform (sources → consumers).""" databases = snap.get("databases", {}) db_apps = by_id.get("db", {}).get("apps", []) db_running = databases.get("running", 0) db_total = max(databases.get("total", 1), 1) def _db_node(db_id: str, label: str, subtitle: str, x: float, y: float, patterns: tuple[str, ...], icon: str) -> dict[str, Any]: matched = [a for a in db_apps if any(p in f"{a.get('name','')} {a.get('image','')}".lower() for p in patterns)] up = sum(1 for a in matched if a.get("state") == "running") total = len(matched) or 1 return _arch_node( db_id, label, subtitle, x, y, "#4c9aed", "sources", "ok" if up == total and up else ("warn" if up else "down"), "atc-db02", "10.0.21.51", [f"{up}/{total} up"], matched or [{"name": label, "state": "running", "image": label.lower(), "ports": []}], up, total, icon, ) # Horizontal columns — nodes stacked vertically per stage (no overlap) C_SRC, C_CDC, C_STR, C_LAKE, C_QRY, C_CON = 11, 27, 43, 59, 75, 91 pg = _db_node("src-postgres", "PostgreSQL", "customers + orders", C_SRC, 12, ("postgres",), "🐘") mysql = _db_node("src-mysql", "MySQL", "inventory + payments", C_SRC, 28, ("mysql",), "🐬") mongo = _db_node("src-mongo", "MongoDB", "profiles + events", C_SRC, 44, ("mongo",), "🍃") cass = _db_node("src-cassandra", "Cassandra", "time-series IoT", C_SRC, 60, ("cassandra",), "💍") airflow_ok = bool(etl.get("airflow_healthy")) airflow = _arch_node( "src-airflow", "Apache Airflow", "Orchestrator", C_SRC, 76, "#e8a838", "sources", "ok" if airflow_ok else "warn", "atc-airflow01", "10.0.21.55", ["SLA green" if airflow_ok else "degraded"], [{"name": "scheduler", "state": "running" if airflow_ok else "down", "image": "airflow", "ports": ["8080"]}], int(airflow_ok), 1, "🌀", ) def _cdc_node(cid: str, label: str, src: str, y: float) -> dict[str, Any]: has = any(src.replace("src-", "") in c.lower() or label.split()[-1].lower() in c.lower() for c in connectors) lag = "420 ms" if has else "—" return _arch_node( cid, f"Debezium {label}", f"CDC · {label}", C_CDC, y, "#e8a838", "cdc", "ok" if has else "warn", "atc-lake01", "10.0.21.50", [f"lag {lag}"], [{"name": c, "state": "running", "image": "connect", "ports": ["8083"]} for c in connectors if label.lower() in c.lower()][:2] or [{"name": f"debezium-{label.lower()}", "state": "running" if has else "down", "image": "connect", "ports": ["8083"]}], len(connectors) if has else 0, 1, "⟿", ) cdc_pg = _cdc_node("cdc-postgres", "PG", "postgres", 16) cdc_mysql = _cdc_node("cdc-mysql", "MySQL", "mysql", 32) cdc_mongo = _cdc_node("cdc-mongo", "Mongo", "mongo", 48) cdc_cass = _cdc_node("cdc-cassandra", "Cassandra", "cassandra", 64) kafka_ok = bool(etl.get("kafka_ui_ok")) kafka = _arch_node( "stream-kafka", "Apache Kafka", "KRaft · 3 brokers", C_STR, 20, "#e8a838", "streaming", "ok" if kafka_ok else "warn", "atc-kafka01", "10.0.21.36", [f"{len(connectors)} topics"], [{"name": "broker", "state": "running" if kafka_ok else "down", "image": "kafka", "ports": ["9092"]}], int(kafka_ok), 1, "📨", {"connectors": connectors[:4]}, ) schema = _arch_node( "stream-schema", "Schema Registry", "Avro schemas", C_STR, 44, "#e8a838", "streaming", "ok" if kafka_ok else "warn", "atc-kafka01", "10.0.21.36", ["compat BACKWARD"], [{"name": "schema-registry", "state": "running" if kafka_ok else "down", "image": "confluent", "ports": ["8081"]}], int(kafka_ok), 1, "📋", ) spark_ok = lake.get("running", 0) > 0 spark = _arch_node( "stream-spark", "Spark Streaming", "Dynamic executors", C_STR, 68, "#e8a838", "streaming", "ok" if spark_ok else "warn", "atc-lake01", "10.0.21.50", ["micro-batch 2.4s"], [a for a in by_id.get("lakehouse", {}).get("apps", []) if "spark" in f"{a.get('name','')} {a.get('image','')}".lower()][:3] or [{"name": "spark-worker", "state": "running" if spark_ok else "down", "image": "spark", "ports": ["8080"]}], lake.get("running", 0), max(lake.get("total", 1), 1), "⚡", ) iceberg = _arch_node( "lake-iceberg", "Iceberg Tables", "bronze → silver → gold", C_LAKE, 28, "#4c9aed", "lakehouse", "ok" if lake.get("trino_ok") else "warn", "atc-lake01", "10.0.21.50", ["Parquet lake"], by_id.get("lakehouse", {}).get("apps", [])[:4], lake.get("running", 0), max(lake.get("total", 1), 1), "🧊", {"trino_ok": lake.get("trino_ok")}, ) s3_node = by_id.get("s3", {}) s3_ok = s3_node.get("level") == "ok" ecs = _arch_node( "lake-s3", "Dell ECS S3", "ObjectScale bucket", C_LAKE, 58, "#4c9aed", "lakehouse", s3_node.get("level", "warn"), "atc-objectscale", "10.0.20.111", ["bucket: data"], s3_node.get("apps", []), s3_node.get("running", 0), max(s3_node.get("total", 1), 1), "🪣", {"bucket": "data", "port": "9020", "consumer_ok": pipeline_active}, ) trino_ok = bool(lake.get("trino_ok")) trino = _arch_node( "query-trino", "Trino", "Federated SQL", C_QRY, 30, "#bc8cff", "query", "ok" if trino_ok else "warn", "atc-lake01", "10.0.21.50", ["5 catalogs"], [a for a in by_id.get("lakehouse", {}).get("apps", []) if "trino" in f"{a.get('name','')} {a.get('image','')}".lower()][:2] or [{"name": "trino", "state": "running" if trino_ok else "down", "image": "trino", "ports": ["8080"]}], int(trino_ok), 1, "🔍", ) dbt = _arch_node( "query-dbt", "dbt on Trino", "Transformations", C_QRY, 58, "#e8a838", "query", "ok" if trino_ok else "warn", "atc-lake01", "10.0.21.50", ["84 models"], [{"name": "dbt-core", "state": "running" if trino_ok else "down", "image": "dbt", "ports": []}], int(trino_ok), 1, "🔧", ) superset_apps = [a for a in docker.get("containers", []) if "superset" in f"{a.get('name','')} {a.get('image','')}".lower()] superset_up = any(a.get("state") == "running" for a in superset_apps) bi = _arch_node( "cons-bi", "BI / Reporting", "Superset", C_CON, 18, "#bc8cff", "consumers", "ok" if superset_up else "warn", "multi-host", "10.0.21.x", ["dashboards"], [_app_row(a) for a in superset_apps[:2]] if superset_apps else [{"name": "superset", "state": "running", "image": "superset", "ports": ["8088"]}], int(superset_up), 1, "📊", ) notebooks = _arch_node( "cons-notebooks", "Notebooks", "Jupyter · DBeaver", C_CON, 44, "#bc8cff", "consumers", "ok", "atc-lake01", "10.0.21.50", ["Trino SQL"], [{"name": "jupyter", "state": "running", "image": "jupyter", "ports": ["8888"]}], 1, 1, "📓", ) gpu_ok = bool(gpu.get("ok")) try: from gpu_config import resolve_gpu_identity gpu_id = resolve_gpu_identity(gpu) except Exception: host = gpu.get("ip") or gpu.get("host") or "10.0.10.106" gpu_id = {"vm": "atc-gpu-prod", "ip": host} ml = _arch_node( "cons-ml", "ML / GenAI", "vLLM cluster", C_CON, 70, "#bc8cff", "consumers", "ok" if gpu_ok else "warn", gpu_id["vm"], gpu_id["ip"], [gpu.get("active_model") or "offline"], [{"name": gpu.get("active_model") or "vllm", "state": "running" if gpu.get("inference_active") else "down", "image": "vllm", "ports": ["8001", "9000"]}], gpu.get("gpu_count", 0) or 0, max(gpu.get("gpu_count", 4) or 4, 1), "🤖", {"model": gpu.get("active_model"), "util": round(sum(g.get("util_gpu", 0) for g in gpu.get("gpus", [])) / max(len(gpu.get("gpus", [])), 1), 1), "ui_url": gpu_id.get("ui_url"), "vllm_url": gpu_id.get("llm_url"), "links": [ {"label": "GPU Lab UI", "url": gpu_id.get("ui_url") or f"http://{gpu_id['ip']}:9000"}, {"label": "vLLM API", "url": gpu_id.get("llm_url") or f"http://{gpu_id['ip']}:8001/v1"}, ]}, ) nodes = [ pg, mysql, mongo, cass, airflow, cdc_pg, cdc_mysql, cdc_mongo, cdc_cass, kafka, schema, spark, iceberg, ecs, trino, dbt, bi, notebooks, ml, ] edges = [ _edge("ar1", "src-postgres", "cdc-postgres", "WAL", "pipeline", True), _edge("ar2", "src-mysql", "cdc-mysql", "binlog", "pipeline", True), _edge("ar3", "src-mongo", "cdc-mongo", "oplog", "pipeline", True), _edge("ar4", "src-cassandra", "cdc-cassandra", "CDC", "pipeline", True), _edge("ar5", "src-airflow", "src-postgres", "seed", "pipeline", airflow_ok), _edge("ar6", "cdc-postgres", "stream-kafka", "topics", "pipeline", bool(connectors)), _edge("ar7", "cdc-mysql", "stream-kafka", "topics", "pipeline", bool(connectors)), _edge("ar8", "cdc-mongo", "stream-kafka", "topics", "pipeline", bool(connectors)), _edge("ar9", "cdc-cassandra", "stream-kafka", "topics", "pipeline", bool(connectors)), _edge("ar10", "stream-kafka", "stream-spark", "consume", "pipeline", kafka_ok), _edge("ar11", "stream-kafka", "stream-schema", "schemas", "infra", kafka_ok), _edge("ar12", "stream-spark", "lake-iceberg", "write", "pipeline", spark_ok), _edge("ar13", "stream-spark", "lake-s3", "persist", "pipeline", pipeline_active), _edge("ar14", "lake-iceberg", "query-trino", "catalog", "query", trino_ok), _edge("ar15", "lake-s3", "query-trino", "S3 tables", "query", trino_ok), _edge("ar16", "query-trino", "query-dbt", "models", "query", trino_ok), _edge("ar17", "query-trino", "cons-bi", "SQL", "query", trino_ok), _edge("ar18", "query-trino", "cons-notebooks", "ad-hoc", "query", trino_ok), _edge("ar19", "lake-iceberg", "cons-ml", "features", "query", gpu_ok), _edge("ar20", "cons-ml", "lake-s3", "training data", "parallel", gpu_ok), ] return { "id": "architecture", "label": "Data Platform Architecture", "subtitle": "Sources → CDC → Streaming → Lakehouse → Query → Consumers", "layers": [ {"id": "sources", "label": "SOURCES", "y": 8, "color": "#4c9aed", "x": 11}, {"id": "cdc", "label": "CDC", "y": 8, "color": "#e8a838", "x": 27}, {"id": "streaming", "label": "STREAMING", "y": 8, "color": "#e8a838", "x": 43}, {"id": "lakehouse", "label": "LAKEHOUSE", "y": 8, "color": "#4c9aed", "x": 59}, {"id": "query", "label": "QUERY", "y": 8, "color": "#bc8cff", "x": 75}, {"id": "consumers", "label": "CONSUMERS", "y": 8, "color": "#bc8cff", "x": 91}, ], "nodes": nodes, "edges": edges, } def _app_row(c: dict[str, Any]) -> dict[str, Any]: img = c.get("image") or "" return { "name": c.get("name", "?"), "state": c.get("state", "unknown"), "image": img.split("/")[-1].split(":")[0][:20], "ports": c.get("ports") or [], "host": c.get("host") or "", }