Add Command Center v2: DQ/RAG integration, S3 browser, Jupyter, GPU matrix.
Mirror mo/atc-GPU layout with config/, docs/, scripts/ for Gitea deploy.
This commit is contained in:
mo
@@ -0,0 +1,268 @@
|
||||
"""Static registry + helpers for topology node metadata."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
NODE_IDS = [
|
||||
"airflow", "db", "debezium", "kafka", "lakehouse", "s3",
|
||||
"docker", "hadoop", "gpu", "command",
|
||||
"mo-commander", "bart-commander", "network-watcher", "mcp-coordinator",
|
||||
]
|
||||
|
||||
NODE_AGENT = {
|
||||
"airflow": "etl-guardian",
|
||||
"db": "data-custodian",
|
||||
"debezium": "etl-guardian",
|
||||
"kafka": "etl-guardian",
|
||||
"lakehouse": "lakehouse-ops",
|
||||
"s3": "lakehouse-ops",
|
||||
"docker": "infra-sentinel",
|
||||
"hadoop": "hadoop-ranger",
|
||||
"gpu": "infra-sentinel",
|
||||
"command": "infra-sentinel",
|
||||
}
|
||||
|
||||
NODE_REGISTRY: dict[str, dict[str, Any]] = {
|
||||
"airflow": {
|
||||
"label": "Airflow",
|
||||
"vm": "atc-airflow01",
|
||||
"vmid": 105,
|
||||
"pve": "pve01",
|
||||
"ip": "10.0.21.55",
|
||||
"ssh": "ssh root@10.0.21.55",
|
||||
"role": "orchestrator",
|
||||
"color": "#4c9aed",
|
||||
"description": "Orchestrates DAG generate_data_all_databases — seeds PostgreSQL, MySQL, MongoDB, Cassandra and Neo4j on db02.",
|
||||
"links": [{"label": "Airflow UI", "url": "http://10.0.21.55:8080"}],
|
||||
"endpoints": [{"name": "web", "host": "10.0.21.55", "port": "8080", "proto": "http"}],
|
||||
"commands": ["dag list", "health check", "trigger generate_data_all_databases"],
|
||||
},
|
||||
"db": {
|
||||
"label": "DB Vault",
|
||||
"vm": "atc-db02",
|
||||
"vmid": 109,
|
||||
"pve": "pve01",
|
||||
"ip": "10.0.21.51",
|
||||
"ssh": "ssh root@10.0.21.51",
|
||||
"role": "sources",
|
||||
"color": "#e8a838",
|
||||
"description": "Source-of-truth databases for CDC. Debezium connectors capture changes from PostgreSQL sales, MySQL HR, MongoDB supply chain and Cassandra telemetry.",
|
||||
"links": [{"label": "Dockhand env 5", "url": "http://10.0.21.45:8082"}],
|
||||
"endpoints": [
|
||||
{"name": "postgres_sales", "host": "10.0.21.51", "port": "5432", "proto": "tcp"},
|
||||
{"name": "mysql_hr", "host": "10.0.21.51", "port": "3306", "proto": "tcp"},
|
||||
{"name": "mongodb_supplychain", "host": "10.0.21.51", "port": "27017", "proto": "tcp"},
|
||||
{"name": "cassandra_telemetry", "host": "10.0.21.51", "port": "9042", "proto": "tcp"},
|
||||
{"name": "neo4j_graph", "host": "10.0.21.51", "port": "7687", "proto": "tcp"},
|
||||
],
|
||||
"commands": ["list containers", "engine status", "connector sources"],
|
||||
},
|
||||
"debezium": {
|
||||
"label": "Debezium CDC",
|
||||
"vm": "atc-lake01",
|
||||
"vmid": 108,
|
||||
"pve": "pve01",
|
||||
"ip": "10.0.21.50",
|
||||
"ssh": "ssh root@10.0.21.50",
|
||||
"role": "cdc",
|
||||
"color": "#c77dff",
|
||||
"description": "Kafka Connect on lake01 runs Debezium connectors — streams row-level changes from source DBs into Kafka topics.",
|
||||
"links": [{"label": "Kafka Connect", "url": "http://10.0.21.50:8083"}],
|
||||
"endpoints": [{"name": "kafka-connect", "host": "10.0.21.50", "port": "8083", "proto": "http"}],
|
||||
"commands": ["list connectors", "connector status", "restart connector"],
|
||||
},
|
||||
"kafka": {
|
||||
"label": "Kafka Bus",
|
||||
"vm": "atc-kafka01",
|
||||
"vmid": 113,
|
||||
"pve": "pve01",
|
||||
"ip": "10.0.21.36",
|
||||
"ssh": "ssh root@10.0.21.36",
|
||||
"role": "bus",
|
||||
"color": "#4c9aed",
|
||||
"description": "Central event bus. CDC topics flow from Debezium to s3-kafka-consumer and Spark on the lakehouse.",
|
||||
"links": [{"label": "Kafka UI", "url": "http://10.0.21.36:9000"}],
|
||||
"endpoints": [
|
||||
{"name": "broker", "host": "10.0.21.36", "port": "9092", "proto": "tcp"},
|
||||
{"name": "kafka-ui", "host": "10.0.21.36", "port": "9000", "proto": "http"},
|
||||
],
|
||||
"commands": ["broker health", "list topics", "consumer lag"],
|
||||
},
|
||||
"lakehouse": {
|
||||
"label": "Lakehouse Hub",
|
||||
"vm": "atc-lake01",
|
||||
"vmid": 108,
|
||||
"pve": "pve01",
|
||||
"ip": "10.0.21.50",
|
||||
"role": "compute",
|
||||
"color": "#e05297",
|
||||
"description": "Spark + Trino + s3-kafka-consumer. Trino federates queries across DB catalogs and Iceberg on ObjectScale S3.",
|
||||
"links": [
|
||||
{"label": "Trino", "url": "http://10.0.21.50:8089"},
|
||||
{"label": "Spark UI", "url": "http://10.0.21.50:8080"},
|
||||
{"label": "Kafka Connect", "url": "http://10.0.21.50:8083"},
|
||||
],
|
||||
"endpoints": [
|
||||
{"name": "trino", "host": "10.0.21.50", "port": "8089", "proto": "http"},
|
||||
{"name": "spark-master", "host": "10.0.21.50", "port": "8080", "proto": "http"},
|
||||
{"name": "spark-submit", "host": "10.0.21.50", "port": "7077", "proto": "tcp"},
|
||||
],
|
||||
"commands": ["trino status", "spark workers", "s3 consumer logs"],
|
||||
},
|
||||
"s3": {
|
||||
"label": "ObjectScale S3",
|
||||
"vm": "atc-objectscale",
|
||||
"vmid": 100,
|
||||
"pve": "pve01",
|
||||
"ip": "10.0.20.111",
|
||||
"ssh": "ssh root@10.0.20.111",
|
||||
"role": "storage",
|
||||
"color": "#d4a017",
|
||||
"description": "Dell ObjectScale S3-compatible storage. Landing zone for s3-kafka-consumer and Trino Iceberg catalog (bucket: data).",
|
||||
"links": [{"label": "S3 API", "url": "http://10.0.20.111:9020"}],
|
||||
"endpoints": [{"name": "s3-api", "host": "10.0.20.111", "port": "9020", "proto": "http"}],
|
||||
"bucket": "data",
|
||||
"commands": ["bucket status", "consumer write rate", "iceberg catalog"],
|
||||
},
|
||||
"docker": {
|
||||
"label": "Docker Rack",
|
||||
"vm": "atc-docker01",
|
||||
"vmid": 115,
|
||||
"pve": "pve01",
|
||||
"ip": "10.0.21.45",
|
||||
"ssh": "ssh root@10.0.21.45",
|
||||
"role": "infra",
|
||||
"color": "#9b72cf",
|
||||
"description": "Platform services — Homepage, Dockhand, Superset, Forgejo, Gitea proxy and monitoring stack.",
|
||||
"links": [
|
||||
{"label": "Homepage", "url": "http://10.0.21.45"},
|
||||
{"label": "Dockhand", "url": "http://10.0.21.45:8082"},
|
||||
{"label": "Superset", "url": "http://10.0.21.45:8088"},
|
||||
],
|
||||
"endpoints": [{"name": "dockhand", "host": "10.0.21.45", "port": "8082", "proto": "http"}],
|
||||
"commands": ["container list", "restart service", "resource usage"],
|
||||
},
|
||||
"hadoop": {
|
||||
"label": "Hadoop HDFS",
|
||||
"vm": "atc-hadoop-m01",
|
||||
"vmid": 210,
|
||||
"pve": "pve02",
|
||||
"ip": "10.0.21.61",
|
||||
"ssh": "ssh root@10.0.21.61",
|
||||
"role": "parallel",
|
||||
"color": "#3fb950",
|
||||
"description": "9-node HDFS cluster (3 masters + 5 datanodes + edge). Parallel storage layer — separate from CDC→S3 pipeline.",
|
||||
"links": [{"label": "NameNode UI", "url": "http://10.0.21.61:9870"}],
|
||||
"endpoints": [
|
||||
{"name": "namenode", "host": "10.0.21.61", "port": "9870", "proto": "http"},
|
||||
{"name": "datanodes", "host": "10.0.21.65-69", "port": "9866", "proto": "http"},
|
||||
],
|
||||
"commands": ["hdfs dfsadmin -report", "datanode status", "block health"],
|
||||
},
|
||||
"gpu": {
|
||||
"label": "GPU Lab",
|
||||
"vm": "atc-gpu-dev",
|
||||
"vmid": 303,
|
||||
"pve": "atc-gpu",
|
||||
"ip": "10.0.20.106",
|
||||
"ssh": "ssh root@10.0.20.106",
|
||||
"role": "inference",
|
||||
"color": "#3fb950",
|
||||
"description": "4× V100 GPU lab. vLLM serves the active model (Llama 3 70B GPTQ) — powers agent reasoning in this Command Center.",
|
||||
"links": [
|
||||
{"label": "GPU Lab UI", "url": "http://10.0.20.106:9000"},
|
||||
{"label": "vLLM API", "url": "http://10.0.20.106:8001/v1"},
|
||||
],
|
||||
"endpoints": [
|
||||
{"name": "gpu-lab", "host": "10.0.20.106", "port": "9000", "proto": "http"},
|
||||
{"name": "vllm", "host": "10.0.20.106", "port": "8001", "proto": "http"},
|
||||
],
|
||||
"commands": ["gpu metrics", "model status", "vram usage"],
|
||||
},
|
||||
"command": {
|
||||
"label": "Command Center",
|
||||
"vm": "MCP · VM304",
|
||||
"vmid": 304,
|
||||
"pve": "atc-gpu",
|
||||
"ip": "10.0.21.33",
|
||||
"ssh": "ssh root@10.0.21.33",
|
||||
"role": "hub",
|
||||
"color": "#4c9aed",
|
||||
"description": "ATC Command Center — FastAPI + React + Redis + Postgres + Caddy. Agent hub & approval inbox.",
|
||||
"links": [
|
||||
{"label": "Dashboard", "url": "http://10.0.21.33/"},
|
||||
{"label": "Dockhand env 13", "url": "http://10.0.21.45:8082"},
|
||||
],
|
||||
"endpoints": [{"name": "api", "host": "10.0.21.33", "port": "80", "proto": "http"}],
|
||||
"commands": ["agent status", "cluster snapshot", "dispatch mission"],
|
||||
},
|
||||
"mo-commander": {
|
||||
"label": "Mo · Command",
|
||||
"vm": "Supervisor Desk",
|
||||
"vmid": 304,
|
||||
"pve": "atc-gpu",
|
||||
"ip": "10.0.21.33",
|
||||
"role": "supervisor",
|
||||
"color": "#4c9aed",
|
||||
"description": "Mo's command desk — receives ALL lab events, agent dispatch, ingress traffic, approvals.",
|
||||
"links": [{"label": "Command Center", "url": "http://10.0.21.33/"}],
|
||||
"endpoints": [{"name": "intel-feed", "host": "10.0.21.33", "port": "80", "proto": "ws"}],
|
||||
"commands": ["events today", "ingress log", "agent status"],
|
||||
},
|
||||
"bart-commander": {
|
||||
"label": "Bart · Ops",
|
||||
"vm": "Supervisor Desk",
|
||||
"vmid": 304,
|
||||
"pve": "atc-gpu",
|
||||
"ip": "10.0.21.33",
|
||||
"role": "supervisor",
|
||||
"color": "#3fb950",
|
||||
"description": "Bart's ops desk — egress monitoring, MCP agent comms, S3 writes, GPU inference output.",
|
||||
"links": [{"label": "Command Center", "url": "http://10.0.21.33/"}],
|
||||
"endpoints": [{"name": "egress-feed", "host": "10.0.21.33", "port": "80", "proto": "ws"}],
|
||||
"commands": ["egress log", "mcp comms", "s3 write rate"],
|
||||
},
|
||||
"network-watcher": {
|
||||
"label": "Network Watcher",
|
||||
"vm": "multi-VLAN",
|
||||
"ip": "10.0.20/21.x",
|
||||
"role": "network",
|
||||
"color": "#58a6ff",
|
||||
"description": "Monitors VLAN 20 (storage/GPU) and VLAN 21 (compute) — data ingress and egress paths.",
|
||||
"links": [],
|
||||
"endpoints": [
|
||||
{"name": "vlan20", "host": "10.0.20.0/24", "port": "-", "proto": "net"},
|
||||
{"name": "vlan21", "host": "10.0.21.0/24", "port": "-", "proto": "net"},
|
||||
],
|
||||
"commands": ["ingress paths", "egress paths", "vlan status"],
|
||||
},
|
||||
"mcp-coordinator": {
|
||||
"label": "MCP Coordinator",
|
||||
"vm": "VM304",
|
||||
"ip": "10.0.21.33",
|
||||
"role": "mcp",
|
||||
"color": "#f778ba",
|
||||
"description": "Routes all MCP agent tool calls. Relays comms between operational agents and supervisor desks.",
|
||||
"links": [{"label": "API", "url": "http://10.0.21.33/api"}],
|
||||
"endpoints": [{"name": "mcp-hub", "host": "10.0.21.33", "port": "3101-3112", "proto": "http"}],
|
||||
"commands": ["agent routes", "mcp status", "relay log"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
NODE_AGENT.update({
|
||||
"mo-commander": "mo-commander",
|
||||
"bart-commander": "bart-commander",
|
||||
"network-watcher": "network-watcher",
|
||||
"mcp-coordinator": "mcp-coordinator",
|
||||
"etl-guardian": "etl-guardian",
|
||||
"lakehouse-ops": "lakehouse-ops",
|
||||
"data-custodian": "data-custodian",
|
||||
"hadoop-ranger": "hadoop-ranger",
|
||||
"infra-sentinel": "infra-sentinel",
|
||||
})
|
||||
|
||||
|
||||
def is_node_id(subject_id: str) -> bool:
|
||||
return subject_id in NODE_REGISTRY
|
||||
Reference in New Issue
Block a user