ATC/atc-agents
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
main
atc-agents/api/node_registry.py
T
mo 9008fbd512 feat: Authentik login + switchable GPU prod target 
Add OIDC auth for Command Center and runtime GPU endpoint selection
pointed at atc-gpu-prod (10.0.10.106), matching what is currently deployed.
2026-07-21 23:20:24 +00:00

310 lines
13 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Static registry + helpers for topology node metadata."""
from __future__ import annotations
from typing import Any
NODE_IDS = [
"airflow", "db", "debezium", "kafka", "lakehouse", "s3",
"docker", "hadoop", "gpu", "command", "openmetadata",
"mo-commander", "bart-commander", "network-watcher", "mcp-coordinator",
]
NODE_AGENT = {
"airflow": "etl-guardian",
"db": "data-custodian",
"debezium": "etl-guardian",
"kafka": "etl-guardian",
"lakehouse": "lakehouse-ops",
"s3": "lakehouse-ops",
"docker": "infra-sentinel",
"hadoop": "hadoop-ranger",
"gpu": "infra-sentinel",
"command": "infra-sentinel",
"openmetadata": "lakehouse-ops",
}
NODE_REGISTRY: dict[str, dict[str, Any]] = {
"airflow": {
"label": "Airflow",
"vm": "atc-airflow01",
"vmid": 105,
"pve": "pve01",
"ip": "10.0.21.55",
"ssh": "ssh root@10.0.21.55",
"role": "orchestrator",
"color": "#4c9aed",
"description": "Orchestrates DAG generate_data_all_databases — seeds PostgreSQL, MySQL, MongoDB, Cassandra and Neo4j on db02.",
"links": [{"label": "Airflow UI", "url": "http://10.0.21.55:8080"}],
"endpoints": [{"name": "web", "host": "10.0.21.55", "port": "8080", "proto": "http"}],
"commands": ["dag list", "health check", "trigger generate_data_all_databases"],
},
"db": {
"label": "DB Vault",
"vm": "atc-db02",
"vmid": 109,
"pve": "pve01",
"ip": "10.0.21.51",
"ssh": "ssh root@10.0.21.51",
"role": "sources",
"color": "#e8a838",
"description": "Source-of-truth databases for CDC. Debezium connectors capture changes from PostgreSQL sales, MySQL HR, MongoDB supply chain and Cassandra telemetry.",
"links": [{"label": "Dockhand env 5", "url": "http://10.0.21.45:8082"}],
"endpoints": [
{"name": "postgres_sales", "host": "10.0.21.51", "port": "5432", "proto": "tcp"},
{"name": "mysql_hr", "host": "10.0.21.51", "port": "3306", "proto": "tcp"},
{"name": "mongodb_supplychain", "host": "10.0.21.51", "port": "27017", "proto": "tcp"},
{"name": "cassandra_telemetry", "host": "10.0.21.51", "port": "9042", "proto": "tcp"},
{"name": "neo4j_graph", "host": "10.0.21.51", "port": "7687", "proto": "tcp"},
],
"commands": ["list containers", "engine status", "connector sources"],
},
"debezium": {
"label": "Debezium CDC",
"vm": "atc-lake01",
"vmid": 108,
"pve": "pve01",
"ip": "10.0.21.50",
"ssh": "ssh root@10.0.21.50",
"role": "cdc",
"color": "#c77dff",
"description": "Kafka Connect on lake01 runs Debezium connectors — streams row-level changes from source DBs into Kafka topics.",
"links": [{"label": "Kafka Connect", "url": "http://10.0.21.50:8083"}],
"endpoints": [{"name": "kafka-connect", "host": "10.0.21.50", "port": "8083", "proto": "http"}],
"commands": ["list connectors", "connector status", "restart connector"],
},
"kafka": {
"label": "Kafka Bus",
"vm": "atc-kafka01",
"vmid": 113,
"pve": "pve01",
"ip": "10.0.21.36",
"ssh": "ssh root@10.0.21.36",
"role": "bus",
"color": "#4c9aed",
"description": "Central event bus. CDC topics flow from Debezium to s3-kafka-consumer and Spark on the lakehouse.",
"links": [{"label": "Kafka UI", "url": "http://10.0.21.36:9000"}],
"endpoints": [
{"name": "broker", "host": "10.0.21.36", "port": "9092", "proto": "tcp"},
{"name": "kafka-ui", "host": "10.0.21.36", "port": "9000", "proto": "http"},
],
"commands": ["broker health", "list topics", "consumer lag"],
},
"lakehouse": {
"label": "Lakehouse Hub",
"vm": "atc-lake01",
"vmid": 108,
"pve": "pve01",
"ip": "10.0.21.50",
"role": "compute",
"color": "#e05297",
"description": "Spark + Trino + s3-kafka-consumer. Trino federates queries across DB catalogs and Iceberg on ObjectScale S3.",
"links": [
{"label": "Trino", "url": "http://10.0.21.50:8089"},
{"label": "Spark UI", "url": "http://10.0.21.50:8080"},
{"label": "Kafka Connect", "url": "http://10.0.21.50:8083"},
],
"endpoints": [
{"name": "trino", "host": "10.0.21.50", "port": "8089", "proto": "http"},
{"name": "spark-master", "host": "10.0.21.50", "port": "8080", "proto": "http"},
{"name": "spark-submit", "host": "10.0.21.50", "port": "7077", "proto": "tcp"},
],
"commands": ["trino status", "spark workers", "s3 consumer logs"],
},
"s3": {
"label": "ObjectScale S3",
"vm": "atc-objectscale",
"vmid": 100,
"pve": "pve01",
"ip": "10.0.20.111",
"ssh": "ssh root@10.0.20.111",
"role": "storage",
"color": "#d4a017",
"description": "Dell ObjectScale S3-compatible storage. Landing zone for s3-kafka-consumer and Trino Iceberg catalog (bucket: data).",
"links": [{"label": "S3 API", "url": "http://10.0.20.111:9020"}],
"endpoints": [{"name": "s3-api", "host": "10.0.20.111", "port": "9020", "proto": "http"}],
"bucket": "data",
"commands": ["bucket status", "consumer write rate", "iceberg catalog"],
},
"docker": {
"label": "Docker Rack",
"vm": "atc-docker01",
"vmid": 115,
"pve": "pve01",
"ip": "10.0.21.45",
"ssh": "ssh root@10.0.21.45",
"role": "infra",
"color": "#9b72cf",
"description": "Platform services — Homepage, Dockhand, Superset, Forgejo, Gitea proxy and monitoring stack.",
"links": [
{"label": "Homepage", "url": "http://10.0.21.45"},
{"label": "Dockhand", "url": "http://10.0.21.45:8082"},
{"label": "Superset", "url": "http://10.0.21.45:8088"},
],
"endpoints": [{"name": "dockhand", "host": "10.0.21.45", "port": "8082", "proto": "http"}],
"commands": ["container list", "restart service", "resource usage"],
},
"hadoop": {
"label": "Hadoop HDFS",
"vm": "atc-hadoop-m01",
"vmid": 210,
"pve": "pve02",
"ip": "10.0.21.61",
"ssh": "ssh root@10.0.21.61",
"role": "parallel",
"color": "#3fb950",
"description": "9-node HDFS cluster (3 masters + 5 datanodes + edge). Parallel storage layer — separate from CDC→S3 pipeline.",
"links": [{"label": "NameNode UI", "url": "http://10.0.21.61:9870"}],
"endpoints": [
{"name": "namenode", "host": "10.0.21.61", "port": "9870", "proto": "http"},
{"name": "datanodes", "host": "10.0.21.65-69", "port": "9866", "proto": "http"},
],
"commands": ["hdfs dfsadmin -report", "datanode status", "block health"],
},
"elastic": {
"label": "Elasticsearch / Kibana",
"vm": "atc-elastic01",
"vmid": 0,
"pve": "atc-gpu",
"ip": "10.0.21.46",
"role": "search",
"color": "#f5a623",
"description": "Elasticsearch cluster atc-lakehouse + Kibana dashboards on port 5601.",
"links": [
{"label": "Kibana", "url": "http://10.0.21.46:5601"},
{"label": "Elasticsearch", "url": "https://10.0.21.46:9200"},
],
"endpoints": [
{"name": "elasticsearch", "host": "10.0.21.46", "port": "9200", "proto": "https"},
{"name": "kibana", "host": "10.0.21.46", "port": "5601", "proto": "http"},
],
"commands": ["cluster health", "cat indices", "kibana status"],
},
"gpu": {
"label": "GPU Lab",
"vm": "atc-gpu-prod",
"vmid": 306,
"pve": "atc-gpu",
"ip": "10.0.10.106",
"ssh": "ssh root@10.0.10.106",
"role": "inference",
"color": "#3fb950",
"description": "4× V100 GPU lab (VM306). vLLM serves the active model (Llama 3 70B GPTQ) — powers agent reasoning in this Command Center.",
"links": [
{"label": "GPU Lab UI", "url": "http://10.0.10.106:9000"},
{"label": "vLLM API", "url": "http://10.0.10.106:8001/v1"},
],
"endpoints": [
{"name": "gpu-lab", "host": "10.0.10.106", "port": "9000", "proto": "http"},
{"name": "vllm", "host": "10.0.10.106", "port": "8001", "proto": "http"},
],
"commands": ["gpu metrics", "model status", "vram usage"],
},
"openmetadata": {
"label": "OpenMetadata",
"vm": "atc-docker02",
"vmid": 0,
"pve": "pve01",
"ip": "10.0.21.47",
"ssh": "ssh root@10.0.21.47",
"role": "governance",
"color": "#7147e8",
"description": "OpenMetadata 1.13 data catalog & lineage. Ingests PostgreSQL, MySQL, MongoDB and Trino; auto-classifies PII (Presidio NER). Single pane for catalog, lineage and governance across the lab.",
"links": [
{"label": "OpenMetadata UI", "url": "http://10.0.21.47:8585"},
{"label": "Ingestion Airflow", "url": "http://10.0.21.47:8080"},
],
"endpoints": [
{"name": "openmetadata", "host": "10.0.21.47", "port": "8585", "proto": "http"},
{"name": "ingestion", "host": "10.0.21.47", "port": "8080", "proto": "http"},
],
"commands": ["catalog status", "run ingestion", "pii classification"],
},
"command": {
"label": "Command Center",
"vm": "MCP · VM304",
"vmid": 304,
"pve": "atc-gpu",
"ip": "10.0.21.33",
"ssh": "ssh root@10.0.21.33",
"role": "hub",
"color": "#4c9aed",
"description": "ATC Command Center — FastAPI + React + Redis + Postgres + Caddy. Agent hub & approval inbox.",
"links": [
{"label": "Dashboard", "url": "http://10.0.21.33/"},
{"label": "Dockhand env 13", "url": "http://10.0.21.45:8082"},
],
"endpoints": [{"name": "api", "host": "10.0.21.33", "port": "80", "proto": "http"}],
"commands": ["agent status", "cluster snapshot", "dispatch mission"],
},
"mo-commander": {
"label": "Mo · Command",
"vm": "Supervisor Desk",
"vmid": 304,
"pve": "atc-gpu",
"ip": "10.0.21.33",
"role": "supervisor",
"color": "#4c9aed",
"description": "Mo's command desk — receives ALL lab events, agent dispatch, ingress traffic, approvals.",
"links": [{"label": "Command Center", "url": "http://10.0.21.33/"}],
"endpoints": [{"name": "intel-feed", "host": "10.0.21.33", "port": "80", "proto": "ws"}],
"commands": ["events today", "ingress log", "agent status"],
},
"bart-commander": {
"label": "Bart · Ops",
"vm": "Supervisor Desk",
"vmid": 304,
"pve": "atc-gpu",
"ip": "10.0.21.33",
"role": "supervisor",
"color": "#3fb950",
"description": "Bart's ops desk — egress monitoring, MCP agent comms, S3 writes, GPU inference output.",
"links": [{"label": "Command Center", "url": "http://10.0.21.33/"}],
"endpoints": [{"name": "egress-feed", "host": "10.0.21.33", "port": "80", "proto": "ws"}],
"commands": ["egress log", "mcp comms", "s3 write rate"],
},
"network-watcher": {
"label": "Network Watcher",
"vm": "multi-VLAN",
"ip": "10.0.20/21.x",
"role": "network",
"color": "#58a6ff",
"description": "Monitors VLAN 20 (storage/GPU) and VLAN 21 (compute) — data ingress and egress paths.",
"links": [],
"endpoints": [
{"name": "vlan20", "host": "10.0.20.0/24", "port": "-", "proto": "net"},
{"name": "vlan21", "host": "10.0.21.0/24", "port": "-", "proto": "net"},
],
"commands": ["ingress paths", "egress paths", "vlan status"],
},
"mcp-coordinator": {
"label": "MCP Coordinator",
"vm": "VM304",
"ip": "10.0.21.33",
"role": "mcp",
"color": "#f778ba",
"description": "Routes all MCP agent tool calls. Relays comms between operational agents and supervisor desks.",
"links": [{"label": "API", "url": "http://10.0.21.33/api"}],
"endpoints": [{"name": "mcp-hub", "host": "10.0.21.33", "port": "3101-3112", "proto": "http"}],
"commands": ["agent routes", "mcp status", "relay log"],
},
}
NODE_AGENT.update({
"mo-commander": "mo-commander",
"bart-commander": "bart-commander",
"network-watcher": "network-watcher",
"mcp-coordinator": "mcp-coordinator",
"etl-guardian": "etl-guardian",
"lakehouse-ops": "lakehouse-ops",
"data-custodian": "data-custodian",
"hadoop-ranger": "hadoop-ranger",
"infra-sentinel": "infra-sentinel",
})
def is_node_id(subject_id: str) -> bool:
return subject_id in NODE_REGISTRY
Reference in New Issue View Git Blame Copy Permalink