ATC/atc-agents
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Nessie catalog UI + Iceberg structure explorer + live Data Flow online/offline

Browse Source 
Add Project Nessie on lake01 with Command Center Iceberg tab (snapshots,
manifests, data files, time-travel SQL). Data Flow pulses only when endpoints
are reachable; offline nodes/edges render red.
This commit is contained in:
mo
2026-07-22 00:31:35 +00:00
parent d5fba208a3
commit b54f5c7a06
9 changed files with 815 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/dataflow.py
+100 -2
View File
@@ -26,6 +26,76 @@ router = APIRouter(prefix="/api/dataflow", tags=["dataflow"])
TRINO_URL = os.getenv("TRINO_URL", "http://10.0.21.50:8089").rstrip("/")
TRINO_USER = os.getenv("TRINO_USER", "mo")
NESSIE_URL = os.getenv("NESSIE_URL", "http://10.0.21.50:19120").rstrip("/")
SRC_DB_HOST = os.getenv("SRC_DB_HOST", "10.0.21.51")
SPARK_UI_URL = os.getenv("SPARK_UI_URL", "http://10.0.21.50:8081").rstrip("/")
KAFKA_UI_URL = os.getenv("KAFKA_UI_URL", "http://10.0.21.36:9000").rstrip("/")
HDFS_NN_URL = os.getenv("HDFS_NN_URL", "http://10.0.21.61:9870").rstrip("/")
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "http://10.0.20.111:9020").rstrip("/")
# TCP ports for source DB reachability on the shared db host
_SOURCE_PORTS = {
"postgres": 5432,
"mysql": 3306,
"mongodb": 27017,
"cassandra": 9042,
"neo4j": 7687,
}
async def _tcp_open(host: str, port: int, timeout: float = 1.2) -> bool:
import asyncio
try:
conn = asyncio.open_connection(host, port)
reader, writer = await asyncio.wait_for(conn, timeout=timeout)
writer.close()
try:
await writer.wait_closed()
except Exception:
pass
return True
except Exception:
return False
async def _http_ok(url: str, timeout: float = 2.0) -> bool:
try:
async with httpx.AsyncClient(timeout=timeout) as client:
r = await client.get(url)
return r.status_code < 500
except Exception:
return False
async def _probe_online() -> dict[str, bool]:
"""Best-effort live reachability for dataflow nodes."""
online: dict[str, bool] = {}
for sid, port in _SOURCE_PORTS.items():
online[sid] = await _tcp_open(SRC_DB_HOST, port)
online["trino"] = await _http_ok(f"{TRINO_URL}/v1/info")
online["nessie"] = await _http_ok(f"{NESSIE_URL}/api/v2/config")
online["spark"] = await _http_ok(f"{SPARK_UI_URL}/json/")
online["kafka"] = await _http_ok(KAFKA_UI_URL)
online["hdfs"] = await _http_ok(HDFS_NN_URL)
# S3: TCP to endpoint host:port
try:
from urllib.parse import urlparse
u = urlparse(S3_ENDPOINT)
online["s3_cdc"] = await _tcp_open(u.hostname or "10.0.20.111", u.port or 9020)
except Exception:
online["s3_cdc"] = False
# lakehouse iceberg nodes follow trino catalog availability
online["iceberg_hadoop"] = online.get("trino", False)
online["iceberg_curated"] = online.get("trino", False)
online["generator"] = True # logical
online["openmetadata"] = await _http_ok("http://10.0.21.47:8585")
online["chromadb"] = True
online["rag"] = True
online["vllm"] = await _http_ok(os.getenv("LLM_URL", "http://10.0.10.106:8001/v1").rstrip("/") + "/models")
online["chat"] = True
return online
# Static landscape (x,y in 0..100). kind drives UI styling.
# Laid out as clean left→right pipeline stages so lineage reads in order:
# producers (col 0) → sources (col 1) → CDC (col 2) → storage/lakehouse (col 3)
@@ -48,6 +118,8 @@ NODES: list[dict[str, Any]] = [
{"id": "s3_cdc", "label": "S3 CDC Archive", "sub": "object store", "kind": "sink", "x": 67, "y": 13},
{"id": "iceberg_curated", "label": "Iceberg · curated_masked", "sub": "masked PII", "kind": "lakehouse", "x": 67, "y": 45},
{"id": "iceberg_hadoop", "label": "Iceberg · hadoop", "sub": "historical_sales_hdfs", "kind": "lakehouse", "x": 67, "y": 80},
{"id": "nessie", "label": "Nessie", "sub": "Iceberg catalog · Git-like", "kind": "catalog", "x": 78, "y": 62,
"url": "http://10.0.21.50:19120"},
# col 4 — query engine
{"id": "trino", "label": "Trino", "sub": "query engine", "kind": "engine", "x": 88, "y": 45},
# governance — bottom centre
@@ -83,6 +155,10 @@ EDGES: list[dict[str, Any]] = [
{"from": "postgres", "to": "iceberg_curated", "kind": "mask", "movement_id": "mask_to_curated"},
{"from": "mysql", "to": "iceberg_curated", "kind": "mask", "movement_id": "mask_to_curated"},
{"from": "mongodb", "to": "iceberg_curated", "kind": "mask", "movement_id": "mask_to_curated"},
{"from": "iceberg_hadoop", "to": "nessie", "kind": "catalog"},
{"from": "iceberg_curated", "to": "nessie", "kind": "catalog"},
{"from": "nessie", "to": "s3_cdc", "kind": "archive"},
{"from": "nessie", "to": "trino", "kind": "query"},
{"from": "iceberg_hadoop", "to": "trino", "kind": "query"},
{"from": "iceberg_curated", "to": "trino", "kind": "query"},
{"from": "kafka", "to": "trino", "kind": "query"},
@@ -198,10 +274,14 @@ async def _build() -> dict[str, Any]:
edge_live = streaming.get("edges") or {}
pii_by_node = {d["node_id"]: d for d in pii.get("datasets", [])}
online = await _probe_online()
nodes = []
for n in NODES:
node = dict(n)
node["level"] = "ok"
is_up = online.get(n["id"], True)
node["online"] = is_up
node["level"] = "ok" if is_up else "err"
metric = None
if n["id"] == "openmetadata":
metric = f"{pii.get('summary', {}).get('pii_columns', 0)} PII cols cataloged"
@@ -217,10 +297,15 @@ async def _build() -> dict[str, Any]:
metric = f"{spark.get('alive_workers', 0)} workers · {used}/{cores} cores · {apps} apps"
node["level"] = "ok" if spark.get("ui_ok") and (spark.get("status") or "").upper() == "ALIVE" else "warn"
elif n["id"] in ("postgres", "mysql", "mongodb", "cassandra", "neo4j"):
metric = f"{cdc.get('by_source', {}).get(n['id'], 0)} CDC/15m"
if not is_up:
metric = "offline"
else:
metric = f"{cdc.get('by_source', {}).get(n['id'], 0)} CDC/15m"
elif n["id"] == "iceberg_hadoop":
c = _iceberg_hadoop_count()
metric = f"{c:,} rows" if c is not None else "iceberg table"
elif n["id"] == "nessie":
metric = "catalog commits" if online.get("nessie") else "offline"
elif n["id"] == "generator":
metric = "Airflow gen DAGs"
elif n["id"] in ("vllm", "rag", "chromadb"):
@@ -283,6 +368,10 @@ async def _build() -> dict[str, Any]:
elif e["kind"] == "archive" and e.get("from") == "kafka" and e.get("to") == "s3_cdc":
# Kafka → S3 CDC archive pulses while the pipeline lands objects in S3
edge["active"] = arch_active
elif e["kind"] == "catalog" and e.get("to") == "nessie":
edge["active"] = bool(online.get("nessie")) and bool(online.get(e.get("from"), True))
elif e["kind"] == "catalog":
edge["active"] = bool(online.get(e.get("from"), True)) and bool(online.get(e.get("to"), True))
elif e["kind"] in ("context", "retrieve", "prompt", "answer"):
# AI serving lane pulses while governed data is being served to the LLM
edge["active"] = bool(_rag_info())
@@ -294,6 +383,15 @@ async def _build() -> dict[str, Any]:
pass
if _flow != "running":
edge["active"] = False
# Realtime online/offline: never pulse if an endpoint is down
frm_up = online.get(e.get("from"), True)
to_up = online.get(e.get("to"), True)
if not (frm_up and to_up):
edge["active"] = False
edge["offline"] = True
else:
edge["offline"] = False
edges.append(edge)
return {