feat(dataflow): add OpenMetadata governance node + catalog edges + UI link
Data Flow graph now shows OpenMetadata as a governance node linked to all sources and Trino (catalog edges); node inspector exposes an 'Open in OpenMetadata' deep link and PII-columns-cataloged metric.
This commit is contained in:
@@ -37,6 +37,8 @@ NODES: list[dict[str, Any]] = [
|
||||
{"id": "iceberg_hadoop", "label": "Iceberg · hadoop", "sub": "historical_sales_hdfs", "kind": "lakehouse", "x": 72, "y": 60},
|
||||
{"id": "iceberg_curated", "label": "Iceberg · curated_masked", "sub": "masked PII", "kind": "lakehouse", "x": 72, "y": 86},
|
||||
{"id": "trino", "label": "Trino", "sub": "query engine", "kind": "engine", "x": 90, "y": 52},
|
||||
{"id": "openmetadata", "label": "OpenMetadata", "sub": "catalog · lineage · PII", "kind": "governance",
|
||||
"x": 50, "y": 92, "url": "http://10.0.21.47:8585"},
|
||||
]
|
||||
|
||||
# Edges. movement_id (optional) links to movements.py so the edge is triggerable.
|
||||
@@ -55,6 +57,10 @@ EDGES: list[dict[str, Any]] = [
|
||||
{"from": "iceberg_hadoop", "to": "trino", "kind": "query"},
|
||||
{"from": "iceberg_curated", "to": "trino", "kind": "query"},
|
||||
{"from": "kafka", "to": "trino", "kind": "query"},
|
||||
{"from": "postgres", "to": "openmetadata", "kind": "catalog"},
|
||||
{"from": "mysql", "to": "openmetadata", "kind": "catalog"},
|
||||
{"from": "mongodb", "to": "openmetadata", "kind": "catalog"},
|
||||
{"from": "trino", "to": "openmetadata", "kind": "catalog"},
|
||||
]
|
||||
|
||||
_cache: dict[str, Any] = {"ts": 0.0, "data": None}
|
||||
@@ -121,6 +127,8 @@ def _build() -> dict[str, Any]:
|
||||
node = dict(n)
|
||||
node["level"] = "ok"
|
||||
metric = None
|
||||
if n["id"] == "openmetadata":
|
||||
metric = f"{pii.get('summary', {}).get('pii_columns', 0)} PII cols cataloged"
|
||||
if n["id"] == "kafka":
|
||||
metric = f"{cdc.get('window_total', 0)} chg/15m · {cdc.get('consumed', 0)} total"
|
||||
node["level"] = "ok" if cdc.get("connected") else "warn"
|
||||
|
||||
Reference in New Issue
Block a user