From c70639c46d04168ffa2cd8357db03283d1b94c34 Mon Sep 17 00:00:00 2001 From: mo Date: Sat, 27 Jun 2026 02:43:22 +0200 Subject: [PATCH] feat(dataflow): add OpenMetadata governance node + catalog edges + UI link Data Flow graph now shows OpenMetadata as a governance node linked to all sources and Trino (catalog edges); node inspector exposes an 'Open in OpenMetadata' deep link and PII-columns-cataloged metric. --- api/dataflow.py | 8 ++++++++ ui/src/components/features/DataFlowView.tsx | 13 +++++++++++++ ui/src/types.ts | 1 + 3 files changed, 22 insertions(+) diff --git a/api/dataflow.py b/api/dataflow.py index 520fe9d..0786377 100644 --- a/api/dataflow.py +++ b/api/dataflow.py @@ -37,6 +37,8 @@ NODES: list[dict[str, Any]] = [ {"id": "iceberg_hadoop", "label": "Iceberg · hadoop", "sub": "historical_sales_hdfs", "kind": "lakehouse", "x": 72, "y": 60}, {"id": "iceberg_curated", "label": "Iceberg · curated_masked", "sub": "masked PII", "kind": "lakehouse", "x": 72, "y": 86}, {"id": "trino", "label": "Trino", "sub": "query engine", "kind": "engine", "x": 90, "y": 52}, + {"id": "openmetadata", "label": "OpenMetadata", "sub": "catalog · lineage · PII", "kind": "governance", + "x": 50, "y": 92, "url": "http://10.0.21.47:8585"}, ] # Edges. movement_id (optional) links to movements.py so the edge is triggerable. @@ -55,6 +57,10 @@ EDGES: list[dict[str, Any]] = [ {"from": "iceberg_hadoop", "to": "trino", "kind": "query"}, {"from": "iceberg_curated", "to": "trino", "kind": "query"}, {"from": "kafka", "to": "trino", "kind": "query"}, + {"from": "postgres", "to": "openmetadata", "kind": "catalog"}, + {"from": "mysql", "to": "openmetadata", "kind": "catalog"}, + {"from": "mongodb", "to": "openmetadata", "kind": "catalog"}, + {"from": "trino", "to": "openmetadata", "kind": "catalog"}, ] _cache: dict[str, Any] = {"ts": 0.0, "data": None} @@ -121,6 +127,8 @@ def _build() -> dict[str, Any]: node = dict(n) node["level"] = "ok" metric = None + if n["id"] == "openmetadata": + metric = f"{pii.get('summary', {}).get('pii_columns', 0)} PII cols cataloged" if n["id"] == "kafka": metric = f"{cdc.get('window_total', 0)} chg/15m · {cdc.get('consumed', 0)} total" node["level"] = "ok" if cdc.get("connected") else "warn" diff --git a/ui/src/components/features/DataFlowView.tsx b/ui/src/components/features/DataFlowView.tsx index 09b44b0..9d16a83 100644 --- a/ui/src/components/features/DataFlowView.tsx +++ b/ui/src/components/features/DataFlowView.tsx @@ -15,6 +15,7 @@ const NODE_KIND: Record = { sink: { ring: 'border-sky-400/60', chip: 'bg-sky-500/15 text-sky-300 border-sky-400/40', dot: '#38bdf8' }, lakehouse: { ring: 'border-blue-400/60', chip: 'bg-blue-500/15 text-blue-300 border-blue-400/40', dot: '#60a5fa' }, engine: { ring: 'border-violet-400/60', chip: 'bg-violet-500/15 text-violet-300 border-violet-400/40', dot: '#a78bfa' }, + governance: { ring: 'border-fuchsia-400/60', chip: 'bg-fuchsia-500/15 text-fuchsia-300 border-fuchsia-400/40', dot: '#d946ef' }, } const EDGE_COLOR: Record = { @@ -24,6 +25,7 @@ const EDGE_COLOR: Record = { movement: '#a78bfa', mask: '#fb7185', query: '#818cf8', + catalog: '#d946ef', } const EDGE_LEGEND: { kind: string; label: string }[] = [ @@ -33,6 +35,7 @@ const EDGE_LEGEND: { kind: string; label: string }[] = [ { kind: 'movement', label: 'ETL movement' }, { kind: 'mask', label: 'PII masking' }, { kind: 'query', label: 'Query' }, + { kind: 'catalog', label: 'Catalog (OpenMetadata)' }, ] type Anchor = { x: number; y: number; w: number; h: number } @@ -314,6 +317,16 @@ export function DataFlowView() { {selNode.metric && (

{selNode.metric}

)} + {selNode.url && ( + + Open in OpenMetadata ↗ + + )} {selNode.pii?.has_pii ? (
diff --git a/ui/src/types.ts b/ui/src/types.ts index 83c08c7..800f99f 100644 --- a/ui/src/types.ts +++ b/ui/src/types.ts @@ -81,6 +81,7 @@ export type DataflowNode = { y: number level: string metric: string | null + url?: string pii?: { has_pii: boolean pii_count: number