feat(dataflow): add OpenMetadata governance node + catalog edges + UI link
Data Flow graph now shows OpenMetadata as a governance node linked to all sources and Trino (catalog edges); node inspector exposes an 'Open in OpenMetadata' deep link and PII-columns-cataloged metric.
This commit is contained in:
@@ -37,6 +37,8 @@ NODES: list[dict[str, Any]] = [
|
|||||||
{"id": "iceberg_hadoop", "label": "Iceberg · hadoop", "sub": "historical_sales_hdfs", "kind": "lakehouse", "x": 72, "y": 60},
|
{"id": "iceberg_hadoop", "label": "Iceberg · hadoop", "sub": "historical_sales_hdfs", "kind": "lakehouse", "x": 72, "y": 60},
|
||||||
{"id": "iceberg_curated", "label": "Iceberg · curated_masked", "sub": "masked PII", "kind": "lakehouse", "x": 72, "y": 86},
|
{"id": "iceberg_curated", "label": "Iceberg · curated_masked", "sub": "masked PII", "kind": "lakehouse", "x": 72, "y": 86},
|
||||||
{"id": "trino", "label": "Trino", "sub": "query engine", "kind": "engine", "x": 90, "y": 52},
|
{"id": "trino", "label": "Trino", "sub": "query engine", "kind": "engine", "x": 90, "y": 52},
|
||||||
|
{"id": "openmetadata", "label": "OpenMetadata", "sub": "catalog · lineage · PII", "kind": "governance",
|
||||||
|
"x": 50, "y": 92, "url": "http://10.0.21.47:8585"},
|
||||||
]
|
]
|
||||||
|
|
||||||
# Edges. movement_id (optional) links to movements.py so the edge is triggerable.
|
# Edges. movement_id (optional) links to movements.py so the edge is triggerable.
|
||||||
@@ -55,6 +57,10 @@ EDGES: list[dict[str, Any]] = [
|
|||||||
{"from": "iceberg_hadoop", "to": "trino", "kind": "query"},
|
{"from": "iceberg_hadoop", "to": "trino", "kind": "query"},
|
||||||
{"from": "iceberg_curated", "to": "trino", "kind": "query"},
|
{"from": "iceberg_curated", "to": "trino", "kind": "query"},
|
||||||
{"from": "kafka", "to": "trino", "kind": "query"},
|
{"from": "kafka", "to": "trino", "kind": "query"},
|
||||||
|
{"from": "postgres", "to": "openmetadata", "kind": "catalog"},
|
||||||
|
{"from": "mysql", "to": "openmetadata", "kind": "catalog"},
|
||||||
|
{"from": "mongodb", "to": "openmetadata", "kind": "catalog"},
|
||||||
|
{"from": "trino", "to": "openmetadata", "kind": "catalog"},
|
||||||
]
|
]
|
||||||
|
|
||||||
_cache: dict[str, Any] = {"ts": 0.0, "data": None}
|
_cache: dict[str, Any] = {"ts": 0.0, "data": None}
|
||||||
@@ -121,6 +127,8 @@ def _build() -> dict[str, Any]:
|
|||||||
node = dict(n)
|
node = dict(n)
|
||||||
node["level"] = "ok"
|
node["level"] = "ok"
|
||||||
metric = None
|
metric = None
|
||||||
|
if n["id"] == "openmetadata":
|
||||||
|
metric = f"{pii.get('summary', {}).get('pii_columns', 0)} PII cols cataloged"
|
||||||
if n["id"] == "kafka":
|
if n["id"] == "kafka":
|
||||||
metric = f"{cdc.get('window_total', 0)} chg/15m · {cdc.get('consumed', 0)} total"
|
metric = f"{cdc.get('window_total', 0)} chg/15m · {cdc.get('consumed', 0)} total"
|
||||||
node["level"] = "ok" if cdc.get("connected") else "warn"
|
node["level"] = "ok" if cdc.get("connected") else "warn"
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ const NODE_KIND: Record<string, { ring: string; chip: string; dot: string }> = {
|
|||||||
sink: { ring: 'border-sky-400/60', chip: 'bg-sky-500/15 text-sky-300 border-sky-400/40', dot: '#38bdf8' },
|
sink: { ring: 'border-sky-400/60', chip: 'bg-sky-500/15 text-sky-300 border-sky-400/40', dot: '#38bdf8' },
|
||||||
lakehouse: { ring: 'border-blue-400/60', chip: 'bg-blue-500/15 text-blue-300 border-blue-400/40', dot: '#60a5fa' },
|
lakehouse: { ring: 'border-blue-400/60', chip: 'bg-blue-500/15 text-blue-300 border-blue-400/40', dot: '#60a5fa' },
|
||||||
engine: { ring: 'border-violet-400/60', chip: 'bg-violet-500/15 text-violet-300 border-violet-400/40', dot: '#a78bfa' },
|
engine: { ring: 'border-violet-400/60', chip: 'bg-violet-500/15 text-violet-300 border-violet-400/40', dot: '#a78bfa' },
|
||||||
|
governance: { ring: 'border-fuchsia-400/60', chip: 'bg-fuchsia-500/15 text-fuchsia-300 border-fuchsia-400/40', dot: '#d946ef' },
|
||||||
}
|
}
|
||||||
|
|
||||||
const EDGE_COLOR: Record<string, string> = {
|
const EDGE_COLOR: Record<string, string> = {
|
||||||
@@ -24,6 +25,7 @@ const EDGE_COLOR: Record<string, string> = {
|
|||||||
movement: '#a78bfa',
|
movement: '#a78bfa',
|
||||||
mask: '#fb7185',
|
mask: '#fb7185',
|
||||||
query: '#818cf8',
|
query: '#818cf8',
|
||||||
|
catalog: '#d946ef',
|
||||||
}
|
}
|
||||||
|
|
||||||
const EDGE_LEGEND: { kind: string; label: string }[] = [
|
const EDGE_LEGEND: { kind: string; label: string }[] = [
|
||||||
@@ -33,6 +35,7 @@ const EDGE_LEGEND: { kind: string; label: string }[] = [
|
|||||||
{ kind: 'movement', label: 'ETL movement' },
|
{ kind: 'movement', label: 'ETL movement' },
|
||||||
{ kind: 'mask', label: 'PII masking' },
|
{ kind: 'mask', label: 'PII masking' },
|
||||||
{ kind: 'query', label: 'Query' },
|
{ kind: 'query', label: 'Query' },
|
||||||
|
{ kind: 'catalog', label: 'Catalog (OpenMetadata)' },
|
||||||
]
|
]
|
||||||
|
|
||||||
type Anchor = { x: number; y: number; w: number; h: number }
|
type Anchor = { x: number; y: number; w: number; h: number }
|
||||||
@@ -314,6 +317,16 @@ export function DataFlowView() {
|
|||||||
{selNode.metric && (
|
{selNode.metric && (
|
||||||
<p className="mt-1 font-mono text-[8px] text-emerald-300">{selNode.metric}</p>
|
<p className="mt-1 font-mono text-[8px] text-emerald-300">{selNode.metric}</p>
|
||||||
)}
|
)}
|
||||||
|
{selNode.url && (
|
||||||
|
<a
|
||||||
|
href={selNode.url}
|
||||||
|
target="_blank"
|
||||||
|
rel="noreferrer"
|
||||||
|
className="mt-1 inline-block rounded border border-fuchsia-400/40 bg-fuchsia-500/15 px-1.5 py-0.5 text-[8px] font-medium text-fuchsia-200 hover:bg-fuchsia-500/25"
|
||||||
|
>
|
||||||
|
Open in OpenMetadata ↗
|
||||||
|
</a>
|
||||||
|
)}
|
||||||
{selNode.pii?.has_pii ? (
|
{selNode.pii?.has_pii ? (
|
||||||
<div className="mt-1.5 border-t border-border pt-1.5">
|
<div className="mt-1.5 border-t border-border pt-1.5">
|
||||||
<div className="mb-1 flex items-center gap-1 text-[9px] font-medium text-rose-300">
|
<div className="mb-1 flex items-center gap-1 text-[9px] font-medium text-rose-300">
|
||||||
|
|||||||
@@ -81,6 +81,7 @@ export type DataflowNode = {
|
|||||||
y: number
|
y: number
|
||||||
level: string
|
level: string
|
||||||
metric: string | null
|
metric: string | null
|
||||||
|
url?: string
|
||||||
pii?: {
|
pii?: {
|
||||||
has_pii: boolean
|
has_pii: boolean
|
||||||
pii_count: number
|
pii_count: number
|
||||||
|
|||||||
Reference in New Issue
Block a user