mo
215ce111f1
- pii_catalog: persistent per-column masking policy (default masked); GET/POST /api/pii/policy and POST /api/pii/lookup which redacts masked values server-side. - get_pii masked flag now reflects the policy; dataflow exposes the dataset key. - Data Flow PII inspector: per-column lock/unlock toggles + mask-all/unmask-all, so operators control exactly which data the assistant may reveal.
489 lines
20 KiB
TypeScript
489 lines
20 KiB
TypeScript
import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from 'react'
|
|
import { GitBranch, Lock, LockOpen, Play, RefreshCw, ShieldAlert, ShieldCheck, Loader2, Bot } from 'lucide-react'
|
|
import type { DataflowEdge, DataflowGraph, DataflowNode } from '../../types'
|
|
import { fetchDataflow, runDataflowMovement, toggleEtlAgent, fetchAgentOpsStatus, setPiiMask } from '../../lib/api'
|
|
import { Badge } from '../ui/Badge'
|
|
import { cn } from '../../lib/utils'
|
|
|
|
/* ── styling maps ───────────────────────────────────────────────── */
|
|
|
|
const NODE_KIND: Record<string, { ring: string; chip: string; dot: string }> = {
|
|
generator: { ring: 'border-amber-400/60', chip: 'bg-amber-500/15 text-amber-300 border-amber-400/40', dot: '#f59e0b' },
|
|
hadoop: { ring: 'border-orange-400/60', chip: 'bg-orange-500/15 text-orange-300 border-orange-400/40', dot: '#fb923c' },
|
|
source: { ring: 'border-emerald-400/60', chip: 'bg-emerald-500/15 text-emerald-300 border-emerald-400/40', dot: '#34d399' },
|
|
stream: { ring: 'border-cyan-400/60', chip: 'bg-cyan-500/15 text-cyan-300 border-cyan-400/40', dot: '#22d3ee' },
|
|
sink: { ring: 'border-sky-400/60', chip: 'bg-sky-500/15 text-sky-300 border-sky-400/40', dot: '#38bdf8' },
|
|
lakehouse: { ring: 'border-blue-400/60', chip: 'bg-blue-500/15 text-blue-300 border-blue-400/40', dot: '#60a5fa' },
|
|
engine: { ring: 'border-violet-400/60', chip: 'bg-violet-500/15 text-violet-300 border-violet-400/40', dot: '#a78bfa' },
|
|
governance: { ring: 'border-fuchsia-400/60', chip: 'bg-fuchsia-500/15 text-fuchsia-300 border-fuchsia-400/40', dot: '#d946ef' },
|
|
}
|
|
|
|
const EDGE_COLOR: Record<string, string> = {
|
|
generate: '#f59e0b',
|
|
cdc: '#22d3ee',
|
|
archive: '#38bdf8',
|
|
movement: '#a78bfa',
|
|
mask: '#fb7185',
|
|
query: '#818cf8',
|
|
catalog: '#d946ef',
|
|
}
|
|
|
|
const EDGE_LEGEND: { kind: string; label: string }[] = [
|
|
{ kind: 'generate', label: 'Generate' },
|
|
{ kind: 'cdc', label: 'CDC capture' },
|
|
{ kind: 'archive', label: 'Archive' },
|
|
{ kind: 'movement', label: 'ETL movement' },
|
|
{ kind: 'mask', label: 'PII masking' },
|
|
{ kind: 'query', label: 'Query' },
|
|
{ kind: 'catalog', label: 'Catalog (OpenMetadata)' },
|
|
]
|
|
|
|
type Anchor = { x: number; y: number; w: number; h: number }
|
|
|
|
function edgePath(a: Anchor, b: Anchor): string {
|
|
const x1 = a.x
|
|
const y1 = a.y
|
|
const x2 = b.x
|
|
const y2 = b.y
|
|
const dx = x2 - x1
|
|
if (Math.abs(dx) < 8) {
|
|
const midY = (y1 + y2) / 2
|
|
return `M ${x1} ${y1} C ${x1} ${midY}, ${x2} ${midY}, ${x2} ${y2}`
|
|
}
|
|
const mx = x1 + dx * 0.5
|
|
return `M ${x1} ${y1} C ${mx} ${y1}, ${mx} ${y2}, ${x2} ${y2}`
|
|
}
|
|
|
|
function fmtDur(s?: number | null): string {
|
|
if (s == null) return ''
|
|
if (s < 60) return `${s.toFixed(0)}s`
|
|
return `${(s / 60).toFixed(1)}m`
|
|
}
|
|
|
|
/* ── component ──────────────────────────────────────────────────── */
|
|
|
|
export function DataFlowView() {
|
|
const [graph, setGraph] = useState<DataflowGraph | null>(null)
|
|
const [loading, setLoading] = useState(true)
|
|
const [refreshing, setRefreshing] = useState(false)
|
|
const [piiOverlay, setPiiOverlay] = useState(true)
|
|
const [selected, setSelected] = useState<string | null>(null)
|
|
const [triggering, setTriggering] = useState<string | null>(null)
|
|
const [etlEnabled, setEtlEnabled] = useState<boolean | null>(null)
|
|
|
|
const canvasRef = useRef<HTMLDivElement>(null)
|
|
const nodeRefs = useRef<Record<string, HTMLButtonElement | null>>({})
|
|
const [anchors, setAnchors] = useState<Record<string, Anchor>>({})
|
|
const [size, setSize] = useState({ w: 800, h: 460 })
|
|
|
|
const setNodeRef = useCallback((id: string) => (el: HTMLButtonElement | null) => {
|
|
nodeRefs.current[id] = el
|
|
}, [])
|
|
|
|
const load = useCallback(async (refresh = false) => {
|
|
const g = await fetchDataflow(refresh)
|
|
if (g) setGraph(g)
|
|
setLoading(false)
|
|
}, [])
|
|
|
|
useEffect(() => {
|
|
load()
|
|
fetchAgentOpsStatus().then((s) => {
|
|
const etl = (s as { etl?: { enabled?: boolean } })?.etl
|
|
if (etl) setEtlEnabled(!!etl.enabled)
|
|
})
|
|
const iv = setInterval(() => load(), 6000)
|
|
return () => clearInterval(iv)
|
|
}, [load])
|
|
|
|
const measure = useCallback(() => {
|
|
const canvas = canvasRef.current
|
|
if (!canvas) return
|
|
const rect = canvas.getBoundingClientRect()
|
|
if (rect.width < 10 || rect.height < 10) return
|
|
setSize({ w: rect.width, h: rect.height })
|
|
const next: Record<string, Anchor> = {}
|
|
for (const [id, el] of Object.entries(nodeRefs.current)) {
|
|
if (!el) continue
|
|
const r = el.getBoundingClientRect()
|
|
next[id] = {
|
|
x: (r.left + r.right) / 2 - rect.left,
|
|
y: (r.top + r.bottom) / 2 - rect.top,
|
|
w: r.width,
|
|
h: r.height,
|
|
}
|
|
}
|
|
setAnchors(next)
|
|
}, [])
|
|
|
|
useLayoutEffect(() => {
|
|
measure()
|
|
const canvas = canvasRef.current
|
|
if (!canvas) return
|
|
const ro = new ResizeObserver(() => measure())
|
|
ro.observe(canvas)
|
|
window.addEventListener('resize', measure)
|
|
return () => {
|
|
ro.disconnect()
|
|
window.removeEventListener('resize', measure)
|
|
}
|
|
}, [measure])
|
|
|
|
useLayoutEffect(() => {
|
|
measure()
|
|
}, [measure, graph])
|
|
|
|
const nodes = graph?.nodes ?? []
|
|
const edges = graph?.edges ?? []
|
|
|
|
const onTrigger = useCallback(async (movementId: string) => {
|
|
setTriggering(movementId)
|
|
try {
|
|
await runDataflowMovement(movementId)
|
|
setTimeout(() => load(true), 800)
|
|
} finally {
|
|
setTimeout(() => setTriggering(null), 1500)
|
|
}
|
|
}, [load])
|
|
|
|
const onToggleEtl = useCallback(async () => {
|
|
const next = !etlEnabled
|
|
setEtlEnabled(next)
|
|
await toggleEtlAgent(next)
|
|
}, [etlEnabled])
|
|
|
|
const onRefresh = useCallback(async () => {
|
|
setRefreshing(true)
|
|
await load(true)
|
|
setTimeout(() => setRefreshing(false), 400)
|
|
}, [load])
|
|
|
|
const [maskBusy, setMaskBusy] = useState<string | null>(null)
|
|
const onToggleMask = useCallback(async (key: string, column: string, masked: boolean) => {
|
|
setMaskBusy(`${key}.${column}`)
|
|
try {
|
|
await setPiiMask(key, column, masked)
|
|
await load(true)
|
|
} finally {
|
|
setMaskBusy(null)
|
|
}
|
|
}, [load])
|
|
|
|
const piiSummary = graph?.pii_summary ?? {}
|
|
const movementEdges = useMemo(
|
|
() => edges.filter((e) => e.movement_id),
|
|
[edges],
|
|
)
|
|
const triggerable = useMemo(() => {
|
|
const seen = new Set<string>()
|
|
const out: DataflowEdge[] = []
|
|
for (const e of movementEdges) {
|
|
if (e.movement_id && !seen.has(e.movement_id)) {
|
|
seen.add(e.movement_id)
|
|
out.push(e)
|
|
}
|
|
}
|
|
return out
|
|
}, [movementEdges])
|
|
|
|
const selNode = nodes.find((n) => n.id === selected) || null
|
|
|
|
return (
|
|
<section className="panel flex h-full min-h-0 flex-1 flex-col overflow-hidden">
|
|
<header
|
|
className="flex shrink-0 flex-col gap-1 border-b border-border px-3 py-2"
|
|
style={{ background: 'var(--topo-header-bg)' }}
|
|
>
|
|
<div className="flex items-center justify-between gap-2">
|
|
<div className="flex min-w-0 items-center gap-2">
|
|
<span className="flex h-6 w-6 shrink-0 items-center justify-center rounded-md bg-docker text-white shadow-docker">
|
|
<GitBranch className="h-3 w-3" />
|
|
</span>
|
|
<div className="min-w-0">
|
|
<h2 className="truncate text-xs font-semibold text-foreground">Data Flow · live lineage</h2>
|
|
<p className="truncate text-[9px] text-foreground-muted">
|
|
Generators → sources → CDC/Kafka → lakehouse · click a node for PII detail · trigger movements below
|
|
</p>
|
|
</div>
|
|
</div>
|
|
<div className="flex shrink-0 flex-wrap items-center justify-end gap-1">
|
|
<Badge variant={graph?.cdc?.connected ? 'success' : 'warning'}>
|
|
{graph?.cdc?.connected ? 'CDC live' : 'CDC offline'}
|
|
</Badge>
|
|
<Badge>{graph?.cdc?.window_total ?? 0} chg/15m</Badge>
|
|
{(piiSummary.unmasked_columns ?? 0) > 0 ? (
|
|
<Badge variant="warning">{piiSummary.unmasked_columns} PII unmasked</Badge>
|
|
) : (
|
|
<Badge variant="accent">{piiSummary.masked_columns ?? 0} PII masked</Badge>
|
|
)}
|
|
<button
|
|
type="button"
|
|
onClick={() => setPiiOverlay((v) => !v)}
|
|
className={cn(
|
|
'rounded border px-1.5 py-0.5 text-[9px] font-medium transition-colors',
|
|
piiOverlay
|
|
? 'border-rose-400/50 bg-rose-500/20 text-rose-200'
|
|
: 'border-border bg-transparent text-foreground-muted hover:text-foreground',
|
|
)}
|
|
>
|
|
PII overlay
|
|
</button>
|
|
<button
|
|
type="button"
|
|
onClick={onToggleEtl}
|
|
className={cn(
|
|
'inline-flex items-center gap-1 rounded border px-1.5 py-0.5 text-[9px] font-medium transition-colors',
|
|
etlEnabled
|
|
? 'border-emerald-400/50 bg-emerald-500/20 text-emerald-200'
|
|
: 'border-border bg-transparent text-foreground-muted hover:text-foreground',
|
|
)}
|
|
>
|
|
<Bot className="h-3 w-3" /> ETL agent {etlEnabled == null ? '' : etlEnabled ? 'on' : 'off'}
|
|
</button>
|
|
<button
|
|
type="button"
|
|
onClick={onRefresh}
|
|
className="inline-flex items-center gap-1 rounded border border-border px-1.5 py-0.5 text-[9px] text-foreground-muted hover:text-foreground"
|
|
>
|
|
<RefreshCw className={cn('h-3 w-3', refreshing && 'animate-spin')} /> Refresh
|
|
</button>
|
|
</div>
|
|
</div>
|
|
<div className="flex flex-wrap gap-x-3 gap-y-0.5">
|
|
{EDGE_LEGEND.map((l) => (
|
|
<span key={l.kind} className="inline-flex items-center gap-1 text-[8px] text-foreground-muted">
|
|
<span className="h-1.5 w-1.5 rounded-full" style={{ background: EDGE_COLOR[l.kind] }} />
|
|
{l.label}
|
|
</span>
|
|
))}
|
|
</div>
|
|
</header>
|
|
|
|
{/* canvas */}
|
|
<div ref={canvasRef} className="topo-canvas relative flex min-h-0 flex-1">
|
|
{loading && (
|
|
<div className="absolute inset-0 z-20 flex items-center justify-center text-[10px] text-foreground-muted">
|
|
<Loader2 className="mr-1 h-3 w-3 animate-spin" /> loading flow…
|
|
</div>
|
|
)}
|
|
<svg
|
|
className="pointer-events-none absolute inset-0 z-0 h-full w-full"
|
|
viewBox={`0 0 ${size.w} ${size.h}`}
|
|
aria-hidden
|
|
>
|
|
{edges.map((edge, i) => {
|
|
const a = anchors[edge.from]
|
|
const b = anchors[edge.to]
|
|
if (!a || !b) return null
|
|
const d = edgePath(a, b)
|
|
const color = EDGE_COLOR[edge.kind] || '#64748b'
|
|
const running = edge.state === 'running'
|
|
const active = edge.active || running
|
|
const dur = 1.6 + (i % 5) * 0.3
|
|
return (
|
|
<g key={`${edge.from}-${edge.to}-${edge.kind}`}>
|
|
<path
|
|
d={d}
|
|
fill="none"
|
|
stroke={color}
|
|
strokeWidth={running ? 2.6 : 1.4}
|
|
strokeOpacity={active ? 0.85 : 0.28}
|
|
strokeDasharray={edge.movement_id && !active ? '4 4' : undefined}
|
|
/>
|
|
{active && (
|
|
<>
|
|
<circle r={running ? 3 : 2.2} fill={color} opacity="0.95">
|
|
<animateMotion dur={`${running ? dur * 0.6 : dur}s`} repeatCount="indefinite" path={d} />
|
|
</circle>
|
|
<circle r="1.3" fill="#ffffff" opacity="0.85">
|
|
<animateMotion dur={`${running ? dur * 0.6 : dur}s`} repeatCount="indefinite" path={d} begin={`${dur * 0.45}s`} />
|
|
</circle>
|
|
</>
|
|
)}
|
|
</g>
|
|
)
|
|
})}
|
|
</svg>
|
|
|
|
{/* nodes */}
|
|
{nodes.map((node) => (
|
|
<NodeCard
|
|
key={node.id}
|
|
node={node}
|
|
piiOverlay={piiOverlay}
|
|
selected={selected === node.id}
|
|
setRef={setNodeRef(node.id)}
|
|
onClick={() => setSelected((s) => (s === node.id ? null : node.id))}
|
|
/>
|
|
))}
|
|
|
|
{/* node inspector */}
|
|
{selNode && (
|
|
<div className="absolute right-2 top-2 z-30 w-52 rounded-lg border border-border bg-surface/95 p-2 shadow-lg backdrop-blur">
|
|
<div className="mb-1 flex items-center justify-between">
|
|
<span className="text-[10px] font-semibold text-foreground">{selNode.label}</span>
|
|
<button onClick={() => setSelected(null)} className="text-[10px] text-foreground-muted hover:text-foreground">✕</button>
|
|
</div>
|
|
<p className="text-[8px] text-foreground-muted">{selNode.sub}</p>
|
|
{selNode.metric && (
|
|
<p className="mt-1 font-mono text-[8px] text-emerald-300">{selNode.metric}</p>
|
|
)}
|
|
{selNode.url && (
|
|
<a
|
|
href={selNode.url}
|
|
target="_blank"
|
|
rel="noreferrer"
|
|
className="mt-1 inline-block rounded border border-fuchsia-400/40 bg-fuchsia-500/15 px-1.5 py-0.5 text-[8px] font-medium text-fuchsia-200 hover:bg-fuchsia-500/25"
|
|
>
|
|
Open in OpenMetadata ↗
|
|
</a>
|
|
)}
|
|
{selNode.pii?.has_pii ? (
|
|
<div className="mt-1.5 border-t border-border pt-1.5">
|
|
<div className="mb-1 flex items-center justify-between gap-1 text-[9px] font-medium text-rose-300">
|
|
<span className="flex items-center gap-1">
|
|
{selNode.pii.all_masked ? <ShieldCheck className="h-3 w-3 text-emerald-300" /> : <ShieldAlert className="h-3 w-3" />}
|
|
{selNode.pii.pii_count} PII column{selNode.pii.pii_count === 1 ? '' : 's'}
|
|
</span>
|
|
{!selNode.pii.masked_layer && (
|
|
<span className="flex gap-1">
|
|
<button
|
|
type="button"
|
|
onClick={() => onToggleMask(selNode.pii!.key, '*', true)}
|
|
className="rounded border border-emerald-400/40 px-1 text-[7px] text-emerald-300 hover:bg-emerald-500/15"
|
|
>mask all</button>
|
|
<button
|
|
type="button"
|
|
onClick={() => onToggleMask(selNode.pii!.key, '*', false)}
|
|
className="rounded border border-rose-400/40 px-1 text-[7px] text-rose-300 hover:bg-rose-500/15"
|
|
>unmask all</button>
|
|
</span>
|
|
)}
|
|
</div>
|
|
<p className="mb-1 text-[7px] leading-tight text-foreground-faint">
|
|
Masked columns are hidden from the assistant — it refuses to reveal them.
|
|
</p>
|
|
<div className="flex flex-col gap-0.5">
|
|
{selNode.pii.columns.map((c) => {
|
|
const locked = c.policy_locked
|
|
const busy = maskBusy === `${selNode.pii!.key}.${c.name}`
|
|
return (
|
|
<button
|
|
key={c.name}
|
|
type="button"
|
|
disabled={locked || busy}
|
|
onClick={() => onToggleMask(selNode.pii!.key, c.name, !c.masked)}
|
|
title={locked ? 'Physically masked in the curated layer' : c.masked ? 'Masked — click to allow the assistant to read it' : 'Visible to the assistant — click to mask'}
|
|
className={cn(
|
|
'flex items-center justify-between gap-1 rounded px-1 py-0.5 text-[8px] transition-colors',
|
|
locked ? 'cursor-default opacity-70' : 'hover:bg-surface-overlay',
|
|
)}
|
|
>
|
|
<span className="truncate font-mono text-foreground-muted">{c.name}</span>
|
|
<span className="flex items-center gap-1">
|
|
<span className="text-blue-300">{c.category}</span>
|
|
<span className={cn('inline-flex items-center gap-0.5', c.masked ? 'text-emerald-300' : 'text-rose-300')}>
|
|
{busy ? <Loader2 className="h-2.5 w-2.5 animate-spin" />
|
|
: c.masked ? <Lock className="h-2.5 w-2.5" /> : <LockOpen className="h-2.5 w-2.5" />}
|
|
{c.masked ? 'masked' : 'visible'}
|
|
</span>
|
|
</span>
|
|
</button>
|
|
)
|
|
})}
|
|
</div>
|
|
</div>
|
|
) : (
|
|
<p className="mt-1 text-[8px] text-foreground-muted">No PII classified.</p>
|
|
)}
|
|
</div>
|
|
)}
|
|
</div>
|
|
|
|
{/* movement control strip */}
|
|
<div className="shrink-0 border-t border-border bg-surface/60 px-3 py-1.5">
|
|
<div className="mb-1 flex items-center gap-1 text-[8px] font-semibold uppercase tracking-wide text-foreground-muted">
|
|
<Play className="h-2.5 w-2.5" /> Data movements
|
|
</div>
|
|
<div className="flex flex-wrap gap-1.5">
|
|
{triggerable.map((e) => {
|
|
const running = e.state === 'running'
|
|
const busy = triggering === e.movement_id || running
|
|
return (
|
|
<button
|
|
key={e.movement_id}
|
|
type="button"
|
|
disabled={busy}
|
|
onClick={() => e.movement_id && onTrigger(e.movement_id)}
|
|
className={cn(
|
|
'inline-flex items-center gap-1 rounded-md border px-2 py-1 text-[9px] font-medium transition-colors disabled:opacity-70',
|
|
running
|
|
? 'border-violet-400/60 bg-violet-500/20 text-violet-200'
|
|
: 'border-border bg-background/40 text-foreground hover:border-emerald-400/50 hover:bg-emerald-500/10',
|
|
)}
|
|
>
|
|
{busy ? <Loader2 className="h-3 w-3 animate-spin" /> : <Play className="h-3 w-3" />}
|
|
<span>{e.movement_id}</span>
|
|
{e.state && e.state !== 'running' && (
|
|
<span className={cn('font-mono', e.state === 'success' ? 'text-emerald-300' : 'text-rose-300')}>
|
|
· {e.state}{e.last_rows != null ? ` ${e.last_rows.toLocaleString()}r` : ''}{e.last_duration_s ? ` ${fmtDur(e.last_duration_s)}` : ''}
|
|
</span>
|
|
)}
|
|
{running && <span className="font-mono text-violet-200">· running…</span>}
|
|
</button>
|
|
)
|
|
})}
|
|
</div>
|
|
</div>
|
|
</section>
|
|
)
|
|
}
|
|
|
|
function NodeCard({
|
|
node, piiOverlay, selected, setRef, onClick,
|
|
}: {
|
|
node: DataflowNode
|
|
piiOverlay: boolean
|
|
selected: boolean
|
|
setRef: (el: HTMLButtonElement | null) => void
|
|
onClick: () => void
|
|
}) {
|
|
const kind = NODE_KIND[node.kind] || NODE_KIND.source
|
|
const pii = node.pii
|
|
const showPii = piiOverlay && pii?.has_pii
|
|
return (
|
|
<button
|
|
ref={setRef}
|
|
type="button"
|
|
onClick={onClick}
|
|
className={cn(
|
|
'absolute z-10 flex w-[116px] -translate-x-1/2 -translate-y-1/2 flex-col items-start gap-0.5 rounded-lg border bg-[#0f1830]/90 px-2 py-1.5 text-left shadow-md backdrop-blur transition-all hover:scale-[1.03]',
|
|
kind.ring,
|
|
selected && 'ring-2 ring-emerald-400/70',
|
|
showPii && !pii?.all_masked && 'ring-2 ring-rose-400/60',
|
|
)}
|
|
style={{ left: `${node.x}%`, top: `${node.y}%` }}
|
|
>
|
|
<div className="flex w-full items-center justify-between gap-1">
|
|
<span className="truncate text-[9px] font-semibold leading-tight text-white">{node.label}</span>
|
|
{showPii && (
|
|
pii?.all_masked
|
|
? <ShieldCheck className="h-3 w-3 shrink-0 text-emerald-300" />
|
|
: <ShieldAlert className="h-3 w-3 shrink-0 text-rose-300" />
|
|
)}
|
|
</div>
|
|
<span className="block w-full truncate text-[7px] leading-tight text-blue-100/70">{node.sub}</span>
|
|
{node.metric && (
|
|
<span className={cn('inline-block max-w-full truncate rounded border px-0.5 py-px font-mono text-[6px] leading-tight', kind.chip)}>
|
|
{node.metric}
|
|
</span>
|
|
)}
|
|
{showPii && (
|
|
<span className="inline-block max-w-full truncate rounded border border-rose-400/40 bg-rose-500/15 px-0.5 py-px text-[6px] font-medium leading-tight text-rose-200">
|
|
{pii?.pii_count} PII · {pii?.categories.slice(0, 2).join(',')}
|
|
</span>
|
|
)}
|
|
</button>
|
|
)
|
|
}
|