feat(ui): Data Flow tab — live lineage graph + PII overlay + movement triggers
New DataFlowView (topology-style): nodes positioned from /api/dataflow with measured-anchor SVG edges and animated particles (active CDC + running movements). PII overlay shows shield badges + per-column detail inspector. Bottom strip triggers ETL movements and toggles the ETL agent. Wired into SideNav (Data Flow) and App routing; added types + api helpers.
This commit is contained in:
@@ -19,6 +19,7 @@ import { StorageView } from './components/features/StorageView'
|
|||||||
import { HdfsView } from './components/features/HdfsView'
|
import { HdfsView } from './components/features/HdfsView'
|
||||||
import { DataGenView } from './components/features/DataGenView'
|
import { DataGenView } from './components/features/DataGenView'
|
||||||
import { ChangesView } from './components/features/ChangesView'
|
import { ChangesView } from './components/features/ChangesView'
|
||||||
|
import { DataFlowView } from './components/features/DataFlowView'
|
||||||
import { SearchView } from './components/features/SearchView'
|
import { SearchView } from './components/features/SearchView'
|
||||||
import { SshTerminal } from './components/features/SshTerminal'
|
import { SshTerminal } from './components/features/SshTerminal'
|
||||||
import { TerminalDock } from './components/features/TerminalDock'
|
import { TerminalDock } from './components/features/TerminalDock'
|
||||||
@@ -131,6 +132,8 @@ export default function App() {
|
|||||||
<DataGenView onPulse={cc.pulseFlow} onOpenPlatform={() => cc.setMainView('platform')} />
|
<DataGenView onPulse={cc.pulseFlow} onOpenPlatform={() => cc.setMainView('platform')} />
|
||||||
) : cc.mainView === 'changes' ? (
|
) : cc.mainView === 'changes' ? (
|
||||||
<ChangesView liveChanges={cc.changes} />
|
<ChangesView liveChanges={cc.changes} />
|
||||||
|
) : cc.mainView === 'dataflow' ? (
|
||||||
|
<DataFlowView />
|
||||||
) : cc.mainView === 'presentation' ? (
|
) : cc.mainView === 'presentation' ? (
|
||||||
<PresentationView />
|
<PresentationView />
|
||||||
) : cc.mainView === 'dataquality' ? (
|
) : cc.mainView === 'dataquality' ? (
|
||||||
|
|||||||
@@ -0,0 +1,429 @@
|
|||||||
|
import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from 'react'
|
||||||
|
import { GitBranch, Play, RefreshCw, ShieldAlert, ShieldCheck, Loader2, Bot } from 'lucide-react'
|
||||||
|
import type { DataflowEdge, DataflowGraph, DataflowNode } from '../../types'
|
||||||
|
import { fetchDataflow, runDataflowMovement, toggleEtlAgent, fetchAgentOpsStatus } from '../../lib/api'
|
||||||
|
import { Badge } from '../ui/Badge'
|
||||||
|
import { cn } from '../../lib/utils'
|
||||||
|
|
||||||
|
/* ── styling maps ───────────────────────────────────────────────── */
|
||||||
|
|
||||||
|
const NODE_KIND: Record<string, { ring: string; chip: string; dot: string }> = {
|
||||||
|
generator: { ring: 'border-amber-400/60', chip: 'bg-amber-500/15 text-amber-300 border-amber-400/40', dot: '#f59e0b' },
|
||||||
|
hadoop: { ring: 'border-orange-400/60', chip: 'bg-orange-500/15 text-orange-300 border-orange-400/40', dot: '#fb923c' },
|
||||||
|
source: { ring: 'border-emerald-400/60', chip: 'bg-emerald-500/15 text-emerald-300 border-emerald-400/40', dot: '#34d399' },
|
||||||
|
stream: { ring: 'border-cyan-400/60', chip: 'bg-cyan-500/15 text-cyan-300 border-cyan-400/40', dot: '#22d3ee' },
|
||||||
|
sink: { ring: 'border-sky-400/60', chip: 'bg-sky-500/15 text-sky-300 border-sky-400/40', dot: '#38bdf8' },
|
||||||
|
lakehouse: { ring: 'border-blue-400/60', chip: 'bg-blue-500/15 text-blue-300 border-blue-400/40', dot: '#60a5fa' },
|
||||||
|
engine: { ring: 'border-violet-400/60', chip: 'bg-violet-500/15 text-violet-300 border-violet-400/40', dot: '#a78bfa' },
|
||||||
|
}
|
||||||
|
|
||||||
|
const EDGE_COLOR: Record<string, string> = {
|
||||||
|
generate: '#f59e0b',
|
||||||
|
cdc: '#22d3ee',
|
||||||
|
archive: '#38bdf8',
|
||||||
|
movement: '#a78bfa',
|
||||||
|
mask: '#fb7185',
|
||||||
|
query: '#818cf8',
|
||||||
|
}
|
||||||
|
|
||||||
|
const EDGE_LEGEND: { kind: string; label: string }[] = [
|
||||||
|
{ kind: 'generate', label: 'Generate' },
|
||||||
|
{ kind: 'cdc', label: 'CDC capture' },
|
||||||
|
{ kind: 'archive', label: 'Archive' },
|
||||||
|
{ kind: 'movement', label: 'ETL movement' },
|
||||||
|
{ kind: 'mask', label: 'PII masking' },
|
||||||
|
{ kind: 'query', label: 'Query' },
|
||||||
|
]
|
||||||
|
|
||||||
|
type Anchor = { x: number; y: number; w: number; h: number }
|
||||||
|
|
||||||
|
function edgePath(a: Anchor, b: Anchor): string {
|
||||||
|
const x1 = a.x
|
||||||
|
const y1 = a.y
|
||||||
|
const x2 = b.x
|
||||||
|
const y2 = b.y
|
||||||
|
const dx = x2 - x1
|
||||||
|
if (Math.abs(dx) < 8) {
|
||||||
|
const midY = (y1 + y2) / 2
|
||||||
|
return `M ${x1} ${y1} C ${x1} ${midY}, ${x2} ${midY}, ${x2} ${y2}`
|
||||||
|
}
|
||||||
|
const mx = x1 + dx * 0.5
|
||||||
|
return `M ${x1} ${y1} C ${mx} ${y1}, ${mx} ${y2}, ${x2} ${y2}`
|
||||||
|
}
|
||||||
|
|
||||||
|
function fmtDur(s?: number | null): string {
|
||||||
|
if (s == null) return ''
|
||||||
|
if (s < 60) return `${s.toFixed(0)}s`
|
||||||
|
return `${(s / 60).toFixed(1)}m`
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ── component ──────────────────────────────────────────────────── */
|
||||||
|
|
||||||
|
export function DataFlowView() {
|
||||||
|
const [graph, setGraph] = useState<DataflowGraph | null>(null)
|
||||||
|
const [loading, setLoading] = useState(true)
|
||||||
|
const [refreshing, setRefreshing] = useState(false)
|
||||||
|
const [piiOverlay, setPiiOverlay] = useState(true)
|
||||||
|
const [selected, setSelected] = useState<string | null>(null)
|
||||||
|
const [triggering, setTriggering] = useState<string | null>(null)
|
||||||
|
const [etlEnabled, setEtlEnabled] = useState<boolean | null>(null)
|
||||||
|
|
||||||
|
const canvasRef = useRef<HTMLDivElement>(null)
|
||||||
|
const nodeRefs = useRef<Record<string, HTMLButtonElement | null>>({})
|
||||||
|
const [anchors, setAnchors] = useState<Record<string, Anchor>>({})
|
||||||
|
const [size, setSize] = useState({ w: 800, h: 460 })
|
||||||
|
|
||||||
|
const setNodeRef = useCallback((id: string) => (el: HTMLButtonElement | null) => {
|
||||||
|
nodeRefs.current[id] = el
|
||||||
|
}, [])
|
||||||
|
|
||||||
|
const load = useCallback(async (refresh = false) => {
|
||||||
|
const g = await fetchDataflow(refresh)
|
||||||
|
if (g) setGraph(g)
|
||||||
|
setLoading(false)
|
||||||
|
}, [])
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
load()
|
||||||
|
fetchAgentOpsStatus().then((s) => {
|
||||||
|
const etl = (s as { etl?: { enabled?: boolean } })?.etl
|
||||||
|
if (etl) setEtlEnabled(!!etl.enabled)
|
||||||
|
})
|
||||||
|
const iv = setInterval(() => load(), 6000)
|
||||||
|
return () => clearInterval(iv)
|
||||||
|
}, [load])
|
||||||
|
|
||||||
|
const measure = useCallback(() => {
|
||||||
|
const canvas = canvasRef.current
|
||||||
|
if (!canvas) return
|
||||||
|
const rect = canvas.getBoundingClientRect()
|
||||||
|
if (rect.width < 10 || rect.height < 10) return
|
||||||
|
setSize({ w: rect.width, h: rect.height })
|
||||||
|
const next: Record<string, Anchor> = {}
|
||||||
|
for (const [id, el] of Object.entries(nodeRefs.current)) {
|
||||||
|
if (!el) continue
|
||||||
|
const r = el.getBoundingClientRect()
|
||||||
|
next[id] = {
|
||||||
|
x: (r.left + r.right) / 2 - rect.left,
|
||||||
|
y: (r.top + r.bottom) / 2 - rect.top,
|
||||||
|
w: r.width,
|
||||||
|
h: r.height,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
setAnchors(next)
|
||||||
|
}, [])
|
||||||
|
|
||||||
|
useLayoutEffect(() => {
|
||||||
|
measure()
|
||||||
|
const canvas = canvasRef.current
|
||||||
|
if (!canvas) return
|
||||||
|
const ro = new ResizeObserver(() => measure())
|
||||||
|
ro.observe(canvas)
|
||||||
|
window.addEventListener('resize', measure)
|
||||||
|
return () => {
|
||||||
|
ro.disconnect()
|
||||||
|
window.removeEventListener('resize', measure)
|
||||||
|
}
|
||||||
|
}, [measure])
|
||||||
|
|
||||||
|
useLayoutEffect(() => {
|
||||||
|
measure()
|
||||||
|
}, [measure, graph])
|
||||||
|
|
||||||
|
const nodes = graph?.nodes ?? []
|
||||||
|
const edges = graph?.edges ?? []
|
||||||
|
|
||||||
|
const onTrigger = useCallback(async (movementId: string) => {
|
||||||
|
setTriggering(movementId)
|
||||||
|
try {
|
||||||
|
await runDataflowMovement(movementId)
|
||||||
|
setTimeout(() => load(true), 800)
|
||||||
|
} finally {
|
||||||
|
setTimeout(() => setTriggering(null), 1500)
|
||||||
|
}
|
||||||
|
}, [load])
|
||||||
|
|
||||||
|
const onToggleEtl = useCallback(async () => {
|
||||||
|
const next = !etlEnabled
|
||||||
|
setEtlEnabled(next)
|
||||||
|
await toggleEtlAgent(next)
|
||||||
|
}, [etlEnabled])
|
||||||
|
|
||||||
|
const onRefresh = useCallback(async () => {
|
||||||
|
setRefreshing(true)
|
||||||
|
await load(true)
|
||||||
|
setTimeout(() => setRefreshing(false), 400)
|
||||||
|
}, [load])
|
||||||
|
|
||||||
|
const piiSummary = graph?.pii_summary ?? {}
|
||||||
|
const movementEdges = useMemo(
|
||||||
|
() => edges.filter((e) => e.movement_id),
|
||||||
|
[edges],
|
||||||
|
)
|
||||||
|
const triggerable = useMemo(() => {
|
||||||
|
const seen = new Set<string>()
|
||||||
|
const out: DataflowEdge[] = []
|
||||||
|
for (const e of movementEdges) {
|
||||||
|
if (e.movement_id && !seen.has(e.movement_id)) {
|
||||||
|
seen.add(e.movement_id)
|
||||||
|
out.push(e)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}, [movementEdges])
|
||||||
|
|
||||||
|
const selNode = nodes.find((n) => n.id === selected) || null
|
||||||
|
|
||||||
|
return (
|
||||||
|
<section className="panel flex h-full min-h-0 flex-1 flex-col overflow-hidden">
|
||||||
|
<header
|
||||||
|
className="flex shrink-0 flex-col gap-1 border-b border-border px-3 py-2"
|
||||||
|
style={{ background: 'var(--topo-header-bg)' }}
|
||||||
|
>
|
||||||
|
<div className="flex items-center justify-between gap-2">
|
||||||
|
<div className="flex min-w-0 items-center gap-2">
|
||||||
|
<span className="flex h-6 w-6 shrink-0 items-center justify-center rounded-md bg-docker text-white shadow-docker">
|
||||||
|
<GitBranch className="h-3 w-3" />
|
||||||
|
</span>
|
||||||
|
<div className="min-w-0">
|
||||||
|
<h2 className="truncate text-xs font-semibold text-foreground">Data Flow · live lineage</h2>
|
||||||
|
<p className="truncate text-[9px] text-foreground-muted">
|
||||||
|
Generators → sources → CDC/Kafka → lakehouse · click a node for PII detail · trigger movements below
|
||||||
|
</p>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div className="flex shrink-0 flex-wrap items-center justify-end gap-1">
|
||||||
|
<Badge variant={graph?.cdc?.connected ? 'success' : 'warning'}>
|
||||||
|
{graph?.cdc?.connected ? 'CDC live' : 'CDC offline'}
|
||||||
|
</Badge>
|
||||||
|
<Badge>{graph?.cdc?.window_total ?? 0} chg/15m</Badge>
|
||||||
|
{(piiSummary.unmasked_columns ?? 0) > 0 ? (
|
||||||
|
<Badge variant="warning">{piiSummary.unmasked_columns} PII unmasked</Badge>
|
||||||
|
) : (
|
||||||
|
<Badge variant="accent">{piiSummary.masked_columns ?? 0} PII masked</Badge>
|
||||||
|
)}
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
onClick={() => setPiiOverlay((v) => !v)}
|
||||||
|
className={cn(
|
||||||
|
'rounded border px-1.5 py-0.5 text-[9px] font-medium transition-colors',
|
||||||
|
piiOverlay
|
||||||
|
? 'border-rose-400/50 bg-rose-500/20 text-rose-200'
|
||||||
|
: 'border-border bg-transparent text-foreground-muted hover:text-foreground',
|
||||||
|
)}
|
||||||
|
>
|
||||||
|
PII overlay
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
onClick={onToggleEtl}
|
||||||
|
className={cn(
|
||||||
|
'inline-flex items-center gap-1 rounded border px-1.5 py-0.5 text-[9px] font-medium transition-colors',
|
||||||
|
etlEnabled
|
||||||
|
? 'border-emerald-400/50 bg-emerald-500/20 text-emerald-200'
|
||||||
|
: 'border-border bg-transparent text-foreground-muted hover:text-foreground',
|
||||||
|
)}
|
||||||
|
>
|
||||||
|
<Bot className="h-3 w-3" /> ETL agent {etlEnabled == null ? '' : etlEnabled ? 'on' : 'off'}
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
onClick={onRefresh}
|
||||||
|
className="inline-flex items-center gap-1 rounded border border-border px-1.5 py-0.5 text-[9px] text-foreground-muted hover:text-foreground"
|
||||||
|
>
|
||||||
|
<RefreshCw className={cn('h-3 w-3', refreshing && 'animate-spin')} /> Refresh
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div className="flex flex-wrap gap-x-3 gap-y-0.5">
|
||||||
|
{EDGE_LEGEND.map((l) => (
|
||||||
|
<span key={l.kind} className="inline-flex items-center gap-1 text-[8px] text-foreground-muted">
|
||||||
|
<span className="h-1.5 w-1.5 rounded-full" style={{ background: EDGE_COLOR[l.kind] }} />
|
||||||
|
{l.label}
|
||||||
|
</span>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
</header>
|
||||||
|
|
||||||
|
{/* canvas */}
|
||||||
|
<div ref={canvasRef} className="topo-canvas relative flex min-h-0 flex-1">
|
||||||
|
{loading && (
|
||||||
|
<div className="absolute inset-0 z-20 flex items-center justify-center text-[10px] text-foreground-muted">
|
||||||
|
<Loader2 className="mr-1 h-3 w-3 animate-spin" /> loading flow…
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
<svg
|
||||||
|
className="pointer-events-none absolute inset-0 z-0 h-full w-full"
|
||||||
|
viewBox={`0 0 ${size.w} ${size.h}`}
|
||||||
|
aria-hidden
|
||||||
|
>
|
||||||
|
{edges.map((edge, i) => {
|
||||||
|
const a = anchors[edge.from]
|
||||||
|
const b = anchors[edge.to]
|
||||||
|
if (!a || !b) return null
|
||||||
|
const d = edgePath(a, b)
|
||||||
|
const color = EDGE_COLOR[edge.kind] || '#64748b'
|
||||||
|
const running = edge.state === 'running'
|
||||||
|
const active = edge.active || running
|
||||||
|
const dur = 1.6 + (i % 5) * 0.3
|
||||||
|
return (
|
||||||
|
<g key={`${edge.from}-${edge.to}-${edge.kind}`}>
|
||||||
|
<path
|
||||||
|
d={d}
|
||||||
|
fill="none"
|
||||||
|
stroke={color}
|
||||||
|
strokeWidth={running ? 2.6 : 1.4}
|
||||||
|
strokeOpacity={active ? 0.85 : 0.28}
|
||||||
|
strokeDasharray={edge.movement_id && !active ? '4 4' : undefined}
|
||||||
|
/>
|
||||||
|
{active && (
|
||||||
|
<>
|
||||||
|
<circle r={running ? 3 : 2.2} fill={color} opacity="0.95">
|
||||||
|
<animateMotion dur={`${running ? dur * 0.6 : dur}s`} repeatCount="indefinite" path={d} />
|
||||||
|
</circle>
|
||||||
|
<circle r="1.3" fill="#ffffff" opacity="0.85">
|
||||||
|
<animateMotion dur={`${running ? dur * 0.6 : dur}s`} repeatCount="indefinite" path={d} begin={`${dur * 0.45}s`} />
|
||||||
|
</circle>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
</g>
|
||||||
|
)
|
||||||
|
})}
|
||||||
|
</svg>
|
||||||
|
|
||||||
|
{/* nodes */}
|
||||||
|
{nodes.map((node) => (
|
||||||
|
<NodeCard
|
||||||
|
key={node.id}
|
||||||
|
node={node}
|
||||||
|
piiOverlay={piiOverlay}
|
||||||
|
selected={selected === node.id}
|
||||||
|
setRef={setNodeRef(node.id)}
|
||||||
|
onClick={() => setSelected((s) => (s === node.id ? null : node.id))}
|
||||||
|
/>
|
||||||
|
))}
|
||||||
|
|
||||||
|
{/* node inspector */}
|
||||||
|
{selNode && (
|
||||||
|
<div className="absolute right-2 top-2 z-30 w-52 rounded-lg border border-border bg-surface/95 p-2 shadow-lg backdrop-blur">
|
||||||
|
<div className="mb-1 flex items-center justify-between">
|
||||||
|
<span className="text-[10px] font-semibold text-foreground">{selNode.label}</span>
|
||||||
|
<button onClick={() => setSelected(null)} className="text-[10px] text-foreground-muted hover:text-foreground">✕</button>
|
||||||
|
</div>
|
||||||
|
<p className="text-[8px] text-foreground-muted">{selNode.sub}</p>
|
||||||
|
{selNode.metric && (
|
||||||
|
<p className="mt-1 font-mono text-[8px] text-emerald-300">{selNode.metric}</p>
|
||||||
|
)}
|
||||||
|
{selNode.pii?.has_pii ? (
|
||||||
|
<div className="mt-1.5 border-t border-border pt-1.5">
|
||||||
|
<div className="mb-1 flex items-center gap-1 text-[9px] font-medium text-rose-300">
|
||||||
|
{selNode.pii.all_masked ? <ShieldCheck className="h-3 w-3 text-emerald-300" /> : <ShieldAlert className="h-3 w-3" />}
|
||||||
|
{selNode.pii.pii_count} PII column{selNode.pii.pii_count === 1 ? '' : 's'}
|
||||||
|
</div>
|
||||||
|
<div className="flex flex-col gap-0.5">
|
||||||
|
{selNode.pii.columns.map((c) => (
|
||||||
|
<div key={c.name} className="flex items-center justify-between gap-1 text-[8px]">
|
||||||
|
<span className="truncate font-mono text-foreground-muted">{c.name}</span>
|
||||||
|
<span className="flex items-center gap-1">
|
||||||
|
<span className="text-blue-300">{c.category}</span>
|
||||||
|
<span className={c.masked ? 'text-emerald-300' : 'text-rose-300'}>
|
||||||
|
{c.masked ? 'masked' : 'raw'}
|
||||||
|
</span>
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
) : (
|
||||||
|
<p className="mt-1 text-[8px] text-foreground-muted">No PII classified.</p>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{/* movement control strip */}
|
||||||
|
<div className="shrink-0 border-t border-border bg-surface/60 px-3 py-1.5">
|
||||||
|
<div className="mb-1 flex items-center gap-1 text-[8px] font-semibold uppercase tracking-wide text-foreground-muted">
|
||||||
|
<Play className="h-2.5 w-2.5" /> Data movements
|
||||||
|
</div>
|
||||||
|
<div className="flex flex-wrap gap-1.5">
|
||||||
|
{triggerable.map((e) => {
|
||||||
|
const running = e.state === 'running'
|
||||||
|
const busy = triggering === e.movement_id || running
|
||||||
|
return (
|
||||||
|
<button
|
||||||
|
key={e.movement_id}
|
||||||
|
type="button"
|
||||||
|
disabled={busy}
|
||||||
|
onClick={() => e.movement_id && onTrigger(e.movement_id)}
|
||||||
|
className={cn(
|
||||||
|
'inline-flex items-center gap-1 rounded-md border px-2 py-1 text-[9px] font-medium transition-colors disabled:opacity-70',
|
||||||
|
running
|
||||||
|
? 'border-violet-400/60 bg-violet-500/20 text-violet-200'
|
||||||
|
: 'border-border bg-background/40 text-foreground hover:border-emerald-400/50 hover:bg-emerald-500/10',
|
||||||
|
)}
|
||||||
|
>
|
||||||
|
{busy ? <Loader2 className="h-3 w-3 animate-spin" /> : <Play className="h-3 w-3" />}
|
||||||
|
<span>{e.movement_id}</span>
|
||||||
|
{e.state && e.state !== 'running' && (
|
||||||
|
<span className={cn('font-mono', e.state === 'success' ? 'text-emerald-300' : 'text-rose-300')}>
|
||||||
|
· {e.state}{e.last_rows != null ? ` ${e.last_rows.toLocaleString()}r` : ''}{e.last_duration_s ? ` ${fmtDur(e.last_duration_s)}` : ''}
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
{running && <span className="font-mono text-violet-200">· running…</span>}
|
||||||
|
</button>
|
||||||
|
)
|
||||||
|
})}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</section>
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
function NodeCard({
|
||||||
|
node, piiOverlay, selected, setRef, onClick,
|
||||||
|
}: {
|
||||||
|
node: DataflowNode
|
||||||
|
piiOverlay: boolean
|
||||||
|
selected: boolean
|
||||||
|
setRef: (el: HTMLButtonElement | null) => void
|
||||||
|
onClick: () => void
|
||||||
|
}) {
|
||||||
|
const kind = NODE_KIND[node.kind] || NODE_KIND.source
|
||||||
|
const pii = node.pii
|
||||||
|
const showPii = piiOverlay && pii?.has_pii
|
||||||
|
return (
|
||||||
|
<button
|
||||||
|
ref={setRef}
|
||||||
|
type="button"
|
||||||
|
onClick={onClick}
|
||||||
|
className={cn(
|
||||||
|
'absolute z-10 flex w-[116px] -translate-x-1/2 -translate-y-1/2 flex-col items-start gap-0.5 rounded-lg border bg-[#0f1830]/90 px-2 py-1.5 text-left shadow-md backdrop-blur transition-all hover:scale-[1.03]',
|
||||||
|
kind.ring,
|
||||||
|
selected && 'ring-2 ring-emerald-400/70',
|
||||||
|
showPii && !pii?.all_masked && 'ring-2 ring-rose-400/60',
|
||||||
|
)}
|
||||||
|
style={{ left: `${node.x}%`, top: `${node.y}%` }}
|
||||||
|
>
|
||||||
|
<div className="flex w-full items-center justify-between gap-1">
|
||||||
|
<span className="truncate text-[9px] font-semibold leading-tight text-white">{node.label}</span>
|
||||||
|
{showPii && (
|
||||||
|
pii?.all_masked
|
||||||
|
? <ShieldCheck className="h-3 w-3 shrink-0 text-emerald-300" />
|
||||||
|
: <ShieldAlert className="h-3 w-3 shrink-0 text-rose-300" />
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
<span className="block w-full truncate text-[7px] leading-tight text-blue-100/70">{node.sub}</span>
|
||||||
|
{node.metric && (
|
||||||
|
<span className={cn('inline-block max-w-full truncate rounded border px-0.5 py-px font-mono text-[6px] leading-tight', kind.chip)}>
|
||||||
|
{node.metric}
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
{showPii && (
|
||||||
|
<span className="inline-block max-w-full truncate rounded border border-rose-400/40 bg-rose-500/15 px-0.5 py-px text-[6px] font-medium leading-tight text-rose-200">
|
||||||
|
{pii?.pii_count} PII · {pii?.categories.slice(0, 2).join(',')}
|
||||||
|
</span>
|
||||||
|
)}
|
||||||
|
</button>
|
||||||
|
)
|
||||||
|
}
|
||||||
@@ -1,4 +1,4 @@
|
|||||||
import { DatabaseZap, HardDrive, Search, LayoutDashboard, MessageSquare, Presentation, Server, TerminalSquare, Cpu, Activity } from 'lucide-react'
|
import { DatabaseZap, HardDrive, Search, LayoutDashboard, MessageSquare, Presentation, Server, TerminalSquare, Cpu, Activity, GitBranch } from 'lucide-react'
|
||||||
import type { GpuStatus, WorkloadData } from '../../types'
|
import type { GpuStatus, WorkloadData } from '../../types'
|
||||||
import type { GpuLiveMetrics } from '../../hooks/useLiveMetrics'
|
import type { GpuLiveMetrics } from '../../hooks/useLiveMetrics'
|
||||||
import { cn } from '../../lib/utils'
|
import { cn } from '../../lib/utils'
|
||||||
@@ -26,6 +26,7 @@ const VIEWS: { id: MainView; label: string; icon: typeof LayoutDashboard }[] = [
|
|||||||
{ id: 'platform', label: 'Data Platform', icon: LayoutDashboard },
|
{ id: 'platform', label: 'Data Platform', icon: LayoutDashboard },
|
||||||
{ id: 'datagen', label: 'Data Generation', icon: Cpu },
|
{ id: 'datagen', label: 'Data Generation', icon: Cpu },
|
||||||
{ id: 'changes', label: 'Live Changes', icon: Activity },
|
{ id: 'changes', label: 'Live Changes', icon: Activity },
|
||||||
|
{ id: 'dataflow', label: 'Data Flow', icon: GitBranch },
|
||||||
{ id: 'presentation', label: 'Presentation', icon: Presentation },
|
{ id: 'presentation', label: 'Presentation', icon: Presentation },
|
||||||
{ id: 'dataquality', label: 'Data Quality', icon: DatabaseZap },
|
{ id: 'dataquality', label: 'Data Quality', icon: DatabaseZap },
|
||||||
{ id: 'knowledge', label: 'Knowledge Chat', icon: MessageSquare },
|
{ id: 'knowledge', label: 'Knowledge Chat', icon: MessageSquare },
|
||||||
|
|||||||
@@ -4,8 +4,11 @@ import type {
|
|||||||
Approval,
|
Approval,
|
||||||
CdcChange,
|
CdcChange,
|
||||||
CdcStats,
|
CdcStats,
|
||||||
|
DataflowGraph,
|
||||||
FeedEntry,
|
FeedEntry,
|
||||||
GpuStatus,
|
GpuStatus,
|
||||||
|
Movement,
|
||||||
|
PiiDataset,
|
||||||
StatusData,
|
StatusData,
|
||||||
TerminalLine,
|
TerminalLine,
|
||||||
WorkloadData,
|
WorkloadData,
|
||||||
@@ -129,6 +132,37 @@ export function runAgentOpOnce(source?: string, op?: string) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function fetchDataflow(refresh = false): Promise<DataflowGraph | null> {
|
||||||
|
return fetchJson<DataflowGraph>(`/api/dataflow${refresh ? '?refresh=true' : ''}`, 25000)
|
||||||
|
}
|
||||||
|
|
||||||
|
export function runDataflowMovement(movementId: string, conf?: Record<string, unknown>) {
|
||||||
|
return fetch(`/api/dataflow/${movementId}/run`, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify(conf ? { conf } : {}),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function fetchMovements(): Promise<Movement[]> {
|
||||||
|
const j = await fetchJson<{ movements?: Movement[] }>('/api/movements', 8000)
|
||||||
|
return j?.movements || []
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function fetchPii(refresh = false): Promise<{ datasets: PiiDataset[]; summary: Record<string, number> } | null> {
|
||||||
|
return fetchJson<{ datasets: PiiDataset[]; summary: Record<string, number> }>(
|
||||||
|
`/api/pii${refresh ? '?refresh=true' : ''}`, 15000,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
export function toggleEtlAgent(enabled?: boolean) {
|
||||||
|
return fetch('/api/agent-ops/etl/toggle', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify(enabled === undefined ? {} : { enabled }),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
export async function decideApproval(id: string, approved: boolean, decidedBy: string, note: string) {
|
export async function decideApproval(id: string, approved: boolean, decidedBy: string, note: string) {
|
||||||
return fetch(`/api/approvals/${id}/decide`, {
|
return fetch(`/api/approvals/${id}/decide`, {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
|
|||||||
@@ -70,6 +70,77 @@ export type CdcStats = {
|
|||||||
consumed: number
|
consumed: number
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type PiiColumn = { name: string; category: string; masked: boolean }
|
||||||
|
|
||||||
|
export type DataflowNode = {
|
||||||
|
id: string
|
||||||
|
label: string
|
||||||
|
sub: string
|
||||||
|
kind: string
|
||||||
|
x: number
|
||||||
|
y: number
|
||||||
|
level: string
|
||||||
|
metric: string | null
|
||||||
|
pii?: {
|
||||||
|
has_pii: boolean
|
||||||
|
pii_count: number
|
||||||
|
all_masked: boolean
|
||||||
|
masked_layer: boolean
|
||||||
|
categories: string[]
|
||||||
|
columns: PiiColumn[]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export type DataflowEdge = {
|
||||||
|
from: string
|
||||||
|
to: string
|
||||||
|
kind: string
|
||||||
|
movement_id?: string
|
||||||
|
state?: string
|
||||||
|
last_rows?: number | null
|
||||||
|
last_duration_s?: number | null
|
||||||
|
active?: boolean
|
||||||
|
}
|
||||||
|
|
||||||
|
export type DataflowGraph = {
|
||||||
|
ok: boolean
|
||||||
|
nodes: DataflowNode[]
|
||||||
|
edges: DataflowEdge[]
|
||||||
|
pii_summary: { datasets?: number; pii_columns?: number; masked_columns?: number; unmasked_columns?: number }
|
||||||
|
cdc: { connected?: boolean; consumed?: number; window_total?: number }
|
||||||
|
ts: number
|
||||||
|
}
|
||||||
|
|
||||||
|
export type Movement = {
|
||||||
|
id: string
|
||||||
|
label: string
|
||||||
|
kind: string
|
||||||
|
dag_id: string
|
||||||
|
agent: string
|
||||||
|
from: string
|
||||||
|
to: string
|
||||||
|
last_run?: {
|
||||||
|
state?: string
|
||||||
|
rows?: number | null
|
||||||
|
duration_s?: number | null
|
||||||
|
ended_at?: string
|
||||||
|
started_at?: string
|
||||||
|
} | null
|
||||||
|
}
|
||||||
|
|
||||||
|
export type PiiDataset = {
|
||||||
|
key: string
|
||||||
|
node_id: string
|
||||||
|
label: string
|
||||||
|
table: string
|
||||||
|
exists: boolean
|
||||||
|
masked_layer: boolean
|
||||||
|
pii_columns: PiiColumn[]
|
||||||
|
pii_count: number
|
||||||
|
has_pii: boolean
|
||||||
|
all_masked: boolean
|
||||||
|
}
|
||||||
|
|
||||||
export type DomainStatus = {
|
export type DomainStatus = {
|
||||||
level: 'ok' | 'warn' | 'down' | 'unknown'
|
level: 'ok' | 'warn' | 'down' | 'unknown'
|
||||||
label: string
|
label: string
|
||||||
|
|||||||
Reference in New Issue
Block a user