feat(ui): Data Flow tab — live lineage graph + PII overlay + movement triggers
New DataFlowView (topology-style): nodes positioned from /api/dataflow with measured-anchor SVG edges and animated particles (active CDC + running movements). PII overlay shows shield badges + per-column detail inspector. Bottom strip triggers ETL movements and toggles the ETL agent. Wired into SideNav (Data Flow) and App routing; added types + api helpers.
This commit is contained in:
mo
@@ -19,6 +19,7 @@ import { StorageView } from './components/features/StorageView'
|
||||
import { HdfsView } from './components/features/HdfsView'
|
||||
import { DataGenView } from './components/features/DataGenView'
|
||||
import { ChangesView } from './components/features/ChangesView'
|
||||
import { DataFlowView } from './components/features/DataFlowView'
|
||||
import { SearchView } from './components/features/SearchView'
|
||||
import { SshTerminal } from './components/features/SshTerminal'
|
||||
import { TerminalDock } from './components/features/TerminalDock'
|
||||
@@ -131,6 +132,8 @@ export default function App() {
|
||||
<DataGenView onPulse={cc.pulseFlow} onOpenPlatform={() => cc.setMainView('platform')} />
|
||||
) : cc.mainView === 'changes' ? (
|
||||
<ChangesView liveChanges={cc.changes} />
|
||||
) : cc.mainView === 'dataflow' ? (
|
||||
<DataFlowView />
|
||||
) : cc.mainView === 'presentation' ? (
|
||||
<PresentationView />
|
||||
) : cc.mainView === 'dataquality' ? (
|
||||
|
||||
@@ -0,0 +1,429 @@
|
||||
import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from 'react'
|
||||
import { GitBranch, Play, RefreshCw, ShieldAlert, ShieldCheck, Loader2, Bot } from 'lucide-react'
|
||||
import type { DataflowEdge, DataflowGraph, DataflowNode } from '../../types'
|
||||
import { fetchDataflow, runDataflowMovement, toggleEtlAgent, fetchAgentOpsStatus } from '../../lib/api'
|
||||
import { Badge } from '../ui/Badge'
|
||||
import { cn } from '../../lib/utils'
|
||||
|
||||
/* ── styling maps ───────────────────────────────────────────────── */
|
||||
|
||||
const NODE_KIND: Record<string, { ring: string; chip: string; dot: string }> = {
|
||||
generator: { ring: 'border-amber-400/60', chip: 'bg-amber-500/15 text-amber-300 border-amber-400/40', dot: '#f59e0b' },
|
||||
hadoop: { ring: 'border-orange-400/60', chip: 'bg-orange-500/15 text-orange-300 border-orange-400/40', dot: '#fb923c' },
|
||||
source: { ring: 'border-emerald-400/60', chip: 'bg-emerald-500/15 text-emerald-300 border-emerald-400/40', dot: '#34d399' },
|
||||
stream: { ring: 'border-cyan-400/60', chip: 'bg-cyan-500/15 text-cyan-300 border-cyan-400/40', dot: '#22d3ee' },
|
||||
sink: { ring: 'border-sky-400/60', chip: 'bg-sky-500/15 text-sky-300 border-sky-400/40', dot: '#38bdf8' },
|
||||
lakehouse: { ring: 'border-blue-400/60', chip: 'bg-blue-500/15 text-blue-300 border-blue-400/40', dot: '#60a5fa' },
|
||||
engine: { ring: 'border-violet-400/60', chip: 'bg-violet-500/15 text-violet-300 border-violet-400/40', dot: '#a78bfa' },
|
||||
}
|
||||
|
||||
const EDGE_COLOR: Record<string, string> = {
|
||||
generate: '#f59e0b',
|
||||
cdc: '#22d3ee',
|
||||
archive: '#38bdf8',
|
||||
movement: '#a78bfa',
|
||||
mask: '#fb7185',
|
||||
query: '#818cf8',
|
||||
}
|
||||
|
||||
const EDGE_LEGEND: { kind: string; label: string }[] = [
|
||||
{ kind: 'generate', label: 'Generate' },
|
||||
{ kind: 'cdc', label: 'CDC capture' },
|
||||
{ kind: 'archive', label: 'Archive' },
|
||||
{ kind: 'movement', label: 'ETL movement' },
|
||||
{ kind: 'mask', label: 'PII masking' },
|
||||
{ kind: 'query', label: 'Query' },
|
||||
]
|
||||
|
||||
type Anchor = { x: number; y: number; w: number; h: number }
|
||||
|
||||
function edgePath(a: Anchor, b: Anchor): string {
|
||||
const x1 = a.x
|
||||
const y1 = a.y
|
||||
const x2 = b.x
|
||||
const y2 = b.y
|
||||
const dx = x2 - x1
|
||||
if (Math.abs(dx) < 8) {
|
||||
const midY = (y1 + y2) / 2
|
||||
return `M ${x1} ${y1} C ${x1} ${midY}, ${x2} ${midY}, ${x2} ${y2}`
|
||||
}
|
||||
const mx = x1 + dx * 0.5
|
||||
return `M ${x1} ${y1} C ${mx} ${y1}, ${mx} ${y2}, ${x2} ${y2}`
|
||||
}
|
||||
|
||||
function fmtDur(s?: number | null): string {
|
||||
if (s == null) return ''
|
||||
if (s < 60) return `${s.toFixed(0)}s`
|
||||
return `${(s / 60).toFixed(1)}m`
|
||||
}
|
||||
|
||||
/* ── component ──────────────────────────────────────────────────── */
|
||||
|
||||
export function DataFlowView() {
|
||||
const [graph, setGraph] = useState<DataflowGraph | null>(null)
|
||||
const [loading, setLoading] = useState(true)
|
||||
const [refreshing, setRefreshing] = useState(false)
|
||||
const [piiOverlay, setPiiOverlay] = useState(true)
|
||||
const [selected, setSelected] = useState<string | null>(null)
|
||||
const [triggering, setTriggering] = useState<string | null>(null)
|
||||
const [etlEnabled, setEtlEnabled] = useState<boolean | null>(null)
|
||||
|
||||
const canvasRef = useRef<HTMLDivElement>(null)
|
||||
const nodeRefs = useRef<Record<string, HTMLButtonElement | null>>({})
|
||||
const [anchors, setAnchors] = useState<Record<string, Anchor>>({})
|
||||
const [size, setSize] = useState({ w: 800, h: 460 })
|
||||
|
||||
const setNodeRef = useCallback((id: string) => (el: HTMLButtonElement | null) => {
|
||||
nodeRefs.current[id] = el
|
||||
}, [])
|
||||
|
||||
const load = useCallback(async (refresh = false) => {
|
||||
const g = await fetchDataflow(refresh)
|
||||
if (g) setGraph(g)
|
||||
setLoading(false)
|
||||
}, [])
|
||||
|
||||
useEffect(() => {
|
||||
load()
|
||||
fetchAgentOpsStatus().then((s) => {
|
||||
const etl = (s as { etl?: { enabled?: boolean } })?.etl
|
||||
if (etl) setEtlEnabled(!!etl.enabled)
|
||||
})
|
||||
const iv = setInterval(() => load(), 6000)
|
||||
return () => clearInterval(iv)
|
||||
}, [load])
|
||||
|
||||
const measure = useCallback(() => {
|
||||
const canvas = canvasRef.current
|
||||
if (!canvas) return
|
||||
const rect = canvas.getBoundingClientRect()
|
||||
if (rect.width < 10 || rect.height < 10) return
|
||||
setSize({ w: rect.width, h: rect.height })
|
||||
const next: Record<string, Anchor> = {}
|
||||
for (const [id, el] of Object.entries(nodeRefs.current)) {
|
||||
if (!el) continue
|
||||
const r = el.getBoundingClientRect()
|
||||
next[id] = {
|
||||
x: (r.left + r.right) / 2 - rect.left,
|
||||
y: (r.top + r.bottom) / 2 - rect.top,
|
||||
w: r.width,
|
||||
h: r.height,
|
||||
}
|
||||
}
|
||||
setAnchors(next)
|
||||
}, [])
|
||||
|
||||
useLayoutEffect(() => {
|
||||
measure()
|
||||
const canvas = canvasRef.current
|
||||
if (!canvas) return
|
||||
const ro = new ResizeObserver(() => measure())
|
||||
ro.observe(canvas)
|
||||
window.addEventListener('resize', measure)
|
||||
return () => {
|
||||
ro.disconnect()
|
||||
window.removeEventListener('resize', measure)
|
||||
}
|
||||
}, [measure])
|
||||
|
||||
useLayoutEffect(() => {
|
||||
measure()
|
||||
}, [measure, graph])
|
||||
|
||||
const nodes = graph?.nodes ?? []
|
||||
const edges = graph?.edges ?? []
|
||||
|
||||
const onTrigger = useCallback(async (movementId: string) => {
|
||||
setTriggering(movementId)
|
||||
try {
|
||||
await runDataflowMovement(movementId)
|
||||
setTimeout(() => load(true), 800)
|
||||
} finally {
|
||||
setTimeout(() => setTriggering(null), 1500)
|
||||
}
|
||||
}, [load])
|
||||
|
||||
const onToggleEtl = useCallback(async () => {
|
||||
const next = !etlEnabled
|
||||
setEtlEnabled(next)
|
||||
await toggleEtlAgent(next)
|
||||
}, [etlEnabled])
|
||||
|
||||
const onRefresh = useCallback(async () => {
|
||||
setRefreshing(true)
|
||||
await load(true)
|
||||
setTimeout(() => setRefreshing(false), 400)
|
||||
}, [load])
|
||||
|
||||
const piiSummary = graph?.pii_summary ?? {}
|
||||
const movementEdges = useMemo(
|
||||
() => edges.filter((e) => e.movement_id),
|
||||
[edges],
|
||||
)
|
||||
const triggerable = useMemo(() => {
|
||||
const seen = new Set<string>()
|
||||
const out: DataflowEdge[] = []
|
||||
for (const e of movementEdges) {
|
||||
if (e.movement_id && !seen.has(e.movement_id)) {
|
||||
seen.add(e.movement_id)
|
||||
out.push(e)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}, [movementEdges])
|
||||
|
||||
const selNode = nodes.find((n) => n.id === selected) || null
|
||||
|
||||
return (
|
||||
<section className="panel flex h-full min-h-0 flex-1 flex-col overflow-hidden">
|
||||
<header
|
||||
className="flex shrink-0 flex-col gap-1 border-b border-border px-3 py-2"
|
||||
style={{ background: 'var(--topo-header-bg)' }}
|
||||
>
|
||||
<div className="flex items-center justify-between gap-2">
|
||||
<div className="flex min-w-0 items-center gap-2">
|
||||
<span className="flex h-6 w-6 shrink-0 items-center justify-center rounded-md bg-docker text-white shadow-docker">
|
||||
<GitBranch className="h-3 w-3" />
|
||||
</span>
|
||||
<div className="min-w-0">
|
||||
<h2 className="truncate text-xs font-semibold text-foreground">Data Flow · live lineage</h2>
|
||||
<p className="truncate text-[9px] text-foreground-muted">
|
||||
Generators → sources → CDC/Kafka → lakehouse · click a node for PII detail · trigger movements below
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex shrink-0 flex-wrap items-center justify-end gap-1">
|
||||
<Badge variant={graph?.cdc?.connected ? 'success' : 'warning'}>
|
||||
{graph?.cdc?.connected ? 'CDC live' : 'CDC offline'}
|
||||
</Badge>
|
||||
<Badge>{graph?.cdc?.window_total ?? 0} chg/15m</Badge>
|
||||
{(piiSummary.unmasked_columns ?? 0) > 0 ? (
|
||||
<Badge variant="warning">{piiSummary.unmasked_columns} PII unmasked</Badge>
|
||||
) : (
|
||||
<Badge variant="accent">{piiSummary.masked_columns ?? 0} PII masked</Badge>
|
||||
)}
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setPiiOverlay((v) => !v)}
|
||||
className={cn(
|
||||
'rounded border px-1.5 py-0.5 text-[9px] font-medium transition-colors',
|
||||
piiOverlay
|
||||
? 'border-rose-400/50 bg-rose-500/20 text-rose-200'
|
||||
: 'border-border bg-transparent text-foreground-muted hover:text-foreground',
|
||||
)}
|
||||
>
|
||||
PII overlay
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={onToggleEtl}
|
||||
className={cn(
|
||||
'inline-flex items-center gap-1 rounded border px-1.5 py-0.5 text-[9px] font-medium transition-colors',
|
||||
etlEnabled
|
||||
? 'border-emerald-400/50 bg-emerald-500/20 text-emerald-200'
|
||||
: 'border-border bg-transparent text-foreground-muted hover:text-foreground',
|
||||
)}
|
||||
>
|
||||
<Bot className="h-3 w-3" /> ETL agent {etlEnabled == null ? '' : etlEnabled ? 'on' : 'off'}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={onRefresh}
|
||||
className="inline-flex items-center gap-1 rounded border border-border px-1.5 py-0.5 text-[9px] text-foreground-muted hover:text-foreground"
|
||||
>
|
||||
<RefreshCw className={cn('h-3 w-3', refreshing && 'animate-spin')} /> Refresh
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex flex-wrap gap-x-3 gap-y-0.5">
|
||||
{EDGE_LEGEND.map((l) => (
|
||||
<span key={l.kind} className="inline-flex items-center gap-1 text-[8px] text-foreground-muted">
|
||||
<span className="h-1.5 w-1.5 rounded-full" style={{ background: EDGE_COLOR[l.kind] }} />
|
||||
{l.label}
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
</header>
|
||||
|
||||
{/* canvas */}
|
||||
<div ref={canvasRef} className="topo-canvas relative flex min-h-0 flex-1">
|
||||
{loading && (
|
||||
<div className="absolute inset-0 z-20 flex items-center justify-center text-[10px] text-foreground-muted">
|
||||
<Loader2 className="mr-1 h-3 w-3 animate-spin" /> loading flow…
|
||||
</div>
|
||||
)}
|
||||
<svg
|
||||
className="pointer-events-none absolute inset-0 z-0 h-full w-full"
|
||||
viewBox={`0 0 ${size.w} ${size.h}`}
|
||||
aria-hidden
|
||||
>
|
||||
{edges.map((edge, i) => {
|
||||
const a = anchors[edge.from]
|
||||
const b = anchors[edge.to]
|
||||
if (!a || !b) return null
|
||||
const d = edgePath(a, b)
|
||||
const color = EDGE_COLOR[edge.kind] || '#64748b'
|
||||
const running = edge.state === 'running'
|
||||
const active = edge.active || running
|
||||
const dur = 1.6 + (i % 5) * 0.3
|
||||
return (
|
||||
<g key={`${edge.from}-${edge.to}-${edge.kind}`}>
|
||||
<path
|
||||
d={d}
|
||||
fill="none"
|
||||
stroke={color}
|
||||
strokeWidth={running ? 2.6 : 1.4}
|
||||
strokeOpacity={active ? 0.85 : 0.28}
|
||||
strokeDasharray={edge.movement_id && !active ? '4 4' : undefined}
|
||||
/>
|
||||
{active && (
|
||||
<>
|
||||
<circle r={running ? 3 : 2.2} fill={color} opacity="0.95">
|
||||
<animateMotion dur={`${running ? dur * 0.6 : dur}s`} repeatCount="indefinite" path={d} />
|
||||
</circle>
|
||||
<circle r="1.3" fill="#ffffff" opacity="0.85">
|
||||
<animateMotion dur={`${running ? dur * 0.6 : dur}s`} repeatCount="indefinite" path={d} begin={`${dur * 0.45}s`} />
|
||||
</circle>
|
||||
</>
|
||||
)}
|
||||
</g>
|
||||
)
|
||||
})}
|
||||
</svg>
|
||||
|
||||
{/* nodes */}
|
||||
{nodes.map((node) => (
|
||||
<NodeCard
|
||||
key={node.id}
|
||||
node={node}
|
||||
piiOverlay={piiOverlay}
|
||||
selected={selected === node.id}
|
||||
setRef={setNodeRef(node.id)}
|
||||
onClick={() => setSelected((s) => (s === node.id ? null : node.id))}
|
||||
/>
|
||||
))}
|
||||
|
||||
{/* node inspector */}
|
||||
{selNode && (
|
||||
<div className="absolute right-2 top-2 z-30 w-52 rounded-lg border border-border bg-surface/95 p-2 shadow-lg backdrop-blur">
|
||||
<div className="mb-1 flex items-center justify-between">
|
||||
<span className="text-[10px] font-semibold text-foreground">{selNode.label}</span>
|
||||
<button onClick={() => setSelected(null)} className="text-[10px] text-foreground-muted hover:text-foreground">✕</button>
|
||||
</div>
|
||||
<p className="text-[8px] text-foreground-muted">{selNode.sub}</p>
|
||||
{selNode.metric && (
|
||||
<p className="mt-1 font-mono text-[8px] text-emerald-300">{selNode.metric}</p>
|
||||
)}
|
||||
{selNode.pii?.has_pii ? (
|
||||
<div className="mt-1.5 border-t border-border pt-1.5">
|
||||
<div className="mb-1 flex items-center gap-1 text-[9px] font-medium text-rose-300">
|
||||
{selNode.pii.all_masked ? <ShieldCheck className="h-3 w-3 text-emerald-300" /> : <ShieldAlert className="h-3 w-3" />}
|
||||
{selNode.pii.pii_count} PII column{selNode.pii.pii_count === 1 ? '' : 's'}
|
||||
</div>
|
||||
<div className="flex flex-col gap-0.5">
|
||||
{selNode.pii.columns.map((c) => (
|
||||
<div key={c.name} className="flex items-center justify-between gap-1 text-[8px]">
|
||||
<span className="truncate font-mono text-foreground-muted">{c.name}</span>
|
||||
<span className="flex items-center gap-1">
|
||||
<span className="text-blue-300">{c.category}</span>
|
||||
<span className={c.masked ? 'text-emerald-300' : 'text-rose-300'}>
|
||||
{c.masked ? 'masked' : 'raw'}
|
||||
</span>
|
||||
</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
) : (
|
||||
<p className="mt-1 text-[8px] text-foreground-muted">No PII classified.</p>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* movement control strip */}
|
||||
<div className="shrink-0 border-t border-border bg-surface/60 px-3 py-1.5">
|
||||
<div className="mb-1 flex items-center gap-1 text-[8px] font-semibold uppercase tracking-wide text-foreground-muted">
|
||||
<Play className="h-2.5 w-2.5" /> Data movements
|
||||
</div>
|
||||
<div className="flex flex-wrap gap-1.5">
|
||||
{triggerable.map((e) => {
|
||||
const running = e.state === 'running'
|
||||
const busy = triggering === e.movement_id || running
|
||||
return (
|
||||
<button
|
||||
key={e.movement_id}
|
||||
type="button"
|
||||
disabled={busy}
|
||||
onClick={() => e.movement_id && onTrigger(e.movement_id)}
|
||||
className={cn(
|
||||
'inline-flex items-center gap-1 rounded-md border px-2 py-1 text-[9px] font-medium transition-colors disabled:opacity-70',
|
||||
running
|
||||
? 'border-violet-400/60 bg-violet-500/20 text-violet-200'
|
||||
: 'border-border bg-background/40 text-foreground hover:border-emerald-400/50 hover:bg-emerald-500/10',
|
||||
)}
|
||||
>
|
||||
{busy ? <Loader2 className="h-3 w-3 animate-spin" /> : <Play className="h-3 w-3" />}
|
||||
<span>{e.movement_id}</span>
|
||||
{e.state && e.state !== 'running' && (
|
||||
<span className={cn('font-mono', e.state === 'success' ? 'text-emerald-300' : 'text-rose-300')}>
|
||||
· {e.state}{e.last_rows != null ? ` ${e.last_rows.toLocaleString()}r` : ''}{e.last_duration_s ? ` ${fmtDur(e.last_duration_s)}` : ''}
|
||||
</span>
|
||||
)}
|
||||
{running && <span className="font-mono text-violet-200">· running…</span>}
|
||||
</button>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
)
|
||||
}
|
||||
|
||||
function NodeCard({
|
||||
node, piiOverlay, selected, setRef, onClick,
|
||||
}: {
|
||||
node: DataflowNode
|
||||
piiOverlay: boolean
|
||||
selected: boolean
|
||||
setRef: (el: HTMLButtonElement | null) => void
|
||||
onClick: () => void
|
||||
}) {
|
||||
const kind = NODE_KIND[node.kind] || NODE_KIND.source
|
||||
const pii = node.pii
|
||||
const showPii = piiOverlay && pii?.has_pii
|
||||
return (
|
||||
<button
|
||||
ref={setRef}
|
||||
type="button"
|
||||
onClick={onClick}
|
||||
className={cn(
|
||||
'absolute z-10 flex w-[116px] -translate-x-1/2 -translate-y-1/2 flex-col items-start gap-0.5 rounded-lg border bg-[#0f1830]/90 px-2 py-1.5 text-left shadow-md backdrop-blur transition-all hover:scale-[1.03]',
|
||||
kind.ring,
|
||||
selected && 'ring-2 ring-emerald-400/70',
|
||||
showPii && !pii?.all_masked && 'ring-2 ring-rose-400/60',
|
||||
)}
|
||||
style={{ left: `${node.x}%`, top: `${node.y}%` }}
|
||||
>
|
||||
<div className="flex w-full items-center justify-between gap-1">
|
||||
<span className="truncate text-[9px] font-semibold leading-tight text-white">{node.label}</span>
|
||||
{showPii && (
|
||||
pii?.all_masked
|
||||
? <ShieldCheck className="h-3 w-3 shrink-0 text-emerald-300" />
|
||||
: <ShieldAlert className="h-3 w-3 shrink-0 text-rose-300" />
|
||||
)}
|
||||
</div>
|
||||
<span className="block w-full truncate text-[7px] leading-tight text-blue-100/70">{node.sub}</span>
|
||||
{node.metric && (
|
||||
<span className={cn('inline-block max-w-full truncate rounded border px-0.5 py-px font-mono text-[6px] leading-tight', kind.chip)}>
|
||||
{node.metric}
|
||||
</span>
|
||||
)}
|
||||
{showPii && (
|
||||
<span className="inline-block max-w-full truncate rounded border border-rose-400/40 bg-rose-500/15 px-0.5 py-px text-[6px] font-medium leading-tight text-rose-200">
|
||||
{pii?.pii_count} PII · {pii?.categories.slice(0, 2).join(',')}
|
||||
</span>
|
||||
)}
|
||||
</button>
|
||||
)
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
import { DatabaseZap, HardDrive, Search, LayoutDashboard, MessageSquare, Presentation, Server, TerminalSquare, Cpu, Activity } from 'lucide-react'
|
||||
import { DatabaseZap, HardDrive, Search, LayoutDashboard, MessageSquare, Presentation, Server, TerminalSquare, Cpu, Activity, GitBranch } from 'lucide-react'
|
||||
import type { GpuStatus, WorkloadData } from '../../types'
|
||||
import type { GpuLiveMetrics } from '../../hooks/useLiveMetrics'
|
||||
import { cn } from '../../lib/utils'
|
||||
@@ -26,6 +26,7 @@ const VIEWS: { id: MainView; label: string; icon: typeof LayoutDashboard }[] = [
|
||||
{ id: 'platform', label: 'Data Platform', icon: LayoutDashboard },
|
||||
{ id: 'datagen', label: 'Data Generation', icon: Cpu },
|
||||
{ id: 'changes', label: 'Live Changes', icon: Activity },
|
||||
{ id: 'dataflow', label: 'Data Flow', icon: GitBranch },
|
||||
{ id: 'presentation', label: 'Presentation', icon: Presentation },
|
||||
{ id: 'dataquality', label: 'Data Quality', icon: DatabaseZap },
|
||||
{ id: 'knowledge', label: 'Knowledge Chat', icon: MessageSquare },
|
||||
|
||||
@@ -4,8 +4,11 @@ import type {
|
||||
Approval,
|
||||
CdcChange,
|
||||
CdcStats,
|
||||
DataflowGraph,
|
||||
FeedEntry,
|
||||
GpuStatus,
|
||||
Movement,
|
||||
PiiDataset,
|
||||
StatusData,
|
||||
TerminalLine,
|
||||
WorkloadData,
|
||||
@@ -129,6 +132,37 @@ export function runAgentOpOnce(source?: string, op?: string) {
|
||||
})
|
||||
}
|
||||
|
||||
export async function fetchDataflow(refresh = false): Promise<DataflowGraph | null> {
|
||||
return fetchJson<DataflowGraph>(`/api/dataflow${refresh ? '?refresh=true' : ''}`, 25000)
|
||||
}
|
||||
|
||||
export function runDataflowMovement(movementId: string, conf?: Record<string, unknown>) {
|
||||
return fetch(`/api/dataflow/${movementId}/run`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(conf ? { conf } : {}),
|
||||
})
|
||||
}
|
||||
|
||||
export async function fetchMovements(): Promise<Movement[]> {
|
||||
const j = await fetchJson<{ movements?: Movement[] }>('/api/movements', 8000)
|
||||
return j?.movements || []
|
||||
}
|
||||
|
||||
export async function fetchPii(refresh = false): Promise<{ datasets: PiiDataset[]; summary: Record<string, number> } | null> {
|
||||
return fetchJson<{ datasets: PiiDataset[]; summary: Record<string, number> }>(
|
||||
`/api/pii${refresh ? '?refresh=true' : ''}`, 15000,
|
||||
)
|
||||
}
|
||||
|
||||
export function toggleEtlAgent(enabled?: boolean) {
|
||||
return fetch('/api/agent-ops/etl/toggle', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(enabled === undefined ? {} : { enabled }),
|
||||
})
|
||||
}
|
||||
|
||||
export async function decideApproval(id: string, approved: boolean, decidedBy: string, note: string) {
|
||||
return fetch(`/api/approvals/${id}/decide`, {
|
||||
method: 'POST',
|
||||
|
||||
@@ -70,6 +70,77 @@ export type CdcStats = {
|
||||
consumed: number
|
||||
}
|
||||
|
||||
export type PiiColumn = { name: string; category: string; masked: boolean }
|
||||
|
||||
export type DataflowNode = {
|
||||
id: string
|
||||
label: string
|
||||
sub: string
|
||||
kind: string
|
||||
x: number
|
||||
y: number
|
||||
level: string
|
||||
metric: string | null
|
||||
pii?: {
|
||||
has_pii: boolean
|
||||
pii_count: number
|
||||
all_masked: boolean
|
||||
masked_layer: boolean
|
||||
categories: string[]
|
||||
columns: PiiColumn[]
|
||||
}
|
||||
}
|
||||
|
||||
export type DataflowEdge = {
|
||||
from: string
|
||||
to: string
|
||||
kind: string
|
||||
movement_id?: string
|
||||
state?: string
|
||||
last_rows?: number | null
|
||||
last_duration_s?: number | null
|
||||
active?: boolean
|
||||
}
|
||||
|
||||
export type DataflowGraph = {
|
||||
ok: boolean
|
||||
nodes: DataflowNode[]
|
||||
edges: DataflowEdge[]
|
||||
pii_summary: { datasets?: number; pii_columns?: number; masked_columns?: number; unmasked_columns?: number }
|
||||
cdc: { connected?: boolean; consumed?: number; window_total?: number }
|
||||
ts: number
|
||||
}
|
||||
|
||||
export type Movement = {
|
||||
id: string
|
||||
label: string
|
||||
kind: string
|
||||
dag_id: string
|
||||
agent: string
|
||||
from: string
|
||||
to: string
|
||||
last_run?: {
|
||||
state?: string
|
||||
rows?: number | null
|
||||
duration_s?: number | null
|
||||
ended_at?: string
|
||||
started_at?: string
|
||||
} | null
|
||||
}
|
||||
|
||||
export type PiiDataset = {
|
||||
key: string
|
||||
node_id: string
|
||||
label: string
|
||||
table: string
|
||||
exists: boolean
|
||||
masked_layer: boolean
|
||||
pii_columns: PiiColumn[]
|
||||
pii_count: number
|
||||
has_pii: boolean
|
||||
all_masked: boolean
|
||||
}
|
||||
|
||||
export type DomainStatus = {
|
||||
level: 'ok' | 'warn' | 'down' | 'unknown'
|
||||
label: string
|
||||
|
||||
Reference in New Issue
Block a user