feat: realtime ETL offload to S3 + live business dashboards

- etl_offload.py: autonomous agent backfills/tails source DBs (PG/MySQL/
  Mongo/Cassandra) to S3 as Parquet in small chunks, accumulates a live
  federated business matrix (/api/etl/status, /api/etl/business, /run, /config).
- storage_s3.py: buffer generated CDC + masked curated rows to S3, overlay
  live last-write into analytics; put_object_bytes for Parquet parts.
- trino_federated.py: capture generated rows + archive to S3; generator_active.
- dataflow.py: pulse generate + kafka/spark->S3 archive edges when active.
- StorageView: realtime ETL ingest panel; TrinoFederationView: realtime
  business KPIs/charts from /api/etl/business.
- ChangesView: top KPIs/charts now overlay the live WS stream on server stats
  so they update in lock-step with the bottom feed; faster 2.5s refresh.
- useCommandCenter: retain 800 live CDC changes.
This commit is contained in:
mo
2026-06-28 23:33:21 +00:00
parent dfd5d4da8a
commit b6d7d3dc74
11 changed files with 1062 additions and 45 deletions
+71 -29
View File
@@ -224,32 +224,59 @@ export function ChangesView({ liveChanges }: { liveChanges: CdcChange[] }) {
const [expanded, setExpanded] = useState<string | null>(null)
const [connected, setConnected] = useState(false)
const [flash, setFlash] = useState(false)
const prevTotal = useRef(0)
// Live overlay: CDC events counted straight off the WebSocket stream since the
// last server stats snapshot. The top KPIs/charts = authoritative server stats
// (refreshed every 2.5s) + this overlay, so they move in lock-step with the
// bottom feed instead of lagging behind it.
const emptyOverlay = { total: 0, by_op: {} as Record<string, number>, by_source: {} as Record<string, number>, by_table: {} as Record<string, number> }
const [overlay, setOverlay] = useState(emptyOverlay)
const lastSeenId = useRef<string | null>(null)
const primed = useRef(false)
const applyStats = useCallback((s: CdcStats | null) => {
if (!s) return
setStats(s)
setOverlay({ total: 0, by_op: {}, by_source: {}, by_table: {} }) // server is now authoritative
}, [])
const load = useCallback(async () => {
const [c, s] = await Promise.all([fetchChanges({ limit: 150 }), fetchChangeStats(15)])
const [c, s] = await Promise.all([fetchChanges({ limit: 200 }), fetchChangeStats(15)])
setSeed(c.changes)
setConnected(c.connected)
if (s) setStats(s)
}, [])
applyStats(s)
}, [applyStats])
useEffect(() => {
load()
const iv = setInterval(() => fetchChangeStats(15).then((s) => s && setStats(s)), 4000)
const iv = setInterval(() => fetchChangeStats(15).then((s) => applyStats(s)), 2500)
return () => clearInterval(iv)
}, [load])
}, [load, applyStats])
// Pulse the header when fresh changes arrive.
// Fold freshly-arrived WS changes into the overlay → instant top-of-page update.
useEffect(() => {
const t = stats?.total ?? 0
if (t > prevTotal.current) {
setFlash(true)
const id = setTimeout(() => setFlash(false), 900)
prevTotal.current = t
return () => clearTimeout(id)
if (!liveChanges.length) return
if (!primed.current) {
primed.current = true
lastSeenId.current = liveChanges[0].id
return
}
prevTotal.current = t
}, [stats?.total])
const idx = liveChanges.findIndex((c) => c.id === lastSeenId.current)
const fresh = idx === -1 ? liveChanges : liveChanges.slice(0, idx)
if (!fresh.length) return
lastSeenId.current = liveChanges[0].id
setOverlay((o) => {
const next = { total: o.total + fresh.length, by_op: { ...o.by_op }, by_source: { ...o.by_source }, by_table: { ...o.by_table } }
for (const c of fresh) {
next.by_op[c.op] = (next.by_op[c.op] || 0) + 1
next.by_source[c.source] = (next.by_source[c.source] || 0) + 1
next.by_table[c.table] = (next.by_table[c.table] || 0) + 1
}
return next
})
setFlash(true)
const id = setTimeout(() => setFlash(false), 800)
return () => clearTimeout(id)
}, [liveChanges])
// Merge live (WS) with seeded backlog, dedupe by id, newest first.
const merged = useMemo(() => {
@@ -264,29 +291,44 @@ export function ChangesView({ liveChanges }: { liveChanges: CdcChange[] }) {
[merged, source, op],
)
const byOp = stats?.by_op || {}
const inserts = byOp.insert || 0
const updates = byOp.update || 0
const deletes = byOp.delete || 0
const total = stats?.total ?? 0
const opVal = (k: string) => (stats?.by_op?.[k] || 0) + (overlay.by_op[k] || 0)
const inserts = opVal('insert')
const updates = opVal('update')
const deletes = opVal('delete')
const total = (stats?.total ?? 0) + overlay.total
const perMin = total / Math.max(1, stats?.window_minutes ?? 15)
const opSegments = useMemo(() => (
['insert', 'update', 'delete', 'snapshot']
.map((k) => ({ label: k, value: byOp[k] || 0, color: opOf(k).color }))
.map((k) => ({ label: k, value: (stats?.by_op?.[k] || 0) + (overlay.by_op[k] || 0), color: opOf(k).color }))
.filter((s) => s.value > 0)
), [byOp])
), [stats?.by_op, overlay])
const sourceRows = useMemo(() => (
Object.entries(stats?.by_source || {}).sort((a, b) => b[1] - a[1])
), [stats?.by_source])
const sourceRows = useMemo(() => {
const m: Record<string, number> = { ...(stats?.by_source || {}) }
for (const [k, v] of Object.entries(overlay.by_source)) m[k] = (m[k] || 0) + v
return Object.entries(m).sort((a, b) => b[1] - a[1])
}, [stats?.by_source, overlay])
const maxSource = Math.max(1, ...sourceRows.map(([, v]) => v))
const tableRows = useMemo(() => (
Object.entries(stats?.by_table || {}).sort((a, b) => b[1] - a[1]).slice(0, 7)
), [stats?.by_table])
const tableRows = useMemo(() => {
const m: Record<string, number> = { ...(stats?.by_table || {}) }
for (const [k, v] of Object.entries(overlay.by_table)) m[k] = (m[k] || 0) + v
return Object.entries(m).sort((a, b) => b[1] - a[1]).slice(0, 7)
}, [stats?.by_table, overlay])
const maxTable = Math.max(1, ...tableRows.map(([, v]) => v))
// Volume chart: bump the current-minute bar with the live overlay so the curve
// visibly rises as changes stream in.
const liveBuckets = useMemo(() => {
const b = (stats?.buckets || []).map((x) => ({ ...x }))
if (overlay.total) {
if (b.length) b[b.length - 1] = { ...b[b.length - 1], n: b[b.length - 1].n + overlay.total }
else b.push({ t: 'now', n: overlay.total })
}
return b
}, [stats?.buckets, overlay.total])
return (
<div className="flex h-full min-h-0 flex-col gap-3 overflow-y-auto scrollbar-thin pr-1">
{/* Header */}
@@ -324,7 +366,7 @@ export function ChangesView({ liveChanges }: { liveChanges: CdcChange[] }) {
<div className="mb-1 flex items-center gap-1.5 text-[10px] uppercase tracking-wide text-foreground-faint">
<TrendingUp className="h-3 w-3" /> Change volume last 15 minutes
</div>
<VolumeArea buckets={stats?.buckets || []} />
<VolumeArea buckets={liveBuckets} />
</div>
<div className="rounded-lg border border-border/60 bg-surface-raised p-3">
<div className="mb-2 flex items-center gap-1.5 text-[10px] uppercase tracking-wide text-foreground-faint">
+116 -1
View File
@@ -184,6 +184,118 @@ function Panel({ title, icon: Icon, children, className, right }: { title: strin
/* ── Main ────────────────────────────────────────────────────────────────── */
type EtlDataset = {
key: string; label: string; engine: string; color: string; parts: number; rows: number; bytes: number
backfilled: boolean; total_source: number | null; last_ts: string | null; last_rows: number
last_key: string | null; error: string | null; progress_pct: number | null
}
type EtlStatus = {
ok: boolean; enabled: boolean; interval_s: number; chunk: number; running_cycle: boolean; cycles: number
last_cycle_rows: number; totals: { parts: number; rows: number; bytes: number }; rate_rows_per_min: number
datasets: EtlDataset[]; series: { t: string; rows: number; bytes: number; orders: number; revenue: number }[]
feed: { ts: string; text: string; level: string }[]
}
function EtlIngestPanel() {
const [etl, setEtl] = useState<EtlStatus | null>(null)
const [busy, setBusy] = useState(false)
const load = useCallback(async () => {
try { const r = await fetch('/api/etl/status'); if (r.ok) setEtl(await r.json()) } catch { /* */ }
}, [])
useEffect(() => { load(); const t = setInterval(load, 5000); return () => clearInterval(t) }, [load])
const runNow = async () => {
setBusy(true)
try { await fetch('/api/etl/run', { method: 'POST' }) } catch { /* */ }
setTimeout(() => { load(); setBusy(false) }, 900)
}
const cfg = async (body: Record<string, unknown>) => {
await fetch('/api/etl/config', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) })
load()
}
const ds = etl?.datasets || []
return (
<Panel
title="Lakehouse ETL · source databases → S3 Parquet (realtime offload)"
icon={Boxes}
right={
<span className="flex items-center gap-2 text-[9px]">
<span className={cn('inline-flex items-center gap-1 rounded-full border px-2 py-0.5',
etl?.running_cycle ? 'border-emerald-500/40 bg-emerald-500/10 text-emerald-300'
: etl?.enabled ? 'border-sky-500/40 bg-sky-500/10 text-sky-300'
: 'border-border text-foreground-faint')}>
<span className={cn('h-1.5 w-1.5 rounded-full', etl?.running_cycle ? 'animate-ping bg-emerald-400' : etl?.enabled ? 'bg-sky-400' : 'bg-foreground-faint')} />
{etl?.running_cycle ? 'OFFLOADING' : etl?.enabled ? 'STREAMING' : 'PAUSED'}
</span>
<span className="text-foreground-faint">{etl?.cycles ?? 0} cycles · {fmtNum(etl?.rate_rows_per_min || 0)} rows/min</span>
</span>
}
>
<div className="mb-2 flex flex-wrap items-center gap-2 text-[10px]">
<span className="text-foreground-muted">A background ETL agent pulls small chunks from every source and writes partitioned Parquet to <span className="font-mono text-docker">s3://data/lake/</span> every</span>
<select value={etl?.interval_s ?? 60} onChange={(e) => cfg({ interval_s: Number(e.target.value) })}
className="rounded border border-border bg-surface px-1.5 py-0.5 font-mono text-foreground">
{[30, 60, 120, 300, 600].map((v) => <option key={v} value={v}>{v >= 60 ? `${v / 60} min` : `${v}s`}</option>)}
</select>
<button type="button" onClick={() => cfg({ enabled: !etl?.enabled })}
className={cn('rounded border px-2 py-0.5', etl?.enabled ? 'border-amber-500/40 text-amber-300' : 'border-emerald-500/40 text-emerald-300')}>
{etl?.enabled ? 'Pause' : 'Resume'}
</button>
<button type="button" onClick={runNow} disabled={busy}
className="inline-flex items-center gap-1 rounded border border-docker/40 bg-docker/10 px-2 py-0.5 text-docker disabled:opacity-50">
{busy ? <Loader2 className="h-3 w-3 animate-spin" /> : <Activity className="h-3 w-3" />} Offload now
</button>
<span className="ml-auto font-mono text-foreground-faint">
{fmtNum(etl?.totals.parts || 0)} parts · {fmtNum(etl?.totals.rows || 0)} rows · {fmtBytes(etl?.totals.bytes || 0)}
</span>
</div>
<div className="grid grid-cols-2 gap-2 lg:grid-cols-4">
{ds.map((d) => (
<div key={d.key} className="rounded-lg border border-border/60 bg-surface-overlay/40 p-2.5">
<div className="flex items-center justify-between">
<span className="flex items-center gap-1.5 text-[10px] font-semibold text-foreground">
<span className="h-2 w-2 rounded-full" style={{ background: d.color }} /> {d.label}
</span>
{d.backfilled
? <span className="rounded bg-emerald-500/15 px-1 py-0.5 text-[8px] font-medium text-emerald-300">TAILING</span>
: <span className="rounded bg-sky-500/15 px-1 py-0.5 text-[8px] font-medium text-sky-300">BACKFILL</span>}
</div>
<p className="mt-0.5 text-[8px] uppercase tracking-wide text-foreground-faint">{d.engine}</p>
<p className="mt-1 font-mono text-base font-bold leading-none text-foreground">{fmtNum(d.rows)}</p>
<p className="text-[9px] text-foreground-faint">rows · {fmtNum(d.parts)} parts · {fmtBytes(d.bytes)}</p>
<div className="mt-1.5 h-1.5 overflow-hidden rounded-full bg-surface">
<div className="h-full rounded-full transition-all" style={{ width: `${d.progress_pct ?? (d.backfilled ? 100 : 3)}%`, background: d.color }} />
</div>
<p className="mt-0.5 flex justify-between text-[8px] text-foreground-faint">
<span>{d.progress_pct != null ? `${d.progress_pct}% of ${fmtNum(d.total_source || 0)}` : 'streaming'}</span>
{d.last_rows ? <span className="text-emerald-400">+{fmtNum(d.last_rows)}</span> : null}
</p>
{d.error && <p className="mt-0.5 truncate text-[8px] text-danger" title={d.error}>{d.error}</p>}
</div>
))}
</div>
<div className="mt-2 grid grid-cols-1 gap-2 lg:grid-cols-3">
<div className="lg:col-span-2">
<p className="mb-1 text-[9px] uppercase tracking-wide text-foreground-faint">Rows offloaded per cycle (realtime)</p>
<Sparkline values={(etl?.series || []).map((p) => p.rows)} color="#34d399" />
</div>
<div>
<p className="mb-1 text-[9px] uppercase tracking-wide text-foreground-faint">ETL agent activity</p>
<div className="max-h-[78px] space-y-0.5 overflow-y-auto scrollbar-thin">
{(etl?.feed || []).slice(0, 6).map((f, i) => (
<p key={i} className="truncate text-[9px] text-foreground-muted" title={f.text}>
<span className="text-foreground-faint">{f.ts.slice(11, 19)}</span> {f.text}
</p>
))}
{!etl?.feed?.length && <Empty label="Warming up…" />}
</div>
</div>
</div>
</Panel>
)
}
export function StorageView() {
const [tab, setTab] = useState<'overview' | 'browser'>('overview')
const [an, setAn] = useState<Analytics | null>(null)
@@ -207,7 +319,7 @@ export function StorageView() {
useEffect(() => {
loadAnalytics()
const t = setInterval(() => loadAnalytics(), 30000)
const t = setInterval(() => loadAnalytics(), 12000)
return () => clearInterval(t)
}, [loadAnalytics])
@@ -264,6 +376,9 @@ export function StorageView() {
<Kpi icon={Clock} label="Last write" value={s?.newest ? new Date(s.newest).toLocaleString(undefined, { month: 'short', day: 'numeric', hour: '2-digit', minute: '2-digit' }) : '—'} accent="#22d3ee" />
</div>
{/* Realtime ETL offload (source → S3 Parquet) */}
<EtlIngestPanel />
{/* Growth + bucket distribution */}
<div className="grid grid-cols-1 gap-3 xl:grid-cols-3">
<Panel title="Data growth (cumulative size · daily ingest)" icon={TrendingUp} className="xl:col-span-2"
@@ -117,6 +117,31 @@ function Kpi({ icon: Icon, label, value, sub, accent }: { icon: typeof Users; la
)
}
function MiniArea({ values, color = '#34d399', label }: { values: number[]; color?: string; label?: string }) {
const w = 280, h = 46, pad = 3
const d = values.length ? values : [0]
const max = Math.max(1, ...d)
const step = d.length > 1 ? (w - pad * 2) / (d.length - 1) : 0
const pts = d.map((v, i) => [pad + i * step, h - pad - (v / max) * (h - pad * 2)] as const)
const line = pts.map(([x, y], i) => `${i ? 'L' : 'M'}${x.toFixed(1)},${y.toFixed(1)}`).join(' ')
const area = `${line} L${pts[pts.length - 1][0].toFixed(1)},${h - pad} L${pts[0][0].toFixed(1)},${h - pad} Z`
return (
<div>
{label && <p className="mb-0.5 text-[9px] uppercase tracking-wide text-foreground-faint">{label}</p>}
<svg viewBox={`0 0 ${w} ${h}`} preserveAspectRatio="none" className="h-11 w-full">
<defs>
<linearGradient id={`ma-${color}`} x1="0" y1="0" x2="0" y2="1">
<stop offset="0%" stopColor={color} stopOpacity="0.4" /><stop offset="100%" stopColor={color} stopOpacity="0" />
</linearGradient>
</defs>
{values.length > 0 && <path d={area} fill={`url(#ma-${color})`} />}
{values.length > 0 && <path d={line} fill="none" stroke={color} strokeWidth="1.5" vectorEffect="non-scaling-stroke" />}
{values.length > 0 && <circle cx={pts[pts.length - 1][0]} cy={pts[pts.length - 1][1]} r="2.5" fill={color}><animate attributeName="r" values="2.5;5;2.5" dur="1.6s" repeatCount="indefinite" /></circle>}
</svg>
</div>
)
}
export function TrinoFederationView({ embedded = false, activeTab }: { embedded?: boolean; activeTab?: SubTab } = {}) {
const [tabState, setTab] = useState<SubTab>('federated')
const tab = embedded ? activeTab ?? 'federated' : tabState
@@ -124,9 +149,26 @@ export function TrinoFederationView({ embedded = false, activeTab }: { embedded?
const [marquee, setMarquee] = useState<any>(null)
const [lake, setLake] = useState<any>(null)
const [dict, setDict] = useState<any>(null)
const [biz, setBiz] = useState<any>(null)
const [loading, setLoading] = useState(false)
const [matRunning, setMatRunning] = useState(false)
const loadBiz = useCallback(async () => {
try {
const r = await fetch('/api/etl/business')
if (r.ok) setBiz(await r.json())
} catch { /* */ }
}, [])
// Poll the live federated business model (built from the ETL lakehouse offload)
// while the federated tab is open, so the graphs move with newly generated data.
useEffect(() => {
if (tab !== 'federated') return
loadBiz()
const t = setInterval(loadBiz, 5000)
return () => clearInterval(t)
}, [tab, loadBiz])
const loadFederated = useCallback(async () => {
setLoading(true)
try {
@@ -198,6 +240,66 @@ export function TrinoFederationView({ embedded = false, activeTab }: { embedded?
{/* ───────── FEDERATED ───────── */}
{tab === 'federated' && (
<>
{/* ───────── REALTIME FEDERATED BUSINESS MODEL ───────── */}
<Panel
title="Realtime federated business model"
subtitle={biz?.generated_at ? `updated ${new Date(biz.generated_at).toLocaleTimeString()}` : 'live'}
icon={Activity}
>
<p className="mb-2 flex items-center gap-1.5 text-[10px] text-foreground-muted">
<span className="h-1.5 w-1.5 animate-ping rounded-full bg-emerald-400" />
Business matrices built continuously from the lakehouse offload across <span className="text-docker">all five data points</span> orders, HR, supply &amp; telemetry and they move as new data is generated &amp; streamed to S3.
</p>
<div className="mb-2 grid grid-cols-2 gap-2 md:grid-cols-3 xl:grid-cols-6">
<Kpi icon={ShoppingCart} label="Orders analyzed" value={fmtNum(biz?.kpis?.orders)} accent="#fbbf24" />
<Kpi icon={DollarSign} label="Revenue" value={fmtMoney(biz?.kpis?.revenue)} sub={`avg ${fmtMoney(biz?.kpis?.avg_order)}`} accent="#34d399" />
<Kpi icon={Users} label="HR events" value={fmtNum(biz?.kpis?.hr_events)} accent="#60a5fa" />
<Kpi icon={Boxes} label="Supply events" value={fmtNum(biz?.kpis?.supply_events)} sub={fmtMoney(biz?.kpis?.supply_amount)} accent="#a78bfa" />
<Kpi icon={Activity} label="Telemetry pts" value={fmtNum(biz?.kpis?.telemetry)} accent="#22d3ee" />
<Kpi icon={Layers} label="Rows in model" value={fmtNum(biz?.kpis?.rows_total)} sub="federated" accent="#dd00a1" />
</div>
<div className="grid grid-cols-1 gap-3 sm:grid-cols-2">
<MiniArea label="Orders ingested / cycle" values={(biz?.ts || []).map((p: any) => p.orders)} color="#fbbf24" />
<MiniArea label="Revenue / cycle (€)" values={(biz?.ts || []).map((p: any) => p.revenue)} color="#34d399" />
</div>
</Panel>
<div className="grid shrink-0 gap-2 lg:grid-cols-2">
<Panel title="Revenue by region" subtitle="live" icon={DollarSign}><BarsH data={biz?.orders_by_region} valueKind="money" colorByIndex /></Panel>
<Panel title="Region matrix — orders · revenue · HR · supply" icon={Network}>
{biz?.region_matrix?.length ? (
<div className="overflow-x-auto">
<table className="w-full text-[10px]">
<thead><tr className="border-b border-border text-left text-foreground-muted">
<th className="py-1 pr-3">Region</th><th className="py-1 pr-3 text-right">Orders</th>
<th className="py-1 pr-3 text-right">Revenue</th><th className="py-1 pr-3 text-right">HR</th><th className="py-1 pr-3 text-right">Supply</th>
</tr></thead>
<tbody>
{biz.region_matrix.map((r: any, i: number) => (
<tr key={i} className="border-b border-border/40">
<td className="py-1 pr-3 font-medium text-foreground">{r.region}</td>
<td className="py-1 pr-3 text-right font-mono text-foreground">{fmtNum(r.orders)}</td>
<td className="py-1 pr-3 text-right font-mono text-emerald-400">{fmtMoney(r.revenue)}</td>
<td className="py-1 pr-3 text-right font-mono text-foreground">{fmtNum(r.hr_events)}</td>
<td className="py-1 pr-3 text-right font-mono text-foreground">{fmtNum(r.supply_events)}</td>
</tr>
))}
</tbody>
</table>
</div>
) : <p className="py-4 text-center text-[10px] text-foreground-faint">Building the model from the lakehouse offload</p>}
</Panel>
<div className="grid gap-2 sm:grid-cols-2">
<Panel title="Orders by status"><Donut data={biz?.orders_by_status} /></Panel>
<Panel title="Revenue by channel"><BarsH data={biz?.orders_by_channel} valueKind="money" colorByIndex /></Panel>
</div>
<Panel title="HR events by department" icon={Users}><BarsH data={biz?.hr_by_department} colorByIndex /></Panel>
<Panel title="Supply value by type" icon={Boxes}><BarsH data={biz?.supply_by_type} valueKind="money" colorByIndex /></Panel>
<Panel title="Telemetry — avg value by metric" icon={Activity}>
<BarsH data={(biz?.telemetry_by_metric || []).map((x: any) => ({ key: x.key, count: x.count, value: x.avg }))} valueKind="num" />
</Panel>
</div>
<div className="grid shrink-0 grid-cols-2 gap-2 md:grid-cols-4">
<Kpi icon={Layers} label="Federated catalogs" value={String(catalogs?.count ?? '—')} sub="one Trino engine" accent="#dd00a1" />
<Kpi icon={ShoppingCart} label="Orders" value={fmtNum(totals.orders)} sub="PostgreSQL (live)" accent="#fbbf24" />
+1 -1
View File
@@ -129,7 +129,7 @@ export function useCommandCenter() {
if (msg.type === 'terminal') appendTerminal(msg.line)
if (msg.type === 'terminal_history' && msg.terminals) setTerminals(msg.terminals)
if (msg.type === 'feed') setFeed((prev) => [msg.entry, ...prev].slice(0, 100))
if (msg.type === 'cdc_change' && msg.entry) setChanges((prev) => [msg.entry, ...prev].slice(0, 400))
if (msg.type === 'cdc_change' && msg.entry) setChanges((prev) => [msg.entry, ...prev].slice(0, 800))
if (msg.type === 'agent_dispatch') {
setSelectedAgentId(msg.agent_id)
setAnims((p) => ({ ...p, [msg.agent_id]: { agentId: msg.agent_id, state: 'walk', zone: msg.zone } }))