feat: realtime ETL offload to S3 + live business dashboards
- etl_offload.py: autonomous agent backfills/tails source DBs (PG/MySQL/ Mongo/Cassandra) to S3 as Parquet in small chunks, accumulates a live federated business matrix (/api/etl/status, /api/etl/business, /run, /config). - storage_s3.py: buffer generated CDC + masked curated rows to S3, overlay live last-write into analytics; put_object_bytes for Parquet parts. - trino_federated.py: capture generated rows + archive to S3; generator_active. - dataflow.py: pulse generate + kafka/spark->S3 archive edges when active. - StorageView: realtime ETL ingest panel; TrinoFederationView: realtime business KPIs/charts from /api/etl/business. - ChangesView: top KPIs/charts now overlay the live WS stream on server stats so they update in lock-step with the bottom feed; faster 2.5s refresh. - useCommandCenter: retain 800 live CDC changes.
This commit is contained in:
mo
@@ -224,32 +224,59 @@ export function ChangesView({ liveChanges }: { liveChanges: CdcChange[] }) {
|
||||
const [expanded, setExpanded] = useState<string | null>(null)
|
||||
const [connected, setConnected] = useState(false)
|
||||
const [flash, setFlash] = useState(false)
|
||||
const prevTotal = useRef(0)
|
||||
// Live overlay: CDC events counted straight off the WebSocket stream since the
|
||||
// last server stats snapshot. The top KPIs/charts = authoritative server stats
|
||||
// (refreshed every 2.5s) + this overlay, so they move in lock-step with the
|
||||
// bottom feed instead of lagging behind it.
|
||||
const emptyOverlay = { total: 0, by_op: {} as Record<string, number>, by_source: {} as Record<string, number>, by_table: {} as Record<string, number> }
|
||||
const [overlay, setOverlay] = useState(emptyOverlay)
|
||||
const lastSeenId = useRef<string | null>(null)
|
||||
const primed = useRef(false)
|
||||
|
||||
const applyStats = useCallback((s: CdcStats | null) => {
|
||||
if (!s) return
|
||||
setStats(s)
|
||||
setOverlay({ total: 0, by_op: {}, by_source: {}, by_table: {} }) // server is now authoritative
|
||||
}, [])
|
||||
|
||||
const load = useCallback(async () => {
|
||||
const [c, s] = await Promise.all([fetchChanges({ limit: 150 }), fetchChangeStats(15)])
|
||||
const [c, s] = await Promise.all([fetchChanges({ limit: 200 }), fetchChangeStats(15)])
|
||||
setSeed(c.changes)
|
||||
setConnected(c.connected)
|
||||
if (s) setStats(s)
|
||||
}, [])
|
||||
applyStats(s)
|
||||
}, [applyStats])
|
||||
|
||||
useEffect(() => {
|
||||
load()
|
||||
const iv = setInterval(() => fetchChangeStats(15).then((s) => s && setStats(s)), 4000)
|
||||
const iv = setInterval(() => fetchChangeStats(15).then((s) => applyStats(s)), 2500)
|
||||
return () => clearInterval(iv)
|
||||
}, [load])
|
||||
}, [load, applyStats])
|
||||
|
||||
// Pulse the header when fresh changes arrive.
|
||||
// Fold freshly-arrived WS changes into the overlay → instant top-of-page update.
|
||||
useEffect(() => {
|
||||
const t = stats?.total ?? 0
|
||||
if (t > prevTotal.current) {
|
||||
setFlash(true)
|
||||
const id = setTimeout(() => setFlash(false), 900)
|
||||
prevTotal.current = t
|
||||
return () => clearTimeout(id)
|
||||
if (!liveChanges.length) return
|
||||
if (!primed.current) {
|
||||
primed.current = true
|
||||
lastSeenId.current = liveChanges[0].id
|
||||
return
|
||||
}
|
||||
prevTotal.current = t
|
||||
}, [stats?.total])
|
||||
const idx = liveChanges.findIndex((c) => c.id === lastSeenId.current)
|
||||
const fresh = idx === -1 ? liveChanges : liveChanges.slice(0, idx)
|
||||
if (!fresh.length) return
|
||||
lastSeenId.current = liveChanges[0].id
|
||||
setOverlay((o) => {
|
||||
const next = { total: o.total + fresh.length, by_op: { ...o.by_op }, by_source: { ...o.by_source }, by_table: { ...o.by_table } }
|
||||
for (const c of fresh) {
|
||||
next.by_op[c.op] = (next.by_op[c.op] || 0) + 1
|
||||
next.by_source[c.source] = (next.by_source[c.source] || 0) + 1
|
||||
next.by_table[c.table] = (next.by_table[c.table] || 0) + 1
|
||||
}
|
||||
return next
|
||||
})
|
||||
setFlash(true)
|
||||
const id = setTimeout(() => setFlash(false), 800)
|
||||
return () => clearTimeout(id)
|
||||
}, [liveChanges])
|
||||
|
||||
// Merge live (WS) with seeded backlog, dedupe by id, newest first.
|
||||
const merged = useMemo(() => {
|
||||
@@ -264,29 +291,44 @@ export function ChangesView({ liveChanges }: { liveChanges: CdcChange[] }) {
|
||||
[merged, source, op],
|
||||
)
|
||||
|
||||
const byOp = stats?.by_op || {}
|
||||
const inserts = byOp.insert || 0
|
||||
const updates = byOp.update || 0
|
||||
const deletes = byOp.delete || 0
|
||||
const total = stats?.total ?? 0
|
||||
const opVal = (k: string) => (stats?.by_op?.[k] || 0) + (overlay.by_op[k] || 0)
|
||||
const inserts = opVal('insert')
|
||||
const updates = opVal('update')
|
||||
const deletes = opVal('delete')
|
||||
const total = (stats?.total ?? 0) + overlay.total
|
||||
const perMin = total / Math.max(1, stats?.window_minutes ?? 15)
|
||||
|
||||
const opSegments = useMemo(() => (
|
||||
['insert', 'update', 'delete', 'snapshot']
|
||||
.map((k) => ({ label: k, value: byOp[k] || 0, color: opOf(k).color }))
|
||||
.map((k) => ({ label: k, value: (stats?.by_op?.[k] || 0) + (overlay.by_op[k] || 0), color: opOf(k).color }))
|
||||
.filter((s) => s.value > 0)
|
||||
), [byOp])
|
||||
), [stats?.by_op, overlay])
|
||||
|
||||
const sourceRows = useMemo(() => (
|
||||
Object.entries(stats?.by_source || {}).sort((a, b) => b[1] - a[1])
|
||||
), [stats?.by_source])
|
||||
const sourceRows = useMemo(() => {
|
||||
const m: Record<string, number> = { ...(stats?.by_source || {}) }
|
||||
for (const [k, v] of Object.entries(overlay.by_source)) m[k] = (m[k] || 0) + v
|
||||
return Object.entries(m).sort((a, b) => b[1] - a[1])
|
||||
}, [stats?.by_source, overlay])
|
||||
const maxSource = Math.max(1, ...sourceRows.map(([, v]) => v))
|
||||
|
||||
const tableRows = useMemo(() => (
|
||||
Object.entries(stats?.by_table || {}).sort((a, b) => b[1] - a[1]).slice(0, 7)
|
||||
), [stats?.by_table])
|
||||
const tableRows = useMemo(() => {
|
||||
const m: Record<string, number> = { ...(stats?.by_table || {}) }
|
||||
for (const [k, v] of Object.entries(overlay.by_table)) m[k] = (m[k] || 0) + v
|
||||
return Object.entries(m).sort((a, b) => b[1] - a[1]).slice(0, 7)
|
||||
}, [stats?.by_table, overlay])
|
||||
const maxTable = Math.max(1, ...tableRows.map(([, v]) => v))
|
||||
|
||||
// Volume chart: bump the current-minute bar with the live overlay so the curve
|
||||
// visibly rises as changes stream in.
|
||||
const liveBuckets = useMemo(() => {
|
||||
const b = (stats?.buckets || []).map((x) => ({ ...x }))
|
||||
if (overlay.total) {
|
||||
if (b.length) b[b.length - 1] = { ...b[b.length - 1], n: b[b.length - 1].n + overlay.total }
|
||||
else b.push({ t: 'now', n: overlay.total })
|
||||
}
|
||||
return b
|
||||
}, [stats?.buckets, overlay.total])
|
||||
|
||||
return (
|
||||
<div className="flex h-full min-h-0 flex-col gap-3 overflow-y-auto scrollbar-thin pr-1">
|
||||
{/* Header */}
|
||||
@@ -324,7 +366,7 @@ export function ChangesView({ liveChanges }: { liveChanges: CdcChange[] }) {
|
||||
<div className="mb-1 flex items-center gap-1.5 text-[10px] uppercase tracking-wide text-foreground-faint">
|
||||
<TrendingUp className="h-3 w-3" /> Change volume — last 15 minutes
|
||||
</div>
|
||||
<VolumeArea buckets={stats?.buckets || []} />
|
||||
<VolumeArea buckets={liveBuckets} />
|
||||
</div>
|
||||
<div className="rounded-lg border border-border/60 bg-surface-raised p-3">
|
||||
<div className="mb-2 flex items-center gap-1.5 text-[10px] uppercase tracking-wide text-foreground-faint">
|
||||
|
||||
Reference in New Issue
Block a user