feat: realtime Live dashboard in Data Explorer + fix panel layout

- New "Live" tab: auto-polls /api/federated/live every 2.5s with
  animated counters, ingestion throughput sparkline, per-source
  write-rate bars, a region scorecard matrix (heat-shaded) and live
  business breakdown charts (region/channel/status/customers/
  telemetry/supply)
- Backend /api/federated/live: instant source estimates (Postgres),
  monotonic max(event_id) for MySQL and Mongo estimated count for
  immediate movement, Cassandra from cached matrix; business aggs
  cached over the small Hadoop lake tables (short TTL)
- Fix embedded Trino panels being squeezed with internal scrollbars
  by making panels/grids shrink-0 so the page scrolls instead
This commit is contained in:
mo
2026-06-28 21:10:19 +00:00
parent 437574f0bb
commit 8c72d1dc63
4 changed files with 466 additions and 9 deletions
@@ -15,15 +15,18 @@ import {
Network,
HardDrive,
ShieldCheck,
Radio,
} from 'lucide-react'
import { cn } from '../../lib/utils'
import { TrinoFederationView, type SubTab } from './TrinoFederationView'
import { LiveDashboard } from './LiveDashboard'
type ExplorerTab = 'business' | SubTab
type ExplorerTab = 'business' | 'live' | SubTab
const TABS: { id: ExplorerTab; label: string; icon: typeof Users; hint: string }[] = [
const TABS: { id: ExplorerTab; label: string; icon: typeof Users; hint: string; live?: boolean }[] = [
{ id: 'business', label: 'Business Overview', icon: BarChart3, hint: 'Customers, orders, workforce, supply chain & telemetry across every source' },
{ id: 'federated', label: 'Federated (Trino)', icon: Network, hint: 'One SQL engine joining PostgreSQL, MySQL & MongoDB live — region scorecard' },
{ id: 'live', label: 'Live', icon: Radio, hint: 'Realtime business activity — live counters, ingestion throughput & region matrix', live: true },
{ id: 'federated', label: 'Federated (Trino)', icon: Network, hint: 'One SQL across all 5 databases + region scorecard joined live' },
{ id: 'lake', label: 'Hadoop Lake', icon: HardDrive, hint: 'All business data mirrored as external Iceberg tables on HDFS' },
{ id: 'dictionary', label: 'Data Dictionary', icon: ShieldCheck, hint: 'Every table & column with PII / masking status — exactly what the assistant sees' },
]
@@ -263,7 +266,7 @@ export function DataExplorerView() {
{/* tab bar */}
<div className="flex shrink-0 flex-wrap gap-1 px-1">
{TABS.map(({ id, label, icon: Icon }) => (
{TABS.map(({ id, label, icon: Icon, live }) => (
<button
key={id}
type="button"
@@ -273,13 +276,24 @@ export function DataExplorerView() {
view === id ? 'bg-docker/15 text-docker' : 'text-foreground-muted hover:bg-surface-overlay',
)}
>
<Icon className="h-3.5 w-3.5" /> {label}
{live ? (
<span className="relative flex h-2 w-2">
<span className="absolute inline-flex h-full w-full animate-ping rounded-full bg-emerald-400 opacity-75" />
<span className="relative inline-flex h-2 w-2 rounded-full bg-emerald-400" />
</span>
) : (
<Icon className="h-3.5 w-3.5" />
)}
{label}
</button>
))}
</div>
{/* Live realtime dashboard */}
{view === 'live' && <LiveDashboard />}
{/* Trino federation / lake / dictionary tabs */}
{view !== 'business' && <TrinoFederationView embedded activeTab={view} />}
{(view === 'federated' || view === 'lake' || view === 'dictionary') && <TrinoFederationView embedded activeTab={view} />}
{/* ───────── BUSINESS OVERVIEW ───────── */}
{view === 'business' && (
@@ -0,0 +1,333 @@
import { useCallback, useEffect, useRef, useState } from 'react'
import { Activity, Pause, Play, ShoppingCart, Users, Boxes, Cpu, DollarSign, Database, Gauge } from 'lucide-react'
import { cn } from '../../lib/utils'
type Bucket = { key: string; count: number; value?: number }
type Source = { key: string; label: string; engine: string; catalog: string; rows: number; color: string }
type RegionRow = { region: string; orders: number; revenue: number; hr_events: number; supply_events: number }
type Live = {
ok: boolean
ts: string
sources: Source[]
totals: { records: number; revenue_est: number; avg_order: number }
business: {
orders_by_region: Bucket[]
orders_by_status: Bucket[]
orders_by_channel: Bucket[]
top_customers: Bucket[]
telemetry_by_metric: Bucket[]
supply_by_type: Bucket[]
region_matrix: RegionRow[]
}
}
const COLORS = ['#34d399', '#60a5fa', '#f472b6', '#fbbf24', '#a78bfa', '#22d3ee', '#fb7185']
const POLL_MS = 2500
function fmtNum(n?: number | string | null) {
if (n == null) return '—'
const v = typeof n === 'number' ? n : Number(n)
if (Number.isNaN(v)) return String(n)
if (Math.abs(v) >= 1e9) return `${(v / 1e9).toFixed(2)}B`
if (Math.abs(v) >= 1e6) return `${(v / 1e6).toFixed(2)}M`
if (Math.abs(v) >= 1e3) return `${(v / 1e3).toFixed(1)}K`
return `${Math.round(v)}`
}
const fmtMoney = (n?: number | string | null) => (n == null ? '—' : `${fmtNum(n)}`)
function useTween(target: number, ms = 700) {
const [disp, setDisp] = useState(target)
const cur = useRef(target)
const startVal = useRef(target)
const start = useRef(0)
const raf = useRef(0)
useEffect(() => {
startVal.current = cur.current
start.current = performance.now()
cancelAnimationFrame(raf.current)
const tick = (now: number) => {
const p = Math.min(1, (now - start.current) / ms)
const e = 1 - Math.pow(1 - p, 3)
const val = startVal.current + (target - startVal.current) * e
cur.current = val
setDisp(val)
if (p < 1) raf.current = requestAnimationFrame(tick)
}
raf.current = requestAnimationFrame(tick)
return () => cancelAnimationFrame(raf.current)
}, [target, ms])
return disp
}
function Spark({ data, color = '#34d399', height = 44 }: { data: number[]; color?: string; height?: number }) {
if (data.length < 2) return <div style={{ height }} className="flex items-center justify-center text-[9px] text-foreground-faint">collecting</div>
const w = 240
const max = Math.max(1, ...data)
const step = w / (data.length - 1)
const coords = data.map((v, i) => [i * step, height - (v / max) * (height - 6) - 3])
const line = coords.map((c, i) => `${i === 0 ? 'M' : 'L'}${c[0].toFixed(1)},${c[1].toFixed(1)}`).join(' ')
const area = `${line} L${w},${height} L0,${height} Z`
const gid = `sg-${color.replace('#', '')}`
return (
<svg viewBox={`0 0 ${w} ${height}`} className="w-full" preserveAspectRatio="none" style={{ height }}>
<defs>
<linearGradient id={gid} x1="0" y1="0" x2="0" y2="1">
<stop offset="0%" stopColor={color} stopOpacity="0.5" />
<stop offset="100%" stopColor={color} stopOpacity="0" />
</linearGradient>
</defs>
<path d={area} fill={`url(#${gid})`} />
<path d={line} fill="none" stroke={color} strokeWidth="1.5" />
<circle cx={coords[coords.length - 1][0]} cy={coords[coords.length - 1][1]} r="2.5" fill={color} />
</svg>
)
}
function Bars({ data, valueKind, colorByIndex }: { data?: Bucket[]; valueKind?: 'money' | 'num'; colorByIndex?: boolean }) {
const d = data || []
const useVal = valueKind != null
const max = Math.max(1, ...d.map((x) => (useVal && x.value != null ? x.value : x.count)))
if (!d.length) return <p className="py-5 text-center text-[10px] text-foreground-faint">No data</p>
return (
<div className="space-y-1.5">
{d.map((x, i) => {
const metric = useVal && x.value != null ? x.value : x.count
const pct = Math.max(2, (metric / max) * 100)
const label = useVal && x.value != null ? (valueKind === 'money' ? fmtMoney(x.value) : fmtNum(x.value)) : fmtNum(x.count)
return (
<div key={x.key ?? i} className="flex items-center gap-2 text-[10px]">
<span className="w-24 shrink-0 truncate text-foreground-muted" title={x.key}>{x.key ?? '—'}</span>
<div className="relative h-3.5 flex-1 overflow-hidden rounded bg-surface-overlay">
<div className="h-full rounded transition-all duration-500" style={{ width: `${pct}%`, backgroundColor: colorByIndex ? COLORS[i % COLORS.length] : '#38bdf8' }} />
</div>
<span className="w-16 shrink-0 text-right font-mono text-foreground">{label}</span>
</div>
)
})}
</div>
)
}
function Donut({ data }: { data?: Bucket[] }) {
const d = data || []
const total = d.reduce((s, x) => s + x.count, 0) || 1
let acc = 0
const r = 42
const c = 2 * Math.PI * r
if (!d.length) return <p className="py-5 text-center text-[10px] text-foreground-faint">No data</p>
return (
<div className="flex items-center gap-4">
<svg viewBox="0 0 100 100" className="h-24 w-24 shrink-0 -rotate-90">
{d.map((x, i) => {
const dash = (x.count / total) * c
const seg = <circle key={x.key ?? i} cx="50" cy="50" r={r} fill="none" stroke={COLORS[i % COLORS.length]} strokeWidth="14" strokeDasharray={`${dash} ${c - dash}`} strokeDashoffset={-acc} />
acc += dash
return seg
})}
</svg>
<div className="min-w-0 flex-1 space-y-1">
{d.slice(0, 7).map((x, i) => (
<div key={x.key ?? i} className="flex items-center gap-1.5 text-[10px]">
<span className="h-2 w-2 shrink-0 rounded-full" style={{ backgroundColor: COLORS[i % COLORS.length] }} />
<span className="flex-1 truncate text-foreground-muted">{x.key ?? '—'}</span>
<span className="font-mono text-foreground">{((x.count / total) * 100).toFixed(0)}%</span>
</div>
))}
</div>
</div>
)
}
function Panel({ title, subtitle, icon: Icon, children, className }: { title: string; subtitle?: string; icon?: typeof Database; children: React.ReactNode; className?: string }) {
return (
<div className={cn('panel flex min-h-0 shrink-0 flex-col p-3', className)}>
<div className="mb-2 flex items-center gap-1.5">
{Icon && <Icon className="h-3.5 w-3.5 text-docker" />}
<h3 className="text-[11px] font-semibold text-foreground">{title}</h3>
{subtitle && <span className="ml-auto text-[9px] text-foreground-faint">{subtitle}</span>}
</div>
{children}
</div>
)
}
function LiveCounter({ value, label, sub, accent, icon: Icon, money }: { value: number; label: string; sub?: string; accent: string; icon: typeof Users; money?: boolean }) {
const tv = useTween(value)
return (
<div className="panel flex items-center gap-3 px-3 py-2.5">
<div className="flex h-9 w-9 shrink-0 items-center justify-center rounded-lg" style={{ backgroundColor: `${accent}1f`, color: accent }}>
<Icon className="h-4 w-4" />
</div>
<div className="min-w-0">
<p className="text-[9px] font-semibold uppercase tracking-wider text-foreground-muted">{label}</p>
<p className="truncate font-mono text-lg font-bold leading-tight text-foreground">{money ? fmtMoney(tv) : fmtNum(tv)}</p>
{sub && <p className="truncate text-[9px] text-foreground-faint">{sub}</p>}
</div>
</div>
)
}
const SRC_ICON: Record<string, typeof Users> = { orders: ShoppingCart, hr_events: Users, supply_events: Boxes, telemetry: Cpu }
export function LiveDashboard() {
const [data, setData] = useState<Live | null>(null)
const [paused, setPaused] = useState(false)
const [err, setErr] = useState(false)
const [totalHist, setTotalHist] = useState<number[]>([])
const [rateBySrc, setRateBySrc] = useState<Record<string, number>>({})
const [added, setAdded] = useState(0)
const prev = useRef<{ ts: number; rows: Record<string, number>; total: number } | null>(null)
const startTotal = useRef<number | null>(null)
const poll = useCallback(async () => {
try {
const r = await fetch('/api/federated/live')
if (!r.ok) { setErr(true); return }
const d: Live = await r.json()
setErr(false)
const now = Date.parse(d.ts) || Date.now()
const total = d.totals.records
if (prev.current) {
const dt = Math.max(0.5, (now - prev.current.ts) / 1000)
const totRate = Math.max(0, (total - prev.current.total) / dt)
setTotalHist((h) => [...h, totRate].slice(-90))
const rmap: Record<string, number> = {}
d.sources.forEach((s) => {
const p = prev.current!.rows[s.key] ?? s.rows
rmap[s.key] = Math.max(0, (s.rows - p) / dt)
})
setRateBySrc(rmap)
}
if (startTotal.current == null) startTotal.current = total
setAdded(Math.max(0, total - (startTotal.current || total)))
prev.current = { ts: now, rows: Object.fromEntries(d.sources.map((s) => [s.key, s.rows])), total }
setData(d)
} catch {
setErr(true)
}
}, [])
useEffect(() => {
poll()
if (paused) return
const t = setInterval(poll, POLL_MS)
return () => clearInterval(t)
}, [poll, paused])
const b = data?.business
const totalRate = totalHist.length ? totalHist[totalHist.length - 1] : 0
const matrix = b?.region_matrix || []
const maxRev = Math.max(1, ...matrix.map((m) => Number(m.revenue) || 0))
return (
<div className="flex min-h-0 flex-col gap-2">
{/* live status bar */}
<div className="flex shrink-0 flex-wrap items-center gap-2 px-1">
<span className="inline-flex items-center gap-1.5 rounded-full border border-emerald-500/40 bg-emerald-500/10 px-2.5 py-1 text-[10px] font-semibold text-emerald-400">
<span className="relative flex h-2 w-2">
{!paused && <span className="absolute inline-flex h-full w-full animate-ping rounded-full bg-emerald-400 opacity-75" />}
<span className="relative inline-flex h-2 w-2 rounded-full bg-emerald-400" />
</span>
{paused ? 'PAUSED' : 'LIVE'}
</span>
<span className="inline-flex items-center gap-1 text-[10px] text-foreground-muted">
<Gauge className="h-3.5 w-3.5" /> {fmtNum(totalRate)} rows/s
</span>
<span className="text-[10px] text-foreground-muted">·</span>
<span className="text-[10px] text-foreground-muted">+{fmtNum(added)} since opened</span>
{err && <span className="text-[10px] text-amber-400">reconnecting</span>}
<span className="ml-auto text-[9px] text-foreground-faint">{data ? `updated ${new Date(data.ts).toLocaleTimeString()}` : 'connecting…'}</span>
<button type="button" onClick={() => setPaused((p) => !p)} className="inline-flex items-center gap-1 rounded-md border border-border px-2 py-1 text-[10px] text-foreground-muted hover:bg-surface-overlay">
{paused ? <Play className="h-3 w-3" /> : <Pause className="h-3 w-3" />} {paused ? 'Resume' : 'Pause'}
</button>
</div>
{/* headline counters */}
<div className="grid shrink-0 grid-cols-2 gap-2 md:grid-cols-3 xl:grid-cols-6">
<LiveCounter value={data?.totals.records || 0} label="Total records" sub="across all engines" accent="#34d399" icon={Database} />
<LiveCounter value={data?.totals.revenue_est || 0} label="Revenue (est.)" sub={`avg ${fmtMoney(data?.totals.avg_order)}/order`} accent="#22d3ee" icon={DollarSign} money />
{(data?.sources || []).map((s) => (
<LiveCounter key={s.key} value={s.rows} label={s.label} sub={`${s.engine} · ${fmtNum(rateBySrc[s.key] || 0)}/s`} accent={s.color} icon={SRC_ICON[s.key] || Activity} />
))}
</div>
{/* throughput + per-source rates */}
<div className="grid shrink-0 gap-2 lg:grid-cols-3">
<Panel title="Ingestion throughput" subtitle="rows/sec · live" icon={Activity} className="lg:col-span-2">
<Spark data={totalHist} color="#34d399" height={90} />
<div className="mt-1 flex justify-between text-[9px] text-foreground-faint">
<span>~{(POLL_MS / 1000) * 90}s window</span>
<span>peak {fmtNum(Math.max(0, ...totalHist))}/s</span>
</div>
</Panel>
<Panel title="Live write rate by source" subtitle="rows/sec" icon={Gauge}>
<div className="space-y-2">
{(data?.sources || []).map((s) => {
const rate = rateBySrc[s.key] || 0
const max = Math.max(1, ...Object.values(rateBySrc))
return (
<div key={s.key} className="flex items-center gap-2 text-[10px]">
<span className="w-20 shrink-0 truncate text-foreground-muted">{s.engine}</span>
<div className="relative h-3 flex-1 overflow-hidden rounded bg-surface-overlay">
<div className="h-full rounded transition-all duration-500" style={{ width: `${Math.max(2, (rate / max) * 100)}%`, backgroundColor: s.color }} />
</div>
<span className="w-14 shrink-0 text-right font-mono text-foreground">{fmtNum(rate)}/s</span>
</div>
)
})}
</div>
</Panel>
</div>
{/* region matrix */}
<Panel title="Region scorecard matrix — business data across regions" subtitle="orders · revenue · workforce · supply" icon={Database}>
{matrix.length ? (
<div className="overflow-x-auto">
<table className="w-full text-[10px]">
<thead>
<tr className="border-b border-border text-left text-foreground-muted">
<th className="py-1 pr-3">Region</th>
<th className="py-1 pr-3 text-right">Orders</th>
<th className="py-1 pr-3 text-right">Revenue</th>
<th className="py-1 pr-3 text-right">HR events</th>
<th className="py-1 pr-3 text-right">Supply events</th>
</tr>
</thead>
<tbody>
{matrix.map((m, i) => {
const heat = (Number(m.revenue) || 0) / maxRev
return (
<tr key={m.region ?? i} className="border-b border-border/40">
<td className="py-1 pr-3 font-medium text-foreground">
<span className="inline-flex items-center gap-1.5"><span className="h-2 w-2 rounded-full" style={{ backgroundColor: COLORS[i % COLORS.length] }} />{m.region}</span>
</td>
<td className="py-1 pr-3 text-right font-mono text-foreground">{fmtNum(m.orders)}</td>
<td className="py-1 pr-3 text-right font-mono" style={{ backgroundColor: `rgba(52,211,153,${(heat * 0.35).toFixed(3)})`, color: '#34d399' }}>{fmtMoney(m.revenue)}</td>
<td className="py-1 pr-3 text-right font-mono text-foreground">{fmtNum(m.hr_events)}</td>
<td className="py-1 pr-3 text-right font-mono text-foreground">{fmtNum(m.supply_events)}</td>
</tr>
)
})}
</tbody>
</table>
</div>
) : (
<p className="py-4 text-center text-[10px] text-foreground-faint">Building matrix</p>
)}
</Panel>
{/* business breakdown charts */}
<div className="grid shrink-0 gap-2 lg:grid-cols-2">
<Panel title="Revenue by region" icon={DollarSign}><Bars data={b?.orders_by_region} valueKind="money" colorByIndex /></Panel>
<Panel title="Revenue by channel" icon={ShoppingCart}><Bars data={b?.orders_by_channel} valueKind="money" colorByIndex /></Panel>
<Panel title="Orders by status"><Donut data={b?.orders_by_status} /></Panel>
<Panel title="Top customers by spend" icon={Users}><Bars data={b?.top_customers} valueKind="money" /></Panel>
<Panel title="Telemetry — avg by metric" icon={Cpu}><Bars data={b?.telemetry_by_metric} valueKind="num" colorByIndex /></Panel>
<Panel title="Supply events by type" icon={Boxes}><Bars data={b?.supply_by_type} colorByIndex /></Panel>
</div>
<p className="shrink-0 px-1 pb-2 text-[9px] text-foreground-faint">
Counters &amp; throughput are live source estimates (Trino over PostgreSQL, MySQL, MongoDB &amp; Cassandra); breakdown charts aggregate the materialized Hadoop lake. Polling every {POLL_MS / 1000}s.
</p>
</div>
)
}
@@ -36,7 +36,7 @@ const fmtMoney = (n?: number | string | null) => (n == null ? '—' : `€${fmtN
function Panel({ title, subtitle, icon: Icon, children }: { title: string; subtitle?: string; icon?: typeof Database; children: React.ReactNode }) {
return (
<div className="panel flex min-h-0 flex-col p-3">
<div className="panel flex min-h-0 shrink-0 flex-col p-3">
<div className="mb-2 flex items-center gap-1.5">
{Icon && <Icon className="h-3.5 w-3.5 text-docker" />}
<h3 className="text-[11px] font-semibold text-foreground">{title}</h3>
@@ -375,10 +375,10 @@ export function TrinoFederationView({ embedded = false, activeTab }: { embedded?
<Kpi icon={ShieldAlert} label="PII columns" value={String(dict?.summary?.pii_columns ?? '—')} accent="#fbbf24" />
<Kpi icon={ShieldCheck} label="Masked" value={String(dict?.summary?.masked_columns ?? '—')} sub="hidden from LLM" accent="#f472b6" />
</div>
<p className="px-1 text-[10px] text-foreground-muted">
<p className="shrink-0 px-1 text-[10px] text-foreground-muted">
This is exactly what the assistant knows about your data every column, its type, and whether it is <span className="text-amber-400">masked</span> or visible.
</p>
<div className="grid gap-2 lg:grid-cols-2">
<div className="grid shrink-0 gap-2 lg:grid-cols-2">
{(dict?.tables || []).map((t: any) => (
<Panel key={t.fqn} title={t.fqn} subtitle={`${t.masked_count}/${t.pii_count} PII masked`} icon={Database}>
<p className="mb-1.5 text-[9px] text-foreground-faint">{t.engine} · {t.desc}</p>