feat: Data Explorer tab + manual data entry with live CDC

Add a dedicated business-data view (separate from infra/search):
- New "Data Explorer" tab: KPIs (customers, employees, products, source
  row totals, sample revenue) + charts driven by Elasticsearch aggregations
  (revenue over time, by region/channel/status, top customers & products,
  HR by department/role/event, supply chain by type, telemetry averages),
  with region + free-text filters.
- Backend /api/search/business endpoint: battery of ES aggregations with
  numeric/date index template (so amount sums and date histograms work),
  source totals via cheap planner estimates; biased the orders sample to
  rows carrying customer names so people are searchable/visible.

Manual data entry on source systems:
- "Insert row" action in the Data Hub browser opens a column-aware form;
  POST /api/sql/row/insert writes to Postgres/MySQL/Mongo/Cassandra/Neo4j.
- Inserts into CDC sources (PG/MySQL/Mongo) are captured by Debezium and
  streamed to Kafka in real time; UI flags this and pulses the data flow.
This commit is contained in:
mo
2026-06-28 17:12:29 +00:00
parent 8a8d779328
commit 8d28695868
7 changed files with 799 additions and 7 deletions
@@ -0,0 +1,323 @@
import { useCallback, useEffect, useMemo, useState } from 'react'
import {
Users,
UserCog,
Package,
ShoppingCart,
DollarSign,
Activity,
Boxes,
RefreshCw,
Search,
Loader2,
TrendingUp,
ExternalLink,
} from 'lucide-react'
import { cn } from '../../lib/utils'
type Bucket = { key: string; count: number; value?: number }
type Business = {
ok: boolean
generated_at: string
filters: { q: string; region: string }
kpis: {
customers: number
employees: number
products: number
orders_indexed: number
revenue_indexed: number
avg_order: number
telemetry_indexed: number
supply_indexed: number
source_totals: { orders?: number; hr_events?: number; supply_events?: number }
}
orders: {
by_region: Bucket[]
by_channel: Bucket[]
by_status: Bucket[]
top_customers: Bucket[]
top_products: Bucket[]
over_time: Bucket[]
}
hr: { by_department: Bucket[]; by_role: Bucket[]; by_event: Bucket[] }
supply: { by_type: Bucket[]; by_region: Bucket[] }
telemetry: { by_metric: Bucket[] }
regions: string[]
}
const REGION_COLORS = ['#34d399', '#60a5fa', '#f472b6', '#fbbf24', '#a78bfa', '#22d3ee', '#fb7185']
function fmtNum(n?: number | null) {
if (n == null) return '—'
if (Math.abs(n) >= 1_000_000_000) return `${(n / 1_000_000_000).toFixed(2)}B`
if (Math.abs(n) >= 1_000_000) return `${(n / 1_000_000).toFixed(1)}M`
if (Math.abs(n) >= 1_000) return `${(n / 1_000).toFixed(1)}K`
return String(n)
}
function fmtMoney(n?: number | null) {
if (n == null) return '—'
return `${fmtNum(n)}`
}
function monthLabel(key: string) {
const m = /^(\d{4})-(\d{2})/.exec(key)
if (!m) return key
const months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
return `${months[Number(m[2]) - 1]} '${m[1].slice(2)}`
}
function Kpi({ icon: Icon, label, value, sub, accent }: { icon: typeof Users; label: string; value: string; sub?: string; accent: string }) {
return (
<div className="panel flex items-center gap-3 px-3 py-2.5">
<div className="flex h-9 w-9 shrink-0 items-center justify-center rounded-lg" style={{ backgroundColor: `${accent}1f`, color: accent }}>
<Icon className="h-4 w-4" />
</div>
<div className="min-w-0">
<p className="text-[9px] font-semibold uppercase tracking-wider text-foreground-muted">{label}</p>
<p className="truncate text-lg font-bold leading-tight text-foreground">{value}</p>
{sub && <p className="truncate text-[9px] text-foreground-faint">{sub}</p>}
</div>
</div>
)
}
function Panel({ title, subtitle, children, icon: Icon }: { title: string; subtitle?: string; children: React.ReactNode; icon?: typeof Users }) {
return (
<div className="panel flex min-h-0 flex-col p-3">
<div className="mb-2 flex items-center gap-1.5">
{Icon && <Icon className="h-3.5 w-3.5 text-docker" />}
<h3 className="text-[11px] font-semibold text-foreground">{title}</h3>
{subtitle && <span className="ml-auto text-[9px] text-foreground-faint">{subtitle}</span>}
</div>
{children}
</div>
)
}
function BarsH({ data, valueKind, colorByIndex }: { data: Bucket[]; valueKind?: 'money' | 'num'; colorByIndex?: boolean }) {
const useVal = valueKind != null
const max = Math.max(1, ...data.map((d) => (useVal && d.value != null ? d.value : d.count)))
if (!data.length) return <p className="py-6 text-center text-[10px] text-foreground-faint">No data</p>
return (
<div className="space-y-1.5">
{data.map((d, i) => {
const metric = useVal && d.value != null ? d.value : d.count
const pct = Math.max(2, (metric / max) * 100)
const color = colorByIndex ? REGION_COLORS[i % REGION_COLORS.length] : '#38bdf8'
const label = useVal && d.value != null ? (valueKind === 'money' ? fmtMoney(d.value) : fmtNum(d.value)) : fmtNum(d.count)
return (
<div key={d.key ?? i} className="flex items-center gap-2 text-[10px]">
<span className="w-28 shrink-0 truncate text-foreground-muted" title={d.key}>{d.key ?? '—'}</span>
<div className="relative h-3.5 flex-1 overflow-hidden rounded bg-surface-overlay">
<div className="h-full rounded" style={{ width: `${pct}%`, backgroundColor: color }} />
</div>
<span className="w-20 shrink-0 text-right font-mono text-foreground">{label}</span>
</div>
)
})}
</div>
)
}
function AreaTrend({ data }: { data: Bucket[] }) {
const pts = data.filter((d) => d.value != null)
if (pts.length < 2) return <p className="py-6 text-center text-[10px] text-foreground-faint">Not enough data</p>
const w = 560
const h = 120
const max = Math.max(...pts.map((p) => p.value || 0))
const min = Math.min(...pts.map((p) => p.value || 0))
const range = max - min || 1
const step = w / (pts.length - 1)
const coords = pts.map((p, i) => [i * step, h - ((((p.value || 0) - min) / range) * (h - 16) + 8)])
const line = coords.map((c, i) => `${i === 0 ? 'M' : 'L'}${c[0].toFixed(1)},${c[1].toFixed(1)}`).join(' ')
const area = `${line} L${w},${h} L0,${h} Z`
return (
<div>
<svg viewBox={`0 0 ${w} ${h}`} className="w-full" preserveAspectRatio="none" style={{ height: 120 }}>
<defs>
<linearGradient id="rev-grad" x1="0" y1="0" x2="0" y2="1">
<stop offset="0%" stopColor="#34d399" stopOpacity="0.45" />
<stop offset="100%" stopColor="#34d399" stopOpacity="0" />
</linearGradient>
</defs>
<path d={area} fill="url(#rev-grad)" />
<path d={line} fill="none" stroke="#34d399" strokeWidth="2" />
{coords.map((c, i) => <circle key={i} cx={c[0]} cy={c[1]} r="2" fill="#34d399" />)}
</svg>
<div className="mt-1 flex justify-between text-[8px] text-foreground-faint">
<span>{monthLabel(pts[0].key)}</span>
<span>{monthLabel(pts[Math.floor(pts.length / 2)].key)}</span>
<span>{monthLabel(pts[pts.length - 1].key)}</span>
</div>
</div>
)
}
function Donut({ data }: { data: Bucket[] }) {
const total = data.reduce((s, d) => s + d.count, 0) || 1
let acc = 0
const r = 42
const c = 2 * Math.PI * r
if (!data.length) return <p className="py-6 text-center text-[10px] text-foreground-faint">No data</p>
return (
<div className="flex items-center gap-4">
<svg viewBox="0 0 100 100" className="h-28 w-28 shrink-0 -rotate-90">
{data.map((d, i) => {
const frac = d.count / total
const dash = frac * c
const seg = (
<circle
key={d.key ?? i}
cx="50" cy="50" r={r} fill="none"
stroke={REGION_COLORS[i % REGION_COLORS.length]} strokeWidth="14"
strokeDasharray={`${dash} ${c - dash}`} strokeDashoffset={-acc}
/>
)
acc += dash
return seg
})}
</svg>
<div className="min-w-0 flex-1 space-y-1">
{data.slice(0, 7).map((d, i) => (
<div key={d.key ?? i} className="flex items-center gap-1.5 text-[10px]">
<span className="h-2 w-2 shrink-0 rounded-full" style={{ backgroundColor: REGION_COLORS[i % REGION_COLORS.length] }} />
<span className="flex-1 truncate text-foreground-muted">{d.key ?? '—'}</span>
<span className="font-mono text-foreground">{((d.count / total) * 100).toFixed(0)}%</span>
</div>
))}
</div>
</div>
)
}
export function DataExplorerView() {
const [data, setData] = useState<Business | null>(null)
const [loading, setLoading] = useState(true)
const [region, setRegion] = useState('')
const [qInput, setQInput] = useState('')
const [q, setQ] = useState('')
const load = useCallback(async () => {
setLoading(true)
try {
const params = new URLSearchParams()
if (q) params.set('q', q)
if (region) params.set('region', region)
const r = await fetch(`/api/search/business?${params.toString()}`)
if (r.ok) setData(await r.json())
} catch { /* */ } finally {
setLoading(false)
}
}, [q, region])
useEffect(() => { load() }, [load])
const k = data?.kpis
const regions = useMemo(() => data?.regions || [], [data])
return (
<div className="scrollbar-thin flex h-full min-h-0 flex-col gap-2 overflow-y-auto p-3">
{/* header */}
<header className="panel flex shrink-0 flex-wrap items-center justify-between gap-3 px-4 py-3">
<div>
<h1 className="flex items-center gap-2 text-base font-semibold text-foreground">
<TrendingUp className="h-5 w-5 text-docker" />
Data Explorer
</h1>
<p className="text-[11px] text-foreground-muted">
Your actual business data customers, orders, employees, supply chain &amp; telemetry across all sources
</p>
</div>
<div className="flex items-center gap-2">
<form
onSubmit={(e) => { e.preventDefault(); setQ(qInput.trim()) }}
className="flex items-center gap-1 rounded-md border border-border bg-surface px-2 py-1"
>
<Search className="h-3.5 w-3.5 text-foreground-muted" />
<input
value={qInput}
onChange={(e) => setQInput(e.target.value)}
placeholder="Filter by name, id, region…"
className="w-48 bg-transparent text-[11px] text-foreground outline-none placeholder:text-foreground-faint"
/>
</form>
<button type="button" onClick={load} className="inline-flex items-center gap-1.5 rounded-md border border-border px-3 py-1.5 text-[11px] text-foreground-muted hover:bg-surface-overlay">
{loading ? <Loader2 className="h-3.5 w-3.5 animate-spin" /> : <RefreshCw className="h-3.5 w-3.5" />} Refresh
</button>
</div>
</header>
{/* region filters */}
<div className="flex shrink-0 flex-wrap items-center gap-1.5 px-1 text-[10px]">
<span className="mr-1 font-semibold uppercase tracking-wider text-foreground-faint">Region</span>
{['', ...regions].map((rg) => (
<button
key={rg || 'all'}
type="button"
onClick={() => setRegion(rg)}
className={cn('rounded-full border px-2.5 py-1 font-medium transition-colors', region === rg ? 'border-docker bg-docker/15 text-docker' : 'border-border text-foreground-muted hover:bg-surface-overlay')}
>
{rg || 'All regions'}
</button>
))}
{(q || region) && (
<button type="button" onClick={() => { setQ(''); setQInput(''); setRegion('') }} className="ml-1 rounded-full border border-amber-500/40 px-2.5 py-1 font-medium text-amber-400 hover:bg-amber-500/10">
Clear filters
</button>
)}
{data && <span className="ml-auto text-[9px] text-foreground-faint">updated {new Date(data.generated_at).toLocaleTimeString()}</span>}
</div>
{/* KPIs */}
<div className="grid shrink-0 grid-cols-2 gap-2 md:grid-cols-4 xl:grid-cols-7">
<Kpi icon={Users} label="Customers" value={fmtNum(k?.customers)} sub="distinct, searchable" accent="#34d399" />
<Kpi icon={UserCog} label="Employees" value={fmtNum(k?.employees)} sub="distinct, searchable" accent="#60a5fa" />
<Kpi icon={Package} label="Products" value={fmtNum(k?.products)} accent="#f472b6" />
<Kpi icon={ShoppingCart} label="Orders (source)" value={fmtNum(k?.source_totals?.orders)} sub={`${fmtNum(k?.orders_indexed)} indexed`} accent="#fbbf24" />
<Kpi icon={DollarSign} label="Revenue (sample)" value={fmtMoney(k?.revenue_indexed)} sub={`avg ${fmtMoney(k?.avg_order)}`} accent="#22d3ee" />
<Kpi icon={Boxes} label="Supply events" value={fmtNum(k?.source_totals?.supply_events)} accent="#a78bfa" />
<Kpi icon={Activity} label="HR events" value={fmtNum(k?.source_totals?.hr_events)} accent="#fb7185" />
</div>
{/* Sales */}
<h2 className="mt-1 shrink-0 px-1 text-[10px] font-semibold uppercase tracking-widest text-foreground-muted">Sales &amp; Customers PostgreSQL</h2>
<div className="grid shrink-0 gap-2 lg:grid-cols-2">
<Panel title="Revenue over time" subtitle="monthly, indexed sample" icon={TrendingUp}>
<AreaTrend data={data?.orders.over_time || []} />
</Panel>
<Panel title="Revenue by region" icon={DollarSign}>
<BarsH data={data?.orders.by_region || []} valueKind="money" colorByIndex />
</Panel>
<Panel title="Top customers by spend" icon={Users}>
<BarsH data={data?.orders.top_customers || []} valueKind="money" />
</Panel>
<div className="grid gap-2 sm:grid-cols-2">
<Panel title="Orders by status">
<Donut data={data?.orders.by_status || []} />
</Panel>
<Panel title="Orders by channel">
<Donut data={data?.orders.by_channel || []} />
</Panel>
</div>
<Panel title="Top products by revenue" icon={Package}>
<BarsH data={data?.orders.top_products || []} valueKind="money" />
</Panel>
</div>
{/* HR */}
<h2 className="mt-1 shrink-0 px-1 text-[10px] font-semibold uppercase tracking-widest text-foreground-muted">Workforce MySQL HR</h2>
<div className="grid shrink-0 gap-2 lg:grid-cols-3">
<Panel title="Employees by department" icon={UserCog}><BarsH data={data?.hr.by_department || []} colorByIndex /></Panel>
<Panel title="By role"><BarsH data={data?.hr.by_role || []} /></Panel>
<Panel title="By event type"><Donut data={data?.hr.by_event || []} /></Panel>
</div>
{/* Supply + telemetry */}
<h2 className="mt-1 shrink-0 px-1 text-[10px] font-semibold uppercase tracking-widest text-foreground-muted">Supply Chain (MongoDB) &amp; Telemetry (Cassandra)</h2>
<div className="grid shrink-0 gap-2 lg:grid-cols-3 pb-2">
<Panel title="Supply events by type" icon={Boxes}><BarsH data={data?.supply.by_type || []} colorByIndex /></Panel>
<Panel title="Supply events by region"><Donut data={data?.supply.by_region || []} /></Panel>
<Panel title="Telemetry — avg value by metric" icon={Activity}><BarsH data={data?.telemetry.by_metric || []} valueKind="num" /></Panel>
</div>
</div>
)
}