feat: Data Explorer tab + manual data entry with live CDC

Add a dedicated business-data view (separate from infra/search):
- New "Data Explorer" tab: KPIs (customers, employees, products, source
  row totals, sample revenue) + charts driven by Elasticsearch aggregations
  (revenue over time, by region/channel/status, top customers & products,
  HR by department/role/event, supply chain by type, telemetry averages),
  with region + free-text filters.
- Backend /api/search/business endpoint: battery of ES aggregations with
  numeric/date index template (so amount sums and date histograms work),
  source totals via cheap planner estimates; biased the orders sample to
  rows carrying customer names so people are searchable/visible.

Manual data entry on source systems:
- "Insert row" action in the Data Hub browser opens a column-aware form;
  POST /api/sql/row/insert writes to Postgres/MySQL/Mongo/Cassandra/Neo4j.
- Inserts into CDC sources (PG/MySQL/Mongo) are captured by Debezium and
  streamed to Kafka in real time; UI flags this and pulses the data flow.
This commit is contained in:
mo
2026-06-28 17:12:29 +00:00
parent 8a8d779328
commit 8d28695868
7 changed files with 799 additions and 7 deletions
@@ -0,0 +1,323 @@
import { useCallback, useEffect, useMemo, useState } from 'react'
import {
Users,
UserCog,
Package,
ShoppingCart,
DollarSign,
Activity,
Boxes,
RefreshCw,
Search,
Loader2,
TrendingUp,
ExternalLink,
} from 'lucide-react'
import { cn } from '../../lib/utils'
type Bucket = { key: string; count: number; value?: number }
type Business = {
ok: boolean
generated_at: string
filters: { q: string; region: string }
kpis: {
customers: number
employees: number
products: number
orders_indexed: number
revenue_indexed: number
avg_order: number
telemetry_indexed: number
supply_indexed: number
source_totals: { orders?: number; hr_events?: number; supply_events?: number }
}
orders: {
by_region: Bucket[]
by_channel: Bucket[]
by_status: Bucket[]
top_customers: Bucket[]
top_products: Bucket[]
over_time: Bucket[]
}
hr: { by_department: Bucket[]; by_role: Bucket[]; by_event: Bucket[] }
supply: { by_type: Bucket[]; by_region: Bucket[] }
telemetry: { by_metric: Bucket[] }
regions: string[]
}
const REGION_COLORS = ['#34d399', '#60a5fa', '#f472b6', '#fbbf24', '#a78bfa', '#22d3ee', '#fb7185']
function fmtNum(n?: number | null) {
if (n == null) return '—'
if (Math.abs(n) >= 1_000_000_000) return `${(n / 1_000_000_000).toFixed(2)}B`
if (Math.abs(n) >= 1_000_000) return `${(n / 1_000_000).toFixed(1)}M`
if (Math.abs(n) >= 1_000) return `${(n / 1_000).toFixed(1)}K`
return String(n)
}
function fmtMoney(n?: number | null) {
if (n == null) return '—'
return `${fmtNum(n)}`
}
function monthLabel(key: string) {
const m = /^(\d{4})-(\d{2})/.exec(key)
if (!m) return key
const months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']
return `${months[Number(m[2]) - 1]} '${m[1].slice(2)}`
}
function Kpi({ icon: Icon, label, value, sub, accent }: { icon: typeof Users; label: string; value: string; sub?: string; accent: string }) {
return (
<div className="panel flex items-center gap-3 px-3 py-2.5">
<div className="flex h-9 w-9 shrink-0 items-center justify-center rounded-lg" style={{ backgroundColor: `${accent}1f`, color: accent }}>
<Icon className="h-4 w-4" />
</div>
<div className="min-w-0">
<p className="text-[9px] font-semibold uppercase tracking-wider text-foreground-muted">{label}</p>
<p className="truncate text-lg font-bold leading-tight text-foreground">{value}</p>
{sub && <p className="truncate text-[9px] text-foreground-faint">{sub}</p>}
</div>
</div>
)
}
function Panel({ title, subtitle, children, icon: Icon }: { title: string; subtitle?: string; children: React.ReactNode; icon?: typeof Users }) {
return (
<div className="panel flex min-h-0 flex-col p-3">
<div className="mb-2 flex items-center gap-1.5">
{Icon && <Icon className="h-3.5 w-3.5 text-docker" />}
<h3 className="text-[11px] font-semibold text-foreground">{title}</h3>
{subtitle && <span className="ml-auto text-[9px] text-foreground-faint">{subtitle}</span>}
</div>
{children}
</div>
)
}
function BarsH({ data, valueKind, colorByIndex }: { data: Bucket[]; valueKind?: 'money' | 'num'; colorByIndex?: boolean }) {
const useVal = valueKind != null
const max = Math.max(1, ...data.map((d) => (useVal && d.value != null ? d.value : d.count)))
if (!data.length) return <p className="py-6 text-center text-[10px] text-foreground-faint">No data</p>
return (
<div className="space-y-1.5">
{data.map((d, i) => {
const metric = useVal && d.value != null ? d.value : d.count
const pct = Math.max(2, (metric / max) * 100)
const color = colorByIndex ? REGION_COLORS[i % REGION_COLORS.length] : '#38bdf8'
const label = useVal && d.value != null ? (valueKind === 'money' ? fmtMoney(d.value) : fmtNum(d.value)) : fmtNum(d.count)
return (
<div key={d.key ?? i} className="flex items-center gap-2 text-[10px]">
<span className="w-28 shrink-0 truncate text-foreground-muted" title={d.key}>{d.key ?? '—'}</span>
<div className="relative h-3.5 flex-1 overflow-hidden rounded bg-surface-overlay">
<div className="h-full rounded" style={{ width: `${pct}%`, backgroundColor: color }} />
</div>
<span className="w-20 shrink-0 text-right font-mono text-foreground">{label}</span>
</div>
)
})}
</div>
)
}
function AreaTrend({ data }: { data: Bucket[] }) {
const pts = data.filter((d) => d.value != null)
if (pts.length < 2) return <p className="py-6 text-center text-[10px] text-foreground-faint">Not enough data</p>
const w = 560
const h = 120
const max = Math.max(...pts.map((p) => p.value || 0))
const min = Math.min(...pts.map((p) => p.value || 0))
const range = max - min || 1
const step = w / (pts.length - 1)
const coords = pts.map((p, i) => [i * step, h - ((((p.value || 0) - min) / range) * (h - 16) + 8)])
const line = coords.map((c, i) => `${i === 0 ? 'M' : 'L'}${c[0].toFixed(1)},${c[1].toFixed(1)}`).join(' ')
const area = `${line} L${w},${h} L0,${h} Z`
return (
<div>
<svg viewBox={`0 0 ${w} ${h}`} className="w-full" preserveAspectRatio="none" style={{ height: 120 }}>
<defs>
<linearGradient id="rev-grad" x1="0" y1="0" x2="0" y2="1">
<stop offset="0%" stopColor="#34d399" stopOpacity="0.45" />
<stop offset="100%" stopColor="#34d399" stopOpacity="0" />
</linearGradient>
</defs>
<path d={area} fill="url(#rev-grad)" />
<path d={line} fill="none" stroke="#34d399" strokeWidth="2" />
{coords.map((c, i) => <circle key={i} cx={c[0]} cy={c[1]} r="2" fill="#34d399" />)}
</svg>
<div className="mt-1 flex justify-between text-[8px] text-foreground-faint">
<span>{monthLabel(pts[0].key)}</span>
<span>{monthLabel(pts[Math.floor(pts.length / 2)].key)}</span>
<span>{monthLabel(pts[pts.length - 1].key)}</span>
</div>
</div>
)
}
function Donut({ data }: { data: Bucket[] }) {
const total = data.reduce((s, d) => s + d.count, 0) || 1
let acc = 0
const r = 42
const c = 2 * Math.PI * r
if (!data.length) return <p className="py-6 text-center text-[10px] text-foreground-faint">No data</p>
return (
<div className="flex items-center gap-4">
<svg viewBox="0 0 100 100" className="h-28 w-28 shrink-0 -rotate-90">
{data.map((d, i) => {
const frac = d.count / total
const dash = frac * c
const seg = (
<circle
key={d.key ?? i}
cx="50" cy="50" r={r} fill="none"
stroke={REGION_COLORS[i % REGION_COLORS.length]} strokeWidth="14"
strokeDasharray={`${dash} ${c - dash}`} strokeDashoffset={-acc}
/>
)
acc += dash
return seg
})}
</svg>
<div className="min-w-0 flex-1 space-y-1">
{data.slice(0, 7).map((d, i) => (
<div key={d.key ?? i} className="flex items-center gap-1.5 text-[10px]">
<span className="h-2 w-2 shrink-0 rounded-full" style={{ backgroundColor: REGION_COLORS[i % REGION_COLORS.length] }} />
<span className="flex-1 truncate text-foreground-muted">{d.key ?? '—'}</span>
<span className="font-mono text-foreground">{((d.count / total) * 100).toFixed(0)}%</span>
</div>
))}
</div>
</div>
)
}
export function DataExplorerView() {
const [data, setData] = useState<Business | null>(null)
const [loading, setLoading] = useState(true)
const [region, setRegion] = useState('')
const [qInput, setQInput] = useState('')
const [q, setQ] = useState('')
const load = useCallback(async () => {
setLoading(true)
try {
const params = new URLSearchParams()
if (q) params.set('q', q)
if (region) params.set('region', region)
const r = await fetch(`/api/search/business?${params.toString()}`)
if (r.ok) setData(await r.json())
} catch { /* */ } finally {
setLoading(false)
}
}, [q, region])
useEffect(() => { load() }, [load])
const k = data?.kpis
const regions = useMemo(() => data?.regions || [], [data])
return (
<div className="scrollbar-thin flex h-full min-h-0 flex-col gap-2 overflow-y-auto p-3">
{/* header */}
<header className="panel flex shrink-0 flex-wrap items-center justify-between gap-3 px-4 py-3">
<div>
<h1 className="flex items-center gap-2 text-base font-semibold text-foreground">
<TrendingUp className="h-5 w-5 text-docker" />
Data Explorer
</h1>
<p className="text-[11px] text-foreground-muted">
Your actual business data customers, orders, employees, supply chain &amp; telemetry across all sources
</p>
</div>
<div className="flex items-center gap-2">
<form
onSubmit={(e) => { e.preventDefault(); setQ(qInput.trim()) }}
className="flex items-center gap-1 rounded-md border border-border bg-surface px-2 py-1"
>
<Search className="h-3.5 w-3.5 text-foreground-muted" />
<input
value={qInput}
onChange={(e) => setQInput(e.target.value)}
placeholder="Filter by name, id, region…"
className="w-48 bg-transparent text-[11px] text-foreground outline-none placeholder:text-foreground-faint"
/>
</form>
<button type="button" onClick={load} className="inline-flex items-center gap-1.5 rounded-md border border-border px-3 py-1.5 text-[11px] text-foreground-muted hover:bg-surface-overlay">
{loading ? <Loader2 className="h-3.5 w-3.5 animate-spin" /> : <RefreshCw className="h-3.5 w-3.5" />} Refresh
</button>
</div>
</header>
{/* region filters */}
<div className="flex shrink-0 flex-wrap items-center gap-1.5 px-1 text-[10px]">
<span className="mr-1 font-semibold uppercase tracking-wider text-foreground-faint">Region</span>
{['', ...regions].map((rg) => (
<button
key={rg || 'all'}
type="button"
onClick={() => setRegion(rg)}
className={cn('rounded-full border px-2.5 py-1 font-medium transition-colors', region === rg ? 'border-docker bg-docker/15 text-docker' : 'border-border text-foreground-muted hover:bg-surface-overlay')}
>
{rg || 'All regions'}
</button>
))}
{(q || region) && (
<button type="button" onClick={() => { setQ(''); setQInput(''); setRegion('') }} className="ml-1 rounded-full border border-amber-500/40 px-2.5 py-1 font-medium text-amber-400 hover:bg-amber-500/10">
Clear filters
</button>
)}
{data && <span className="ml-auto text-[9px] text-foreground-faint">updated {new Date(data.generated_at).toLocaleTimeString()}</span>}
</div>
{/* KPIs */}
<div className="grid shrink-0 grid-cols-2 gap-2 md:grid-cols-4 xl:grid-cols-7">
<Kpi icon={Users} label="Customers" value={fmtNum(k?.customers)} sub="distinct, searchable" accent="#34d399" />
<Kpi icon={UserCog} label="Employees" value={fmtNum(k?.employees)} sub="distinct, searchable" accent="#60a5fa" />
<Kpi icon={Package} label="Products" value={fmtNum(k?.products)} accent="#f472b6" />
<Kpi icon={ShoppingCart} label="Orders (source)" value={fmtNum(k?.source_totals?.orders)} sub={`${fmtNum(k?.orders_indexed)} indexed`} accent="#fbbf24" />
<Kpi icon={DollarSign} label="Revenue (sample)" value={fmtMoney(k?.revenue_indexed)} sub={`avg ${fmtMoney(k?.avg_order)}`} accent="#22d3ee" />
<Kpi icon={Boxes} label="Supply events" value={fmtNum(k?.source_totals?.supply_events)} accent="#a78bfa" />
<Kpi icon={Activity} label="HR events" value={fmtNum(k?.source_totals?.hr_events)} accent="#fb7185" />
</div>
{/* Sales */}
<h2 className="mt-1 shrink-0 px-1 text-[10px] font-semibold uppercase tracking-widest text-foreground-muted">Sales &amp; Customers PostgreSQL</h2>
<div className="grid shrink-0 gap-2 lg:grid-cols-2">
<Panel title="Revenue over time" subtitle="monthly, indexed sample" icon={TrendingUp}>
<AreaTrend data={data?.orders.over_time || []} />
</Panel>
<Panel title="Revenue by region" icon={DollarSign}>
<BarsH data={data?.orders.by_region || []} valueKind="money" colorByIndex />
</Panel>
<Panel title="Top customers by spend" icon={Users}>
<BarsH data={data?.orders.top_customers || []} valueKind="money" />
</Panel>
<div className="grid gap-2 sm:grid-cols-2">
<Panel title="Orders by status">
<Donut data={data?.orders.by_status || []} />
</Panel>
<Panel title="Orders by channel">
<Donut data={data?.orders.by_channel || []} />
</Panel>
</div>
<Panel title="Top products by revenue" icon={Package}>
<BarsH data={data?.orders.top_products || []} valueKind="money" />
</Panel>
</div>
{/* HR */}
<h2 className="mt-1 shrink-0 px-1 text-[10px] font-semibold uppercase tracking-widest text-foreground-muted">Workforce MySQL HR</h2>
<div className="grid shrink-0 gap-2 lg:grid-cols-3">
<Panel title="Employees by department" icon={UserCog}><BarsH data={data?.hr.by_department || []} colorByIndex /></Panel>
<Panel title="By role"><BarsH data={data?.hr.by_role || []} /></Panel>
<Panel title="By event type"><Donut data={data?.hr.by_event || []} /></Panel>
</div>
{/* Supply + telemetry */}
<h2 className="mt-1 shrink-0 px-1 text-[10px] font-semibold uppercase tracking-widest text-foreground-muted">Supply Chain (MongoDB) &amp; Telemetry (Cassandra)</h2>
<div className="grid shrink-0 gap-2 lg:grid-cols-3 pb-2">
<Panel title="Supply events by type" icon={Boxes}><BarsH data={data?.supply.by_type || []} colorByIndex /></Panel>
<Panel title="Supply events by region"><Donut data={data?.supply.by_region || []} /></Panel>
<Panel title="Telemetry — avg value by metric" icon={Activity}><BarsH data={data?.telemetry.by_metric || []} valueKind="num" /></Panel>
</div>
</div>
)
}
+123 -1
View File
@@ -1,16 +1,20 @@
import { useCallback, useEffect, useRef, useState } from 'react'
import {
Activity,
AlertCircle,
CheckCircle2,
ChevronRight,
Database,
FolderTree,
Loader2,
Network,
Play,
Plus,
RefreshCw,
Server,
Table2,
TerminalSquare,
X,
} from 'lucide-react'
import { Badge } from '../ui/Badge'
import { DbBrandIcon, dbBrandColor } from '../ui/DbBrandIcon'
@@ -94,8 +98,47 @@ export function DataSourcesView({ focusEngine, onPulse }: Props) {
const [sampleLoading, setSampleLoading] = useState(false)
const [page, setPage] = useState(1)
const [pageSize, setPageSize] = useState(100)
const [showInsert, setShowInsert] = useState(false)
const [insertVals, setInsertVals] = useState<Record<string, string>>({})
const [inserting, setInserting] = useState(false)
const [insertMsg, setInsertMsg] = useState<{ ok: boolean; text: string } | null>(null)
const meta = getSourceMeta(active)
const cdcEngine = active === 'postgres' || active === 'mysql' || active === 'mongodb'
const submitInsert = useCallback(async () => {
if (!selectedObject) return
setInserting(true)
setInsertMsg(null)
try {
const values: Record<string, unknown> = {}
Object.entries(insertVals).forEach(([key, val]) => { if (val !== '') values[key] = val })
if (Object.keys(values).length === 0) {
setInsertMsg({ ok: false, text: 'Enter at least one value' })
setInserting(false)
return
}
const r = await fetch('/api/sql/row/insert', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ engine: active, object: selectedObject.fqn, values }),
})
const j = await r.json()
if (j.ok) {
setInsertMsg({ ok: true, text: cdcEngine ? 'Inserted — Debezium CDC is streaming this change to Kafka → see Live Changes' : 'Inserted successfully' })
onPulse?.()
loadSample(active, selectedObject, page, pageSize)
setTimeout(() => { setShowInsert(false); setInsertMsg(null) }, 1800)
} else {
setInsertMsg({ ok: false, text: j.error || 'Insert failed' })
}
} catch {
setInsertMsg({ ok: false, text: 'Insert request failed' })
} finally {
setInserting(false)
}
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [selectedObject, insertVals, active, cdcEngine, page, pageSize])
useEffect(() => {
if (focusEngine) setActive(focusEngine)
@@ -308,12 +351,22 @@ export function DataSourcesView({ focusEngine, onPulse }: Props) {
</div>
<div className="flex h-full min-h-0 min-w-0 flex-1 flex-col overflow-hidden">
<div className="flex shrink-0 items-center border-b border-border/60 px-3 py-2">
<div className="flex shrink-0 items-center justify-between gap-2 border-b border-border/60 px-3 py-2">
<span className="text-[10px] font-semibold text-foreground">
{selectedObject ? (
<>Data: <span className="font-mono text-docker">{selectedObject.fqn}</span></>
) : 'Select an object'}
</span>
{selectedObject && (
<button
type="button"
onClick={() => { setInsertVals({}); setInsertMsg(null); setShowInsert(true) }}
className="inline-flex items-center gap-1 rounded-md border border-emerald-500/40 bg-emerald-500/10 px-2.5 py-1 text-[10px] font-medium text-emerald-400 hover:bg-emerald-500/20"
>
<Plus className="h-3 w-3" /> Insert row
{cdcEngine && <span className="ml-1 rounded bg-emerald-500/20 px-1 text-[8px]"> CDC</span>}
</button>
)}
</div>
<DataBrowserGrid
engine={active}
@@ -347,6 +400,75 @@ export function DataSourcesView({ focusEngine, onPulse }: Props) {
{subTab === 'shell' && <DbShell initialCommand={meta.shellCommand} />}
</div>
</div>
{showInsert && selectedObject && (
<div className="fixed inset-0 z-50 flex items-center justify-center bg-black/60 p-4" onClick={() => setShowInsert(false)}>
<div className="flex max-h-[85vh] w-full max-w-lg flex-col overflow-hidden rounded-xl border border-border bg-surface-raised shadow-2xl" onClick={(e) => e.stopPropagation()}>
<div className="flex shrink-0 items-center justify-between border-b border-border px-4 py-3" style={{ backgroundColor: `${dbBrandColor(active)}12` }}>
<div>
<h3 className="flex items-center gap-2 text-sm font-semibold text-foreground">
<DbBrandIcon engine={active} size={16} /> Insert row
</h3>
<p className="font-mono text-[10px] text-foreground-muted">{selectedObject.fqn}</p>
</div>
<button type="button" onClick={() => setShowInsert(false)} className="rounded p-1 text-foreground-muted hover:bg-surface-overlay">
<X className="h-4 w-4" />
</button>
</div>
{cdcEngine && (
<div className="shrink-0 border-b border-border bg-emerald-500/10 px-4 py-1.5 text-[10px] text-emerald-300">
<Activity className="mr-1 inline h-3 w-3" /> This source has CDC the new row is captured by Debezium and streamed to Kafka in real time.
</div>
)}
<div className="scrollbar-thin min-h-0 flex-1 space-y-2 overflow-y-auto p-4">
{(sample?.columns || []).map((col) => {
const isPk = (sample?.primary_keys || []).includes(col)
return (
<div key={col} className="flex items-center gap-2">
<label className="flex w-36 shrink-0 items-center gap-1 truncate text-[10px] font-medium text-foreground-muted" title={col}>
{col}
{isPk && <span className="rounded bg-amber-500/20 px-1 text-[8px] text-amber-400">PK</span>}
</label>
<input
value={insertVals[col] ?? ''}
onChange={(e) => setInsertVals((v) => ({ ...v, [col]: e.target.value }))}
placeholder="(leave blank to skip)"
className="flex-1 rounded-md border border-border bg-surface px-2 py-1.5 text-[11px] text-foreground outline-none focus:border-docker/50 placeholder:text-foreground-faint"
/>
</div>
)
})}
{!sample?.columns?.length && (
<p className="py-6 text-center text-[10px] text-foreground-faint">No column metadata open a table in the browser first.</p>
)}
</div>
{insertMsg && (
<div className={cn('flex shrink-0 items-center gap-1.5 px-4 py-2 text-[10px]', insertMsg.ok ? 'bg-emerald-500/10 text-emerald-300' : 'bg-red-500/10 text-red-300')}>
{insertMsg.ok ? <CheckCircle2 className="h-3.5 w-3.5" /> : <AlertCircle className="h-3.5 w-3.5" />}
{insertMsg.text}
</div>
)}
<div className="flex shrink-0 items-center justify-end gap-2 border-t border-border px-4 py-3">
<button type="button" onClick={() => setShowInsert(false)} className="rounded-md border border-border px-3 py-1.5 text-[11px] text-foreground-muted hover:bg-surface-overlay">
Cancel
</button>
<button
type="button"
onClick={submitInsert}
disabled={inserting}
className="inline-flex items-center gap-1.5 rounded-md bg-emerald-500/90 px-3 py-1.5 text-[11px] font-medium text-white hover:bg-emerald-500 disabled:opacity-60"
>
{inserting ? <Loader2 className="h-3.5 w-3.5 animate-spin" /> : <Plus className="h-3.5 w-3.5" />}
Insert {cdcEngine ? '→ stream via CDC' : ''}
</button>
</div>
</div>
</div>
)}
</div>
)
}