feat: continuous live generator + vLLM/RAG lane in Data Flow

Live dashboard now feels truly real-time: - Background generator streams randomly-sized bursts of real rows into PostgreSQL, MySQL, MongoDB & Cassandra every ~4s (CDC picks them up). Throughput rises and falls; counters move in lock-step (base snapshot + generated). Runs only while the Live tab is polling (heartbeat-gated) so source tables do not grow unbounded; on/off toggle exposed in the UI. - New /api/federated/live/generator toggle; /live returns per-tick activity (last burst sizes, orders by region/status, event feed). - LiveDashboard: live-activity panel, orders-per-tick sparkline, event stream feed, burst-by-region/status charts, generator status + control. Data Flow graph now explains how data reaches the assistant: - Added ChromaDB -> RAG (LangChain) -> vLLM Gateway -> Knowledge Chat lane, with Trino / OpenMetadata / curated-masked feeding LLM context. Live model & embed metrics pulled from the RAG /config. New node/edge kinds + legend.
2026-06-28 21:37:41 +00:00
parent 8c72d1dc63
commit 9059006cc2
4 changed files with 429 additions and 26 deletions
@@ -18,6 +18,10 @@ const NODE_KIND: Record<string, { ring: string; chip: string; dot: string }> = {
  compute: { ring: 'border-violet-400/60', chip: 'bg-violet-500/15 text-violet-300 border-violet-400/40', dot: '#a78bfa' },
  engine: { ring: 'border-violet-400/60', chip: 'bg-violet-500/15 text-violet-300 border-violet-400/40', dot: '#a78bfa' },
  governance: { ring: 'border-fuchsia-400/60', chip: 'bg-fuchsia-500/15 text-fuchsia-300 border-fuchsia-400/40', dot: '#d946ef' },
+  vector: { ring: 'border-teal-400/60', chip: 'bg-teal-500/15 text-teal-300 border-teal-400/40', dot: '#2dd4bf' },
+  rag: { ring: 'border-pink-400/60', chip: 'bg-pink-500/15 text-pink-300 border-pink-400/40', dot: '#f472b6' },
+  llm: { ring: 'border-rose-400/70', chip: 'bg-rose-500/15 text-rose-200 border-rose-400/50', dot: '#fb7185' },
+  chat: { ring: 'border-indigo-400/60', chip: 'bg-indigo-500/15 text-indigo-300 border-indigo-400/40', dot: '#818cf8' },
 }

 const EDGE_COLOR: Record<string, string> = {
@@ -28,6 +32,10 @@ const EDGE_COLOR: Record<string, string> = {
  mask: '#fb7185',
  query: '#818cf8',
  catalog: '#d946ef',
+  context: '#2dd4bf',
+  retrieve: '#f472b6',
+  prompt: '#fb7185',
+  answer: '#818cf8',
 }

 const EDGE_LEGEND: { kind: string; label: string }[] = [
@@ -38,6 +46,10 @@ const EDGE_LEGEND: { kind: string; label: string }[] = [
  { kind: 'mask', label: 'PII masking' },
  { kind: 'query', label: 'Query' },
  { kind: 'catalog', label: 'Catalog (OpenMetadata)' },
+  { kind: 'context', label: 'LLM context' },
+  { kind: 'retrieve', label: 'Vector retrieve' },
+  { kind: 'prompt', label: 'Prompt' },
+  { kind: 'answer', label: 'Answer → chat' },
 ]

 type Anchor = { x: number; y: number; w: number; h: number }
@@ -1,14 +1,26 @@
 import { useCallback, useEffect, useRef, useState } from 'react'
-import { Activity, Pause, Play, ShoppingCart, Users, Boxes, Cpu, DollarSign, Database, Gauge } from 'lucide-react'
+import { Activity, Pause, Play, ShoppingCart, Users, Boxes, Cpu, DollarSign, Database, Gauge, Zap, Sparkles, Radio } from 'lucide-react'
 import { cn } from '../../lib/utils'

 type Bucket = { key: string; count: number; value?: number }
-type Source = { key: string; label: string; engine: string; catalog: string; rows: number; color: string }
+type Source = { key: string; label: string; engine: string; catalog: string; rows: number; added?: number; color: string }
 type RegionRow = { region: string; orders: number; revenue: number; hr_events: number; supply_events: number }
+type Gen = {
+  enabled: boolean
+  running: boolean
+  interval: number
+  counts: { orders: number; hr_events: number; supply_events: number; telemetry: number }
+  last_batch: { orders: number; hr_events: number; supply_events: number; telemetry: number }
+  tick_value: number
+  by_region: Bucket[]
+  by_status: Bucket[]
+  feed: { ts: string; text: string }[]
+}
 type Live = {
  ok: boolean
  ts: string
  sources: Source[]
+  generator?: Gen
  totals: { records: number; revenue_est: number; avg_order: number }
  business: {
    orders_by_region: Bucket[]
@@ -174,9 +186,11 @@ export function LiveDashboard() {
  const [paused, setPaused] = useState(false)
  const [err, setErr] = useState(false)
  const [totalHist, setTotalHist] = useState<number[]>([])
+  const [ordHist, setOrdHist] = useState<number[]>([])
  const [rateBySrc, setRateBySrc] = useState<Record<string, number>>({})
  const [added, setAdded] = useState(0)
-  const prev = useRef<{ ts: number; rows: Record<string, number>; total: number } | null>(null)
+  const [genBusy, setGenBusy] = useState(false)
+  const prev = useRef<{ ts: number; rows: Record<string, number>; total: number; genOrders: number } | null>(null)
  const startTotal = useRef<number | null>(null)

  const poll = useCallback(async () => {
@@ -187,10 +201,13 @@ export function LiveDashboard() {
      setErr(false)
      const now = Date.parse(d.ts) || Date.now()
      const total = d.totals.records
+      const genOrders = d.generator?.counts.orders ?? 0
      if (prev.current) {
        const dt = Math.max(0.5, (now - prev.current.ts) / 1000)
        const totRate = Math.max(0, (total - prev.current.total) / dt)
        setTotalHist((h) => [...h, totRate].slice(-90))
+        // orders added between polls — fluctuates up and down with each batch
+        setOrdHist((h) => [...h, Math.max(0, genOrders - prev.current!.genOrders)].slice(-60))
        const rmap: Record<string, number> = {}
        d.sources.forEach((s) => {
          const p = prev.current!.rows[s.key] ?? s.rows
@@ -200,13 +217,23 @@ export function LiveDashboard() {
      }
      if (startTotal.current == null) startTotal.current = total
      setAdded(Math.max(0, total - (startTotal.current || total)))
-      prev.current = { ts: now, rows: Object.fromEntries(d.sources.map((s) => [s.key, s.rows])), total }
+      prev.current = { ts: now, rows: Object.fromEntries(d.sources.map((s) => [s.key, s.rows])), total, genOrders }
      setData(d)
    } catch {
      setErr(true)
    }
  }, [])

+  const toggleGen = useCallback(async (enabled: boolean) => {
+    setGenBusy(true)
+    try {
+      await fetch('/api/federated/live/generator', {
+        method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ enabled }),
+      })
+      await poll()
+    } catch { /* ignore */ } finally { setGenBusy(false) }
+  }, [poll])
+
  useEffect(() => {
    poll()
    if (paused) return
@@ -215,7 +242,10 @@ export function LiveDashboard() {
  }, [poll, paused])

  const b = data?.business
+  const gen = data?.generator
+  const lb = gen?.last_batch
  const totalRate = totalHist.length ? totalHist[totalHist.length - 1] : 0
+  const ordPeak = Math.max(1, ...ordHist)
  const matrix = b?.region_matrix || []
  const maxRev = Math.max(1, ...matrix.map((m) => Number(m.revenue) || 0))

@@ -236,7 +266,20 @@ export function LiveDashboard() {
        <span className="text-[10px] text-foreground-muted">·</span>
        <span className="text-[10px] text-foreground-muted">+{fmtNum(added)} since opened</span>
        {err && <span className="text-[10px] text-amber-400">reconnecting…</span>}
+        {gen && (
+          <span className={cn('inline-flex items-center gap-1 rounded-full border px-2 py-0.5 text-[10px] font-medium',
+            gen.running ? 'border-amber-500/40 bg-amber-500/10 text-amber-300' : 'border-border text-foreground-faint')}>
+            <Zap className={cn('h-3 w-3', gen.running && 'animate-pulse')} /> stream {gen.running ? `every ${gen.interval}s` : 'idle'}
+          </span>
+        )}
        <span className="ml-auto text-[9px] text-foreground-faint">{data ? `updated ${new Date(data.ts).toLocaleTimeString()}` : 'connecting…'}</span>
+        {gen && (
+          <button type="button" disabled={genBusy} onClick={() => toggleGen(!gen.enabled)}
+            className={cn('inline-flex items-center gap-1 rounded-md border px-2 py-1 text-[10px] transition-colors disabled:opacity-60',
+              gen.enabled ? 'border-amber-500/40 bg-amber-500/10 text-amber-300 hover:bg-amber-500/20' : 'border-border text-foreground-muted hover:bg-surface-overlay')}>
+            <Zap className="h-3 w-3" /> {gen.enabled ? 'Generator on' : 'Generator off'}
+          </button>
+        )}
        <button type="button" onClick={() => setPaused((p) => !p)} className="inline-flex items-center gap-1 rounded-md border border-border px-2 py-1 text-[10px] text-foreground-muted hover:bg-surface-overlay">
          {paused ? <Play className="h-3 w-3" /> : <Pause className="h-3 w-3" />} {paused ? 'Resume' : 'Pause'}
        </button>
@@ -251,6 +294,52 @@ export function LiveDashboard() {
        ))}
      </div>

+      {/* live activity — the per-tick pulse: orders & events streaming into the sources right now */}
+      {gen && (
+        <div className="grid shrink-0 gap-2 lg:grid-cols-3">
+          <Panel title="Live activity — last burst" subtitle={gen.running ? `every ${gen.interval}s` : 'paused'} icon={Sparkles} className="lg:col-span-2">
+            <div className="mb-2 grid grid-cols-2 gap-2 sm:grid-cols-4">
+              {([
+                { k: 'orders', label: 'orders', color: '#fbbf24', icon: ShoppingCart },
+                { k: 'telemetry', label: 'telemetry', color: '#22d3ee', icon: Cpu },
+                { k: 'hr_events', label: 'HR', color: '#60a5fa', icon: Users },
+                { k: 'supply_events', label: 'supply', color: '#a78bfa', icon: Boxes },
+              ] as const).map(({ k, label, color, icon: Icon }) => {
+                const v = lb ? (lb as Record<string, number>)[k] : 0
+                return (
+                  <div key={k} className="rounded-lg border border-border bg-surface-overlay/40 px-2 py-1.5">
+                    <span className="flex items-center gap-1 text-[9px] uppercase tracking-wider text-foreground-muted"><Icon className="h-3 w-3" style={{ color }} />{label}</span>
+                    <span className="font-mono text-base font-bold" style={{ color }}>+{fmtNum(v)}</span>
+                  </div>
+                )
+              })}
+            </div>
+            <Spark data={ordHist} color="#fbbf24" height={56} />
+            <div className="mt-1 flex justify-between text-[9px] text-foreground-faint">
+              <span>orders per {(POLL_MS / 1000).toFixed(1)}s tick — watch it rise &amp; fall</span>
+              <span>peak {fmtNum(ordPeak)} · €{fmtNum(gen.tick_value)} last burst</span>
+            </div>
+          </Panel>
+          <Panel title="Event stream" subtitle="newest first" icon={Radio}>
+            <div className="max-h-40 space-y-1 overflow-y-auto">
+              {(gen.feed || []).length ? gen.feed.map((f, i) => (
+                <div key={`${f.ts}-${i}`} className={cn('rounded border-l-2 px-2 py-1 text-[9px] font-mono', i === 0 ? 'border-amber-400 bg-amber-500/10 text-amber-200' : 'border-border bg-surface-overlay/30 text-foreground-muted')}>
+                  <span className="text-foreground-faint">{new Date(f.ts).toLocaleTimeString()}</span> · {f.text}
+                </div>
+              )) : <p className="py-4 text-center text-[10px] text-foreground-faint">waiting for the next burst…</p>}
+            </div>
+          </Panel>
+        </div>
+      )}
+
+      {/* orders by region this burst (changes every tick) */}
+      {gen && (gen.by_region?.length || gen.by_status?.length) ? (
+        <div className="grid shrink-0 gap-2 lg:grid-cols-2">
+          <Panel title="Orders this burst — by region" subtitle="live distribution" icon={Database}><Bars data={gen.by_region} colorByIndex /></Panel>
+          <Panel title="Orders this burst — by status" subtitle="live distribution" icon={ShoppingCart}><Donut data={gen.by_status} /></Panel>
+        </div>
+      ) : null}
+
      {/* throughput + per-source rates */}
      <div className="grid shrink-0 gap-2 lg:grid-cols-3">
        <Panel title="Ingestion throughput" subtitle="rows/sec · live" icon={Activity} className="lg:col-span-2">
@@ -326,7 +415,7 @@ export function LiveDashboard() {
        <Panel title="Supply events by type" icon={Boxes}><Bars data={b?.supply_by_type} colorByIndex /></Panel>
      </div>
      <p className="shrink-0 px-1 pb-2 text-[9px] text-foreground-faint">
-        Counters &amp; throughput are live source estimates (Trino over PostgreSQL, MySQL, MongoDB &amp; Cassandra); breakdown charts aggregate the materialized Hadoop lake. Polling every {POLL_MS / 1000}s.
+        While this tab is open a live generator streams randomly-sized bursts of real rows into PostgreSQL, MySQL, MongoDB &amp; Cassandra (picked up by CDC) — counters &amp; the activity feed move every {gen?.interval ?? 4}s; the region scorecard &amp; breakdown charts aggregate the materialized Hadoop lake. Polling every {POLL_MS / 1000}s.
      </p>
    </div>
  )