feat: pulse generator edges on data gen + New & Changed Data dashboard

Generated data is now visible flowing into the source systems on the Data
Flow graph: a manual "Generate data" burst (and the continuous live loop)
stamps a last_tick, and a new generator_active() helper lights up the
generator -> postgres/mysql/mongodb/cassandra edges for ~12s. Neo4j stays
dark since the live generator does not write to it.

Redesigned the Changes tab into a "New & Changed Data" dashboard driven by
the live CDC stream: animated KPI cards (new/updated/deleted + throughput),
a change-volume area chart, an operation-mix donut, per-system and
top-table breakdown bars, plus the existing filterable change feed.

api/dataflow.py

@@ -173,6 +173,11 @@ async def _build() -> dict[str, Any]:
         cdc = cdc_snapshot(15)
     except Exception:
         cdc = {"connected": False, "consumed": 0, "by_source": {}, "window_total": 0}
+    try:
+        from trino_federated import generator_active
+        gen_active = generator_active()
+    except Exception:
+        gen_active = False
     try:
         from pii_catalog import get_pii
         pii = get_pii()
@@ -255,7 +260,12 @@ async def _build() -> dict[str, Any]:
             edge["last_rows"] = lr.get("rows")
             edge["last_duration_s"] = lr.get("duration_s")
             edge["active"] = lr.get("state") == "running"
-        if e["kind"] == "cdc":
+        if e["kind"] == "generate":
+            # The live generator (continuous loop or the 'Generate data' burst)
+            # writes into these four sources; pulse the edge while it is active.
+            if e["to"] in ("postgres", "mysql", "mongodb", "cassandra"):
+                edge["active"] = gen_active or bool(edge.get("active"))
+        elif e["kind"] == "cdc":
             edge["active"] = cdc.get("by_source", {}).get(e["from"], 0) > 0
         elif e.get("from") == "hdfs" and e.get("to") == "kafka":
             edge["active"] = bool(edge_live.get("hdfs→kafka"))

api/trino_federated.py

@@ -534,6 +534,9 @@ def _generate_once(orders: int, hr: int, supply: int, tel: int) -> dict[str, Any
         c = _GEN["counts"]
         for k in out:
             c[k] += out[k]
+        _GEN["last_batch"] = {"orders": out["orders"], "hr_events": out["hr_events"],
+                              "supply_events": out["supply_events"], "telemetry": out["telemetry"]}
+        _GEN["last_tick"] = time.time()  # marks recent activity → Data Flow CDC edges pulse
         if out["orders"]:
             _GEN["by_region"] = by_r
             _GEN["by_status"] = by_s
@@ -609,6 +612,13 @@ def _gen_loop():
 threading.Thread(target=_gen_loop, daemon=True, name="live-generator").start()
 
 
+def generator_active(window_s: float = 12.0) -> bool:
+    """True when the live generator (continuous loop OR a manual 'Generate data'
+    burst) wrote rows very recently. The Data Flow graph uses this to pulse the
+    generator→source edges so generated data is visible flowing into the sources."""
+    return (time.time() - float(_GEN.get("last_tick") or 0.0)) < window_s
+
+
 @router.post("/live/generator")
 async def toggle_generator(body: dict = Body(default={})):
     if "enabled" in body:

ui/src/components/features/ChangesView.tsx

@@ -1,14 +1,14 @@
-import { useCallback, useEffect, useMemo, useState } from 'react'
-import { Activity, Radio, RefreshCw } from 'lucide-react'
+import { useCallback, useEffect, useMemo, useRef, useState } from 'react'
+import { Activity, Radio, RefreshCw, PlusCircle, Pencil, Trash2, Database, Layers, TrendingUp } from 'lucide-react'
 import { fetchChanges, fetchChangeStats } from '../../lib/api'
 import type { CdcChange, CdcStats } from '../../types'
 import { cn } from '../../lib/utils'
 
-const OP_STYLE: Record<string, { label: string; cls: string }> = {
-  insert: { label: 'INSERT', cls: 'bg-emerald-500/15 text-emerald-300 border-emerald-500/30' },
-  update: { label: 'UPDATE', cls: 'bg-amber-500/15 text-amber-300 border-amber-500/30' },
-  delete: { label: 'DELETE', cls: 'bg-rose-500/15 text-rose-300 border-rose-500/30' },
-  snapshot: { label: 'SNAPSHOT', cls: 'bg-sky-500/15 text-sky-300 border-sky-500/30' },
+const OP_STYLE: Record<string, { label: string; cls: string; color: string }> = {
+  insert: { label: 'INSERT', cls: 'bg-emerald-500/15 text-emerald-300 border-emerald-500/30', color: '#34d399' },
+  update: { label: 'UPDATE', cls: 'bg-amber-500/15 text-amber-300 border-amber-500/30', color: '#fbbf24' },
+  delete: { label: 'DELETE', cls: 'bg-rose-500/15 text-rose-300 border-rose-500/30', color: '#fb7185' },
+  snapshot: { label: 'SNAPSHOT', cls: 'bg-sky-500/15 text-sky-300 border-sky-500/30', color: '#38bdf8' },
 }
 
 const SOURCE_COLOR: Record<string, string> = {
@@ -23,7 +23,7 @@ const SOURCES = ['all', 'postgres', 'mysql', 'mongodb', 'cassandra', 'neo4j']
 const OPS = ['all', 'insert', 'update', 'delete']
 
 function opOf(o: string) {
-  return OP_STYLE[o] || { label: o.toUpperCase(), cls: 'bg-slate-500/15 text-slate-300 border-slate-500/30' }
+  return OP_STYLE[o] || { label: o.toUpperCase(), cls: 'bg-slate-500/15 text-slate-300 border-slate-500/30', color: '#94a3b8' }
 }
 
 function timeAgo(ts: string) {
@@ -34,6 +34,147 @@ function timeAgo(ts: string) {
   return `${Math.floor(d / 3600000)}h ago`
 }
 
+// Smoothly animates a number toward its target so counters tick up nicely.
+function useTween(target: number, ms = 700) {
+  const [val, setVal] = useState(target)
+  const from = useRef(target)
+  const start = useRef(0)
+  const raf = useRef(0)
+  useEffect(() => {
+    from.current = val
+    start.current = performance.now()
+    const step = (now: number) => {
+      const t = Math.min(1, (now - start.current) / ms)
+      const eased = 1 - Math.pow(1 - t, 3)
+      setVal(from.current + (target - from.current) * eased)
+      if (t < 1) raf.current = requestAnimationFrame(step)
+    }
+    raf.current = requestAnimationFrame(step)
+    return () => cancelAnimationFrame(raf.current)
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [target, ms])
+  return val
+}
+
+function KpiCard({ label, value, accent, icon: Icon, sub }: {
+  label: string; value: number; accent: string; icon: typeof PlusCircle; sub?: string
+}) {
+  const v = useTween(value)
+  return (
+    <div className="relative overflow-hidden rounded-lg border border-border/60 bg-surface-raised p-3">
+      <div className="absolute -right-3 -top-3 h-14 w-14 rounded-full opacity-[0.12] blur-xl" style={{ background: accent }} />
+      <div className="flex items-center gap-1.5 text-[9px] uppercase tracking-wide text-foreground-faint">
+        <Icon className="h-3 w-3" style={{ color: accent }} /> {label}
+      </div>
+      <div className="mt-1 text-2xl font-semibold tabular-nums text-foreground" style={{ textShadow: `0 0 18px ${accent}22` }}>
+        {Math.round(v).toLocaleString()}
+      </div>
+      {sub && <div className="text-[10px] text-foreground-muted">{sub}</div>}
+    </div>
+  )
+}
+
+// SVG donut for the operation mix.
+function Donut({ segments, total }: { segments: { label: string; value: number; color: string }[]; total: number }) {
+  const R = 42
+  const C = 2 * Math.PI * R
+  let offset = 0
+  return (
+    <div className="flex items-center gap-4">
+      <svg viewBox="0 0 110 110" className="h-28 w-28 shrink-0 -rotate-90">
+        <circle cx="55" cy="55" r={R} fill="none" stroke="rgba(148,163,184,0.12)" strokeWidth="13" />
+        {total > 0 && segments.map((s) => {
+          const frac = s.value / total
+          const dash = frac * C
+          const el = (
+            <circle key={s.label} cx="55" cy="55" r={R} fill="none" stroke={s.color} strokeWidth="13"
+              strokeDasharray={`${dash} ${C - dash}`} strokeDashoffset={-offset} strokeLinecap="butt"
+              style={{ transition: 'stroke-dasharray .6s ease, stroke-dashoffset .6s ease' }} />
+          )
+          offset += dash
+          return el
+        })}
+        <g className="rotate-90" style={{ transformOrigin: '55px 55px' }}>
+          <text x="55" y="51" textAnchor="middle" className="fill-foreground text-[16px] font-semibold tabular-nums">{total.toLocaleString()}</text>
+          <text x="55" y="65" textAnchor="middle" className="fill-foreground-faint text-[7px] uppercase tracking-wider">changes</text>
+        </g>
+      </svg>
+      <div className="flex-1 space-y-1.5">
+        {segments.map((s) => (
+          <div key={s.label} className="flex items-center gap-2 text-[11px]">
+            <span className="h-2.5 w-2.5 rounded-sm" style={{ background: s.color }} />
+            <span className="capitalize text-foreground-muted">{s.label}</span>
+            <span className="ml-auto font-mono text-foreground">{s.value.toLocaleString()}</span>
+            <span className="w-9 text-right font-mono text-foreground-faint">{total ? Math.round((s.value / total) * 100) : 0}%</span>
+          </div>
+        ))}
+      </div>
+    </div>
+  )
+}
+
+// Smooth area chart of per-minute change volume.
+function VolumeArea({ buckets }: { buckets: { t: string; n: number }[] }) {
+  const w = 600
+  const h = 120
+  const pad = 6
+  const data = buckets.length ? buckets : [{ t: '', n: 0 }]
+  const max = Math.max(1, ...data.map((b) => b.n))
+  const stepX = data.length > 1 ? (w - pad * 2) / (data.length - 1) : 0
+  const pts = data.map((b, i) => {
+    const x = pad + i * stepX
+    const y = h - pad - (b.n / max) * (h - pad * 2)
+    return [x, y] as const
+  })
+  const line = pts.map(([x, y], i) => `${i ? 'L' : 'M'}${x.toFixed(1)},${y.toFixed(1)}`).join(' ')
+  const area = `${line} L${pts[pts.length - 1][0].toFixed(1)},${h - pad} L${pts[0][0].toFixed(1)},${h - pad} Z`
+  const last = data[data.length - 1]
+  return (
+    <div>
+      <div className="relative">
+        <svg viewBox={`0 0 ${w} ${h}`} preserveAspectRatio="none" className="h-28 w-full">
+          <defs>
+            <linearGradient id="cdcvol" x1="0" y1="0" x2="0" y2="1">
+              <stop offset="0%" stopColor="#38bdf8" stopOpacity="0.45" />
+              <stop offset="100%" stopColor="#38bdf8" stopOpacity="0" />
+            </linearGradient>
+          </defs>
+          {[0.25, 0.5, 0.75].map((g) => (
+            <line key={g} x1={pad} x2={w - pad} y1={pad + g * (h - pad * 2)} y2={pad + g * (h - pad * 2)} stroke="rgba(148,163,184,0.08)" strokeWidth="1" />
+          ))}
+          {buckets.length > 0 && <path d={area} fill="url(#cdcvol)" />}
+          {buckets.length > 0 && <path d={line} fill="none" stroke="#38bdf8" strokeWidth="2" vectorEffect="non-scaling-stroke" />}
+          {buckets.length > 0 && (
+            <circle cx={pts[pts.length - 1][0]} cy={pts[pts.length - 1][1]} r="3.5" fill="#38bdf8">
+              <animate attributeName="r" values="3.5;6;3.5" dur="1.6s" repeatCount="indefinite" />
+            </circle>
+          )}
+        </svg>
+        <div className="pointer-events-none absolute left-1.5 top-1 text-[9px] font-mono text-foreground-faint">{max}/min</div>
+      </div>
+      <div className="mt-1 flex justify-between text-[9px] font-mono text-foreground-faint">
+        <span>{data[0]?.t || '—'}</span>
+        <span className="text-docker">now · {last?.n ?? 0}/min</span>
+      </div>
+      {buckets.length === 0 && <div className="mt-1 text-center text-[10px] text-foreground-faint">No changes in the window yet…</div>}
+    </div>
+  )
+}
+
+function BarRow({ label, value, max, color }: { label: string; value: number; max: number; color: string }) {
+  return (
+    <div className="flex items-center gap-2">
+      <span className="flex w-24 shrink-0 items-center gap-1.5 text-[11px] capitalize text-foreground-muted">
+        <span className="h-2 w-2 shrink-0 rounded-full" style={{ background: color }} /> {label}
+      </span>
+      <div className="h-2.5 flex-1 overflow-hidden rounded-full bg-surface">
+        <div className="h-full rounded-full transition-all duration-500" style={{ width: `${Math.max(3, (value / max) * 100)}%`, background: color }} />
+      </div>
+      <span className="w-12 shrink-0 text-right font-mono text-[11px] text-foreground">{value.toLocaleString()}</span>
+    </div>
+  )
+}
+
 function Diff({ change }: { change: CdcChange }) {
   const keys = useMemo(() => {
     const set = new Set<string>()
@@ -82,6 +223,8 @@ export function ChangesView({ liveChanges }: { liveChanges: CdcChange[] }) {
   const [op, setOp] = useState('all')
   const [expanded, setExpanded] = useState<string | null>(null)
   const [connected, setConnected] = useState(false)
+  const [flash, setFlash] = useState(false)
+  const prevTotal = useRef(0)
 
   const load = useCallback(async () => {
     const [c, s] = await Promise.all([fetchChanges({ limit: 150 }), fetchChangeStats(15)])
@@ -92,10 +235,22 @@ export function ChangesView({ liveChanges }: { liveChanges: CdcChange[] }) {
 
   useEffect(() => {
     load()
-    const iv = setInterval(() => fetchChangeStats(15).then((s) => s && setStats(s)), 5000)
+    const iv = setInterval(() => fetchChangeStats(15).then((s) => s && setStats(s)), 4000)
     return () => clearInterval(iv)
   }, [load])
 
+  // Pulse the header when fresh changes arrive.
+  useEffect(() => {
+    const t = stats?.total ?? 0
+    if (t > prevTotal.current) {
+      setFlash(true)
+      const id = setTimeout(() => setFlash(false), 900)
+      prevTotal.current = t
+      return () => clearTimeout(id)
+    }
+    prevTotal.current = t
+  }, [stats?.total])
+
   // Merge live (WS) with seeded backlog, dedupe by id, newest first.
   const merged = useMemo(() => {
     const byId = new Map<string, CdcChange>()
@@ -109,18 +264,39 @@ export function ChangesView({ liveChanges }: { liveChanges: CdcChange[] }) {
     [merged, source, op],
   )
 
-  const maxBucket = Math.max(1, ...(stats?.buckets || []).map((b) => b.n))
+  const byOp = stats?.by_op || {}
+  const inserts = byOp.insert || 0
+  const updates = byOp.update || 0
+  const deletes = byOp.delete || 0
+  const total = stats?.total ?? 0
+  const perMin = total / Math.max(1, stats?.window_minutes ?? 15)
+
+  const opSegments = useMemo(() => (
+    ['insert', 'update', 'delete', 'snapshot']
+      .map((k) => ({ label: k, value: byOp[k] || 0, color: opOf(k).color }))
+      .filter((s) => s.value > 0)
+  ), [byOp])
+
+  const sourceRows = useMemo(() => (
+    Object.entries(stats?.by_source || {}).sort((a, b) => b[1] - a[1])
+  ), [stats?.by_source])
+  const maxSource = Math.max(1, ...sourceRows.map(([, v]) => v))
+
+  const tableRows = useMemo(() => (
+    Object.entries(stats?.by_table || {}).sort((a, b) => b[1] - a[1]).slice(0, 7)
+  ), [stats?.by_table])
+  const maxTable = Math.max(1, ...tableRows.map(([, v]) => v))
 
   return (
-    <div className="flex h-full min-h-0 flex-col gap-3">
+    <div className="flex h-full min-h-0 flex-col gap-3 overflow-y-auto scrollbar-thin pr-1">
       {/* Header */}
-      <div className="flex items-center justify-between">
+      <div className="flex shrink-0 items-center justify-between">
         <div>
           <h1 className="flex items-center gap-2 text-base font-semibold text-foreground">
-            <Activity className="h-4 w-4 text-docker" /> Live Changes · CDC Stream
+            <Activity className={cn('h-4 w-4 text-docker', flash && 'animate-pulse')} /> New &amp; Changed Data
           </h1>
           <p className="text-[11px] text-foreground-muted">
-            Real-time Debezium change data capture from all source databases via Kafka
+            Live Debezium change data capture — every insert, update &amp; delete across all source databases, streamed via Kafka
           </p>
         </div>
         <div className="flex items-center gap-3">
@@ -134,53 +310,60 @@ export function ChangesView({ liveChanges }: { liveChanges: CdcChange[] }) {
         </div>
       </div>
 
-      {/* Stat cards */}
+      {/* KPI row */}
       <div className="grid shrink-0 grid-cols-2 gap-2 lg:grid-cols-4">
-        <div className="rounded-lg border border-border/60 bg-surface-raised p-3">
-          <div className="text-[9px] uppercase tracking-wide text-foreground-faint">Changes / 15 min</div>
-          <div className="text-xl font-semibold text-foreground">{stats?.total ?? 0}</div>
-        </div>
-        <div className="rounded-lg border border-border/60 bg-surface-raised p-3">
-          <div className="text-[9px] uppercase tracking-wide text-foreground-faint">Total consumed</div>
-          <div className="text-xl font-semibold text-foreground">{stats?.consumed ?? 0}</div>
-        </div>
-        <div className="rounded-lg border border-border/60 bg-surface-raised p-3">
-          <div className="text-[9px] uppercase tracking-wide text-foreground-faint">By operation</div>
-          <div className="mt-1 flex flex-wrap gap-1">
-            {Object.entries(stats?.by_op || {}).map(([k, v]) => (
-              <span key={k} className={cn('rounded border px-1.5 py-0.5 text-[9px]', opOf(k).cls)}>{opOf(k).label} {v}</span>
-            ))}
-            {!Object.keys(stats?.by_op || {}).length && <span className="text-[10px] text-foreground-faint">—</span>}
+        <KpiCard label="New records" value={inserts} accent="#34d399" icon={PlusCircle} sub="inserts · last 15m" />
+        <KpiCard label="Updates" value={updates} accent="#fbbf24" icon={Pencil} sub="modified rows · 15m" />
+        <KpiCard label="Deletes" value={deletes} accent="#fb7185" icon={Trash2} sub="removed rows · 15m" />
+        <KpiCard label="Throughput" value={Math.round(perMin)} accent="#38bdf8" icon={TrendingUp} sub={`changes/min · ${(stats?.consumed ?? 0).toLocaleString()} total consumed`} />
+      </div>
+
+      {/* Volume + operation mix */}
+      <div className="grid shrink-0 grid-cols-1 gap-2 lg:grid-cols-3">
+        <div className="rounded-lg border border-border/60 bg-surface-raised p-3 lg:col-span-2">
+          <div className="mb-1 flex items-center gap-1.5 text-[10px] uppercase tracking-wide text-foreground-faint">
+            <TrendingUp className="h-3 w-3" /> Change volume — last 15 minutes
           </div>
+          <VolumeArea buckets={stats?.buckets || []} />
         </div>
         <div className="rounded-lg border border-border/60 bg-surface-raised p-3">
-          <div className="text-[9px] uppercase tracking-wide text-foreground-faint">By source</div>
-          <div className="mt-1 flex flex-wrap gap-1">
-            {Object.entries(stats?.by_source || {}).map(([k, v]) => (
-              <span key={k} className="flex items-center gap-1 rounded border border-border/60 px-1.5 py-0.5 text-[9px] text-foreground-muted">
-                <span className="h-2 w-2 rounded-full" style={{ background: SOURCE_COLOR[k] || '#94a3b8' }} />{k} {v}
-              </span>
-            ))}
-            {!Object.keys(stats?.by_source || {}).length && <span className="text-[10px] text-foreground-faint">—</span>}
+          <div className="mb-2 flex items-center gap-1.5 text-[10px] uppercase tracking-wide text-foreground-faint">
+            <Layers className="h-3 w-3" /> Operation mix
           </div>
+          {opSegments.length ? <Donut segments={opSegments} total={total} /> : (
+            <div className="flex h-28 items-center justify-center text-[10px] text-foreground-faint">No changes yet…</div>
+          )}
         </div>
       </div>
 
-      {/* Volume sparkbars */}
-      <div className="shrink-0 rounded-lg border border-border/60 bg-surface-raised p-3">
-        <div className="mb-1.5 text-[9px] uppercase tracking-wide text-foreground-faint">Change volume per minute (last 15m)</div>
-        <div className="flex h-16 items-end gap-0.5">
-          {(stats?.buckets || []).map((b) => (
-            <div key={b.t} className="group relative flex-1" title={`${b.t}: ${b.n}`}>
-              <div className="w-full rounded-t bg-docker/70 transition-all group-hover:bg-docker" style={{ height: `${Math.max(4, (b.n / maxBucket) * 100)}%` }} />
-            </div>
-          ))}
-          {!(stats?.buckets || []).length && <div className="text-[10px] text-foreground-faint">No changes in the window yet…</div>}
+      {/* By system + top tables */}
+      <div className="grid shrink-0 grid-cols-1 gap-2 lg:grid-cols-2">
+        <div className="rounded-lg border border-border/60 bg-surface-raised p-3">
+          <div className="mb-2 flex items-center gap-1.5 text-[10px] uppercase tracking-wide text-foreground-faint">
+            <Database className="h-3 w-3" /> New &amp; changed by system
+          </div>
+          <div className="space-y-2">
+            {sourceRows.length ? sourceRows.map(([k, v]) => (
+              <BarRow key={k} label={k} value={v} max={maxSource} color={SOURCE_COLOR[k] || '#94a3b8'} />
+            )) : <div className="text-[10px] text-foreground-faint">No source activity in the window…</div>}
+          </div>
+        </div>
+        <div className="rounded-lg border border-border/60 bg-surface-raised p-3">
+          <div className="mb-2 flex items-center gap-1.5 text-[10px] uppercase tracking-wide text-foreground-faint">
+            <Layers className="h-3 w-3" /> Most active tables / collections
+          </div>
+          <div className="space-y-2">
+            {tableRows.length ? tableRows.map(([k, v]) => (
+              <BarRow key={k} label={k} value={v} max={maxTable} color="#818cf8" />
+            )) : <div className="text-[10px] text-foreground-faint">No table activity yet…</div>}
+          </div>
         </div>
       </div>
 
       {/* Filters */}
       <div className="flex shrink-0 flex-wrap items-center gap-2">
+        <span className="text-[10px] font-semibold uppercase tracking-wide text-foreground-muted">Latest changes</span>
+        <span className="mx-1 h-3 w-px bg-border/60" />
         <span className="text-[9px] uppercase tracking-wide text-foreground-faint">Source</span>
         {SOURCES.map((s) => (
           <button key={s} type="button" onClick={() => setSource(s)}
@@ -200,10 +383,10 @@ export function ChangesView({ liveChanges }: { liveChanges: CdcChange[] }) {
       </div>
 
       {/* Live list */}
-      <div className="min-h-0 flex-1 overflow-y-auto scrollbar-thin rounded-lg border border-border/60 bg-surface-raised">
+      <div className="min-h-[180px] rounded-lg border border-border/60 bg-surface-raised">
         {filtered.length === 0 && (
           <div className="p-6 text-center text-[11px] text-foreground-faint">
-            Waiting for changes… trigger data generation or agent DML to see live CDC events.
+            Waiting for changes… trigger data generation (Data Flow → Generate data) or agent DML to see live CDC events.
           </div>
         )}
         {filtered.map((c) => (