feat: neo4j in topology + agent-driven datagen with activity log; fix all-sources generate

ui/src/components/features/DataGenView.tsx

@@ -1,5 +1,5 @@
 import { useCallback, useEffect, useRef, useState } from 'react'
-import { Database, Boxes, Activity, Cpu, Network, Layers, Play, Loader2, CheckCircle2, XCircle, RefreshCw } from 'lucide-react'
+import { Database, Boxes, Activity, Cpu, Network, Layers, Play, Loader2, CheckCircle2, XCircle, RefreshCw, Bot, ScrollText } from 'lucide-react'
 import { cn } from '../../lib/utils'
 import { subTabActive, subTabIdle } from '../../lib/tabActive'
 
@@ -26,6 +26,8 @@ const SOURCES: SourceMeta[] = [
 ]
 
 type RunInfo = { run_id?: string; state?: string; start?: string; end?: string; conf?: { rows?: number } }
+type AgentInfo = { agent_id: string; agent_name: string }
+type ActivityItem = { id: string; ts?: string; agent_id: string; agent_name: string; message: string; level: string }
 
 type Props = {
   onPulse: () => void
@@ -43,6 +45,8 @@ export function DataGenView({ onPulse, onOpenPlatform }: Props) {
   const [runs, setRuns] = useState<Record<string, RunInfo[]>>({})
   const [counts, setCounts] = useState<Record<string, number | null>>({})
   const [msg, setMsg] = useState<string | null>(null)
+  const [agentMap, setAgentMap] = useState<Record<string, AgentInfo>>({})
+  const [activity, setActivity] = useState<ActivityItem[]>([])
   const pollRef = useRef<Record<string, ReturnType<typeof setInterval>>>({})
 
   const meta = SOURCES.find((s) => s.key === active)!
@@ -55,6 +59,14 @@ export function DataGenView({ onPulse, onOpenPlatform }: Props) {
     } catch { /* */ }
   }, [])
 
+  const loadActivity = useCallback(async () => {
+    try {
+      const r = await fetch('/api/pipeline/activity?limit=25')
+      const j = await r.json()
+      if (j.ok) setActivity(j.activity || [])
+    } catch { /* */ }
+  }, [])
+
   const loadRuns = useCallback(async (source: SourceKey) => {
     try {
       const r = await fetch(`/api/pipeline/runs/${source}?limit=5`)
@@ -68,9 +80,13 @@ export function DataGenView({ onPulse, onOpenPlatform }: Props) {
 
   useEffect(() => {
     loadCounts()
+    loadActivity()
     SOURCES.forEach((s) => loadRuns(s.key))
-    return () => { Object.values(pollRef.current).forEach(clearInterval) }
-  }, [loadCounts, loadRuns])
+    fetch('/api/pipeline/agents').then((r) => r.json()).then((j) => { if (j.ok) setAgentMap(j.agents || {}) }).catch(() => {})
+    const act = setInterval(loadActivity, 8000)
+    const poll = pollRef.current
+    return () => { Object.values(poll).forEach(clearInterval); clearInterval(act) }
+  }, [loadCounts, loadRuns, loadActivity])
 
   const startPolling = useCallback((source: SourceKey) => {
     if (pollRef.current[source]) clearInterval(pollRef.current[source])
@@ -87,7 +103,7 @@ export function DataGenView({ onPulse, onOpenPlatform }: Props) {
     }, 3000)
   }, [loadRuns, loadCounts])
 
-  const generate = useCallback(async (source: SourceKey) => {
+  const generate = useCallback(async (source: SourceKey, autonomous = false) => {
     setMsg(null)
     setBusy((b) => ({ ...b, [source]: true }))
     onPulse()
@@ -95,7 +111,7 @@ export function DataGenView({ onPulse, onOpenPlatform }: Props) {
       const r = await fetch(`/api/pipeline/generate/${source}`, {
         method: 'POST',
         headers: { 'Content-Type': 'application/json' },
-        body: JSON.stringify({ rows: rows[source] }),
+        body: JSON.stringify({ rows: rows[source], autonomous }),
       })
       const j = await r.json()
       if (!j.ok) {
@@ -103,13 +119,15 @@ export function DataGenView({ onPulse, onOpenPlatform }: Props) {
         setBusy((b) => ({ ...b, [source]: false }))
         return
       }
-      setMsg(`${source}: gestart (run ${String(j.run_id).slice(-8)})`)
+      const who = autonomous ? `${j.agent_name || 'Agent'} genereert zelf` : 'gestart'
+      setMsg(`${source}: ${who} (run ${String(j.run_id).slice(-8)})`)
       startPolling(source)
+      setTimeout(loadActivity, 800)
     } catch {
       setMsg('API niet bereikbaar')
       setBusy((b) => ({ ...b, [source]: false }))
     }
-  }, [rows, onPulse, startPolling])
+  }, [rows, onPulse, startPolling, loadActivity])
 
   const stateBadge = (state?: string) => {
     if (state === 'success') return <span className="inline-flex items-center gap-1 text-emerald-400"><CheckCircle2 className="h-3 w-3" /> success</span>
@@ -170,6 +188,11 @@ export function DataGenView({ onPulse, onOpenPlatform }: Props) {
               ) : (
                 <span className="rounded bg-white/10 px-1.5 py-0.5 text-[9px] text-foreground-muted">geen CDC-stream</span>
               )}
+              {agentMap[active] && (
+                <span className="inline-flex items-center gap-1 rounded bg-violet-500/15 px-1.5 py-0.5 text-[9px] text-violet-300">
+                  <Bot className="h-2.5 w-2.5" /> {agentMap[active].agent_name}
+                </span>
+              )}
             </div>
             <p className="mb-3 text-[11px] text-foreground-muted">{meta.desc} Doel: <span className="font-mono">{meta.target}</span></p>
 
@@ -194,6 +217,16 @@ export function DataGenView({ onPulse, onOpenPlatform }: Props) {
                 {busy[active] ? <Loader2 className="h-4 w-4 animate-spin" /> : <Play className="h-4 w-4" />}
                 Genereer data
               </button>
+              <button
+                type="button"
+                disabled={!!busy[active]}
+                onClick={() => generate(active, true)}
+                title="Laat de verantwoordelijke agent zelf data genereren (wordt gelogd)"
+                className="inline-flex items-center gap-2 rounded-md border border-violet-400/50 px-3 py-2 text-[12px] font-medium text-violet-300 hover:bg-violet-500/10 disabled:opacity-40"
+              >
+                <Bot className="h-4 w-4" />
+                Laat agent genereren
+              </button>
               {active !== 'all' && COUNT_KEYS.includes(active) && (
                 <div className="text-[11px] text-foreground-muted">
                   Huidige rijen: <span className="font-mono text-foreground">{counts[active]?.toLocaleString() ?? '…'}</span>
@@ -242,6 +275,27 @@ export function DataGenView({ onPulse, onOpenPlatform }: Props) {
               ))}
             </div>
           </div>
+
+          <div className="rounded-lg border border-border p-4">
+            <h4 className="mb-2 flex items-center gap-1.5 text-[11px] font-semibold uppercase tracking-wide text-foreground-faint">
+              <ScrollText className="h-3.5 w-3.5" /> Agent-activiteit (wat de agents deden)
+            </h4>
+            {activity.length === 0 ? (
+              <p className="text-[11px] text-foreground-faint">Nog geen agent-acties gelogd.</p>
+            ) : (
+              <ul className="space-y-1">
+                {activity.map((a) => (
+                  <li key={a.id} className="flex items-start gap-2 text-[11px]">
+                    <span className="mt-0.5 text-foreground-faint">{a.ts?.slice(11, 19) || ''}</span>
+                    <span className="inline-flex items-center gap-1 rounded bg-violet-500/15 px-1 text-[9px] text-violet-300">
+                      <Bot className="h-2.5 w-2.5" />{a.agent_name}
+                    </span>
+                    <span className={cn('flex-1', a.level === 'err' ? 'text-danger' : 'text-foreground-muted')}>{a.message}</span>
+                  </li>
+                ))}
+              </ul>
+            )}
+          </div>
         </div>
       </div>
     </div>

ui/src/components/features/PlatformTopology.tsx

@@ -34,6 +34,7 @@ const STAGES: TopoStage[] = [
       { id: 'mysql', label: 'MySQL', sub: 'Replica set', metricKey: 'mysql' },
       { id: 'mongodb', label: 'MongoDB', sub: 'Document store', metricKey: 'mongodb' },
       { id: 'cassandra', label: 'Cassandra', sub: 'Wide-column', metricKey: 'cassandra' },
+      { id: 'neo4j', label: 'Neo4j', sub: 'Graph store', metricKey: 'neo4j' },
     ],
   },
   {
@@ -78,11 +79,13 @@ const FLOW_EDGES: FlowEdge[] = [
   { from: 'airflow', to: 'mysql', kind: 'orchestration', label: 'Daily Python gen' },
   { from: 'airflow', to: 'mongodb', kind: 'orchestration', label: 'Daily Python gen' },
   { from: 'airflow', to: 'cassandra', kind: 'orchestration', label: 'Daily Python gen' },
+  { from: 'airflow', to: 'neo4j', kind: 'orchestration', label: 'Daily Python gen' },
   // CDC capture from sources
   { from: 'postgresql', to: 'debezium', kind: 'cdc', label: 'CDC' },
   { from: 'mysql', to: 'debezium', kind: 'cdc', label: 'CDC' },
   { from: 'mongodb', to: 'debezium', kind: 'cdc', label: 'CDC' },
   { from: 'cassandra', to: 'debezium', kind: 'cdc', label: 'CDC' },
+  { from: 'neo4j', to: 'debezium', kind: 'cdc', label: 'CDC' },
   // Streaming bus
   { from: 'debezium', to: 'kafka', kind: 'stream', label: 'Events' },
   { from: 'airflow', to: 'kafka', kind: 'orchestration', label: 'DAG trigger' },
@@ -144,7 +147,7 @@ const PARTICLE_FILL: Record<FlowKind, string> = {
 }
 
 const NODE_CLICK_MAP: Record<string, string> = {
-  postgresql: 'src-postgres', mysql: 'src-mysql', mongodb: 'src-mongo', cassandra: 'src-cassandra',
+  postgresql: 'src-postgres', mysql: 'src-mysql', mongodb: 'src-mongo', cassandra: 'src-cassandra', neo4j: 'src-neo4j',
   debezium: 'cdc-postgres', kafka: 'stream-kafka', airflow: 'src-airflow', spark: 'stream-spark',
   trino: 'query-trino', iceberg: 'lake-iceberg', s3: 'lake-s3', hadoop: 'hadoop', bi: 'cons-bi',
   jupyter: 'cons-notebooks', elasticsearch: 'cons-elastic', kibana: 'cons-kibana', llm: 'cons-ml',
@@ -187,7 +190,7 @@ type MetricState = Record<string, string>
 
 function seedMetrics(): MetricState {
   return {
-    postgresql: '12.4k rows/s', mysql: '8.1k rows/s', mongodb: '2.3k docs/s', cassandra: '5.6k ops/s',
+    postgresql: '12.4k rows/s', mysql: '8.1k rows/s', mongodb: '2.3k docs/s', cassandra: '5.6k ops/s', neo4j: '1.4k nodes/s',
     debezium: '4 connectors active', kafka: '142 MB/s', airflow: '18 DAGs · daily 02:00 UTC',
     spark: '6 executors live', trino: '3 queries active', iceberg: '847 tables · 2.1 TB', s3: '14.2 TB stored',
     hadoop: 'Historical archive',
@@ -235,6 +238,7 @@ function jitterMetric(key: string, current: string, workload: WorkloadData | nul
     mysql: () => `${(8.1 + n() * 0.6).toFixed(1)}k rows/s`,
     mongodb: () => `${(2.3 + n() * 0.3).toFixed(1)}k docs/s`,
     cassandra: () => `${(5.6 + n() * 0.5).toFixed(1)}k ops/s`,
+    neo4j: () => `${(1.4 + n() * 0.3).toFixed(1)}k nodes/s`,
     debezium: () => `${Math.max(3, Math.round(4 + n()))} connectors active`,
     kafka: () => `${Math.max(80, Math.round(142 + n() * 18))} MB/s`,
     airflow: () => `${Math.max(12, Math.round(18 + n() * 2))} DAGs · daily 02:00 UTC`,

ui/src/lib/infraCatalog.ts

@@ -47,7 +47,7 @@ export const INFRA_CATALOG: InfraNode[] = [
       { label: 'PostgreSQL', url: 'postgresql://10.0.21.51:5432/postgres', port: '5432' },
       { label: 'MongoDB', url: 'mongodb://10.0.21.51:27017/', port: '27017' },
     ],
-    topoIds: ['postgresql', 'mysql', 'mongodb', 'cassandra', 'src-postgres', 'src-mysql', 'src-mongo', 'src-cassandra'],
+    topoIds: ['postgresql', 'mysql', 'mongodb', 'cassandra', 'neo4j', 'src-postgres', 'src-mysql', 'src-mongo', 'src-cassandra', 'src-neo4j'],
   },
   {
     id: 'airflow',