diff --git a/ui/src/App.tsx b/ui/src/App.tsx index 78de735..17050c7 100644 --- a/ui/src/App.tsx +++ b/ui/src/App.tsx @@ -17,6 +17,7 @@ import { DataQualityView } from './components/features/DataQualityView' import { KnowledgeChatView } from './components/features/KnowledgeChatView' import { StorageView } from './components/features/StorageView' import { HdfsView } from './components/features/HdfsView' +import { DataGenView } from './components/features/DataGenView' import { SearchView } from './components/features/SearchView' import { SshTerminal } from './components/features/SshTerminal' import { TerminalDock } from './components/features/TerminalDock' @@ -123,7 +124,10 @@ export default function App() { animations={cc.anims} selectedNodeId={cc.selectedNodeId} onNodeClick={cc.selectNode} + pulse={cc.genPulse} /> + ) : cc.mainView === 'datagen' ? ( + cc.setMainView('platform')} /> ) : cc.mainView === 'presentation' ? ( ) : cc.mainView === 'dataquality' ? ( diff --git a/ui/src/components/features/DataGenView.tsx b/ui/src/components/features/DataGenView.tsx new file mode 100644 index 0000000..a49ef85 --- /dev/null +++ b/ui/src/components/features/DataGenView.tsx @@ -0,0 +1,249 @@ +import { useCallback, useEffect, useRef, useState } from 'react' +import { Database, Boxes, Activity, Cpu, Network, Layers, Play, Loader2, CheckCircle2, XCircle, RefreshCw } from 'lucide-react' +import { cn } from '../../lib/utils' +import { subTabActive, subTabIdle } from '../../lib/tabActive' + +type SourceKey = 'all' | 'postgres' | 'mysql' | 'mongodb' | 'cassandra' | 'neo4j' + +type SourceMeta = { + key: SourceKey + label: string + icon: typeof Database + accent: string + target: string + cdc: boolean + desc: string + defaultRows: number +} + +const SOURCES: SourceMeta[] = [ + { key: 'all', label: 'All sources', icon: Layers, accent: 'text-violet-400', target: 'alle 5 databases', cdc: true, desc: 'Genereer tegelijk in alle databases.', defaultRows: 5000 }, + { key: 'postgres', label: 'PostgreSQL', icon: Database, accent: 'text-sky-400', target: 'sales_orders', cdc: true, desc: 'Sales orders. CDC via Debezium -> Kafka -> S3.', defaultRows: 5000 }, + { key: 'mysql', label: 'MySQL', icon: Database, accent: 'text-amber-400', target: 'employee_events', cdc: true, desc: 'HR employee events. CDC via Debezium -> Kafka -> S3.', defaultRows: 5000 }, + { key: 'mongodb', label: 'MongoDB', icon: Boxes, accent: 'text-emerald-400', target: 'supplychain.events', cdc: true, desc: 'Supply chain events. CDC via Debezium -> Kafka -> S3.', defaultRows: 5000 }, + { key: 'cassandra', label: 'Cassandra', icon: Activity, accent: 'text-cyan-400', target: 'device_metrics', cdc: false, desc: 'Telemetry metrics. Zichtbaar via Trino.', defaultRows: 5000 }, + { key: 'neo4j', label: 'Neo4j', icon: Network, accent: 'text-pink-400', target: 'Product/Supplier graph', cdc: false, desc: 'Graafdata (producten, leveranciers, relaties).', defaultRows: 2000 }, +] + +type RunInfo = { run_id?: string; state?: string; start?: string; end?: string; conf?: { rows?: number } } + +type Props = { + onPulse: () => void + onOpenPlatform: () => void +} + +const COUNT_KEYS: SourceKey[] = ['postgres', 'mysql', 'mongodb', 'cassandra'] + +export function DataGenView({ onPulse, onOpenPlatform }: Props) { + const [active, setActive] = useState('all') + const [rows, setRows] = useState>( + Object.fromEntries(SOURCES.map((s) => [s.key, s.defaultRows])) as Record, + ) + const [busy, setBusy] = useState>({}) + const [runs, setRuns] = useState>({}) + const [counts, setCounts] = useState>({}) + const [msg, setMsg] = useState(null) + const pollRef = useRef>>({}) + + const meta = SOURCES.find((s) => s.key === active)! + + const loadCounts = useCallback(async () => { + try { + const r = await fetch('/api/pipeline/sync') + const j = await r.json() + if (j.ok) setCounts(j.counts || {}) + } catch { /* */ } + }, []) + + const loadRuns = useCallback(async (source: SourceKey) => { + try { + const r = await fetch(`/api/pipeline/runs/${source}?limit=5`) + const j = await r.json() + if (j.ok) setRuns((prev) => ({ ...prev, [source]: j.runs || [] })) + return (j.runs || [])[0] as RunInfo | undefined + } catch { + return undefined + } + }, []) + + useEffect(() => { + loadCounts() + SOURCES.forEach((s) => loadRuns(s.key)) + return () => { Object.values(pollRef.current).forEach(clearInterval) } + }, [loadCounts, loadRuns]) + + const startPolling = useCallback((source: SourceKey) => { + if (pollRef.current[source]) clearInterval(pollRef.current[source]) + pollRef.current[source] = setInterval(async () => { + const latest = await loadRuns(source) + if (latest && (latest.state === 'success' || latest.state === 'failed')) { + clearInterval(pollRef.current[source]) + delete pollRef.current[source] + setBusy((b) => ({ ...b, [source]: false })) + loadCounts() + if (latest.state === 'success') setMsg(`${source}: klaar — nieuwe data gegenereerd`) + else setMsg(`${source}: run mislukt — check Airflow logs`) + } + }, 3000) + }, [loadRuns, loadCounts]) + + const generate = useCallback(async (source: SourceKey) => { + setMsg(null) + setBusy((b) => ({ ...b, [source]: true })) + onPulse() + try { + const r = await fetch(`/api/pipeline/generate/${source}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ rows: rows[source] }), + }) + const j = await r.json() + if (!j.ok) { + setMsg(`Fout: ${j.error || 'kon niet triggeren'}`) + setBusy((b) => ({ ...b, [source]: false })) + return + } + setMsg(`${source}: gestart (run ${String(j.run_id).slice(-8)})`) + startPolling(source) + } catch { + setMsg('API niet bereikbaar') + setBusy((b) => ({ ...b, [source]: false })) + } + }, [rows, onPulse, startPolling]) + + const stateBadge = (state?: string) => { + if (state === 'success') return success + if (state === 'failed') return failed + if (state === 'running' || state === 'queued') return {state} + return {state || '—'} + } + + const latest = (runs[active] || [])[0] + + return ( +
+
+
+

+ Data Generation +

+

+ Genereer per database nieuwe data en pulse de hele flow: bron -> Debezium -> Kafka -> S3 +

+
+
+ + +
+
+ +
+ {SOURCES.map((s) => { + const Icon = s.icon + return ( + + ) + })} +
+ +
+
+
+
+ +

{meta.label}

+ {meta.cdc ? ( + CDC actief + ) : ( + geen CDC-stream + )} +
+

{meta.desc} Doel: {meta.target}

+ +
+ + + {active !== 'all' && COUNT_KEYS.includes(active) && ( +
+ Huidige rijen: {counts[active]?.toLocaleString() ?? '…'} +
+ )} +
+ {msg &&

{msg}

} +
+ +
+

Recente runs — {meta.label}

+ {latest ? ( + + + + + + + + + + + {(runs[active] || []).map((run) => ( + + + + + + + ))} + +
StateRowsStartEind
{stateBadge(run.state)}{run.conf?.rows ?? '—'}{run.start?.slice(11, 19) || '—'}{run.end?.slice(11, 19) || '—'}
+ ) : ( +

Nog geen runs.

+ )} +
+ +
+

Live tellingen (Trino)

+
+ {COUNT_KEYS.map((k) => ( +
+
{k}
+
{counts[k]?.toLocaleString() ?? '…'}
+
+ ))} +
+
+
+
+
+ ) +} diff --git a/ui/src/components/features/PlatformTopology.tsx b/ui/src/components/features/PlatformTopology.tsx index 8c138e4..4e87bfc 100644 --- a/ui/src/components/features/PlatformTopology.tsx +++ b/ui/src/components/features/PlatformTopology.tsx @@ -253,9 +253,10 @@ type Props = { animations: Record selectedNodeId: string | null onNodeClick: (nodeId: string) => void + pulse?: boolean } -export function PlatformTopology({ workload, animations, selectedNodeId, onNodeClick }: Props) { +export function PlatformTopology({ workload, animations, selectedNodeId, onNodeClick, pulse = false }: Props) { const [metrics, setMetrics] = useState(seedMetrics) const canvasRef = useRef(null) const nodeRefs = useRef>({}) @@ -292,7 +293,7 @@ export function PlatformTopology({ workload, animations, selectedNodeId, onNodeC [animations], ) - const edgesLive = pipelineActive || anyBusy + const edgesLive = pulse || pipelineActive || anyBusy useEffect(() => { fetch("/api/search/health").then(r => r.json()).then(j => { @@ -343,7 +344,7 @@ export function PlatformTopology({ workload, animations, selectedNodeId, onNodeC }, [measureAnchors, metrics, llmLabel]) return ( -
+
1 || (Math.abs(b.y - a.y) > 48 && Math.abs(toX - fromX) > 120) const d = flowPath(fromX, a.y, toX, b.y, { backward, longArc }) const live = edgesLive - const dur = 1.8 + (i % 5) * 0.35 + const dur = (pulse ? 0.85 : 1.8) + (i % 5) * (pulse ? 0.12 : 0.35) return ( diff --git a/ui/src/components/layout/SideNav.tsx b/ui/src/components/layout/SideNav.tsx index 0aab8b7..559c5de 100644 --- a/ui/src/components/layout/SideNav.tsx +++ b/ui/src/components/layout/SideNav.tsx @@ -1,4 +1,4 @@ -import { DatabaseZap, HardDrive, Search, LayoutDashboard, MessageSquare, Presentation, Server, TerminalSquare } from 'lucide-react' +import { DatabaseZap, HardDrive, Search, LayoutDashboard, MessageSquare, Presentation, Server, TerminalSquare, Cpu } from 'lucide-react' import type { GpuStatus, WorkloadData } from '../../types' import type { GpuLiveMetrics } from '../../hooks/useLiveMetrics' import { cn } from '../../lib/utils' @@ -6,7 +6,7 @@ import { viewTabActive, viewTabIdle } from '../../lib/tabActive' import { GpuMatrixPanel } from '../features/GpuMatrixPanel' import { LabHealthPanel } from '../features/LabHealthPanel' -type MainView = 'platform' | 'presentation' | 'dataquality' | 'knowledge' | 'storage' | 'hdfs' | 'search' | 'approvals' +type MainView = 'platform' | 'presentation' | 'dataquality' | 'knowledge' | 'storage' | 'hdfs' | 'search' | 'datagen' | 'approvals' type Props = { workload: WorkloadData | null @@ -24,6 +24,7 @@ type Props = { const VIEWS: { id: MainView; label: string; icon: typeof LayoutDashboard }[] = [ { id: 'platform', label: 'Data Platform', icon: LayoutDashboard }, + { id: 'datagen', label: 'Data Generation', icon: Cpu }, { id: 'presentation', label: 'Presentation', icon: Presentation }, { id: 'dataquality', label: 'Data Quality', icon: DatabaseZap }, { id: 'knowledge', label: 'Knowledge Chat', icon: MessageSquare }, diff --git a/ui/src/hooks/useCommandCenter.ts b/ui/src/hooks/useCommandCenter.ts index 4dcec49..e25f163 100644 --- a/ui/src/hooks/useCommandCenter.ts +++ b/ui/src/hooks/useCommandCenter.ts @@ -68,7 +68,14 @@ export function useCommandCenter() { const [selectedNode, setSelectedNode] = useState(null) const [nodeDetail, setNodeDetail] = useState(null) const [nodeBusy, setNodeBusy] = useState(false) - const [mainView, setMainView] = useState<'platform' | 'approvals' | 'presentation' | 'dataquality' | 'knowledge' | 'storage' | 'hdfs' | 'search'>('platform') + const [mainView, setMainView] = useState<'platform' | 'approvals' | 'presentation' | 'dataquality' | 'knowledge' | 'storage' | 'hdfs' | 'search' | 'datagen'>('platform') + const [genPulse, setGenPulse] = useState(false) + const genPulseTimer = useRef | null>(null) + const pulseFlow = useCallback(() => { + setGenPulse(true) + if (genPulseTimer.current) clearTimeout(genPulseTimer.current) + genPulseTimer.current = setTimeout(() => setGenPulse(false), 60000) + }, []) const [approvalHighlight, setApprovalHighlight] = useState(false) const [workbenchMode, setWorkbenchMode] = useState<'agent' | 'sql-postgres' | 'sql-mysql' | 'sql-mongodb' | 'sql-trino' | null>(null) const [chatExpanded, setChatExpanded] = useState(false) @@ -336,6 +343,8 @@ export function useCommandCenter() { nodeBusy, mainView, setMainView, + genPulse, + pulseFlow, approvalHighlight, setApprovalHighlight, workbenchMode, diff --git a/ui/src/styles/globals.css b/ui/src/styles/globals.css index 26df2e9..23b391b 100644 --- a/ui/src/styles/globals.css +++ b/ui/src/styles/globals.css @@ -107,6 +107,21 @@ .topo-edge-query.topo-edge-live { stroke: #818cf8; filter: drop-shadow(0 0 3px rgba(129, 140, 248, 0.45)); } .topo-edge-serve.topo-edge-live { stroke: #34d399; filter: drop-shadow(0 0 3px rgba(52, 211, 153, 0.45)); } + /* Whole-flow pulse triggered by a data-generation run */ + .topo-pulsing .topo-edge-live { + stroke-width: 3; + animation: flow-dash 0.6s linear infinite; + } + .topo-pulsing .topo-edge-glow { + stroke: rgba(124, 231, 135, 0.35); + stroke-width: 6; + animation: topo-pulse-glow 1.4s ease-in-out infinite; + } + @keyframes topo-pulse-glow { + 0%, 100% { opacity: 0.4; } + 50% { opacity: 1; } + } + .topo-edge-active { stroke: url(#topo-flow-gradient); stroke-width: 2.5;