feat: Spark Workbench everywhere, autonomous Hadoop offload & LLM masking-aware

- Data Hub with Hadoop tab (HDFS/Iceberg browser, Spark, pipeline)
- Databricks-style Lakehouse Workbench (Trino engine, live exec matrix,
  materialize to Iceberg/S3); reused & embedded in every source-DB UI
- HDFS -> Kafka -> Spark -> Iceberg/S3 pipeline; WebHDFS hostname resolver
- Data Flow master pulse switch (Run/Pause/Stop) gating animated edges
- Data Custodian autonomous Hadoop offload loop (batch counterpart to CDC),
  pulsing source -> HDFS edges; toggle in Data Flow
- LLM now autonomously aware of all latest platform changes (live platform
  context) and enforces masking policy: never reveals masked PII, still
  answers helpfully with aggregates/explanations

ui/src/components/features/DataSourcesView.tsx

@@ -6,15 +6,20 @@ import {
   FolderTree,
   Loader2,
   Network,
+  Play,
   RefreshCw,
   Server,
   Table2,
   TerminalSquare,
 } from 'lucide-react'
 import { Badge } from '../ui/Badge'
+import { DbBrandIcon, dbBrandColor } from '../ui/DbBrandIcon'
+import { DataBrowserGrid } from './DataBrowserGrid'
+import { DataGenPanel } from './DataGenPanel'
 import { DbShell } from './DbShell'
 import { Neo4jGraphView } from './Neo4jGraphView'
 import { SqlWorkbench } from './SqlWorkbench'
+import { LakehouseWorkbench } from './SparkView'
 import {
   getSourceMeta,
   SOURCE_CATALOG,
@@ -39,16 +44,33 @@ type SampleResponse = {
   row_count?: number
   elapsed_ms?: number
   error?: string
+  primary_keys?: string[]
+  cdc?: boolean
+  editable?: boolean
+  offset?: number
+  limit?: number
+  total_count?: number | null
 }
 
 type Props = {
   focusEngine?: SourceEngine | null
+  onPulse?: () => void
 }
 
-const SUB_TABS: { id: SourceSubTab; label: string; icon: typeof FolderTree; neo4jOnly?: boolean }[] = [
+const ENGINE_CATALOG: Record<SourceEngine, string | undefined> = {
+  postgres: 'postgres_sales',
+  mysql: 'mysql_hr',
+  mongodb: 'mongodb_supplychain',
+  cassandra: 'cassandra_telemetry',
+  neo4j: undefined,
+}
+
+const SUB_TABS: { id: SourceSubTab; label: string; icon: typeof FolderTree; neo4jOnly?: boolean; hideNeo4j?: boolean }[] = [
   { id: 'browser', label: 'Browser', icon: FolderTree },
   { id: 'graph', label: 'Graph', icon: Network, neo4jOnly: true },
   { id: 'console', label: 'Query Console', icon: Database },
+  { id: 'workbench', label: 'Workbench', icon: Activity, hideNeo4j: true },
+  { id: 'generate', label: 'Generate', icon: Play },
   { id: 'shell', label: 'Shell', icon: TerminalSquare },
 ]
 
@@ -59,7 +81,7 @@ function fmtCount(n?: number | null) {
   return String(n)
 }
 
-export function DataSourcesView({ focusEngine }: Props) {
+export function DataSourcesView({ focusEngine, onPulse }: Props) {
   const [active, setActive] = useState<SourceEngine>(focusEngine || 'postgres')
   const [subTab, setSubTab] = useState<SourceSubTab>('browser')
   const [health, setHealth] = useState<HealthMap>({})
@@ -68,6 +90,8 @@ export function DataSourcesView({ focusEngine }: Props) {
   const [selectedObject, setSelectedObject] = useState<CatalogObject | null>(null)
   const [sample, setSample] = useState<SampleResponse | null>(null)
   const [sampleLoading, setSampleLoading] = useState(false)
+  const [page, setPage] = useState(1)
+  const [pageSize, setPageSize] = useState(100)
 
   const meta = getSourceMeta(active)
 
@@ -104,11 +128,14 @@ export function DataSourcesView({ focusEngine }: Props) {
     }
   }, [])
 
-  const loadSample = useCallback(async (engine: SourceEngine, obj: CatalogObject) => {
+  const loadSample = useCallback(async (engine: SourceEngine, obj: CatalogObject, pg = page, ps = pageSize) => {
     setSampleLoading(true)
     setSample(null)
+    const offset = (pg - 1) * ps
     try {
-      const r = await fetch(`/api/sql/sample/${engine}?object=${encodeURIComponent(obj.fqn)}&limit=50`)
+      const r = await fetch(
+        `/api/sql/sample/${engine}?object=${encodeURIComponent(obj.fqn)}&limit=${ps}&offset=${offset}`,
+      )
       const j = await r.json()
       setSample(j)
     } catch {
@@ -116,7 +143,7 @@ export function DataSourcesView({ focusEngine }: Props) {
     } finally {
       setSampleLoading(false)
     }
-  }, [])
+  }, [page, pageSize])
 
   useEffect(() => { loadHealth() }, [loadHealth])
   useEffect(() => {
@@ -124,21 +151,26 @@ export function DataSourcesView({ focusEngine }: Props) {
   }, [active, subTab, loadCatalog])
 
   useEffect(() => {
-    if (selectedObject && subTab === 'browser') loadSample(active, selectedObject)
-  }, [selectedObject, active, subTab, loadSample])
+    setPage(1)
+  }, [selectedObject?.fqn, active])
+
+  useEffect(() => {
+    if (selectedObject && subTab === 'browser') loadSample(active, selectedObject, page, pageSize)
+  }, [selectedObject, active, subTab, page, pageSize, loadSample])
 
   const refreshAll = () => {
     loadHealth()
-    if (subTab === 'browser') loadCatalog(active)
-    else if (subTab === 'graph' && active === 'neo4j') { /* Neo4jGraphView reloads itself */ }
-    else if (selectedObject) loadSample(active, selectedObject)
+    if (subTab === 'browser') {
+      loadCatalog(active)
+      if (selectedObject) loadSample(active, selectedObject, page, pageSize)
+    } else if (subTab === 'graph' && active === 'neo4j') { /* Neo4jGraphView reloads itself */ }
+    else if (selectedObject) loadSample(active, selectedObject, page, pageSize)
   }
 
-  const visibleSubTabs = SUB_TABS.filter((t) => !t.neo4jOnly || active === 'neo4j')
+  const visibleSubTabs = SUB_TABS.filter((t) => (!t.neo4jOnly || active === 'neo4j') && !(t.hideNeo4j && active === 'neo4j'))
 
   return (
     <div className="flex h-full min-h-0 flex-col gap-2 p-3">
-      {/* Header */}
       <header className="panel flex shrink-0 flex-wrap items-center justify-between gap-3 px-4 py-3">
         <div>
           <h1 className="flex items-center gap-2 text-base font-semibold text-foreground">
@@ -146,7 +178,7 @@ export function DataSourcesView({ focusEngine }: Props) {
             Data Sources UI
           </h1>
           <p className="text-[11px] text-foreground-muted">
-            Enterprise data browser — schema exploration, query console &amp; interactive shells for all source databases
+            Browse, filter, edit &amp; query all source databases — edits on CDC sources flow to Live Changes
           </p>
         </div>
         <button type="button" onClick={refreshAll} className={cn('inline-flex items-center gap-1.5 rounded-md px-3 py-1.5 text-[11px]', subTabIdle)}>
@@ -154,170 +186,159 @@ export function DataSourcesView({ focusEngine }: Props) {
         </button>
       </header>
 
-      <div className="flex min-h-0 flex-1 gap-2 overflow-hidden">
-        {/* Left rail — database cards */}
-        <aside className="panel flex w-[220px] shrink-0 flex-col overflow-hidden">
-          <div className="shrink-0 border-b border-border px-3 py-2">
-            <p className="text-[9px] font-semibold uppercase tracking-widest text-foreground-faint">Source Databases</p>
-          </div>
-          <div className="scrollbar-thin min-h-0 flex-1 space-y-1 overflow-y-auto p-2">
-            {SOURCE_CATALOG.map((src) => {
-              const Icon = src.icon
-              const up = health[src.engine]?.ok
-              const selected = active === src.engine
-              return (
-                <button
-                  key={src.engine}
-                  type="button"
-                  onClick={() => { setActive(src.engine); setSubTab('browser') }}
-                  className={cn(
-                    'flex w-full flex-col gap-1 rounded-lg border p-2.5 text-left transition-all',
-                    selected ? cn(src.border, src.accentBg, 'shadow-sm') : 'border-transparent hover:border-border hover:bg-surface-overlay',
-                  )}
-                >
-                  <div className="flex items-center justify-between">
-                    <span className={cn('flex items-center gap-1.5 text-[12px] font-semibold', selected ? src.accent : 'text-foreground')}>
-                      <Icon className="h-4 w-4" />
-                      {src.label}
-                    </span>
-                    <span className={cn('h-2 w-2 rounded-full', up === true ? 'bg-emerald-400 shadow-[0_0_6px_rgba(52,211,153,0.6)]' : up === false ? 'bg-red-400' : 'bg-foreground-faint')} title={up ? 'Online' : up === false ? 'Offline' : 'Unknown'} />
-                  </div>
-                  <p className="text-[9px] leading-snug text-foreground-muted">{src.description}</p>
-                  <div className="flex flex-wrap gap-1">
-                    <Badge variant="default">{src.host}:{src.port}</Badge>
-                    {src.cdc && <Badge variant="accent">CDC</Badge>}
-                  </div>
-                </button>
-              )
-            })}
-          </div>
-        </aside>
-
-        {/* Main panel */}
-        <div className="panel flex min-h-0 min-w-0 flex-1 flex-col overflow-hidden">
-          {/* Engine header */}
-          <div className={cn('flex shrink-0 flex-wrap items-center justify-between gap-2 border-b border-border px-4 py-2.5', meta.accentBg)}>
-            <div>
-              <h2 className={cn('flex items-center gap-2 text-sm font-semibold', meta.accent)}>
-                <meta.icon className="h-4 w-4" />
-                {meta.label}
-                {catalog?.version && (
-                  <span className="font-mono text-[10px] font-normal text-foreground-muted">v{catalog.version.split(' ')[0]?.slice(0, 20)}</span>
+      {/* Horizontal database selector */}
+      <div className="panel shrink-0 px-3 py-2">
+        <p className="mb-2 text-[9px] font-semibold uppercase tracking-widest text-foreground-muted">Source Databases</p>
+        <div className="scroll-x-stable scrollbar-thin flex gap-2 pb-1">
+          {SOURCE_CATALOG.map((src) => {
+            const up = health[src.engine]?.ok
+            const selected = active === src.engine
+            const brand = dbBrandColor(src.engine)
+            return (
+              <button
+                key={src.engine}
+                type="button"
+                onClick={() => setActive(src.engine)}
+                className={cn(
+                  'flex min-w-[148px] shrink-0 flex-col gap-1.5 rounded-lg border px-3 py-2.5 text-left transition-all',
+                  selected
+                    ? 'shadow-md'
+                    : 'border-border/60 hover:border-border hover:bg-surface-overlay',
                 )}
-              </h2>
-              <p className="font-mono text-[10px] text-foreground-muted">
-                {meta.host}:{meta.port} · {meta.database} · container {meta.container}
-              </p>
-            </div>
-            <div className="flex gap-1">
-              {visibleSubTabs.map(({ id, label, icon: Icon }) => (
-                <button
-                  key={id}
-                  type="button"
-                  onClick={() => setSubTab(id)}
-                  className={cn('inline-flex items-center gap-1 rounded-md px-2.5 py-1 text-[10px] font-medium', subTab === id ? subTabActive : subTabIdle)}
-                >
-                  <Icon className="h-3 w-3" /> {label}
-                </button>
-              ))}
-            </div>
-          </div>
-
-          {/* Sub-tab content */}
-          <div className="min-h-0 flex-1 overflow-hidden">
-            {subTab === 'browser' && (
-              <div className="flex h-full min-h-0">
-                {/* Object tree */}
-                <div className="flex w-[280px] shrink-0 flex-col border-r border-border/60">
-                  <div className="flex shrink-0 items-center justify-between border-b border-border/60 px-3 py-2">
-                    <span className="text-[10px] font-semibold uppercase tracking-wider text-foreground-faint">Objects</span>
-                    {catalogLoading && <Loader2 className="h-3 w-3 animate-spin text-foreground-muted" />}
-                  </div>
-                  <div className="scrollbar-thin min-h-0 flex-1 overflow-y-auto p-1.5">
-                    {catalog?.objects?.map((obj) => (
-                      <button
-                        key={obj.fqn}
-                        type="button"
-                        onClick={() => setSelectedObject(obj)}
-                        className={cn(
-                          'mb-0.5 flex w-full items-center gap-1.5 rounded px-2 py-1.5 text-left text-[10px] transition-colors',
-                          selectedObject?.fqn === obj.fqn ? subTabActive : 'hover:bg-surface-overlay',
-                        )}
-                      >
-                        {obj.type === 'node_label' ? <Activity className="h-3 w-3 shrink-0 text-pink-400" /> : <Table2 className="h-3 w-3 shrink-0 text-foreground-muted" />}
-                        <div className="min-w-0 flex-1">
-                          <p className="truncate font-medium text-foreground">{obj.name}</p>
-                          <p className="truncate font-mono text-[8px] text-foreground-faint">{obj.schema}{obj.type === 'relationship' ? ' · rel' : ''}</p>
-                        </div>
-                        <span className="shrink-0 font-mono text-[9px] text-foreground-muted">{fmtCount(obj.row_count)}</span>
-                        <ChevronRight className="h-3 w-3 shrink-0 text-foreground-faint" />
-                      </button>
-                    ))}
-                    {!catalogLoading && !catalog?.objects?.length && (
-                      <p className="p-4 text-center text-[10px] text-foreground-faint">No objects found</p>
+                style={selected ? { borderColor: brand, backgroundColor: `${brand}18`, boxShadow: `0 0 0 1px ${brand}40` } : undefined}
+              >
+                <div className="flex items-center justify-between gap-2">
+                  <span className="flex items-center gap-2 text-[12px] font-semibold text-foreground">
+                    <DbBrandIcon engine={src.engine} size={22} />
+                    {src.label}
+                  </span>
+                  <span
+                    className={cn(
+                      'h-2 w-2 shrink-0 rounded-full',
+                      up === true ? 'bg-emerald-400 shadow-[0_0_6px_rgba(52,211,153,0.6)]' : up === false ? 'bg-red-400' : 'bg-foreground-faint',
                     )}
-                  </div>
+                    title={up ? 'Online' : up === false ? 'Offline' : 'Unknown'}
+                  />
                 </div>
+                <p className="line-clamp-2 text-[9px] leading-snug text-foreground-muted">{src.description}</p>
+                <div className="flex flex-wrap gap-1">
+                  <Badge variant="default">{src.host}:{src.port}</Badge>
+                  {src.cdc && <Badge variant="accent">CDC</Badge>}
+                </div>
+              </button>
+            )
+          })}
+        </div>
+      </div>
 
-                {/* Sample data grid */}
-                <div className="flex min-h-0 min-w-0 flex-1 flex-col">
-                  <div className="flex shrink-0 items-center justify-between border-b border-border/60 px-3 py-2">
-                    <span className="text-[10px] font-semibold text-foreground">
-                      {selectedObject ? (
-                        <>Sample: <span className="font-mono text-docker">{selectedObject.fqn}</span></>
-                      ) : 'Select an object'}
-                    </span>
-                    {sampleLoading && <Loader2 className="h-3 w-3 animate-spin" />}
-                  </div>
-                  <div className="scrollbar-thin min-h-0 flex-1 overflow-auto p-2">
-                    {sample?.ok && sample.columns && (
-                      <>
-                        <p className="mb-1 font-mono text-[9px] text-foreground-faint">
-                          {sample.row_count} rows · {sample.elapsed_ms}ms
-                        </p>
-                        <table className="w-full text-left font-mono text-[10px]">
-                          <thead>
-                            <tr className="sticky top-0 border-b border-border bg-surface-raised text-docker">
-                              {sample.columns.map((c) => <th key={c} className="px-2 py-1">{c}</th>)}
-                            </tr>
-                          </thead>
-                          <tbody>
-                            {sample.rows?.map((row, i) => (
-                              <tr key={i} className="border-b border-border/30 hover:bg-white/5">
-                                {row.map((cell, j) => (
-                                  <td key={j} className="max-w-[200px] truncate px-2 py-1 text-foreground-muted">
-                                    {cell === null || cell === undefined ? 'NULL' : String(cell)}
-                                  </td>
-                                ))}
-                              </tr>
-                            ))}
-                          </tbody>
-                        </table>
-                      </>
-                    )}
-                    {sample && !sample.ok && (
-                      <p className="p-4 text-[11px] text-danger">{sample.error || 'Failed to load sample'}</p>
-                    )}
-                    {!selectedObject && !sampleLoading && (
-                      <p className="py-8 text-center text-[11px] text-foreground-faint">Select a table, collection or label to preview data</p>
-                    )}
-                  </div>
+      {/* Main panel — full width */}
+      <div className="panel flex min-h-0 min-w-0 flex-1 flex-col overflow-hidden">
+        <div
+          className="flex shrink-0 flex-wrap items-center justify-between gap-2 border-b border-border px-4 py-2.5"
+          style={{ backgroundColor: `${dbBrandColor(active)}12` }}
+        >
+          <div>
+            <h2 className="flex items-center gap-2 text-sm font-semibold text-foreground">
+              <DbBrandIcon engine={active} size={18} />
+              {meta.label}
+              {catalog?.version && (
+                <span className="font-mono text-[10px] font-normal text-foreground-muted">v{catalog.version.split(' ')[0]?.slice(0, 20)}</span>
+              )}
+            </h2>
+            <p className="font-mono text-[10px] text-foreground-muted">
+              {meta.host}:{meta.port} · {meta.database} · container {meta.container}
+            </p>
+          </div>
+          <div className="flex flex-wrap gap-1">
+            {visibleSubTabs.map(({ id, label, icon: Icon }) => (
+              <button
+                key={id}
+                type="button"
+                onClick={() => setSubTab(id)}
+                className={cn('inline-flex items-center gap-1 rounded-md px-2.5 py-1 text-[10px] font-medium', subTab === id ? subTabActive : subTabIdle)}
+              >
+                <Icon className="h-3 w-3" /> {label}
+              </button>
+            ))}
+          </div>
+        </div>
+
+        <div className="flex min-h-0 flex-1 flex-col overflow-hidden">
+          {subTab === 'browser' && (
+            <div className="flex h-full min-h-0 flex-1 overflow-hidden">
+              <div className="flex w-[260px] shrink-0 flex-col border-r border-border/60">
+                <div className="flex shrink-0 items-center justify-between border-b border-border/60 px-3 py-2">
+                  <span className="text-[10px] font-semibold uppercase tracking-wider text-foreground-muted">Objects</span>
+                  {catalogLoading && <Loader2 className="h-3 w-3 animate-spin text-foreground-muted" />}
+                </div>
+                <div className="scrollbar-thin min-h-0 flex-1 overflow-y-auto p-1.5">
+                  {catalog?.objects?.map((obj) => (
+                    <button
+                      key={obj.fqn}
+                      type="button"
+                      onClick={() => setSelectedObject(obj)}
+                      className={cn(
+                        'mb-0.5 flex w-full items-center gap-1.5 rounded px-2 py-1.5 text-left text-[10px] transition-colors',
+                        selectedObject?.fqn === obj.fqn ? subTabActive : 'hover:bg-surface-overlay',
+                      )}
+                    >
+                      {obj.type === 'node_label' ? (
+                        <Activity className="h-3 w-3 shrink-0" style={{ color: dbBrandColor('neo4j') }} />
+                      ) : (
+                        <Table2 className="h-3 w-3 shrink-0 text-foreground-muted" />
+                      )}
+                      <div className="min-w-0 flex-1">
+                        <p className="truncate font-medium text-foreground">{obj.name}</p>
+                        <p className="truncate font-mono text-[8px] text-foreground-muted">{obj.schema}{obj.type === 'relationship' ? ' · rel' : ''}</p>
+                      </div>
+                      <span className="shrink-0 font-mono text-[9px] text-foreground-muted">{fmtCount(obj.row_count)}</span>
+                      <ChevronRight className="h-3 w-3 shrink-0 text-foreground-muted" />
+                    </button>
+                  ))}
+                  {!catalogLoading && !catalog?.objects?.length && (
+                    <p className="p-4 text-center text-[10px] text-foreground-muted">No objects found</p>
+                  )}
                 </div>
               </div>
-            )}
 
-            {subTab === 'graph' && active === 'neo4j' && (
-              <Neo4jGraphView />
-            )}
+              <div className="flex h-full min-h-0 min-w-0 flex-1 flex-col overflow-hidden">
+                <div className="flex shrink-0 items-center border-b border-border/60 px-3 py-2">
+                  <span className="text-[10px] font-semibold text-foreground">
+                    {selectedObject ? (
+                      <>Data: <span className="font-mono text-docker">{selectedObject.fqn}</span></>
+                    ) : 'Select an object'}
+                  </span>
+                </div>
+                <DataBrowserGrid
+                  engine={active}
+                  objectFqn={selectedObject?.fqn || ''}
+                  sample={selectedObject ? sample : null}
+                  loading={sampleLoading}
+                  page={page}
+                  pageSize={pageSize}
+                  onPageChange={setPage}
+                  onPageSizeChange={(ps) => { setPageSize(ps); setPage(1) }}
+                  onReload={() => selectedObject && loadSample(active, selectedObject, page, pageSize)}
+                />
+              </div>
+            </div>
+          )}
 
-            {subTab === 'console' && (
-              <SqlWorkbench engine={active} />
-            )}
+          {subTab === 'graph' && active === 'neo4j' && <Neo4jGraphView />}
 
-            {subTab === 'shell' && (
-              <DbShell initialCommand={meta.shellCommand} />
-            )}
-          </div>
+          {subTab === 'console' && <SqlWorkbench engine={active} />}
+
+          {subTab === 'workbench' && (
+            <div className="flex min-h-0 flex-1 overflow-hidden">
+              <LakehouseWorkbench lockedCatalog={ENGINE_CATALOG[active]} />
+            </div>
+          )}
+
+          {subTab === 'generate' && onPulse && (
+            <DataGenPanel embedded focusSource={active} onPulse={onPulse} />
+          )}
+
+          {subTab === 'shell' && <DbShell initialCommand={meta.shellCommand} />}
         </div>
       </div>
     </div>