feat: Trino federation + Hadoop external tables + LLM data catalog

Trino Federation tab (3 sub-views):
- Federated: catalog landscape + a single cross-source SQL that joins
  PostgreSQL + MySQL + MongoDB (region scorecard) — the federation proof,
  computed in the background and cached (large full scans take ~2 min).
- Hadoop Lake: all federated business data materialized as external Iceberg
  tables on HDFS (iceberg.hadoop.*_ext, ~120k rows each) with live, fast
  business analytics (revenue by region/channel, top customers, HR by
  department, supply by type, telemetry averages). Includes a one-click
  "rebuild external tables" job.
- Data Dictionary: every business table + column with masked / visible PII
  badges and categories.

Backend trino_federated.py: /catalogs, /marquee(+refresh), /lake,
/materialize(+status), /dictionary. Name-based PII detection flags raw PII
in derived/lake tables as visible vs physically-masked curated layer.

LLM context: platform_context now emits a full BUSINESS DATA CATALOG section
(tables, columns, types, source row counts, federated scorecard) with exact
per-column masked/visible status, so the assistant knows the data in detail
and what is masked vs not.

ui/src/components/features/TrinoFederationView.tsx

@@ -0,0 +1,373 @@
+import { useCallback, useEffect, useState } from 'react'
+import {
+  Network,
+  Database,
+  Boxes,
+  RefreshCw,
+  Loader2,
+  DollarSign,
+  ShoppingCart,
+  Users,
+  Activity,
+  Layers,
+  ShieldCheck,
+  ShieldAlert,
+  Server,
+  HardDrive,
+  Play,
+} from 'lucide-react'
+import { cn } from '../../lib/utils'
+
+type Bucket = { key: string; count: number; value?: number }
+type SubTab = 'federated' | 'lake' | 'dictionary'
+
+const COLORS = ['#34d399', '#60a5fa', '#f472b6', '#fbbf24', '#a78bfa', '#22d3ee', '#fb7185']
+
+function fmtNum(n?: number | string | null) {
+  if (n == null) return '—'
+  const v = typeof n === 'number' ? n : Number(n)
+  if (Number.isNaN(v)) return String(n)
+  if (Math.abs(v) >= 1e9) return `${(v / 1e9).toFixed(2)}B`
+  if (Math.abs(v) >= 1e6) return `${(v / 1e6).toFixed(1)}M`
+  if (Math.abs(v) >= 1e3) return `${(v / 1e3).toFixed(1)}K`
+  return String(v)
+}
+const fmtMoney = (n?: number | string | null) => (n == null ? '—' : `€${fmtNum(n)}`)
+
+function Panel({ title, subtitle, icon: Icon, children }: { title: string; subtitle?: string; icon?: typeof Database; children: React.ReactNode }) {
+  return (
+    <div className="panel flex min-h-0 flex-col p-3">
+      <div className="mb-2 flex items-center gap-1.5">
+        {Icon && <Icon className="h-3.5 w-3.5 text-docker" />}
+        <h3 className="text-[11px] font-semibold text-foreground">{title}</h3>
+        {subtitle && <span className="ml-auto text-[9px] text-foreground-faint">{subtitle}</span>}
+      </div>
+      {children}
+    </div>
+  )
+}
+
+function BarsH({ data, valueKind, colorByIndex }: { data?: Bucket[]; valueKind?: 'money' | 'num'; colorByIndex?: boolean }) {
+  const d = data || []
+  const useVal = valueKind != null
+  const max = Math.max(1, ...d.map((x) => (useVal && x.value != null ? x.value : x.count)))
+  if (!d.length) return <p className="py-5 text-center text-[10px] text-foreground-faint">No data</p>
+  return (
+    <div className="space-y-1.5">
+      {d.map((x, i) => {
+        const metric = useVal && x.value != null ? x.value : x.count
+        const pct = Math.max(2, (metric / max) * 100)
+        const label = useVal && x.value != null ? (valueKind === 'money' ? fmtMoney(x.value) : fmtNum(x.value)) : fmtNum(x.count)
+        return (
+          <div key={x.key ?? i} className="flex items-center gap-2 text-[10px]">
+            <span className="w-28 shrink-0 truncate text-foreground-muted" title={x.key}>{x.key ?? '—'}</span>
+            <div className="relative h-3.5 flex-1 overflow-hidden rounded bg-surface-overlay">
+              <div className="h-full rounded" style={{ width: `${pct}%`, backgroundColor: colorByIndex ? COLORS[i % COLORS.length] : '#38bdf8' }} />
+            </div>
+            <span className="w-20 shrink-0 text-right font-mono text-foreground">{label}</span>
+          </div>
+        )
+      })}
+    </div>
+  )
+}
+
+function Donut({ data }: { data?: Bucket[] }) {
+  const d = data || []
+  const total = d.reduce((s, x) => s + x.count, 0) || 1
+  let acc = 0
+  const r = 42
+  const c = 2 * Math.PI * r
+  if (!d.length) return <p className="py-5 text-center text-[10px] text-foreground-faint">No data</p>
+  return (
+    <div className="flex items-center gap-4">
+      <svg viewBox="0 0 100 100" className="h-24 w-24 shrink-0 -rotate-90">
+        {d.map((x, i) => {
+          const dash = (x.count / total) * c
+          const seg = <circle key={x.key ?? i} cx="50" cy="50" r={r} fill="none" stroke={COLORS[i % COLORS.length]} strokeWidth="14" strokeDasharray={`${dash} ${c - dash}`} strokeDashoffset={-acc} />
+          acc += dash
+          return seg
+        })}
+      </svg>
+      <div className="min-w-0 flex-1 space-y-1">
+        {d.slice(0, 7).map((x, i) => (
+          <div key={x.key ?? i} className="flex items-center gap-1.5 text-[10px]">
+            <span className="h-2 w-2 shrink-0 rounded-full" style={{ backgroundColor: COLORS[i % COLORS.length] }} />
+            <span className="flex-1 truncate text-foreground-muted">{x.key ?? '—'}</span>
+            <span className="font-mono text-foreground">{((x.count / total) * 100).toFixed(0)}%</span>
+          </div>
+        ))}
+      </div>
+    </div>
+  )
+}
+
+function Kpi({ icon: Icon, label, value, sub, accent }: { icon: typeof Users; label: string; value: string; sub?: string; accent: string }) {
+  return (
+    <div className="panel flex items-center gap-3 px-3 py-2.5">
+      <div className="flex h-9 w-9 shrink-0 items-center justify-center rounded-lg" style={{ backgroundColor: `${accent}1f`, color: accent }}>
+        <Icon className="h-4 w-4" />
+      </div>
+      <div className="min-w-0">
+        <p className="text-[9px] font-semibold uppercase tracking-wider text-foreground-muted">{label}</p>
+        <p className="truncate text-lg font-bold leading-tight text-foreground">{value}</p>
+        {sub && <p className="truncate text-[9px] text-foreground-faint">{sub}</p>}
+      </div>
+    </div>
+  )
+}
+
+export function TrinoFederationView() {
+  const [tab, setTab] = useState<SubTab>('federated')
+  const [catalogs, setCatalogs] = useState<any>(null)
+  const [marquee, setMarquee] = useState<any>(null)
+  const [lake, setLake] = useState<any>(null)
+  const [dict, setDict] = useState<any>(null)
+  const [loading, setLoading] = useState(false)
+  const [matRunning, setMatRunning] = useState(false)
+
+  const loadFederated = useCallback(async () => {
+    setLoading(true)
+    try {
+      const [c, m] = await Promise.all([
+        fetch('/api/federated/catalogs').then((r) => (r.ok ? r.json() : null)),
+        fetch('/api/federated/marquee').then((r) => (r.ok ? r.json() : null)),
+      ])
+      setCatalogs(c)
+      setMarquee(m)
+    } catch { /* */ } finally { setLoading(false) }
+  }, [])
+
+  const loadLake = useCallback(async () => {
+    setLoading(true)
+    try {
+      const r = await fetch('/api/federated/lake')
+      if (r.ok) setLake(await r.json())
+      const ms = await fetch('/api/federated/materialize/status').then((x) => (x.ok ? x.json() : null))
+      setMatRunning(!!ms?.running)
+    } catch { /* */ } finally { setLoading(false) }
+  }, [])
+
+  const loadDict = useCallback(async () => {
+    setLoading(true)
+    try {
+      const r = await fetch('/api/federated/dictionary')
+      if (r.ok) setDict(await r.json())
+    } catch { /* */ } finally { setLoading(false) }
+  }, [])
+
+  useEffect(() => {
+    if (tab === 'federated') loadFederated()
+    else if (tab === 'lake') loadLake()
+    else loadDict()
+  }, [tab, loadFederated, loadLake, loadDict])
+
+  // poll marquee while it is computing
+  useEffect(() => {
+    if (tab !== 'federated' || !marquee?.running) return
+    const t = setTimeout(loadFederated, 5000)
+    return () => clearTimeout(t)
+  }, [tab, marquee, loadFederated])
+
+  // poll materialize while running
+  useEffect(() => {
+    if (tab !== 'lake' || !matRunning) return
+    const t = setTimeout(loadLake, 6000)
+    return () => clearTimeout(t)
+  }, [tab, matRunning, loadLake])
+
+  const rebuildLake = async () => {
+    await fetch('/api/federated/materialize', { method: 'POST' })
+    setMatRunning(true)
+    setTimeout(loadLake, 2000)
+  }
+  const refreshMarquee = async () => {
+    await fetch('/api/federated/marquee/refresh', { method: 'POST' })
+    setTimeout(loadFederated, 1500)
+  }
+
+  const m = marquee?.marquee
+  const totals = catalogs?.source_totals || {}
+
+  return (
+    <div className="scrollbar-thin flex h-full min-h-0 flex-col gap-2 overflow-y-auto p-3">
+      <header className="panel flex shrink-0 flex-wrap items-center justify-between gap-3 px-4 py-3">
+        <div>
+          <h1 className="flex items-center gap-2 text-base font-semibold text-foreground">
+            <Network className="h-5 w-5 text-docker" />
+            Trino Federation &amp; Hadoop Lakehouse
+          </h1>
+          <p className="text-[11px] text-foreground-muted">
+            One SQL engine over every source — federated business analytics, mirrored into Hadoop as external Iceberg tables
+          </p>
+        </div>
+        <div className="flex flex-wrap gap-1">
+          {([
+            { id: 'federated', label: 'Federated', icon: Network },
+            { id: 'lake', label: 'Hadoop Lake', icon: HardDrive },
+            { id: 'dictionary', label: 'Data Dictionary', icon: ShieldCheck },
+          ] as { id: SubTab; label: string; icon: typeof Network }[]).map(({ id, label, icon: Icon }) => (
+            <button key={id} type="button" onClick={() => setTab(id)}
+              className={cn('inline-flex items-center gap-1.5 rounded-md px-3 py-1.5 text-[11px] font-medium transition-colors', tab === id ? 'bg-docker/15 text-docker' : 'text-foreground-muted hover:bg-surface-overlay')}>
+              <Icon className="h-3.5 w-3.5" /> {label}
+            </button>
+          ))}
+        </div>
+      </header>
+
+      {/* ───────── FEDERATED ───────── */}
+      {tab === 'federated' && (
+        <>
+          <div className="grid shrink-0 grid-cols-2 gap-2 md:grid-cols-4">
+            <Kpi icon={Layers} label="Federated catalogs" value={String(catalogs?.count ?? '—')} sub="one Trino engine" accent="#dd00a1" />
+            <Kpi icon={ShoppingCart} label="Orders" value={fmtNum(totals.orders)} sub="PostgreSQL (live)" accent="#fbbf24" />
+            <Kpi icon={Users} label="HR events" value={fmtNum(totals.hr_events)} sub="MySQL (live)" accent="#60a5fa" />
+            <Kpi icon={Boxes} label="Supply events" value={fmtNum(totals.supply_events)} sub="MongoDB (live)" accent="#a78bfa" />
+          </div>
+
+          <Panel title="Federated catalog landscape" subtitle={`${catalogs?.count ?? 0} catalogs`} icon={Database}>
+            <div className="flex flex-wrap gap-2">
+              {(catalogs?.catalogs || []).map((c: any) => (
+                <div key={c.catalog} className="flex min-w-[150px] flex-col gap-0.5 rounded-lg border px-3 py-2" style={{ borderColor: `${c.color}55`, backgroundColor: `${c.color}12` }}>
+                  <span className="flex items-center gap-1.5 text-[11px] font-semibold text-foreground">
+                    <Server className="h-3 w-3" style={{ color: c.color }} /> {c.label}
+                  </span>
+                  <span className="font-mono text-[9px] text-foreground-muted">{c.catalog}</span>
+                  <span className="text-[9px] text-foreground-faint">{c.desc}</span>
+                  {c.rows != null && <span className="mt-0.5 font-mono text-[10px] text-foreground">{fmtNum(c.rows)} rows</span>}
+                </div>
+              ))}
+            </div>
+          </Panel>
+
+          <Panel
+            title="Cross-source federated query — one SQL, three databases"
+            subtitle={m?.elapsed_ms != null ? `${(m.elapsed_ms / 1000).toFixed(1)}s` : marquee?.running ? 'computing…' : ''}
+            icon={Network}
+          >
+            <div className="mb-2 flex flex-wrap items-center gap-1.5">
+              {(m?.catalogs || ['postgres_sales', 'mysql_hr', 'mongodb_supplychain']).map((c: string) => (
+                <span key={c} className="rounded-full bg-docker/15 px-2 py-0.5 text-[9px] font-medium text-docker">{c}</span>
+              ))}
+              <button type="button" onClick={refreshMarquee} className="ml-auto inline-flex items-center gap-1 rounded border border-border px-2 py-0.5 text-[9px] text-foreground-muted hover:bg-surface-overlay">
+                {marquee?.running ? <Loader2 className="h-3 w-3 animate-spin" /> : <RefreshCw className="h-3 w-3" />} re-run
+              </button>
+            </div>
+            <pre className="mb-2 overflow-x-auto rounded-md border border-border bg-surface p-2 font-mono text-[9px] leading-relaxed text-foreground-muted">{marquee?.sql || m?.sql}</pre>
+            {marquee?.running && !m?.rows?.length ? (
+              <p className="py-4 text-center text-[10px] text-foreground-muted"><Loader2 className="mr-1 inline h-3 w-3 animate-spin" /> Federating across live sources… (~1–2 min, cached afterwards)</p>
+            ) : m?.rows?.length ? (
+              <div className="overflow-x-auto">
+                <table className="w-full text-[10px]">
+                  <thead>
+                    <tr className="border-b border-border text-left text-foreground-muted">
+                      <th className="py-1 pr-3">Region</th>
+                      <th className="py-1 pr-3 text-right">Orders</th>
+                      <th className="py-1 pr-3 text-right">Revenue</th>
+                      <th className="py-1 pr-3 text-right">HR events</th>
+                      <th className="py-1 pr-3 text-right">Supply events</th>
+                    </tr>
+                  </thead>
+                  <tbody>
+                    {m.rows.map((r: any, i: number) => (
+                      <tr key={i} className="border-b border-border/40">
+                        <td className="py-1 pr-3 font-medium text-foreground">{r.region}</td>
+                        <td className="py-1 pr-3 text-right font-mono text-foreground">{fmtNum(r.orders)}</td>
+                        <td className="py-1 pr-3 text-right font-mono text-emerald-400">{fmtMoney(r.revenue)}</td>
+                        <td className="py-1 pr-3 text-right font-mono text-foreground">{fmtNum(r.hr_events)}</td>
+                        <td className="py-1 pr-3 text-right font-mono text-foreground">{fmtNum(r.supply_events)}</td>
+                      </tr>
+                    ))}
+                  </tbody>
+                </table>
+                {m.generated_at && <p className="mt-1 text-[9px] text-foreground-faint">as of {new Date(m.generated_at).toLocaleString()} · joined live across PostgreSQL + MySQL + MongoDB</p>}
+              </div>
+            ) : (
+              <p className="py-4 text-center text-[10px] text-foreground-faint">{m?.error || 'No result yet — click re-run.'}</p>
+            )}
+          </Panel>
+        </>
+      )}
+
+      {/* ───────── HADOOP LAKE ───────── */}
+      {tab === 'lake' && (
+        <>
+          <div className="flex shrink-0 flex-wrap items-center justify-between gap-2 px-1">
+            <p className="text-[11px] text-foreground-muted">
+              All federated business data materialized as <span className="text-docker">external Iceberg tables on HDFS</span> — queried live (fast).
+            </p>
+            <button type="button" onClick={rebuildLake} disabled={matRunning}
+              className="inline-flex items-center gap-1.5 rounded-md border border-emerald-500/40 bg-emerald-500/10 px-3 py-1.5 text-[11px] font-medium text-emerald-400 hover:bg-emerald-500/20 disabled:opacity-60">
+              {matRunning ? <Loader2 className="h-3.5 w-3.5 animate-spin" /> : <Play className="h-3.5 w-3.5" />}
+              {matRunning ? 'Materializing…' : 'Rebuild external tables'}
+            </button>
+          </div>
+
+          <Panel title="Hadoop external tables (iceberg.hadoop.*)" subtitle={`${fmtNum(lake?.total_rows)} rows total`} icon={HardDrive}>
+            <div className="grid grid-cols-2 gap-2 md:grid-cols-3 lg:grid-cols-4">
+              {(lake?.tables || []).map((t: any) => (
+                <div key={t.table} className="rounded-lg border border-border/60 bg-surface-overlay/40 px-3 py-2">
+                  <p className="flex items-center gap-1 truncate font-mono text-[10px] text-foreground"><Boxes className="h-3 w-3 text-docker" />{t.table}</p>
+                  <p className="font-mono text-[13px] font-bold text-foreground">{fmtNum(t.rows)}</p>
+                </div>
+              ))}
+              {!lake?.tables?.length && <p className="col-span-full py-4 text-center text-[10px] text-foreground-faint">No external tables yet — click “Rebuild external tables”.</p>}
+            </div>
+          </Panel>
+
+          <div className="grid shrink-0 gap-2 lg:grid-cols-2">
+            <Panel title="Revenue by region" icon={DollarSign}><BarsH data={lake?.orders?.by_region} valueKind="money" colorByIndex /></Panel>
+            <Panel title="Top customers by spend" icon={Users}><BarsH data={lake?.orders?.top_customers} valueKind="money" /></Panel>
+            <div className="grid gap-2 sm:grid-cols-2">
+              <Panel title="Orders by status"><Donut data={lake?.orders?.by_status} /></Panel>
+              <Panel title="Revenue by channel"><BarsH data={lake?.orders?.by_channel} valueKind="money" colorByIndex /></Panel>
+            </div>
+            <Panel title="Employees by department" icon={Users}><BarsH data={lake?.hr?.by_department} colorByIndex /></Panel>
+            <Panel title="Supply events by type" icon={Boxes}><BarsH data={lake?.supply?.by_type} colorByIndex /></Panel>
+            <Panel title="Telemetry — avg value by metric" icon={Activity}><BarsH data={lake?.telemetry?.by_metric} valueKind="num" /></Panel>
+          </div>
+        </>
+      )}
+
+      {/* ───────── DICTIONARY ───────── */}
+      {tab === 'dictionary' && (
+        <>
+          <div className="grid shrink-0 grid-cols-2 gap-2 md:grid-cols-4">
+            <Kpi icon={Layers} label="Tables" value={String(dict?.summary?.tables ?? '—')} accent="#5b8def" />
+            <Kpi icon={Database} label="Columns" value={String(dict?.summary?.columns ?? '—')} accent="#34d399" />
+            <Kpi icon={ShieldAlert} label="PII columns" value={String(dict?.summary?.pii_columns ?? '—')} accent="#fbbf24" />
+            <Kpi icon={ShieldCheck} label="Masked" value={String(dict?.summary?.masked_columns ?? '—')} sub="hidden from LLM" accent="#f472b6" />
+          </div>
+          <p className="px-1 text-[10px] text-foreground-muted">
+            This is exactly what the assistant knows about your data — every column, its type, and whether it is <span className="text-amber-400">masked</span> or visible.
+          </p>
+          <div className="grid gap-2 lg:grid-cols-2">
+            {(dict?.tables || []).map((t: any) => (
+              <Panel key={t.fqn} title={t.fqn} subtitle={`${t.masked_count}/${t.pii_count} PII masked`} icon={Database}>
+                <p className="mb-1.5 text-[9px] text-foreground-faint">{t.engine} · {t.desc}</p>
+                <div className="flex flex-wrap gap-1">
+                  {t.columns.map((c: any) => (
+                    <span key={c.name}
+                      className={cn('inline-flex items-center gap-1 rounded border px-1.5 py-0.5 font-mono text-[9px]',
+                        c.masked ? 'border-amber-500/40 bg-amber-500/10 text-amber-300'
+                          : c.pii ? 'border-rose-500/40 bg-rose-500/10 text-rose-300'
+                            : 'border-border/60 text-foreground-muted')}
+                      title={`${c.type}${c.category ? ` · ${c.category}` : ''}${c.masked ? ' · MASKED' : c.pii ? ' · PII visible' : ''}`}>
+                      {c.masked && <ShieldCheck className="h-2.5 w-2.5" />}
+                      {!c.masked && c.pii && <ShieldAlert className="h-2.5 w-2.5" />}
+                      {c.name}
+                    </span>
+                  ))}
+                </div>
+              </Panel>
+            ))}
+          </div>
+        </>
+      )}
+
+      {loading && !catalogs && !lake && !dict && (
+        <div className="flex flex-1 items-center justify-center text-foreground-muted"><Loader2 className="h-5 w-5 animate-spin" /></div>
+      )}
+    </div>
+  )
+}

ui/src/components/layout/SideNav.tsx

@@ -1,5 +1,5 @@
 import { useEffect, useState } from 'react'
-import { Database, DatabaseZap, HardDrive, Search, LayoutDashboard, MessageSquare, Server, TerminalSquare, Activity, GitBranch, ExternalLink, LineChart } from 'lucide-react'
+import { Database, DatabaseZap, HardDrive, Search, LayoutDashboard, MessageSquare, Server, TerminalSquare, Activity, GitBranch, ExternalLink, LineChart, Network } from 'lucide-react'
 import type { GpuStatus, WorkloadData } from '../../types'
 import type { GpuLiveMetrics } from '../../hooks/useLiveMetrics'
 import { cn } from '../../lib/utils'
@@ -7,7 +7,7 @@ import { viewTabActive, viewTabIdle } from '../../lib/tabActive'
 import { GpuMatrixPanel } from '../features/GpuMatrixPanel'
 import { LabHealthPanel } from '../features/LabHealthPanel'
 
-type MainView = 'platform' | 'dataquality' | 'knowledge' | 'storage' | 'hdfs' | 'search' | 'approvals' | 'changes' | 'dataflow' | 'datasources' | 'dataexplorer'
+type MainView = 'platform' | 'dataquality' | 'knowledge' | 'storage' | 'hdfs' | 'search' | 'approvals' | 'changes' | 'dataflow' | 'datasources' | 'dataexplorer' | 'trino'
 
 type Props = {
   workload: WorkloadData | null
@@ -27,6 +27,7 @@ const VIEWS: { id: MainView; label: string; icon: typeof LayoutDashboard }[] = [
   { id: 'platform', label: 'Data Platform', icon: LayoutDashboard },
   { id: 'datasources', label: 'Data Hub', icon: Database },
   { id: 'dataexplorer', label: 'Data Explorer', icon: LineChart },
+  { id: 'trino', label: 'Trino Federation', icon: Network },
   { id: 'changes', label: 'Live Changes', icon: Activity },
   { id: 'dataflow', label: 'Data Flow', icon: GitBranch },
   { id: 'dataquality', label: 'Data Quality', icon: DatabaseZap },