feat: Trino federation + Hadoop external tables + LLM data catalog

Trino Federation tab (3 sub-views):
- Federated: catalog landscape + a single cross-source SQL that joins
  PostgreSQL + MySQL + MongoDB (region scorecard) — the federation proof,
  computed in the background and cached (large full scans take ~2 min).
- Hadoop Lake: all federated business data materialized as external Iceberg
  tables on HDFS (iceberg.hadoop.*_ext, ~120k rows each) with live, fast
  business analytics (revenue by region/channel, top customers, HR by
  department, supply by type, telemetry averages). Includes a one-click
  "rebuild external tables" job.
- Data Dictionary: every business table + column with masked / visible PII
  badges and categories.

Backend trino_federated.py: /catalogs, /marquee(+refresh), /lake,
/materialize(+status), /dictionary. Name-based PII detection flags raw PII
in derived/lake tables as visible vs physically-masked curated layer.

LLM context: platform_context now emits a full BUSINESS DATA CATALOG section
(tables, columns, types, source row counts, federated scorecard) with exact
per-column masked/visible status, so the assistant knows the data in detail
and what is masked vs not.
This commit is contained in:
mo
2026-06-28 18:01:25 +00:00
parent 8d28695868
commit ea3e59cf9c
8 changed files with 855 additions and 5 deletions
+3
View File
@@ -19,6 +19,7 @@ import { ChangesView } from './components/features/ChangesView'
import { DataFlowView } from './components/features/DataFlowView'
import { SearchView } from './components/features/SearchView'
import { DataExplorerView } from './components/features/DataExplorerView'
import { TrinoFederationView } from './components/features/TrinoFederationView'
import { DataHubView } from './components/features/DataHubView'
import { SshTerminal } from './components/features/SshTerminal'
import { TerminalDock } from './components/features/TerminalDock'
@@ -144,6 +145,8 @@ export default function App() {
<SearchView />
) : cc.mainView === 'dataexplorer' ? (
<DataExplorerView />
) : cc.mainView === 'trino' ? (
<TrinoFederationView />
) : (
<ApprovalInbox agents={cc.agents} livePending={cc.approvals} onDecide={cc.decide} />
)}
@@ -0,0 +1,373 @@
import { useCallback, useEffect, useState } from 'react'
import {
Network,
Database,
Boxes,
RefreshCw,
Loader2,
DollarSign,
ShoppingCart,
Users,
Activity,
Layers,
ShieldCheck,
ShieldAlert,
Server,
HardDrive,
Play,
} from 'lucide-react'
import { cn } from '../../lib/utils'
type Bucket = { key: string; count: number; value?: number }
type SubTab = 'federated' | 'lake' | 'dictionary'
const COLORS = ['#34d399', '#60a5fa', '#f472b6', '#fbbf24', '#a78bfa', '#22d3ee', '#fb7185']
function fmtNum(n?: number | string | null) {
if (n == null) return '—'
const v = typeof n === 'number' ? n : Number(n)
if (Number.isNaN(v)) return String(n)
if (Math.abs(v) >= 1e9) return `${(v / 1e9).toFixed(2)}B`
if (Math.abs(v) >= 1e6) return `${(v / 1e6).toFixed(1)}M`
if (Math.abs(v) >= 1e3) return `${(v / 1e3).toFixed(1)}K`
return String(v)
}
const fmtMoney = (n?: number | string | null) => (n == null ? '—' : `${fmtNum(n)}`)
function Panel({ title, subtitle, icon: Icon, children }: { title: string; subtitle?: string; icon?: typeof Database; children: React.ReactNode }) {
return (
<div className="panel flex min-h-0 flex-col p-3">
<div className="mb-2 flex items-center gap-1.5">
{Icon && <Icon className="h-3.5 w-3.5 text-docker" />}
<h3 className="text-[11px] font-semibold text-foreground">{title}</h3>
{subtitle && <span className="ml-auto text-[9px] text-foreground-faint">{subtitle}</span>}
</div>
{children}
</div>
)
}
function BarsH({ data, valueKind, colorByIndex }: { data?: Bucket[]; valueKind?: 'money' | 'num'; colorByIndex?: boolean }) {
const d = data || []
const useVal = valueKind != null
const max = Math.max(1, ...d.map((x) => (useVal && x.value != null ? x.value : x.count)))
if (!d.length) return <p className="py-5 text-center text-[10px] text-foreground-faint">No data</p>
return (
<div className="space-y-1.5">
{d.map((x, i) => {
const metric = useVal && x.value != null ? x.value : x.count
const pct = Math.max(2, (metric / max) * 100)
const label = useVal && x.value != null ? (valueKind === 'money' ? fmtMoney(x.value) : fmtNum(x.value)) : fmtNum(x.count)
return (
<div key={x.key ?? i} className="flex items-center gap-2 text-[10px]">
<span className="w-28 shrink-0 truncate text-foreground-muted" title={x.key}>{x.key ?? '—'}</span>
<div className="relative h-3.5 flex-1 overflow-hidden rounded bg-surface-overlay">
<div className="h-full rounded" style={{ width: `${pct}%`, backgroundColor: colorByIndex ? COLORS[i % COLORS.length] : '#38bdf8' }} />
</div>
<span className="w-20 shrink-0 text-right font-mono text-foreground">{label}</span>
</div>
)
})}
</div>
)
}
function Donut({ data }: { data?: Bucket[] }) {
const d = data || []
const total = d.reduce((s, x) => s + x.count, 0) || 1
let acc = 0
const r = 42
const c = 2 * Math.PI * r
if (!d.length) return <p className="py-5 text-center text-[10px] text-foreground-faint">No data</p>
return (
<div className="flex items-center gap-4">
<svg viewBox="0 0 100 100" className="h-24 w-24 shrink-0 -rotate-90">
{d.map((x, i) => {
const dash = (x.count / total) * c
const seg = <circle key={x.key ?? i} cx="50" cy="50" r={r} fill="none" stroke={COLORS[i % COLORS.length]} strokeWidth="14" strokeDasharray={`${dash} ${c - dash}`} strokeDashoffset={-acc} />
acc += dash
return seg
})}
</svg>
<div className="min-w-0 flex-1 space-y-1">
{d.slice(0, 7).map((x, i) => (
<div key={x.key ?? i} className="flex items-center gap-1.5 text-[10px]">
<span className="h-2 w-2 shrink-0 rounded-full" style={{ backgroundColor: COLORS[i % COLORS.length] }} />
<span className="flex-1 truncate text-foreground-muted">{x.key ?? '—'}</span>
<span className="font-mono text-foreground">{((x.count / total) * 100).toFixed(0)}%</span>
</div>
))}
</div>
</div>
)
}
function Kpi({ icon: Icon, label, value, sub, accent }: { icon: typeof Users; label: string; value: string; sub?: string; accent: string }) {
return (
<div className="panel flex items-center gap-3 px-3 py-2.5">
<div className="flex h-9 w-9 shrink-0 items-center justify-center rounded-lg" style={{ backgroundColor: `${accent}1f`, color: accent }}>
<Icon className="h-4 w-4" />
</div>
<div className="min-w-0">
<p className="text-[9px] font-semibold uppercase tracking-wider text-foreground-muted">{label}</p>
<p className="truncate text-lg font-bold leading-tight text-foreground">{value}</p>
{sub && <p className="truncate text-[9px] text-foreground-faint">{sub}</p>}
</div>
</div>
)
}
export function TrinoFederationView() {
const [tab, setTab] = useState<SubTab>('federated')
const [catalogs, setCatalogs] = useState<any>(null)
const [marquee, setMarquee] = useState<any>(null)
const [lake, setLake] = useState<any>(null)
const [dict, setDict] = useState<any>(null)
const [loading, setLoading] = useState(false)
const [matRunning, setMatRunning] = useState(false)
const loadFederated = useCallback(async () => {
setLoading(true)
try {
const [c, m] = await Promise.all([
fetch('/api/federated/catalogs').then((r) => (r.ok ? r.json() : null)),
fetch('/api/federated/marquee').then((r) => (r.ok ? r.json() : null)),
])
setCatalogs(c)
setMarquee(m)
} catch { /* */ } finally { setLoading(false) }
}, [])
const loadLake = useCallback(async () => {
setLoading(true)
try {
const r = await fetch('/api/federated/lake')
if (r.ok) setLake(await r.json())
const ms = await fetch('/api/federated/materialize/status').then((x) => (x.ok ? x.json() : null))
setMatRunning(!!ms?.running)
} catch { /* */ } finally { setLoading(false) }
}, [])
const loadDict = useCallback(async () => {
setLoading(true)
try {
const r = await fetch('/api/federated/dictionary')
if (r.ok) setDict(await r.json())
} catch { /* */ } finally { setLoading(false) }
}, [])
useEffect(() => {
if (tab === 'federated') loadFederated()
else if (tab === 'lake') loadLake()
else loadDict()
}, [tab, loadFederated, loadLake, loadDict])
// poll marquee while it is computing
useEffect(() => {
if (tab !== 'federated' || !marquee?.running) return
const t = setTimeout(loadFederated, 5000)
return () => clearTimeout(t)
}, [tab, marquee, loadFederated])
// poll materialize while running
useEffect(() => {
if (tab !== 'lake' || !matRunning) return
const t = setTimeout(loadLake, 6000)
return () => clearTimeout(t)
}, [tab, matRunning, loadLake])
const rebuildLake = async () => {
await fetch('/api/federated/materialize', { method: 'POST' })
setMatRunning(true)
setTimeout(loadLake, 2000)
}
const refreshMarquee = async () => {
await fetch('/api/federated/marquee/refresh', { method: 'POST' })
setTimeout(loadFederated, 1500)
}
const m = marquee?.marquee
const totals = catalogs?.source_totals || {}
return (
<div className="scrollbar-thin flex h-full min-h-0 flex-col gap-2 overflow-y-auto p-3">
<header className="panel flex shrink-0 flex-wrap items-center justify-between gap-3 px-4 py-3">
<div>
<h1 className="flex items-center gap-2 text-base font-semibold text-foreground">
<Network className="h-5 w-5 text-docker" />
Trino Federation &amp; Hadoop Lakehouse
</h1>
<p className="text-[11px] text-foreground-muted">
One SQL engine over every source federated business analytics, mirrored into Hadoop as external Iceberg tables
</p>
</div>
<div className="flex flex-wrap gap-1">
{([
{ id: 'federated', label: 'Federated', icon: Network },
{ id: 'lake', label: 'Hadoop Lake', icon: HardDrive },
{ id: 'dictionary', label: 'Data Dictionary', icon: ShieldCheck },
] as { id: SubTab; label: string; icon: typeof Network }[]).map(({ id, label, icon: Icon }) => (
<button key={id} type="button" onClick={() => setTab(id)}
className={cn('inline-flex items-center gap-1.5 rounded-md px-3 py-1.5 text-[11px] font-medium transition-colors', tab === id ? 'bg-docker/15 text-docker' : 'text-foreground-muted hover:bg-surface-overlay')}>
<Icon className="h-3.5 w-3.5" /> {label}
</button>
))}
</div>
</header>
{/* ───────── FEDERATED ───────── */}
{tab === 'federated' && (
<>
<div className="grid shrink-0 grid-cols-2 gap-2 md:grid-cols-4">
<Kpi icon={Layers} label="Federated catalogs" value={String(catalogs?.count ?? '—')} sub="one Trino engine" accent="#dd00a1" />
<Kpi icon={ShoppingCart} label="Orders" value={fmtNum(totals.orders)} sub="PostgreSQL (live)" accent="#fbbf24" />
<Kpi icon={Users} label="HR events" value={fmtNum(totals.hr_events)} sub="MySQL (live)" accent="#60a5fa" />
<Kpi icon={Boxes} label="Supply events" value={fmtNum(totals.supply_events)} sub="MongoDB (live)" accent="#a78bfa" />
</div>
<Panel title="Federated catalog landscape" subtitle={`${catalogs?.count ?? 0} catalogs`} icon={Database}>
<div className="flex flex-wrap gap-2">
{(catalogs?.catalogs || []).map((c: any) => (
<div key={c.catalog} className="flex min-w-[150px] flex-col gap-0.5 rounded-lg border px-3 py-2" style={{ borderColor: `${c.color}55`, backgroundColor: `${c.color}12` }}>
<span className="flex items-center gap-1.5 text-[11px] font-semibold text-foreground">
<Server className="h-3 w-3" style={{ color: c.color }} /> {c.label}
</span>
<span className="font-mono text-[9px] text-foreground-muted">{c.catalog}</span>
<span className="text-[9px] text-foreground-faint">{c.desc}</span>
{c.rows != null && <span className="mt-0.5 font-mono text-[10px] text-foreground">{fmtNum(c.rows)} rows</span>}
</div>
))}
</div>
</Panel>
<Panel
title="Cross-source federated query — one SQL, three databases"
subtitle={m?.elapsed_ms != null ? `${(m.elapsed_ms / 1000).toFixed(1)}s` : marquee?.running ? 'computing…' : ''}
icon={Network}
>
<div className="mb-2 flex flex-wrap items-center gap-1.5">
{(m?.catalogs || ['postgres_sales', 'mysql_hr', 'mongodb_supplychain']).map((c: string) => (
<span key={c} className="rounded-full bg-docker/15 px-2 py-0.5 text-[9px] font-medium text-docker">{c}</span>
))}
<button type="button" onClick={refreshMarquee} className="ml-auto inline-flex items-center gap-1 rounded border border-border px-2 py-0.5 text-[9px] text-foreground-muted hover:bg-surface-overlay">
{marquee?.running ? <Loader2 className="h-3 w-3 animate-spin" /> : <RefreshCw className="h-3 w-3" />} re-run
</button>
</div>
<pre className="mb-2 overflow-x-auto rounded-md border border-border bg-surface p-2 font-mono text-[9px] leading-relaxed text-foreground-muted">{marquee?.sql || m?.sql}</pre>
{marquee?.running && !m?.rows?.length ? (
<p className="py-4 text-center text-[10px] text-foreground-muted"><Loader2 className="mr-1 inline h-3 w-3 animate-spin" /> Federating across live sources (~12 min, cached afterwards)</p>
) : m?.rows?.length ? (
<div className="overflow-x-auto">
<table className="w-full text-[10px]">
<thead>
<tr className="border-b border-border text-left text-foreground-muted">
<th className="py-1 pr-3">Region</th>
<th className="py-1 pr-3 text-right">Orders</th>
<th className="py-1 pr-3 text-right">Revenue</th>
<th className="py-1 pr-3 text-right">HR events</th>
<th className="py-1 pr-3 text-right">Supply events</th>
</tr>
</thead>
<tbody>
{m.rows.map((r: any, i: number) => (
<tr key={i} className="border-b border-border/40">
<td className="py-1 pr-3 font-medium text-foreground">{r.region}</td>
<td className="py-1 pr-3 text-right font-mono text-foreground">{fmtNum(r.orders)}</td>
<td className="py-1 pr-3 text-right font-mono text-emerald-400">{fmtMoney(r.revenue)}</td>
<td className="py-1 pr-3 text-right font-mono text-foreground">{fmtNum(r.hr_events)}</td>
<td className="py-1 pr-3 text-right font-mono text-foreground">{fmtNum(r.supply_events)}</td>
</tr>
))}
</tbody>
</table>
{m.generated_at && <p className="mt-1 text-[9px] text-foreground-faint">as of {new Date(m.generated_at).toLocaleString()} · joined live across PostgreSQL + MySQL + MongoDB</p>}
</div>
) : (
<p className="py-4 text-center text-[10px] text-foreground-faint">{m?.error || 'No result yet — click re-run.'}</p>
)}
</Panel>
</>
)}
{/* ───────── HADOOP LAKE ───────── */}
{tab === 'lake' && (
<>
<div className="flex shrink-0 flex-wrap items-center justify-between gap-2 px-1">
<p className="text-[11px] text-foreground-muted">
All federated business data materialized as <span className="text-docker">external Iceberg tables on HDFS</span> queried live (fast).
</p>
<button type="button" onClick={rebuildLake} disabled={matRunning}
className="inline-flex items-center gap-1.5 rounded-md border border-emerald-500/40 bg-emerald-500/10 px-3 py-1.5 text-[11px] font-medium text-emerald-400 hover:bg-emerald-500/20 disabled:opacity-60">
{matRunning ? <Loader2 className="h-3.5 w-3.5 animate-spin" /> : <Play className="h-3.5 w-3.5" />}
{matRunning ? 'Materializing…' : 'Rebuild external tables'}
</button>
</div>
<Panel title="Hadoop external tables (iceberg.hadoop.*)" subtitle={`${fmtNum(lake?.total_rows)} rows total`} icon={HardDrive}>
<div className="grid grid-cols-2 gap-2 md:grid-cols-3 lg:grid-cols-4">
{(lake?.tables || []).map((t: any) => (
<div key={t.table} className="rounded-lg border border-border/60 bg-surface-overlay/40 px-3 py-2">
<p className="flex items-center gap-1 truncate font-mono text-[10px] text-foreground"><Boxes className="h-3 w-3 text-docker" />{t.table}</p>
<p className="font-mono text-[13px] font-bold text-foreground">{fmtNum(t.rows)}</p>
</div>
))}
{!lake?.tables?.length && <p className="col-span-full py-4 text-center text-[10px] text-foreground-faint">No external tables yet click Rebuild external tables.</p>}
</div>
</Panel>
<div className="grid shrink-0 gap-2 lg:grid-cols-2">
<Panel title="Revenue by region" icon={DollarSign}><BarsH data={lake?.orders?.by_region} valueKind="money" colorByIndex /></Panel>
<Panel title="Top customers by spend" icon={Users}><BarsH data={lake?.orders?.top_customers} valueKind="money" /></Panel>
<div className="grid gap-2 sm:grid-cols-2">
<Panel title="Orders by status"><Donut data={lake?.orders?.by_status} /></Panel>
<Panel title="Revenue by channel"><BarsH data={lake?.orders?.by_channel} valueKind="money" colorByIndex /></Panel>
</div>
<Panel title="Employees by department" icon={Users}><BarsH data={lake?.hr?.by_department} colorByIndex /></Panel>
<Panel title="Supply events by type" icon={Boxes}><BarsH data={lake?.supply?.by_type} colorByIndex /></Panel>
<Panel title="Telemetry — avg value by metric" icon={Activity}><BarsH data={lake?.telemetry?.by_metric} valueKind="num" /></Panel>
</div>
</>
)}
{/* ───────── DICTIONARY ───────── */}
{tab === 'dictionary' && (
<>
<div className="grid shrink-0 grid-cols-2 gap-2 md:grid-cols-4">
<Kpi icon={Layers} label="Tables" value={String(dict?.summary?.tables ?? '—')} accent="#5b8def" />
<Kpi icon={Database} label="Columns" value={String(dict?.summary?.columns ?? '—')} accent="#34d399" />
<Kpi icon={ShieldAlert} label="PII columns" value={String(dict?.summary?.pii_columns ?? '—')} accent="#fbbf24" />
<Kpi icon={ShieldCheck} label="Masked" value={String(dict?.summary?.masked_columns ?? '—')} sub="hidden from LLM" accent="#f472b6" />
</div>
<p className="px-1 text-[10px] text-foreground-muted">
This is exactly what the assistant knows about your data every column, its type, and whether it is <span className="text-amber-400">masked</span> or visible.
</p>
<div className="grid gap-2 lg:grid-cols-2">
{(dict?.tables || []).map((t: any) => (
<Panel key={t.fqn} title={t.fqn} subtitle={`${t.masked_count}/${t.pii_count} PII masked`} icon={Database}>
<p className="mb-1.5 text-[9px] text-foreground-faint">{t.engine} · {t.desc}</p>
<div className="flex flex-wrap gap-1">
{t.columns.map((c: any) => (
<span key={c.name}
className={cn('inline-flex items-center gap-1 rounded border px-1.5 py-0.5 font-mono text-[9px]',
c.masked ? 'border-amber-500/40 bg-amber-500/10 text-amber-300'
: c.pii ? 'border-rose-500/40 bg-rose-500/10 text-rose-300'
: 'border-border/60 text-foreground-muted')}
title={`${c.type}${c.category ? ` · ${c.category}` : ''}${c.masked ? ' · MASKED' : c.pii ? ' · PII visible' : ''}`}>
{c.masked && <ShieldCheck className="h-2.5 w-2.5" />}
{!c.masked && c.pii && <ShieldAlert className="h-2.5 w-2.5" />}
{c.name}
</span>
))}
</div>
</Panel>
))}
</div>
</>
)}
{loading && !catalogs && !lake && !dict && (
<div className="flex flex-1 items-center justify-center text-foreground-muted"><Loader2 className="h-5 w-5 animate-spin" /></div>
)}
</div>
)
}
+3 -2
View File
@@ -1,5 +1,5 @@
import { useEffect, useState } from 'react'
import { Database, DatabaseZap, HardDrive, Search, LayoutDashboard, MessageSquare, Server, TerminalSquare, Activity, GitBranch, ExternalLink, LineChart } from 'lucide-react'
import { Database, DatabaseZap, HardDrive, Search, LayoutDashboard, MessageSquare, Server, TerminalSquare, Activity, GitBranch, ExternalLink, LineChart, Network } from 'lucide-react'
import type { GpuStatus, WorkloadData } from '../../types'
import type { GpuLiveMetrics } from '../../hooks/useLiveMetrics'
import { cn } from '../../lib/utils'
@@ -7,7 +7,7 @@ import { viewTabActive, viewTabIdle } from '../../lib/tabActive'
import { GpuMatrixPanel } from '../features/GpuMatrixPanel'
import { LabHealthPanel } from '../features/LabHealthPanel'
type MainView = 'platform' | 'dataquality' | 'knowledge' | 'storage' | 'hdfs' | 'search' | 'approvals' | 'changes' | 'dataflow' | 'datasources' | 'dataexplorer'
type MainView = 'platform' | 'dataquality' | 'knowledge' | 'storage' | 'hdfs' | 'search' | 'approvals' | 'changes' | 'dataflow' | 'datasources' | 'dataexplorer' | 'trino'
type Props = {
workload: WorkloadData | null
@@ -27,6 +27,7 @@ const VIEWS: { id: MainView; label: string; icon: typeof LayoutDashboard }[] = [
{ id: 'platform', label: 'Data Platform', icon: LayoutDashboard },
{ id: 'datasources', label: 'Data Hub', icon: Database },
{ id: 'dataexplorer', label: 'Data Explorer', icon: LineChart },
{ id: 'trino', label: 'Trino Federation', icon: Network },
{ id: 'changes', label: 'Live Changes', icon: Activity },
{ id: 'dataflow', label: 'Data Flow', icon: GitBranch },
{ id: 'dataquality', label: 'Data Quality', icon: DatabaseZap },
+1 -1
View File
@@ -60,7 +60,7 @@ export function useCommandCenter() {
const [selectedNode, setSelectedNode] = useState<TopologyNode | null>(null)
const [nodeDetail, setNodeDetail] = useState<NodeDetail | null>(null)
const [nodeBusy, setNodeBusy] = useState(false)
const [mainView, setMainView] = useState<'platform' | 'approvals' | 'dataquality' | 'knowledge' | 'storage' | 'hdfs' | 'search' | 'changes' | 'dataflow' | 'datasources' | 'dataexplorer'>('platform')
const [mainView, setMainView] = useState<'platform' | 'approvals' | 'dataquality' | 'knowledge' | 'storage' | 'hdfs' | 'search' | 'changes' | 'dataflow' | 'datasources' | 'dataexplorer' | 'trino'>('platform')
const [changes, setChanges] = useState<CdcChange[]>([])
const [genPulse, setGenPulse] = useState(false)
const genPulseTimer = useRef<ReturnType<typeof setTimeout> | null>(null)