feat: Spark Workbench everywhere, autonomous Hadoop offload & LLM masking-aware
- Data Hub with Hadoop tab (HDFS/Iceberg browser, Spark, pipeline) - Databricks-style Lakehouse Workbench (Trino engine, live exec matrix, materialize to Iceberg/S3); reused & embedded in every source-DB UI - HDFS -> Kafka -> Spark -> Iceberg/S3 pipeline; WebHDFS hostname resolver - Data Flow master pulse switch (Run/Pause/Stop) gating animated edges - Data Custodian autonomous Hadoop offload loop (batch counterpart to CDC), pulsing source -> HDFS edges; toggle in Data Flow - LLM now autonomously aware of all latest platform changes (live platform context) and enforces masking policy: never reveals masked PII, still answers helpfully with aggregates/explanations
This commit is contained in:
mo
@@ -0,0 +1,592 @@
|
||||
import { useCallback, useEffect, useRef, useState } from 'react'
|
||||
import {
|
||||
Activity,
|
||||
Cpu,
|
||||
Database,
|
||||
ExternalLink,
|
||||
Gauge,
|
||||
HardDrive,
|
||||
Layers,
|
||||
Loader2,
|
||||
Play,
|
||||
Plus,
|
||||
RefreshCw,
|
||||
Save,
|
||||
Server,
|
||||
Square,
|
||||
Table2,
|
||||
Trash2,
|
||||
Zap,
|
||||
} from 'lucide-react'
|
||||
import type { SparkLive, SparkRun, SparkRunStats } from '../../types'
|
||||
import {
|
||||
cancelSparkRun,
|
||||
createSparkRun,
|
||||
fetchSparkCatalogs,
|
||||
fetchSparkColumns,
|
||||
fetchSparkLive,
|
||||
fetchSparkRun,
|
||||
fetchSparkRuns,
|
||||
fetchSparkSchemas,
|
||||
fetchSparkTables,
|
||||
} from '../../lib/api'
|
||||
import { cn } from '../../lib/utils'
|
||||
import { subTabActive, subTabIdle } from '../../lib/tabActive'
|
||||
|
||||
type Tab = 'workbench' | 'cluster' | 'runs' | 'ui'
|
||||
type Operation = 'preview' | 'filter' | 'aggregate' | 'profile' | 'join' | 'sql'
|
||||
type Column = { name: string; type: string }
|
||||
type Metric = { fn: string; col: string; alias?: string }
|
||||
|
||||
const OPERATIONS: { id: Operation; label: string; desc: string }[] = [
|
||||
{ id: 'preview', label: 'Preview', desc: 'Sample rows from a table' },
|
||||
{ id: 'filter', label: 'Filter', desc: 'WHERE predicate on a table' },
|
||||
{ id: 'aggregate', label: 'Aggregate', desc: 'Group by + sum/avg/count/min/max' },
|
||||
{ id: 'profile', label: 'Profile', desc: 'Row count, distinct & non-null per column' },
|
||||
{ id: 'join', label: 'Join', desc: 'Join two tables on keys' },
|
||||
{ id: 'sql', label: 'SQL', desc: 'Run arbitrary distributed SQL' },
|
||||
]
|
||||
|
||||
const AGG_FNS = ['count', 'sum', 'avg', 'min', 'max', 'approx_distinct', 'count_distinct']
|
||||
|
||||
function fmtNum(n?: number | null) {
|
||||
if (n == null) return '—'
|
||||
if (n >= 1e9) return `${(n / 1e9).toFixed(2)}B`
|
||||
if (n >= 1e6) return `${(n / 1e6).toFixed(2)}M`
|
||||
if (n >= 1e3) return `${(n / 1e3).toFixed(1)}K`
|
||||
return String(n)
|
||||
}
|
||||
function fmtBytes(n?: number | null) {
|
||||
if (n == null) return '—'
|
||||
let v = n
|
||||
for (const u of ['B', 'KB', 'MB', 'GB', 'TB']) {
|
||||
if (v < 1024) return `${v.toFixed(u === 'B' ? 0 : 1)} ${u}`
|
||||
v /= 1024
|
||||
}
|
||||
return `${v.toFixed(1)} PB`
|
||||
}
|
||||
function fmtMs(n?: number | null) {
|
||||
if (n == null) return '—'
|
||||
if (n < 1000) return `${n} ms`
|
||||
if (n < 60000) return `${(n / 1000).toFixed(1)} s`
|
||||
return `${(n / 60000).toFixed(1)} m`
|
||||
}
|
||||
|
||||
const STATE_COLOR: Record<string, string> = {
|
||||
QUEUED: 'text-amber-300 bg-amber-500/15',
|
||||
RUNNING: 'text-sky-300 bg-sky-500/15',
|
||||
FINISHED: 'text-emerald-300 bg-emerald-500/15',
|
||||
FAILED: 'text-rose-300 bg-rose-500/15',
|
||||
CANCELED: 'text-foreground-muted bg-surface-overlay',
|
||||
}
|
||||
|
||||
export function SparkView({ embedded }: { embedded?: boolean }) {
|
||||
const [tab, setTab] = useState<Tab>('workbench')
|
||||
const [live, setLive] = useState<SparkLive | null>(null)
|
||||
|
||||
const loadLive = useCallback(async () => {
|
||||
const l = await fetchSparkLive()
|
||||
if (l) setLive(l)
|
||||
}, [])
|
||||
|
||||
useEffect(() => {
|
||||
loadLive()
|
||||
const iv = setInterval(loadLive, 4000)
|
||||
return () => clearInterval(iv)
|
||||
}, [loadLive])
|
||||
|
||||
const spark = live?.spark
|
||||
const alive = spark?.ui_ok && (spark?.status || '').toUpperCase() === 'ALIVE'
|
||||
const activeRuns = live?.active_runs ?? []
|
||||
|
||||
return (
|
||||
<div className={cn('flex min-h-0 flex-1 flex-col gap-2', embedded ? 'p-2' : 'p-3')}>
|
||||
<header className="panel flex shrink-0 flex-wrap items-center justify-between gap-3 px-4 py-3">
|
||||
<div>
|
||||
<h2 className="flex items-center gap-2 text-sm font-semibold text-foreground">
|
||||
<Zap className="h-4 w-4 text-amber-400" />
|
||||
Spark Lakehouse Workbench
|
||||
</h2>
|
||||
<p className="text-[10px] text-foreground-muted">
|
||||
Select data · transform on the distributed engine · materialize to Iceberg / S3 · live cluster metrics
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<LiveChip label="Cluster" value={alive ? 'ALIVE' : 'down'} ok={!!alive} />
|
||||
<LiveChip label="Cores" value={`${spark?.cores_used ?? 0}/${spark?.cores ?? 0}`} ok={(spark?.cores_used ?? 0) > 0} />
|
||||
<LiveChip label="Active jobs" value={String(activeRuns.length)} ok={activeRuns.length > 0} />
|
||||
<a href="/spark-ui/" target="_blank" rel="noreferrer" className="inline-flex items-center gap-1 rounded-md border border-border px-2 py-1 text-[10px] text-docker hover:bg-docker/10">
|
||||
<ExternalLink className="h-3 w-3" /> Native UI
|
||||
</a>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<div className="panel flex shrink-0 gap-1 px-2 py-1.5">
|
||||
{(['workbench', 'cluster', 'runs', 'ui'] as Tab[]).map((t) => (
|
||||
<button key={t} type="button" onClick={() => setTab(t)} className={cn('rounded-md px-2.5 py-1 text-[10px] font-medium capitalize', tab === t ? subTabActive : subTabIdle)}>
|
||||
{t === 'ui' ? 'Spark UI' : t}
|
||||
{t === 'runs' && activeRuns.length > 0 && <span className="ml-1 rounded-full bg-sky-500/30 px-1 text-[8px] text-sky-200">{activeRuns.length}</span>}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
|
||||
{tab === 'workbench' && <LakehouseWorkbench />}
|
||||
{tab === 'cluster' && <ClusterPanel live={live} />}
|
||||
{tab === 'runs' && <RunsPanel live={live} />}
|
||||
{tab === 'ui' && (
|
||||
<div className="panel min-h-0 flex-1 overflow-hidden p-1">
|
||||
<iframe title="Spark Master UI" src="/spark-ui/" className="h-full min-h-[420px] w-full rounded-md border-0 bg-black" />
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
/* ── Workbench: data selector + operation builder + live run ─────────────── */
|
||||
export function LakehouseWorkbench({ lockedCatalog }: { lockedCatalog?: string }) {
|
||||
const [catalogs, setCatalogs] = useState<string[]>([])
|
||||
const [catalog, setCatalog] = useState(lockedCatalog || 'iceberg')
|
||||
const [schemas, setSchemas] = useState<string[]>([])
|
||||
const [schema, setSchema] = useState('')
|
||||
const [tables, setTables] = useState<{ name: string; fqn: string }[]>([])
|
||||
const [table, setTable] = useState('')
|
||||
const [columns, setColumns] = useState<Column[]>([])
|
||||
|
||||
const [op, setOp] = useState<Operation>('preview')
|
||||
const [limit, setLimit] = useState(200)
|
||||
const [where, setWhere] = useState('')
|
||||
const [groupBy, setGroupBy] = useState<string[]>([])
|
||||
const [metrics, setMetrics] = useState<Metric[]>([{ fn: 'count', col: '*' }])
|
||||
const [profileCols, setProfileCols] = useState<string[]>([])
|
||||
const [sql, setSql] = useState('SELECT region, count(*) AS orders, sum(amount) AS revenue\nFROM iceberg.hadoop.historical_sales_hdfs\nGROUP BY region\nORDER BY revenue DESC')
|
||||
const [rightTable, setRightTable] = useState('')
|
||||
const [leftKey, setLeftKey] = useState('')
|
||||
const [rightKey, setRightKey] = useState('')
|
||||
const [joinType, setJoinType] = useState('INNER')
|
||||
|
||||
const [matEnabled, setMatEnabled] = useState(false)
|
||||
const [matSchema, setMatSchema] = useState('hadoop')
|
||||
const [matTable, setMatTable] = useState('')
|
||||
const [matMode, setMatMode] = useState('create')
|
||||
|
||||
const [run, setRun] = useState<SparkRun | null>(null)
|
||||
const [submitting, setSubmitting] = useState(false)
|
||||
const [err, setErr] = useState<string | null>(null)
|
||||
const pollRef = useRef<ReturnType<typeof setInterval> | null>(null)
|
||||
|
||||
useEffect(() => {
|
||||
if (lockedCatalog) { setCatalog(lockedCatalog); return }
|
||||
fetchSparkCatalogs().then((c) => {
|
||||
setCatalogs(c)
|
||||
if (c.length && !c.includes(catalog)) setCatalog(c[0])
|
||||
})
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [lockedCatalog])
|
||||
|
||||
useEffect(() => {
|
||||
if (!catalog) return
|
||||
setSchema(''); setTables([]); setTable(''); setColumns([])
|
||||
fetchSparkSchemas(catalog).then((s) => { setSchemas(s); if (s.length) setSchema(s[0]) })
|
||||
}, [catalog])
|
||||
|
||||
useEffect(() => {
|
||||
if (!catalog || !schema) return
|
||||
setTable(''); setColumns([])
|
||||
fetchSparkTables(catalog, schema).then((t) => { setTables(t); if (t.length) setTable(t[0].fqn) })
|
||||
}, [catalog, schema])
|
||||
|
||||
useEffect(() => {
|
||||
if (!table) { setColumns([]); return }
|
||||
fetchSparkColumns(table).then(setColumns)
|
||||
setGroupBy([]); setProfileCols([])
|
||||
}, [table])
|
||||
|
||||
useEffect(() => () => { if (pollRef.current) clearInterval(pollRef.current) }, [])
|
||||
|
||||
const startPolling = useCallback((runId: string) => {
|
||||
if (pollRef.current) clearInterval(pollRef.current)
|
||||
pollRef.current = setInterval(async () => {
|
||||
const j = await fetchSparkRun(runId)
|
||||
if (j?.run) {
|
||||
setRun(j.run)
|
||||
if (['FINISHED', 'FAILED', 'CANCELED'].includes(j.run.state)) {
|
||||
if (pollRef.current) clearInterval(pollRef.current)
|
||||
}
|
||||
}
|
||||
}, 700)
|
||||
}, [])
|
||||
|
||||
const onRun = async () => {
|
||||
setErr(null); setSubmitting(true); setRun(null)
|
||||
const body: Record<string, unknown> = { operation: op, limit }
|
||||
if (op !== 'sql' && op !== 'join') body.table = table
|
||||
if (op === 'filter') body.where = where
|
||||
if (op === 'aggregate') { body.table = table; body.group_by = groupBy; body.metrics = metrics }
|
||||
if (op === 'profile') { body.table = table; body.columns = profileCols.length ? profileCols : columns.slice(0, 8).map((c) => c.name) }
|
||||
if (op === 'sql') body.sql = sql
|
||||
if (op === 'join') {
|
||||
body.left = table; body.right = rightTable
|
||||
body.left_key = leftKey; body.right_key = rightKey; body.join_type = joinType
|
||||
}
|
||||
if (matEnabled && matTable) body.materialize = { enabled: true, schema: matSchema, table: matTable, mode: matMode }
|
||||
try {
|
||||
const r = await createSparkRun(body)
|
||||
if (!r.ok || !r.run_id) { setErr(r.error || 'Failed to submit'); setSubmitting(false); return }
|
||||
startPolling(r.run_id)
|
||||
} catch {
|
||||
setErr('Submit failed')
|
||||
} finally {
|
||||
setSubmitting(false)
|
||||
}
|
||||
}
|
||||
|
||||
const onCancel = async () => { if (run) await cancelSparkRun(run.id) }
|
||||
|
||||
const colNames = columns.map((c) => c.name)
|
||||
|
||||
return (
|
||||
<div className="flex min-h-0 flex-1 gap-2 overflow-hidden">
|
||||
{/* data selector */}
|
||||
<aside className="panel flex w-60 shrink-0 flex-col gap-2 overflow-y-auto p-3">
|
||||
<h3 className="flex items-center gap-1.5 text-[11px] font-semibold text-foreground">
|
||||
<Database className="h-3.5 w-3.5 text-emerald-400" /> Data
|
||||
</h3>
|
||||
{!lockedCatalog && (
|
||||
<Field label="Catalog">
|
||||
<select value={catalog} onChange={(e) => setCatalog(e.target.value)} className={selectCls}>
|
||||
{catalogs.map((c) => <option key={c} value={c}>{c}</option>)}
|
||||
</select>
|
||||
</Field>
|
||||
)}
|
||||
<Field label="Schema">
|
||||
<select value={schema} onChange={(e) => setSchema(e.target.value)} className={selectCls}>
|
||||
{schemas.map((s) => <option key={s} value={s}>{s}</option>)}
|
||||
</select>
|
||||
</Field>
|
||||
<Field label="Table">
|
||||
<select value={table} onChange={(e) => setTable(e.target.value)} className={selectCls}>
|
||||
{tables.map((t) => <option key={t.fqn} value={t.fqn}>{t.name}</option>)}
|
||||
</select>
|
||||
</Field>
|
||||
<div className="min-h-0 flex-1">
|
||||
<p className="mb-1 text-[9px] font-semibold uppercase tracking-wider text-foreground-muted">
|
||||
{columns.length} columns
|
||||
</p>
|
||||
<div className="scrollbar-thin space-y-0.5 overflow-y-auto">
|
||||
{columns.map((c) => (
|
||||
<div key={c.name} className="flex items-center justify-between gap-1 rounded px-1.5 py-0.5 text-[9px] hover:bg-surface-overlay">
|
||||
<span className="truncate font-mono text-foreground">{c.name}</span>
|
||||
<span className="shrink-0 text-[8px] text-foreground-faint">{c.type}</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</aside>
|
||||
|
||||
{/* builder + run */}
|
||||
<div className="flex min-h-0 flex-1 flex-col gap-2 overflow-hidden">
|
||||
<div className="panel shrink-0 space-y-2 p-3">
|
||||
<div className="flex flex-wrap gap-1">
|
||||
{OPERATIONS.map((o) => (
|
||||
<button key={o.id} type="button" onClick={() => setOp(o.id)} title={o.desc}
|
||||
className={cn('rounded-md px-2.5 py-1 text-[10px] font-medium', op === o.id ? subTabActive : subTabIdle)}>
|
||||
{o.label}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
|
||||
{op === 'sql' && (
|
||||
<textarea value={sql} onChange={(e) => setSql(e.target.value)} rows={4} spellCheck={false}
|
||||
className="w-full rounded-md border border-border bg-[#0d1117] p-2 font-mono text-[11px] text-emerald-100 outline-none" />
|
||||
)}
|
||||
|
||||
{op === 'filter' && (
|
||||
<Field label="WHERE predicate">
|
||||
<input value={where} onChange={(e) => setWhere(e.target.value)} placeholder="amount > 1000 AND region = 'EU'" className={inputCls} />
|
||||
</Field>
|
||||
)}
|
||||
|
||||
{op === 'aggregate' && (
|
||||
<div className="space-y-2">
|
||||
<Field label="Group by">
|
||||
<MultiChips options={colNames} selected={groupBy} onToggle={(c) =>
|
||||
setGroupBy((g) => g.includes(c) ? g.filter((x) => x !== c) : [...g, c])} />
|
||||
</Field>
|
||||
<div>
|
||||
<p className="mb-1 text-[9px] font-semibold uppercase tracking-wider text-foreground-muted">Metrics</p>
|
||||
{metrics.map((m, i) => (
|
||||
<div key={i} className="mb-1 flex items-center gap-1">
|
||||
<select value={m.fn} onChange={(e) => setMetrics((ms) => ms.map((x, j) => j === i ? { ...x, fn: e.target.value } : x))} className={cn(selectCls, 'w-32')}>
|
||||
{AGG_FNS.map((f) => <option key={f} value={f}>{f}</option>)}
|
||||
</select>
|
||||
<select value={m.col} onChange={(e) => setMetrics((ms) => ms.map((x, j) => j === i ? { ...x, col: e.target.value } : x))} className={cn(selectCls, 'flex-1')}>
|
||||
<option value="*">*</option>
|
||||
{colNames.map((c) => <option key={c} value={c}>{c}</option>)}
|
||||
</select>
|
||||
<button type="button" onClick={() => setMetrics((ms) => ms.filter((_, j) => j !== i))} className="rounded p-1 text-foreground-muted hover:text-rose-300">
|
||||
<Trash2 className="h-3 w-3" />
|
||||
</button>
|
||||
</div>
|
||||
))}
|
||||
<button type="button" onClick={() => setMetrics((ms) => [...ms, { fn: 'sum', col: colNames[0] || '*' }])}
|
||||
className="inline-flex items-center gap-1 rounded border border-border px-2 py-0.5 text-[9px] text-foreground-muted hover:text-foreground">
|
||||
<Plus className="h-3 w-3" /> Add metric
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{op === 'profile' && (
|
||||
<Field label="Columns (default: first 8)">
|
||||
<MultiChips options={colNames} selected={profileCols} onToggle={(c) =>
|
||||
setProfileCols((g) => g.includes(c) ? g.filter((x) => x !== c) : [...g, c])} />
|
||||
</Field>
|
||||
)}
|
||||
|
||||
{op === 'join' && (
|
||||
<div className="grid grid-cols-2 gap-2">
|
||||
<Field label="Right table (fqn)">
|
||||
<input value={rightTable} onChange={(e) => setRightTable(e.target.value)} placeholder="postgres_sales.public.customers" className={inputCls} />
|
||||
</Field>
|
||||
<Field label="Join type">
|
||||
<select value={joinType} onChange={(e) => setJoinType(e.target.value)} className={selectCls}>
|
||||
{['INNER', 'LEFT', 'RIGHT', 'FULL'].map((t) => <option key={t} value={t}>{t}</option>)}
|
||||
</select>
|
||||
</Field>
|
||||
<Field label="Left key">
|
||||
<select value={leftKey} onChange={(e) => setLeftKey(e.target.value)} className={selectCls}>
|
||||
<option value="">—</option>
|
||||
{colNames.map((c) => <option key={c} value={c}>{c}</option>)}
|
||||
</select>
|
||||
</Field>
|
||||
<Field label="Right key">
|
||||
<input value={rightKey} onChange={(e) => setRightKey(e.target.value)} placeholder="customer_id" className={inputCls} />
|
||||
</Field>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* materialize + run row */}
|
||||
<div className="flex flex-wrap items-end gap-2 border-t border-border pt-2">
|
||||
<label className="flex items-center gap-1.5 text-[10px] text-foreground">
|
||||
<input type="checkbox" checked={matEnabled} onChange={(e) => setMatEnabled(e.target.checked)} />
|
||||
<Save className="h-3 w-3 text-violet-300" /> Materialize → Iceberg/S3
|
||||
</label>
|
||||
{matEnabled && (
|
||||
<>
|
||||
<input value={matSchema} onChange={(e) => setMatSchema(e.target.value)} placeholder="schema" className={cn(inputCls, 'w-24')} />
|
||||
<input value={matTable} onChange={(e) => setMatTable(e.target.value)} placeholder="new_table" className={cn(inputCls, 'w-32')} />
|
||||
<select value={matMode} onChange={(e) => setMatMode(e.target.value)} className={cn(selectCls, 'w-28')}>
|
||||
<option value="create">create</option>
|
||||
<option value="replace">replace</option>
|
||||
<option value="insert">insert into</option>
|
||||
</select>
|
||||
</>
|
||||
)}
|
||||
{!matEnabled && (
|
||||
<Field label="Limit" inline>
|
||||
<input type="number" value={limit} onChange={(e) => setLimit(Math.max(1, Math.min(500, +e.target.value)))} className={cn(inputCls, 'w-20')} />
|
||||
</Field>
|
||||
)}
|
||||
<div className="ml-auto flex gap-2">
|
||||
{run && run.state === 'RUNNING' && (
|
||||
<button type="button" onClick={onCancel} className="inline-flex items-center gap-1 rounded-md border border-rose-400/50 bg-rose-500/15 px-3 py-1.5 text-[11px] text-rose-200 hover:bg-rose-500/25">
|
||||
<Square className="h-3.5 w-3.5" /> Cancel
|
||||
</button>
|
||||
)}
|
||||
<button type="button" onClick={onRun} disabled={submitting}
|
||||
className="inline-flex items-center gap-1.5 rounded-md border border-amber-400/50 bg-amber-500/15 px-4 py-1.5 text-[11px] font-medium text-amber-200 hover:bg-amber-500/25 disabled:opacity-50">
|
||||
{submitting ? <Loader2 className="h-4 w-4 animate-spin" /> : <Play className="h-4 w-4" />}
|
||||
Run on Spark
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
{err && <p className="text-[11px] text-rose-300">{err}</p>}
|
||||
</div>
|
||||
|
||||
{/* live run + results */}
|
||||
<div className="panel flex min-h-0 flex-1 flex-col overflow-hidden">
|
||||
{run ? <RunLive run={run} /> : (
|
||||
<div className="flex flex-1 items-center justify-center text-[11px] text-foreground-muted">
|
||||
Build an operation and Run to see the live execution matrix.
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function RunLive({ run }: { run: SparkRun }) {
|
||||
const s = run.stats || {}
|
||||
const pct = s.progress_pct ?? (run.state === 'FINISHED' ? 100 : 0)
|
||||
return (
|
||||
<div className="flex min-h-0 flex-1 flex-col">
|
||||
<div className="shrink-0 border-b border-border p-3">
|
||||
<div className="mb-2 flex items-center justify-between gap-2">
|
||||
<div className="min-w-0">
|
||||
<p className="truncate text-[11px] font-semibold text-foreground">{run.label}</p>
|
||||
{run.target && <p className="truncate font-mono text-[9px] text-violet-300">→ {run.target}</p>}
|
||||
</div>
|
||||
<span className={cn('rounded-full px-2 py-0.5 text-[10px] font-medium', STATE_COLOR[run.state] || STATE_COLOR.QUEUED)}>
|
||||
{run.state}
|
||||
</span>
|
||||
</div>
|
||||
<div className="h-1.5 w-full overflow-hidden rounded-full bg-surface-overlay">
|
||||
<div className="h-full rounded-full bg-gradient-to-r from-sky-400 to-emerald-400 transition-all" style={{ width: `${pct}%` }} />
|
||||
</div>
|
||||
<SparkMatrix stats={s} />
|
||||
{run.error && <p className="mt-2 rounded bg-rose-500/10 p-2 font-mono text-[10px] text-rose-300">{run.error}</p>}
|
||||
</div>
|
||||
<div className="min-h-0 flex-1 overflow-auto">
|
||||
{run.columns && run.columns.length > 0 ? (
|
||||
<table className="w-full border-collapse text-[10px]">
|
||||
<thead className="sticky top-0 bg-surface">
|
||||
<tr>{run.columns.map((c) => <th key={c} className="border-b border-border px-2 py-1 text-left font-semibold text-foreground-muted">{c}</th>)}</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{(run.rows || []).map((row, i) => (
|
||||
<tr key={i} className="hover:bg-surface-overlay/40">
|
||||
{row.map((cell, j) => <td key={j} className="border-b border-border/50 px-2 py-1 font-mono text-foreground">{cell == null ? '∅' : String(cell)}</td>)}
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
) : run.state === 'FINISHED' && run.target ? (
|
||||
<p className="p-3 text-[11px] text-emerald-300">✓ Materialized to {run.target}{run.update_type ? ` (${run.update_type})` : ''}</p>
|
||||
) : null}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function SparkMatrix({ stats }: { stats: SparkRunStats }) {
|
||||
const cells: { label: string; value: string; icon: typeof Cpu }[] = [
|
||||
{ label: 'Splits', value: `${fmtNum(stats.completed_splits)}/${fmtNum(stats.total_splits)}`, icon: Layers },
|
||||
{ label: 'Running', value: fmtNum(stats.running_splits), icon: Activity },
|
||||
{ label: 'Rows', value: fmtNum(stats.processed_rows), icon: Table2 },
|
||||
{ label: 'Input', value: fmtBytes(stats.processed_bytes), icon: HardDrive },
|
||||
{ label: 'CPU', value: fmtMs(stats.cpu_time_ms), icon: Cpu },
|
||||
{ label: 'Wall', value: fmtMs(stats.elapsed_ms ?? stats.wall_time_ms), icon: Gauge },
|
||||
{ label: 'Peak mem', value: fmtBytes(stats.peak_memory_bytes), icon: Server },
|
||||
{ label: 'Nodes', value: fmtNum(stats.nodes), icon: Cpu },
|
||||
]
|
||||
return (
|
||||
<div className="mt-2 grid grid-cols-4 gap-1.5 lg:grid-cols-8">
|
||||
{cells.map((c) => (
|
||||
<div key={c.label} className="rounded-md border border-border bg-surface-overlay/30 px-2 py-1.5">
|
||||
<div className="flex items-center gap-1 text-[8px] uppercase tracking-wider text-foreground-muted">
|
||||
<c.icon className="h-2.5 w-2.5" /> {c.label}
|
||||
</div>
|
||||
<p className="font-mono text-[11px] font-semibold tabular-nums text-foreground">{c.value}</p>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
/* ── Cluster panel ───────────────────────────────────────────────────────── */
|
||||
function ClusterPanel({ live }: { live: SparkLive | null }) {
|
||||
const spark = live?.spark
|
||||
return (
|
||||
<div className="panel grid min-h-0 flex-1 gap-3 overflow-y-auto p-4 sm:grid-cols-2 lg:grid-cols-4">
|
||||
<Metric label="Workers" value={String(spark?.alive_workers ?? 0)} icon={Server} />
|
||||
<Metric label="Cores" value={`${spark?.cores_used ?? 0} / ${spark?.cores ?? 0}`} icon={Cpu} />
|
||||
<Metric label="Memory" value={`${fmtNum(spark?.memory_used_mb)} / ${fmtNum(spark?.memory_mb)} MB`} icon={Activity} />
|
||||
<Metric label="Active jobs" value={String(live?.active_runs?.length ?? 0)} icon={Zap} />
|
||||
<div className="col-span-full">
|
||||
<h3 className="mb-2 text-[11px] font-semibold uppercase tracking-wider text-foreground-muted">Workers</h3>
|
||||
<div className="grid gap-2 sm:grid-cols-2 lg:grid-cols-3">
|
||||
{(spark?.workers || []).map((w) => (
|
||||
<div key={w.id} className="rounded-lg border border-border bg-surface-overlay/30 p-3">
|
||||
<p className="truncate font-mono text-[10px] text-foreground">{w.host || w.id}</p>
|
||||
<p className="text-[9px] text-foreground-muted">{w.cores_used}/{w.cores} cores · {w.memory_mb} MB · {w.state}</p>
|
||||
</div>
|
||||
))}
|
||||
{!spark?.workers?.length && <p className="text-[10px] text-foreground-muted">No worker details</p>}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
/* ── Runs history ────────────────────────────────────────────────────────── */
|
||||
function RunsPanel({ live }: { live: SparkLive | null }) {
|
||||
const [runs, setRuns] = useState<SparkRun[]>([])
|
||||
useEffect(() => {
|
||||
fetchSparkRuns().then(setRuns)
|
||||
const iv = setInterval(() => fetchSparkRuns().then(setRuns), 3000)
|
||||
return () => clearInterval(iv)
|
||||
}, [])
|
||||
const list = runs.length ? runs : (live?.recent_runs ?? [])
|
||||
return (
|
||||
<div className="panel min-h-0 flex-1 overflow-y-auto p-3">
|
||||
{list.length === 0 ? (
|
||||
<p className="p-4 text-center text-[11px] text-foreground-muted">No runs yet. Submit an operation from the Workbench.</p>
|
||||
) : (
|
||||
<div className="space-y-1.5">
|
||||
{list.map((r) => (
|
||||
<div key={r.id} className="flex items-center justify-between gap-3 rounded-lg border border-border bg-surface-overlay/20 px-3 py-2">
|
||||
<div className="min-w-0">
|
||||
<p className="truncate text-[11px] font-medium text-foreground">{r.label}</p>
|
||||
<p className="truncate font-mono text-[9px] text-foreground-muted">{r.sql.slice(0, 90)}</p>
|
||||
</div>
|
||||
<div className="flex shrink-0 items-center gap-3 text-[9px] text-foreground-muted">
|
||||
<span>{fmtNum(r.stats?.processed_rows)} rows</span>
|
||||
<span>{fmtMs(r.stats?.elapsed_ms)}</span>
|
||||
<span className={cn('rounded-full px-2 py-0.5 font-medium', STATE_COLOR[r.state] || STATE_COLOR.QUEUED)}>{r.state}</span>
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
/* ── small UI helpers ────────────────────────────────────────────────────── */
|
||||
const selectCls = 'w-full rounded-md border border-border bg-surface px-2 py-1 text-[10px] text-foreground outline-none'
|
||||
const inputCls = 'rounded-md border border-border bg-surface px-2 py-1 text-[10px] text-foreground outline-none'
|
||||
|
||||
function Field({ label, children, inline }: { label: string; children: React.ReactNode; inline?: boolean }) {
|
||||
return (
|
||||
<label className={cn('text-[9px] font-semibold uppercase tracking-wider text-foreground-muted', inline ? 'flex items-center gap-1.5' : 'block')}>
|
||||
{label}
|
||||
<div className={inline ? '' : 'mt-1'}>{children}</div>
|
||||
</label>
|
||||
)
|
||||
}
|
||||
|
||||
function MultiChips({ options, selected, onToggle }: { options: string[]; selected: string[]; onToggle: (c: string) => void }) {
|
||||
return (
|
||||
<div className="flex max-h-24 flex-wrap gap-1 overflow-y-auto">
|
||||
{options.map((o) => (
|
||||
<button key={o} type="button" onClick={() => onToggle(o)}
|
||||
className={cn('rounded border px-1.5 py-0.5 font-mono text-[9px]', selected.includes(o) ? 'border-emerald-400/50 bg-emerald-500/15 text-emerald-200' : 'border-border text-foreground-muted hover:bg-surface-overlay')}>
|
||||
{o}
|
||||
</button>
|
||||
))}
|
||||
{!options.length && <span className="text-[9px] text-foreground-faint">select a table</span>}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function Metric({ label, value, icon: Icon }: { label: string; value: string; icon: typeof Cpu }) {
|
||||
return (
|
||||
<div className="rounded-lg border border-border bg-surface-overlay/25 p-3">
|
||||
<div className="mb-1 flex items-center gap-1.5 text-[9px] uppercase tracking-wider text-foreground-muted">
|
||||
<Icon className="h-3 w-3" /> {label}
|
||||
</div>
|
||||
<p className="text-lg font-semibold tabular-nums text-foreground">{value}</p>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function LiveChip({ label, value, ok }: { label: string; value: string; ok: boolean }) {
|
||||
return (
|
||||
<span className="inline-flex items-center gap-1 rounded-full border border-border px-2 py-0.5 text-[9px]">
|
||||
<span className={cn('h-1.5 w-1.5 rounded-full', ok ? 'bg-emerald-400' : 'bg-amber-400')} />
|
||||
<span className="text-foreground-muted">{label}</span>
|
||||
<span className="font-mono font-semibold text-foreground">{value}</span>
|
||||
</span>
|
||||
)
|
||||
}
|
||||
Reference in New Issue
Block a user