feat: Spark Workbench everywhere, autonomous Hadoop offload & LLM masking-aware
- Data Hub with Hadoop tab (HDFS/Iceberg browser, Spark, pipeline) - Databricks-style Lakehouse Workbench (Trino engine, live exec matrix, materialize to Iceberg/S3); reused & embedded in every source-DB UI - HDFS -> Kafka -> Spark -> Iceberg/S3 pipeline; WebHDFS hostname resolver - Data Flow master pulse switch (Run/Pause/Stop) gating animated edges - Data Custodian autonomous Hadoop offload loop (batch counterpart to CDC), pulsing source -> HDFS edges; toggle in Data Flow - LLM now autonomously aware of all latest platform changes (live platform context) and enforces masking policy: never reveals masked PII, still answers helpfully with aggregates/explanations
This commit is contained in:
mo
@@ -146,9 +146,56 @@ const POSITIONS: Record<string, Record<string, { x: number; y: number }>> = {
|
||||
},
|
||||
}
|
||||
|
||||
export function ArchitectureDiagram({ animation }: { animation: string }) {
|
||||
const COMPACT_POSITIONS: Record<string, Record<string, { x: number; y: number }>> = {
|
||||
'full-stack': {
|
||||
user: { x: 14, y: 18 },
|
||||
caddy: { x: 32, y: 18 },
|
||||
ui: { x: 50, y: 18 },
|
||||
api: { x: 12, y: 48 },
|
||||
dq: { x: 34, y: 48 },
|
||||
rag: { x: 56, y: 48 },
|
||||
docling: { x: 34, y: 78 },
|
||||
chroma: { x: 72, y: 48 },
|
||||
llm: { x: 88, y: 78 },
|
||||
lake: { x: 12, y: 78 },
|
||||
},
|
||||
'rag-flow': {
|
||||
upload: { x: 10, y: 22 },
|
||||
store: { x: 10, y: 72 },
|
||||
docling: { x: 28, y: 22 },
|
||||
chunk: { x: 46, y: 22 },
|
||||
embed: { x: 46, y: 47 },
|
||||
chroma: { x: 46, y: 72 },
|
||||
query: { x: 72, y: 22 },
|
||||
retrieve: { x: 72, y: 47 },
|
||||
llm: { x: 90, y: 72 },
|
||||
},
|
||||
'dq-flow': {
|
||||
data: { x: 8, y: 50 },
|
||||
docling: { x: 24, y: 22 },
|
||||
pandas: { x: 24, y: 78 },
|
||||
ge: { x: 46, y: 32 },
|
||||
soda: { x: 46, y: 68 },
|
||||
maturity: { x: 68, y: 50 },
|
||||
report: { x: 88, y: 50 },
|
||||
},
|
||||
'lakehouse': {
|
||||
pg: { x: 6, y: 22 },
|
||||
mysql: { x: 6, y: 42 },
|
||||
mongo: { x: 6, y: 62 },
|
||||
debezium: { x: 24, y: 42 },
|
||||
kafka: { x: 38, y: 42 },
|
||||
spark: { x: 52, y: 42 },
|
||||
iceberg: { x: 66, y: 42 },
|
||||
trino: { x: 80, y: 42 },
|
||||
bi: { x: 92, y: 62 },
|
||||
},
|
||||
}
|
||||
|
||||
export function ArchitectureDiagram({ animation, compact = false, present = false }: { animation: string; compact?: boolean; present?: boolean }) {
|
||||
const flow = FLOWS[animation] || FLOWS['full-stack']
|
||||
const positions = POSITIONS[animation] || POSITIONS['full-stack']
|
||||
const positions = (compact ? COMPACT_POSITIONS[animation] : POSITIONS[animation])
|
||||
|| (compact ? COMPACT_POSITIONS['full-stack'] : POSITIONS['full-stack'])
|
||||
const [tick, setTick] = useState(0)
|
||||
|
||||
useEffect(() => {
|
||||
@@ -159,7 +206,12 @@ export function ArchitectureDiagram({ animation }: { animation: string }) {
|
||||
const activeEdge = tick % flow.edges.length
|
||||
|
||||
return (
|
||||
<div className="relative mx-auto mb-6 h-[280px] w-full max-w-4xl rounded-xl border border-docker/30 bg-surface-overlay/60 p-2 md:h-[320px]">
|
||||
<div className={cn(
|
||||
'relative mx-auto w-full rounded-xl border border-docker/30 bg-surface-overlay/60 p-2',
|
||||
present ? 'mb-8 h-[340px] max-w-5xl md:h-[400px]'
|
||||
: compact ? 'mb-3 h-[200px] max-w-none'
|
||||
: 'mb-6 h-[280px] max-w-4xl md:h-[320px]',
|
||||
)}>
|
||||
<svg className="absolute inset-0 h-full w-full" viewBox="0 0 100 100" preserveAspectRatio="none">
|
||||
{flow.edges.map((edge, i) => {
|
||||
const from = positions[edge.from]
|
||||
@@ -195,15 +247,19 @@ export function ArchitectureDiagram({ animation }: { animation: string }) {
|
||||
<div
|
||||
key={node.id}
|
||||
className={cn(
|
||||
'absolute -translate-x-1/2 -translate-y-1/2 rounded-lg border px-2 py-1 text-center transition-all duration-500',
|
||||
'absolute -translate-x-1/2 -translate-y-1/2 rounded-lg border text-center transition-all duration-500',
|
||||
compact ? 'px-1 py-0.5' : present ? 'px-3 py-2' : 'px-2 py-1',
|
||||
lit ? 'scale-105 border-docker shadow-docker bg-docker/20' : 'border-border bg-surface-raised/90',
|
||||
)}
|
||||
style={{ left: `${pos.x}%`, top: `${pos.y}%`, minWidth: '72px' }}
|
||||
style={{ left: `${pos.x}%`, top: `${pos.y}%`, minWidth: compact ? '52px' : present ? '88px' : '72px', maxWidth: compact ? '64px' : present ? '120px' : '96px' }}
|
||||
>
|
||||
<p className="text-[9px] font-semibold leading-tight text-foreground md:text-[10px]" style={{ color: lit ? node.color : undefined }}>
|
||||
<p className={cn(
|
||||
'font-semibold leading-tight text-foreground',
|
||||
compact ? 'text-[6px]' : present ? 'text-[11px] md:text-xs' : 'text-[9px] md:text-[10px]',
|
||||
)} style={{ color: lit ? node.color : undefined }}>
|
||||
{node.label}
|
||||
</p>
|
||||
{node.sub && <p className="text-[7px] text-foreground-faint md:text-[8px]">{node.sub}</p>}
|
||||
{node.sub && <p className={cn('text-foreground-faint', compact ? 'text-[5px] leading-none' : present ? 'text-[9px] md:text-[10px]' : 'text-[7px] md:text-[8px]')}>{node.sub}</p>}
|
||||
</div>
|
||||
)
|
||||
})}
|
||||
|
||||
@@ -0,0 +1,376 @@
|
||||
import { useCallback, useMemo, useState } from 'react'
|
||||
import { Check, Filter, Loader2, Pencil, RotateCcw, Save, Search, X } from 'lucide-react'
|
||||
import type { SourceEngine } from '../../lib/dataSourceCatalog'
|
||||
import { cn } from '../../lib/utils'
|
||||
import { subTabActive, subTabIdle } from '../../lib/tabActive'
|
||||
|
||||
type SampleData = {
|
||||
ok: boolean
|
||||
columns?: string[]
|
||||
rows?: unknown[][]
|
||||
row_count?: number
|
||||
elapsed_ms?: number
|
||||
error?: string
|
||||
primary_keys?: string[]
|
||||
cdc?: boolean
|
||||
editable?: boolean
|
||||
offset?: number
|
||||
limit?: number
|
||||
total_count?: number | null
|
||||
}
|
||||
|
||||
type DataEngine = SourceEngine | 'hadoop'
|
||||
|
||||
type Props = {
|
||||
engine: DataEngine
|
||||
objectFqn: string
|
||||
sample: SampleData | null
|
||||
loading: boolean
|
||||
page: number
|
||||
pageSize: number
|
||||
onPageChange: (page: number) => void
|
||||
onPageSizeChange: (size: number) => void
|
||||
onReload: () => void
|
||||
}
|
||||
|
||||
const PAGE_SIZES = [100, 250, 500]
|
||||
|
||||
function cellStr(v: unknown) {
|
||||
if (v === null || v === undefined) return ''
|
||||
return String(v)
|
||||
}
|
||||
|
||||
function rowMatches(row: unknown[], columns: string[], filters: Record<string, string>, global: string) {
|
||||
if (global) {
|
||||
const hay = row.map(cellStr).join(' ').toLowerCase()
|
||||
if (!hay.includes(global.toLowerCase())) return false
|
||||
}
|
||||
for (const col of columns) {
|
||||
const f = filters[col]?.trim()
|
||||
if (!f) continue
|
||||
const idx = columns.indexOf(col)
|
||||
const val = cellStr(row[idx]).toLowerCase()
|
||||
if (!val.includes(f.toLowerCase())) return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
export function DataBrowserGrid({
|
||||
engine,
|
||||
objectFqn,
|
||||
sample,
|
||||
loading,
|
||||
page,
|
||||
pageSize,
|
||||
onPageChange,
|
||||
onPageSizeChange,
|
||||
onReload,
|
||||
}: Props) {
|
||||
const [globalFilter, setGlobalFilter] = useState('')
|
||||
const [colFilters, setColFilters] = useState<Record<string, string>>({})
|
||||
const [editMode, setEditMode] = useState(false)
|
||||
const [drafts, setDrafts] = useState<Record<number, Record<string, string>>>({})
|
||||
const [savingRow, setSavingRow] = useState<number | null>(null)
|
||||
const [msg, setMsg] = useState<{ text: string; ok: boolean } | null>(null)
|
||||
|
||||
const columns = sample?.columns || []
|
||||
const rows = sample?.rows || []
|
||||
const pks = sample?.primary_keys || []
|
||||
const editable = sample?.editable && pks.length > 0
|
||||
const totalCount = sample?.total_count ?? null
|
||||
const offset = sample?.offset ?? (page - 1) * pageSize
|
||||
const totalPages = totalCount != null ? Math.max(1, Math.ceil(totalCount / pageSize)) : null
|
||||
const rowFrom = rows.length ? offset + 1 : 0
|
||||
const rowTo = offset + rows.length
|
||||
|
||||
const filteredRows = useMemo(() => {
|
||||
return rows
|
||||
.map((row, idx) => ({ row, idx }))
|
||||
.filter(({ row }) => rowMatches(row, columns, colFilters, globalFilter))
|
||||
}, [rows, columns, colFilters, globalFilter])
|
||||
|
||||
const pkValuesForRow = useCallback(
|
||||
(row: unknown[]) => {
|
||||
const pk: Record<string, unknown> = {}
|
||||
for (const k of pks) {
|
||||
const i = columns.indexOf(k)
|
||||
if (i >= 0) pk[k] = row[i]
|
||||
}
|
||||
return pk
|
||||
},
|
||||
[columns, pks],
|
||||
)
|
||||
|
||||
const getDraft = (rowIdx: number, col: string, original: unknown) => {
|
||||
if (drafts[rowIdx]?.[col] !== undefined) return drafts[rowIdx][col]
|
||||
return cellStr(original)
|
||||
}
|
||||
|
||||
const setDraft = (rowIdx: number, col: string, val: string) => {
|
||||
setDrafts((d) => ({ ...d, [rowIdx]: { ...d[rowIdx], [col]: val } }))
|
||||
}
|
||||
|
||||
const rowDirty = (rowIdx: number, row: unknown[]) => {
|
||||
const d = drafts[rowIdx]
|
||||
if (!d) return false
|
||||
return columns.some((col, j) => {
|
||||
if (pks.includes(col)) return false
|
||||
return d[col] !== undefined && d[col] !== cellStr(row[j])
|
||||
})
|
||||
}
|
||||
|
||||
const saveRow = async (rowIdx: number, row: unknown[]) => {
|
||||
const pk = pkValuesForRow(row)
|
||||
if (!Object.keys(pk).length) {
|
||||
setMsg({ text: 'No primary key — cannot save', ok: false })
|
||||
return
|
||||
}
|
||||
const changes: Record<string, unknown> = {}
|
||||
const d = drafts[rowIdx] || {}
|
||||
for (const col of columns) {
|
||||
if (pks.includes(col)) continue
|
||||
if (d[col] !== undefined && d[col] !== cellStr(row[columns.indexOf(col)])) {
|
||||
changes[col] = d[col] === '' ? null : d[col]
|
||||
}
|
||||
}
|
||||
if (!Object.keys(changes).length) return
|
||||
|
||||
setSavingRow(rowIdx)
|
||||
setMsg(null)
|
||||
try {
|
||||
const r = await fetch('/api/sql/row/update', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ engine, object: objectFqn, pk, changes }),
|
||||
})
|
||||
const j = await r.json()
|
||||
if (!j.ok) {
|
||||
setMsg({ text: j.error || 'Save failed', ok: false })
|
||||
return
|
||||
}
|
||||
setMsg({
|
||||
text: j.cdc
|
||||
? 'Saved — change will appear in Live Changes via CDC'
|
||||
: 'Saved (this source has no CDC stream)',
|
||||
ok: true,
|
||||
})
|
||||
setDrafts((d) => {
|
||||
const next = { ...d }
|
||||
delete next[rowIdx]
|
||||
return next
|
||||
})
|
||||
setTimeout(onReload, 600)
|
||||
} catch {
|
||||
setMsg({ text: 'API unreachable', ok: false })
|
||||
} finally {
|
||||
setSavingRow(null)
|
||||
}
|
||||
}
|
||||
|
||||
const clearFilters = () => {
|
||||
setGlobalFilter('')
|
||||
setColFilters({})
|
||||
}
|
||||
|
||||
if (!sample?.ok && !loading) {
|
||||
return (
|
||||
<p className="p-4 text-[11px] text-danger">{sample?.error || 'Failed to load data'}</p>
|
||||
)
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="flex h-full min-h-0 flex-1 flex-col overflow-hidden">
|
||||
{/* Toolbar */}
|
||||
<div className="flex shrink-0 flex-wrap items-center gap-2 border-b border-border/60 px-3 py-2">
|
||||
<div className="relative min-w-[160px] flex-1">
|
||||
<Search className="absolute left-2 top-1/2 h-3 w-3 -translate-y-1/2 text-foreground-muted" />
|
||||
<input
|
||||
type="text"
|
||||
placeholder="Search all columns…"
|
||||
value={globalFilter}
|
||||
onChange={(e) => setGlobalFilter(e.target.value)}
|
||||
className="w-full rounded border border-border bg-surface-overlay py-1 pl-7 pr-2 text-[10px] text-foreground"
|
||||
/>
|
||||
</div>
|
||||
{(globalFilter || Object.values(colFilters).some(Boolean)) && (
|
||||
<button type="button" onClick={clearFilters} className={cn('inline-flex items-center gap-1 rounded-md px-2 py-1 text-[10px]', subTabIdle)}>
|
||||
<X className="h-3 w-3" /> Clear filters
|
||||
</button>
|
||||
)}
|
||||
{editable && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => { setEditMode((v) => !v); setDrafts({}) }}
|
||||
className={cn('inline-flex items-center gap-1 rounded-md px-2 py-1 text-[10px]', editMode ? subTabActive : subTabIdle)}
|
||||
>
|
||||
<Pencil className="h-3 w-3" /> {editMode ? 'Editing' : 'Edit rows'}
|
||||
</button>
|
||||
)}
|
||||
{sample?.cdc && (
|
||||
<span className="rounded bg-emerald-500/15 px-2 py-0.5 text-[9px] text-emerald-300">CDC → Live Changes</span>
|
||||
)}
|
||||
{loading && <Loader2 className="h-3.5 w-3.5 animate-spin text-foreground-muted" />}
|
||||
</div>
|
||||
|
||||
{msg && (
|
||||
<p className={cn('shrink-0 px-3 py-1.5 text-[10px]', msg.ok ? 'text-emerald-300' : 'text-danger')}>{msg.text}</p>
|
||||
)}
|
||||
|
||||
<div className="scrollbar-thin min-h-0 flex-1 overflow-x-auto overflow-y-auto overscroll-contain p-2">
|
||||
{columns.length > 0 && (
|
||||
<>
|
||||
<p className="mb-1 font-mono text-[9px] text-foreground-muted">
|
||||
{filteredRows.length} shown
|
||||
{totalCount != null
|
||||
? ` · rows ${rowFrom.toLocaleString()}–${rowTo.toLocaleString()} of ${totalCount.toLocaleString()}`
|
||||
: ` · page ${page}${totalPages ? ` of ${totalPages.toLocaleString()}` : ''}`}
|
||||
{sample?.elapsed_ms != null && ` · ${sample.elapsed_ms}ms`}
|
||||
{pks.length > 0 && ` · PK: ${pks.join(', ')}`}
|
||||
</p>
|
||||
<table className="w-full text-left font-mono text-[10px]">
|
||||
<thead>
|
||||
<tr className="sticky top-0 z-10 border-b border-border bg-surface-raised text-docker">
|
||||
{editMode && editable && <th className="w-8 px-1 py-1" />}
|
||||
{columns.map((c) => (
|
||||
<th key={c} className="px-2 py-1">
|
||||
<div className="flex flex-col gap-0.5">
|
||||
<span className={cn(pks.includes(c) && 'text-amber-300')}>{c}{pks.includes(c) ? ' (PK)' : ''}</span>
|
||||
<div className="relative">
|
||||
<Filter className="pointer-events-none absolute left-1 top-1/2 h-2.5 w-2.5 -translate-y-1/2 text-foreground-faint" />
|
||||
<input
|
||||
type="text"
|
||||
placeholder="filter"
|
||||
value={colFilters[c] || ''}
|
||||
onChange={(e) => setColFilters((f) => ({ ...f, [c]: e.target.value }))}
|
||||
className="w-full min-w-[60px] rounded border border-border/60 bg-surface py-0.5 pl-5 pr-1 text-[9px] font-normal text-foreground"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</th>
|
||||
))}
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{filteredRows.map(({ row, idx }) => {
|
||||
const dirty = rowDirty(idx, row)
|
||||
return (
|
||||
<tr key={idx} className={cn('border-b border-border/30', dirty && 'bg-violet-500/10')}>
|
||||
{editMode && editable && (
|
||||
<td className="px-1 py-1">
|
||||
{dirty && (
|
||||
<button
|
||||
type="button"
|
||||
disabled={savingRow === idx}
|
||||
onClick={() => saveRow(idx, row)}
|
||||
title="Save row"
|
||||
className="rounded p-0.5 text-emerald-400 hover:bg-emerald-500/20 disabled:opacity-40"
|
||||
>
|
||||
{savingRow === idx ? <Loader2 className="h-3.5 w-3.5 animate-spin" /> : <Save className="h-3.5 w-3.5" />}
|
||||
</button>
|
||||
)}
|
||||
</td>
|
||||
)}
|
||||
{row.map((cell, j) => {
|
||||
const col = columns[j]
|
||||
const isPk = pks.includes(col)
|
||||
if (editMode && editable && !isPk) {
|
||||
return (
|
||||
<td key={j} className="max-w-[180px] px-1 py-0.5">
|
||||
<input
|
||||
type="text"
|
||||
value={getDraft(idx, col, cell)}
|
||||
onChange={(e) => setDraft(idx, col, e.target.value)}
|
||||
className="w-full rounded border border-border/60 bg-surface-overlay px-1.5 py-0.5 text-[10px] text-foreground"
|
||||
/>
|
||||
</td>
|
||||
)
|
||||
}
|
||||
return (
|
||||
<td key={j} className={cn('max-w-[200px] truncate px-2 py-1', isPk ? 'text-amber-200' : 'text-foreground')}>
|
||||
{cell === null || cell === undefined ? 'NULL' : String(cell)}
|
||||
</td>
|
||||
)
|
||||
})}
|
||||
</tr>
|
||||
)
|
||||
})}
|
||||
</tbody>
|
||||
</table>
|
||||
{filteredRows.length === 0 && (
|
||||
<p className="py-6 text-center text-[11px] text-foreground-muted">No rows match filters</p>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
{!columns.length && !loading && (
|
||||
<p className="py-8 text-center text-[11px] text-foreground-muted">Select a table, collection or label to preview data</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Pagination */}
|
||||
{columns.length > 0 && (
|
||||
<div className="flex shrink-0 flex-wrap items-center justify-between gap-2 border-t border-border/60 bg-surface-raised px-3 py-2">
|
||||
<div className="flex items-center gap-2 text-[10px] text-foreground-muted">
|
||||
<span>Rows per page</span>
|
||||
<select
|
||||
value={pageSize}
|
||||
onChange={(e) => onPageSizeChange(Number(e.target.value))}
|
||||
className="rounded border border-border bg-surface-overlay px-2 py-0.5 text-[10px] text-foreground"
|
||||
>
|
||||
{PAGE_SIZES.map((s) => (
|
||||
<option key={s} value={s}>{s}</option>
|
||||
))}
|
||||
</select>
|
||||
</div>
|
||||
<div className="flex flex-wrap items-center gap-1">
|
||||
<button
|
||||
type="button"
|
||||
disabled={page <= 1 || loading}
|
||||
onClick={() => onPageChange(1)}
|
||||
className={cn('rounded px-2 py-1 text-[10px]', subTabIdle, 'disabled:opacity-30')}
|
||||
>
|
||||
First
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
disabled={page <= 1 || loading}
|
||||
onClick={() => onPageChange(page - 1)}
|
||||
className={cn('rounded px-2 py-1 text-[10px]', subTabIdle, 'disabled:opacity-30')}
|
||||
>
|
||||
Prev
|
||||
</button>
|
||||
<span className="px-2 font-mono text-[10px] text-foreground">
|
||||
Page {page.toLocaleString()}{totalPages != null ? ` / ${totalPages.toLocaleString()}` : ''}
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
disabled={loading || (totalPages != null ? page >= totalPages : rows.length < pageSize)}
|
||||
onClick={() => onPageChange(page + 1)}
|
||||
className={cn('rounded px-2 py-1 text-[10px]', subTabIdle, 'disabled:opacity-30')}
|
||||
>
|
||||
Next
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
disabled={totalPages == null || page >= totalPages || loading}
|
||||
onClick={() => totalPages && onPageChange(totalPages)}
|
||||
className={cn('rounded px-2 py-1 text-[10px]', subTabIdle, 'disabled:opacity-30')}
|
||||
>
|
||||
Last
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{editMode && editable && (
|
||||
<div className="flex shrink-0 items-center gap-2 border-t border-border/60 px-3 py-1.5 text-[9px] text-foreground-muted">
|
||||
<Check className="h-3 w-3 text-emerald-400" />
|
||||
Edit cells, then click <Save className="inline h-3 w-3" /> on a row to save.
|
||||
{sample?.cdc && ' Changes on CDC sources appear in Live Changes within seconds.'}
|
||||
<button type="button" onClick={() => { setDrafts({}); onReload() }} className="ml-auto inline-flex items-center gap-1 text-docker hover:underline">
|
||||
<RotateCcw className="h-3 w-3" /> Reset
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -1,7 +1,8 @@
|
||||
import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from 'react'
|
||||
import { GitBranch, Lock, LockOpen, Play, RefreshCw, ShieldAlert, ShieldCheck, Loader2, Bot } from 'lucide-react'
|
||||
import { GitBranch, Lock, LockOpen, Play, Pause, Square, RefreshCw, ShieldAlert, ShieldCheck, Loader2, Bot } from 'lucide-react'
|
||||
import type { DataflowEdge, DataflowGraph, DataflowNode } from '../../types'
|
||||
import { fetchDataflow, runDataflowMovement, toggleEtlAgent, fetchAgentOpsStatus, setPiiMask } from '../../lib/api'
|
||||
import { fetchDataflow, runDataflowMovement, toggleEtlAgent, toggleCustodianOffload, fetchAgentOpsStatus, setPiiMask, setStreamingFlow } from '../../lib/api'
|
||||
import { SparkKafkaPanel } from './SparkKafkaPanel'
|
||||
import { Badge } from '../ui/Badge'
|
||||
import { cn } from '../../lib/utils'
|
||||
|
||||
@@ -14,6 +15,7 @@ const NODE_KIND: Record<string, { ring: string; chip: string; dot: string }> = {
|
||||
stream: { ring: 'border-cyan-400/60', chip: 'bg-cyan-500/15 text-cyan-300 border-cyan-400/40', dot: '#22d3ee' },
|
||||
sink: { ring: 'border-sky-400/60', chip: 'bg-sky-500/15 text-sky-300 border-sky-400/40', dot: '#38bdf8' },
|
||||
lakehouse: { ring: 'border-blue-400/60', chip: 'bg-blue-500/15 text-blue-300 border-blue-400/40', dot: '#60a5fa' },
|
||||
compute: { ring: 'border-violet-400/60', chip: 'bg-violet-500/15 text-violet-300 border-violet-400/40', dot: '#a78bfa' },
|
||||
engine: { ring: 'border-violet-400/60', chip: 'bg-violet-500/15 text-violet-300 border-violet-400/40', dot: '#a78bfa' },
|
||||
governance: { ring: 'border-fuchsia-400/60', chip: 'bg-fuchsia-500/15 text-fuchsia-300 border-fuchsia-400/40', dot: '#d946ef' },
|
||||
}
|
||||
@@ -70,6 +72,7 @@ export function DataFlowView() {
|
||||
const [selected, setSelected] = useState<string | null>(null)
|
||||
const [triggering, setTriggering] = useState<string | null>(null)
|
||||
const [etlEnabled, setEtlEnabled] = useState<boolean | null>(null)
|
||||
const [custEnabled, setCustEnabled] = useState<boolean | null>(null)
|
||||
|
||||
const canvasRef = useRef<HTMLDivElement>(null)
|
||||
const nodeRefs = useRef<Record<string, HTMLButtonElement | null>>({})
|
||||
@@ -91,6 +94,8 @@ export function DataFlowView() {
|
||||
fetchAgentOpsStatus().then((s) => {
|
||||
const etl = (s as { etl?: { enabled?: boolean } })?.etl
|
||||
if (etl) setEtlEnabled(!!etl.enabled)
|
||||
const cust = (s as { custodian?: { enabled?: boolean } })?.custodian
|
||||
if (cust) setCustEnabled(!!cust.enabled)
|
||||
})
|
||||
const iv = setInterval(() => load(), 6000)
|
||||
return () => clearInterval(iv)
|
||||
@@ -152,12 +157,24 @@ export function DataFlowView() {
|
||||
await toggleEtlAgent(next)
|
||||
}, [etlEnabled])
|
||||
|
||||
const onToggleCust = useCallback(async () => {
|
||||
const next = !custEnabled
|
||||
setCustEnabled(next)
|
||||
await toggleCustodianOffload(next)
|
||||
}, [custEnabled])
|
||||
|
||||
const onRefresh = useCallback(async () => {
|
||||
setRefreshing(true)
|
||||
await load(true)
|
||||
setTimeout(() => setRefreshing(false), 400)
|
||||
}, [load])
|
||||
|
||||
const flowMode = (graph as unknown as { flow?: string })?.flow ?? 'running'
|
||||
const onFlow = useCallback(async (action: 'pause' | 'resume' | 'stop') => {
|
||||
await setStreamingFlow(action)
|
||||
setTimeout(() => load(true), 300)
|
||||
}, [load])
|
||||
|
||||
const [maskBusy, setMaskBusy] = useState<string | null>(null)
|
||||
const onToggleMask = useCallback(async (key: string, column: string, masked: boolean) => {
|
||||
setMaskBusy(`${key}.${column}`)
|
||||
@@ -202,7 +219,7 @@ export function DataFlowView() {
|
||||
<div className="min-w-0">
|
||||
<h2 className="truncate text-xs font-semibold text-foreground">Data Flow · live lineage</h2>
|
||||
<p className="truncate text-[9px] text-foreground-muted">
|
||||
Generators → sources → CDC/Kafka → lakehouse · click a node for PII detail · trigger movements below
|
||||
Generators → sources → CDC/Kafka → Spark → lakehouse · pulses show live flow · Spark/Kafka panel below
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
@@ -240,6 +257,33 @@ export function DataFlowView() {
|
||||
>
|
||||
<Bot className="h-3 w-3" /> ETL agent {etlEnabled == null ? '' : etlEnabled ? 'on' : 'off'}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={onToggleCust}
|
||||
title="Data Custodian: autonomous batch offload of source data into the Hadoop Iceberg lake"
|
||||
className={cn(
|
||||
'inline-flex items-center gap-1 rounded border px-1.5 py-0.5 text-[9px] font-medium transition-colors',
|
||||
custEnabled
|
||||
? 'border-orange-400/50 bg-orange-500/20 text-orange-200'
|
||||
: 'border-border bg-transparent text-foreground-muted hover:text-foreground',
|
||||
)}
|
||||
>
|
||||
<Bot className="h-3 w-3" /> Hadoop offload {custEnabled == null ? '' : custEnabled ? 'on' : 'off'}
|
||||
</button>
|
||||
<div className="inline-flex items-center gap-0.5 rounded border border-border p-0.5" title="Master pulse control">
|
||||
<button type="button" onClick={() => onFlow('resume')}
|
||||
className={cn('inline-flex items-center gap-1 rounded px-1.5 py-0.5 text-[9px] font-medium transition-colors', flowMode === 'running' ? 'bg-emerald-500/25 text-emerald-200' : 'text-foreground-muted hover:text-foreground')}>
|
||||
<Play className="h-3 w-3" /> Run
|
||||
</button>
|
||||
<button type="button" onClick={() => onFlow('pause')}
|
||||
className={cn('inline-flex items-center gap-1 rounded px-1.5 py-0.5 text-[9px] font-medium transition-colors', flowMode === 'paused' ? 'bg-amber-500/25 text-amber-200' : 'text-foreground-muted hover:text-foreground')}>
|
||||
<Pause className="h-3 w-3" /> Pause
|
||||
</button>
|
||||
<button type="button" onClick={() => onFlow('stop')}
|
||||
className={cn('inline-flex items-center gap-1 rounded px-1.5 py-0.5 text-[9px] font-medium transition-colors', flowMode === 'stopped' ? 'bg-rose-500/25 text-rose-200' : 'text-foreground-muted hover:text-foreground')}>
|
||||
<Square className="h-3 w-3" /> Stop
|
||||
</button>
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
onClick={onRefresh}
|
||||
@@ -333,11 +377,19 @@ export function DataFlowView() {
|
||||
href={selNode.url}
|
||||
target="_blank"
|
||||
rel="noreferrer"
|
||||
className="mt-1 inline-block rounded border border-fuchsia-400/40 bg-fuchsia-500/15 px-1.5 py-0.5 text-[8px] font-medium text-fuchsia-200 hover:bg-fuchsia-500/25"
|
||||
className={cn(
|
||||
'mt-1 inline-block rounded border px-1.5 py-0.5 text-[8px] font-medium hover:opacity-90',
|
||||
selNode.id === 'openmetadata'
|
||||
? 'border-fuchsia-400/40 bg-fuchsia-500/15 text-fuchsia-200'
|
||||
: 'border-docker/40 bg-docker/15 text-docker',
|
||||
)}
|
||||
>
|
||||
Open in OpenMetadata ↗
|
||||
Open {selNode.label} ↗
|
||||
</a>
|
||||
)}
|
||||
{(selNode.id === 'spark' || selNode.id === 'kafka') && (
|
||||
<p className="mt-1 text-[8px] text-docker/80">See Spark/Kafka panel below for full UI + job control.</p>
|
||||
)}
|
||||
{selNode.pii?.has_pii ? (
|
||||
<div className="mt-1.5 border-t border-border pt-1.5">
|
||||
<div className="mb-1 flex items-center justify-between gap-1 text-[9px] font-medium text-rose-300">
|
||||
@@ -400,6 +452,13 @@ export function DataFlowView() {
|
||||
)}
|
||||
</div>
|
||||
|
||||
<SparkKafkaPanel
|
||||
embedded
|
||||
selectedNodeId={selected}
|
||||
streaming={graph?.streaming ?? null}
|
||||
onRefreshGraph={() => load(true)}
|
||||
/>
|
||||
|
||||
{/* movement control strip */}
|
||||
<div className="shrink-0 border-t border-border bg-surface/60 px-3 py-1.5">
|
||||
<div className="mb-1 flex items-center gap-1 text-[8px] font-semibold uppercase tracking-wide text-foreground-muted">
|
||||
|
||||
@@ -0,0 +1,370 @@
|
||||
import { useCallback, useEffect, useRef, useState } from 'react'
|
||||
import { Database, Boxes, Activity, Network, Layers, Play, Loader2, CheckCircle2, XCircle, RefreshCw, Bot, ScrollText } from 'lucide-react'
|
||||
import { cn } from '../../lib/utils'
|
||||
import { subTabActive, subTabIdle } from '../../lib/tabActive'
|
||||
|
||||
export type SourceKey = 'all' | 'postgres' | 'mysql' | 'mongodb' | 'cassandra' | 'neo4j'
|
||||
|
||||
type SourceMeta = {
|
||||
key: SourceKey
|
||||
label: string
|
||||
icon: typeof Database
|
||||
accent: string
|
||||
target: string
|
||||
cdc: boolean
|
||||
desc: string
|
||||
defaultRows: number
|
||||
}
|
||||
|
||||
export const DATA_GEN_SOURCES: SourceMeta[] = [
|
||||
{ key: 'all', label: 'All sources', icon: Layers, accent: 'text-violet-400', target: 'all 5 databases', cdc: true, desc: 'Generate across all databases at once.', defaultRows: 5000 },
|
||||
{ key: 'postgres', label: 'PostgreSQL', icon: Database, accent: 'text-sky-400', target: 'sales_orders', cdc: true, desc: 'Sales orders. CDC via Debezium -> Kafka -> S3.', defaultRows: 5000 },
|
||||
{ key: 'mysql', label: 'MySQL', icon: Database, accent: 'text-amber-400', target: 'employee_events', cdc: true, desc: 'HR employee events. CDC via Debezium -> Kafka -> S3.', defaultRows: 5000 },
|
||||
{ key: 'mongodb', label: 'MongoDB', icon: Boxes, accent: 'text-emerald-400', target: 'supplychain.events', cdc: true, desc: 'Supply chain events. CDC via Debezium -> Kafka -> S3.', defaultRows: 5000 },
|
||||
{ key: 'cassandra', label: 'Cassandra', icon: Activity, accent: 'text-cyan-400', target: 'device_metrics', cdc: false, desc: 'Telemetry metrics. Queryable via Trino.', defaultRows: 5000 },
|
||||
{ key: 'neo4j', label: 'Neo4j', icon: Network, accent: 'text-pink-400', target: 'Product/Supplier graph', cdc: false, desc: 'Graph data (products, suppliers, relationships).', defaultRows: 2000 },
|
||||
]
|
||||
|
||||
type RunInfo = { run_id?: string; state?: string; start?: string; end?: string; conf?: { rows?: number } }
|
||||
type AgentInfo = { agent_id: string; agent_name: string }
|
||||
type ActivityItem = { id: string; ts?: string; agent_id: string; agent_name: string; message: string; level: string }
|
||||
|
||||
type Props = {
|
||||
onPulse: () => void
|
||||
/** Lock to one source (embedded in Data Sources UI) */
|
||||
focusSource?: Exclude<SourceKey, 'all'>
|
||||
embedded?: boolean
|
||||
onOpenPlatform?: () => void
|
||||
}
|
||||
|
||||
const COUNT_KEYS: SourceKey[] = ['postgres', 'mysql', 'mongodb', 'cassandra']
|
||||
|
||||
export function DataGenPanel({ onPulse, focusSource, embedded = false, onOpenPlatform }: Props) {
|
||||
const [active, setActive] = useState<SourceKey>(focusSource || 'all')
|
||||
const [rows, setRows] = useState<Record<SourceKey, number>>(
|
||||
Object.fromEntries(DATA_GEN_SOURCES.map((s) => [s.key, s.defaultRows])) as Record<SourceKey, number>,
|
||||
)
|
||||
const [busy, setBusy] = useState<Record<string, boolean>>({})
|
||||
const [runs, setRuns] = useState<Record<string, RunInfo[]>>({})
|
||||
const [counts, setCounts] = useState<Record<string, number | null>>({})
|
||||
const [msg, setMsg] = useState<string | null>(null)
|
||||
const [agentMap, setAgentMap] = useState<Record<string, AgentInfo>>({})
|
||||
const [activity, setActivity] = useState<ActivityItem[]>([])
|
||||
const pollRef = useRef<Record<string, ReturnType<typeof setInterval>>>({})
|
||||
|
||||
useEffect(() => {
|
||||
if (focusSource) setActive(focusSource)
|
||||
}, [focusSource])
|
||||
|
||||
const meta = DATA_GEN_SOURCES.find((s) => s.key === active)!
|
||||
|
||||
const loadCounts = useCallback(async () => {
|
||||
try {
|
||||
const r = await fetch('/api/pipeline/sync')
|
||||
const j = await r.json()
|
||||
if (j.ok) setCounts(j.counts || {})
|
||||
} catch { /* */ }
|
||||
}, [])
|
||||
|
||||
const loadActivity = useCallback(async () => {
|
||||
try {
|
||||
const r = await fetch('/api/pipeline/activity?limit=25')
|
||||
const j = await r.json()
|
||||
if (j.ok) setActivity(j.activity || [])
|
||||
} catch { /* */ }
|
||||
}, [])
|
||||
|
||||
const loadRuns = useCallback(async (source: SourceKey) => {
|
||||
try {
|
||||
const r = await fetch(`/api/pipeline/runs/${source}?limit=5`)
|
||||
const j = await r.json()
|
||||
if (j.ok) setRuns((prev) => ({ ...prev, [source]: j.runs || [] }))
|
||||
return (j.runs || [])[0] as RunInfo | undefined
|
||||
} catch {
|
||||
return undefined
|
||||
}
|
||||
}, [])
|
||||
|
||||
useEffect(() => {
|
||||
loadCounts()
|
||||
loadActivity()
|
||||
const keys: SourceKey[] = focusSource ? [focusSource, 'all'] : DATA_GEN_SOURCES.map((s) => s.key)
|
||||
keys.forEach((k) => loadRuns(k))
|
||||
fetch('/api/pipeline/agents').then((r) => r.json()).then((j) => { if (j.ok) setAgentMap(j.agents || {}) }).catch(() => {})
|
||||
const act = setInterval(loadActivity, 8000)
|
||||
const poll = pollRef.current
|
||||
return () => { Object.values(poll).forEach(clearInterval); clearInterval(act) }
|
||||
}, [loadCounts, loadRuns, loadActivity, focusSource])
|
||||
|
||||
const startPolling = useCallback((source: SourceKey) => {
|
||||
if (pollRef.current[source]) clearInterval(pollRef.current[source])
|
||||
pollRef.current[source] = setInterval(async () => {
|
||||
const latest = await loadRuns(source)
|
||||
if (latest && (latest.state === 'success' || latest.state === 'failed')) {
|
||||
clearInterval(pollRef.current[source])
|
||||
delete pollRef.current[source]
|
||||
setBusy((b) => ({ ...b, [source]: false }))
|
||||
loadCounts()
|
||||
if (latest.state === 'success') setMsg(`${source}: complete — new data generated`)
|
||||
else setMsg(`${source}: run failed — check Airflow logs`)
|
||||
}
|
||||
}, 3000)
|
||||
}, [loadRuns, loadCounts])
|
||||
|
||||
const generate = useCallback(async (source: SourceKey, autonomous = false) => {
|
||||
setMsg(null)
|
||||
setBusy((b) => ({ ...b, [source]: true }))
|
||||
onPulse()
|
||||
try {
|
||||
const r = await fetch(`/api/pipeline/generate/${source}`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ rows: rows[source], autonomous }),
|
||||
})
|
||||
const j = await r.json()
|
||||
if (!j.ok) {
|
||||
setMsg(`Error: ${j.error || 'could not trigger run'}`)
|
||||
setBusy((b) => ({ ...b, [source]: false }))
|
||||
return
|
||||
}
|
||||
const who = autonomous ? `${j.agent_name || 'Agent'} generating autonomously` : 'started'
|
||||
setMsg(`${source}: ${who} (run ${String(j.run_id).slice(-8)})`)
|
||||
startPolling(source)
|
||||
setTimeout(loadActivity, 800)
|
||||
} catch {
|
||||
setMsg('API unreachable')
|
||||
setBusy((b) => ({ ...b, [source]: false }))
|
||||
}
|
||||
}, [rows, onPulse, startPolling, loadActivity])
|
||||
|
||||
const stateBadge = (state?: string) => {
|
||||
if (state === 'success') return <span className="inline-flex items-center gap-1 text-emerald-400"><CheckCircle2 className="h-3 w-3" /> success</span>
|
||||
if (state === 'failed') return <span className="inline-flex items-center gap-1 text-danger"><XCircle className="h-3 w-3" /> failed</span>
|
||||
if (state === 'running' || state === 'queued') return <span className="inline-flex items-center gap-1 text-amber-400"><Loader2 className="h-3 w-3 animate-spin" /> {state}</span>
|
||||
return <span className="text-foreground-faint">{state || '—'}</span>
|
||||
}
|
||||
|
||||
const latest = (runs[active] || [])[0]
|
||||
const showSourceTabs = !embedded && !focusSource
|
||||
|
||||
return (
|
||||
<div className={cn('flex h-full min-h-0 flex-col', embedded ? '' : 'panel flex-1 overflow-hidden')}>
|
||||
{!embedded && (
|
||||
<header className="flex shrink-0 flex-wrap items-center justify-between gap-2 border-b border-border px-4 py-3">
|
||||
<div>
|
||||
<h2 className="text-sm font-semibold text-foreground">Data Generation</h2>
|
||||
<p className="text-[10px] text-foreground-muted">
|
||||
Generate new data per database — source → Debezium → Kafka → S3
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex gap-2">
|
||||
<button type="button" onClick={() => { loadCounts(); DATA_GEN_SOURCES.forEach((s) => loadRuns(s.key)) }} className={cn('rounded-md px-3 py-1.5 text-[11px]', subTabIdle)}>
|
||||
<RefreshCw className="inline h-3 w-3" /> Refresh
|
||||
</button>
|
||||
{onOpenPlatform && (
|
||||
<button type="button" onClick={onOpenPlatform} className={cn('rounded-md px-3 py-1.5 text-[11px]', subTabActive)}>
|
||||
View topology pulse
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</header>
|
||||
)}
|
||||
|
||||
{showSourceTabs && (
|
||||
<div className="flex shrink-0 flex-wrap gap-1 border-b border-border px-3 py-2">
|
||||
{DATA_GEN_SOURCES.map((s) => {
|
||||
const Icon = s.icon
|
||||
return (
|
||||
<button
|
||||
key={s.key}
|
||||
type="button"
|
||||
onClick={() => setActive(s.key)}
|
||||
className={cn('inline-flex items-center gap-1.5 rounded-md px-3 py-1.5 text-[11px] font-medium', active === s.key ? subTabActive : subTabIdle)}
|
||||
>
|
||||
<Icon className={cn('h-3.5 w-3.5', s.accent)} />
|
||||
{s.label}
|
||||
{busy[s.key] && <Loader2 className="h-3 w-3 animate-spin text-amber-400" />}
|
||||
</button>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="scrollbar-thin min-h-0 flex-1 overflow-y-auto p-4">
|
||||
<div className={cn('space-y-4', embedded ? 'max-w-none' : 'max-w-2xl')}>
|
||||
{embedded && (
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<button
|
||||
type="button"
|
||||
disabled={!!busy.all}
|
||||
onClick={() => generate('all')}
|
||||
className="inline-flex items-center gap-1.5 rounded-md border border-violet-400/40 bg-violet-500/10 px-3 py-1.5 text-[10px] font-medium text-violet-300 hover:bg-violet-500/20 disabled:opacity-40"
|
||||
>
|
||||
{busy.all ? <Loader2 className="h-3 w-3 animate-spin" /> : <Layers className="h-3 w-3" />}
|
||||
Generate all databases
|
||||
</button>
|
||||
<button type="button" onClick={() => { loadCounts(); loadRuns(active); loadActivity() }} className={cn('rounded-md px-2 py-1 text-[10px]', subTabIdle)}>
|
||||
<RefreshCw className="inline h-3 w-3" /> Refresh
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="rounded-lg border border-border bg-surface-overlay/40 p-4">
|
||||
<div className="mb-1 flex flex-wrap items-center gap-2">
|
||||
<meta.icon className={cn('h-5 w-5', meta.accent)} />
|
||||
<h3 className="text-sm font-semibold text-foreground">{meta.label}</h3>
|
||||
{meta.cdc ? (
|
||||
<span className="rounded bg-emerald-500/15 px-1.5 py-0.5 text-[9px] text-emerald-400">CDC active</span>
|
||||
) : (
|
||||
<span className="rounded bg-white/10 px-1.5 py-0.5 text-[9px] text-foreground-muted">no CDC stream</span>
|
||||
)}
|
||||
{agentMap[active] && (
|
||||
<span className="inline-flex items-center gap-1 rounded bg-violet-500/15 px-1.5 py-0.5 text-[9px] text-violet-300">
|
||||
<Bot className="h-2.5 w-2.5" /> {agentMap[active].agent_name}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
<p className="mb-3 text-[11px] text-foreground-muted">{meta.desc} Target: <span className="font-mono text-foreground">{meta.target}</span></p>
|
||||
|
||||
<div className="flex flex-wrap items-end gap-3">
|
||||
<label className="flex flex-col gap-1 text-[10px] uppercase tracking-wide text-foreground-muted">
|
||||
Row count
|
||||
<input
|
||||
type="number"
|
||||
min={1}
|
||||
max={2000000}
|
||||
value={rows[active]}
|
||||
onChange={(e) => setRows((r) => ({ ...r, [active]: Number(e.target.value) }))}
|
||||
className="w-40 rounded border border-border bg-background px-2 py-1.5 text-[12px] text-foreground"
|
||||
/>
|
||||
</label>
|
||||
<button
|
||||
type="button"
|
||||
disabled={!!busy[active]}
|
||||
onClick={() => generate(active)}
|
||||
className="inline-flex items-center gap-2 rounded-md bg-violet-500/90 px-4 py-2 text-[12px] font-semibold text-black hover:bg-violet-400 disabled:opacity-40"
|
||||
>
|
||||
{busy[active] ? <Loader2 className="h-4 w-4 animate-spin" /> : <Play className="h-4 w-4" />}
|
||||
Generate data
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
disabled={!!busy[active]}
|
||||
onClick={() => generate(active, true)}
|
||||
title="Let the assigned agent generate data autonomously"
|
||||
className="inline-flex items-center gap-2 rounded-md border border-violet-400/50 px-3 py-2 text-[12px] font-medium text-violet-300 hover:bg-violet-500/10 disabled:opacity-40"
|
||||
>
|
||||
<Bot className="h-4 w-4" />
|
||||
Let agent generate
|
||||
</button>
|
||||
{active !== 'all' && COUNT_KEYS.includes(active) && (
|
||||
<div className="text-[11px] text-foreground-muted">
|
||||
Current rows: <span className="font-mono text-foreground">{counts[active]?.toLocaleString() ?? '…'}</span>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
{msg && <p className="mt-3 text-[11px] text-foreground-muted">{msg}</p>}
|
||||
</div>
|
||||
|
||||
<div className="rounded-lg border border-border p-4">
|
||||
<h4 className="mb-2 text-[11px] font-semibold uppercase tracking-wide text-foreground-muted">Recent runs — {meta.label}</h4>
|
||||
{latest ? (
|
||||
<table className="w-full text-left text-[11px]">
|
||||
<thead>
|
||||
<tr className="border-b border-border text-[9px] uppercase text-foreground-muted">
|
||||
<th className="py-1 pr-2">State</th>
|
||||
<th className="py-1 pr-2">Rows</th>
|
||||
<th className="py-1 pr-2">Start</th>
|
||||
<th className="py-1 pr-2">End</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{(runs[active] || []).map((run) => (
|
||||
<tr key={run.run_id} className="border-b border-border/40">
|
||||
<td className="py-1 pr-2">{stateBadge(run.state)}</td>
|
||||
<td className="py-1 pr-2 font-mono text-foreground">{run.conf?.rows ?? '—'}</td>
|
||||
<td className="py-1 pr-2 text-foreground-muted">{run.start?.slice(11, 19) || '—'}</td>
|
||||
<td className="py-1 pr-2 text-foreground-muted">{run.end?.slice(11, 19) || '—'}</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
) : (
|
||||
<p className="text-[11px] text-foreground-muted">No runs yet.</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{!embedded && (
|
||||
<>
|
||||
<div className="rounded-lg border border-border p-4">
|
||||
<h4 className="mb-2 text-[11px] font-semibold uppercase tracking-wide text-foreground-muted">Live counts (Trino)</h4>
|
||||
<div className="grid grid-cols-2 gap-2 sm:grid-cols-4">
|
||||
{COUNT_KEYS.map((k) => (
|
||||
<div key={k} className="rounded border border-border/60 px-2 py-1.5">
|
||||
<div className="text-[9px] uppercase text-foreground-muted">{k}</div>
|
||||
<div className="font-mono text-[12px] text-foreground">{counts[k]?.toLocaleString() ?? '…'}</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="rounded-lg border border-border p-4">
|
||||
<h4 className="mb-2 flex items-center gap-1.5 text-[11px] font-semibold uppercase tracking-wide text-foreground-faint">
|
||||
<ScrollText className="h-3.5 w-3.5" /> Agent activity
|
||||
</h4>
|
||||
{activity.length === 0 ? (
|
||||
<p className="text-[11px] text-foreground-muted">No agent actions logged yet.</p>
|
||||
) : (
|
||||
<ul className="space-y-1">
|
||||
{activity.map((a) => (
|
||||
<li key={a.id} className="flex items-start gap-2 text-[11px]">
|
||||
<span className="mt-0.5 text-foreground-muted">{a.ts?.slice(11, 19) || ''}</span>
|
||||
<span className="inline-flex items-center gap-1 rounded bg-violet-500/15 px-1 text-[9px] text-violet-300">
|
||||
<Bot className="h-2.5 w-2.5" />{a.agent_name}
|
||||
</span>
|
||||
<span className={cn('flex-1', a.level === 'err' ? 'text-danger' : 'text-foreground-muted')}>{a.message}</span>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
)}
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
|
||||
{embedded && (
|
||||
<div className="grid gap-4 lg:grid-cols-2">
|
||||
<div className="rounded-lg border border-border p-4">
|
||||
<h4 className="mb-2 text-[11px] font-semibold uppercase tracking-wide text-foreground-muted">Live counts (Trino)</h4>
|
||||
<div className="grid grid-cols-2 gap-2">
|
||||
{COUNT_KEYS.map((k) => (
|
||||
<div key={k} className="rounded border border-border/60 px-2 py-1.5">
|
||||
<div className="text-[9px] uppercase text-foreground-muted">{k}</div>
|
||||
<div className="font-mono text-[12px] text-foreground">{counts[k]?.toLocaleString() ?? '…'}</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
<div className="rounded-lg border border-border p-4">
|
||||
<h4 className="mb-2 flex items-center gap-1.5 text-[11px] font-semibold uppercase tracking-wide text-foreground-faint">
|
||||
<ScrollText className="h-3.5 w-3.5" /> Agent activity
|
||||
</h4>
|
||||
{activity.length === 0 ? (
|
||||
<p className="text-[11px] text-foreground-muted">No agent actions yet.</p>
|
||||
) : (
|
||||
<ul className="scrollbar-thin max-h-[140px] space-y-1 overflow-y-auto">
|
||||
{activity.slice(0, 8).map((a) => (
|
||||
<li key={a.id} className="flex items-start gap-2 text-[10px]">
|
||||
<span className="text-foreground-muted">{a.ts?.slice(11, 19) || ''}</span>
|
||||
<span className="flex-1 text-foreground">{a.message}</span>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -1,303 +1,11 @@
|
||||
import { useCallback, useEffect, useRef, useState } from 'react'
|
||||
import { Database, Boxes, Activity, Cpu, Network, Layers, Play, Loader2, CheckCircle2, XCircle, RefreshCw, Bot, ScrollText } from 'lucide-react'
|
||||
import { cn } from '../../lib/utils'
|
||||
import { subTabActive, subTabIdle } from '../../lib/tabActive'
|
||||
|
||||
type SourceKey = 'all' | 'postgres' | 'mysql' | 'mongodb' | 'cassandra' | 'neo4j'
|
||||
|
||||
type SourceMeta = {
|
||||
key: SourceKey
|
||||
label: string
|
||||
icon: typeof Database
|
||||
accent: string
|
||||
target: string
|
||||
cdc: boolean
|
||||
desc: string
|
||||
defaultRows: number
|
||||
}
|
||||
|
||||
const SOURCES: SourceMeta[] = [
|
||||
{ key: 'all', label: 'All sources', icon: Layers, accent: 'text-violet-400', target: 'alle 5 databases', cdc: true, desc: 'Genereer tegelijk in alle databases.', defaultRows: 5000 },
|
||||
{ key: 'postgres', label: 'PostgreSQL', icon: Database, accent: 'text-sky-400', target: 'sales_orders', cdc: true, desc: 'Sales orders. CDC via Debezium -> Kafka -> S3.', defaultRows: 5000 },
|
||||
{ key: 'mysql', label: 'MySQL', icon: Database, accent: 'text-amber-400', target: 'employee_events', cdc: true, desc: 'HR employee events. CDC via Debezium -> Kafka -> S3.', defaultRows: 5000 },
|
||||
{ key: 'mongodb', label: 'MongoDB', icon: Boxes, accent: 'text-emerald-400', target: 'supplychain.events', cdc: true, desc: 'Supply chain events. CDC via Debezium -> Kafka -> S3.', defaultRows: 5000 },
|
||||
{ key: 'cassandra', label: 'Cassandra', icon: Activity, accent: 'text-cyan-400', target: 'device_metrics', cdc: false, desc: 'Telemetry metrics. Zichtbaar via Trino.', defaultRows: 5000 },
|
||||
{ key: 'neo4j', label: 'Neo4j', icon: Network, accent: 'text-pink-400', target: 'Product/Supplier graph', cdc: false, desc: 'Graafdata (producten, leveranciers, relaties).', defaultRows: 2000 },
|
||||
]
|
||||
|
||||
type RunInfo = { run_id?: string; state?: string; start?: string; end?: string; conf?: { rows?: number } }
|
||||
type AgentInfo = { agent_id: string; agent_name: string }
|
||||
type ActivityItem = { id: string; ts?: string; agent_id: string; agent_name: string; message: string; level: string }
|
||||
import { DataGenPanel } from './DataGenPanel'
|
||||
|
||||
type Props = {
|
||||
onPulse: () => void
|
||||
onOpenPlatform: () => void
|
||||
}
|
||||
|
||||
const COUNT_KEYS: SourceKey[] = ['postgres', 'mysql', 'mongodb', 'cassandra']
|
||||
|
||||
/** Standalone page wrapper — prefer Data Sources UI → Generate tab */
|
||||
export function DataGenView({ onPulse, onOpenPlatform }: Props) {
|
||||
const [active, setActive] = useState<SourceKey>('all')
|
||||
const [rows, setRows] = useState<Record<SourceKey, number>>(
|
||||
Object.fromEntries(SOURCES.map((s) => [s.key, s.defaultRows])) as Record<SourceKey, number>,
|
||||
)
|
||||
const [busy, setBusy] = useState<Record<string, boolean>>({})
|
||||
const [runs, setRuns] = useState<Record<string, RunInfo[]>>({})
|
||||
const [counts, setCounts] = useState<Record<string, number | null>>({})
|
||||
const [msg, setMsg] = useState<string | null>(null)
|
||||
const [agentMap, setAgentMap] = useState<Record<string, AgentInfo>>({})
|
||||
const [activity, setActivity] = useState<ActivityItem[]>([])
|
||||
const pollRef = useRef<Record<string, ReturnType<typeof setInterval>>>({})
|
||||
|
||||
const meta = SOURCES.find((s) => s.key === active)!
|
||||
|
||||
const loadCounts = useCallback(async () => {
|
||||
try {
|
||||
const r = await fetch('/api/pipeline/sync')
|
||||
const j = await r.json()
|
||||
if (j.ok) setCounts(j.counts || {})
|
||||
} catch { /* */ }
|
||||
}, [])
|
||||
|
||||
const loadActivity = useCallback(async () => {
|
||||
try {
|
||||
const r = await fetch('/api/pipeline/activity?limit=25')
|
||||
const j = await r.json()
|
||||
if (j.ok) setActivity(j.activity || [])
|
||||
} catch { /* */ }
|
||||
}, [])
|
||||
|
||||
const loadRuns = useCallback(async (source: SourceKey) => {
|
||||
try {
|
||||
const r = await fetch(`/api/pipeline/runs/${source}?limit=5`)
|
||||
const j = await r.json()
|
||||
if (j.ok) setRuns((prev) => ({ ...prev, [source]: j.runs || [] }))
|
||||
return (j.runs || [])[0] as RunInfo | undefined
|
||||
} catch {
|
||||
return undefined
|
||||
}
|
||||
}, [])
|
||||
|
||||
useEffect(() => {
|
||||
loadCounts()
|
||||
loadActivity()
|
||||
SOURCES.forEach((s) => loadRuns(s.key))
|
||||
fetch('/api/pipeline/agents').then((r) => r.json()).then((j) => { if (j.ok) setAgentMap(j.agents || {}) }).catch(() => {})
|
||||
const act = setInterval(loadActivity, 8000)
|
||||
const poll = pollRef.current
|
||||
return () => { Object.values(poll).forEach(clearInterval); clearInterval(act) }
|
||||
}, [loadCounts, loadRuns, loadActivity])
|
||||
|
||||
const startPolling = useCallback((source: SourceKey) => {
|
||||
if (pollRef.current[source]) clearInterval(pollRef.current[source])
|
||||
pollRef.current[source] = setInterval(async () => {
|
||||
const latest = await loadRuns(source)
|
||||
if (latest && (latest.state === 'success' || latest.state === 'failed')) {
|
||||
clearInterval(pollRef.current[source])
|
||||
delete pollRef.current[source]
|
||||
setBusy((b) => ({ ...b, [source]: false }))
|
||||
loadCounts()
|
||||
if (latest.state === 'success') setMsg(`${source}: klaar — nieuwe data gegenereerd`)
|
||||
else setMsg(`${source}: run mislukt — check Airflow logs`)
|
||||
}
|
||||
}, 3000)
|
||||
}, [loadRuns, loadCounts])
|
||||
|
||||
const generate = useCallback(async (source: SourceKey, autonomous = false) => {
|
||||
setMsg(null)
|
||||
setBusy((b) => ({ ...b, [source]: true }))
|
||||
onPulse()
|
||||
try {
|
||||
const r = await fetch(`/api/pipeline/generate/${source}`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ rows: rows[source], autonomous }),
|
||||
})
|
||||
const j = await r.json()
|
||||
if (!j.ok) {
|
||||
setMsg(`Fout: ${j.error || 'kon niet triggeren'}`)
|
||||
setBusy((b) => ({ ...b, [source]: false }))
|
||||
return
|
||||
}
|
||||
const who = autonomous ? `${j.agent_name || 'Agent'} genereert zelf` : 'gestart'
|
||||
setMsg(`${source}: ${who} (run ${String(j.run_id).slice(-8)})`)
|
||||
startPolling(source)
|
||||
setTimeout(loadActivity, 800)
|
||||
} catch {
|
||||
setMsg('API niet bereikbaar')
|
||||
setBusy((b) => ({ ...b, [source]: false }))
|
||||
}
|
||||
}, [rows, onPulse, startPolling, loadActivity])
|
||||
|
||||
const stateBadge = (state?: string) => {
|
||||
if (state === 'success') return <span className="inline-flex items-center gap-1 text-emerald-400"><CheckCircle2 className="h-3 w-3" /> success</span>
|
||||
if (state === 'failed') return <span className="inline-flex items-center gap-1 text-danger"><XCircle className="h-3 w-3" /> failed</span>
|
||||
if (state === 'running' || state === 'queued') return <span className="inline-flex items-center gap-1 text-amber-400"><Loader2 className="h-3 w-3 animate-spin" /> {state}</span>
|
||||
return <span className="text-foreground-faint">{state || '—'}</span>
|
||||
}
|
||||
|
||||
const latest = (runs[active] || [])[0]
|
||||
|
||||
return (
|
||||
<div className="panel flex h-full min-h-0 flex-1 flex-col overflow-hidden">
|
||||
<header className="flex shrink-0 flex-wrap items-center justify-between gap-2 border-b border-border px-4 py-3">
|
||||
<div>
|
||||
<h2 className="flex items-center gap-2 text-sm font-semibold text-foreground">
|
||||
<Cpu className="h-4 w-4 text-violet-400" /> Data Generation
|
||||
</h2>
|
||||
<p className="text-[10px] text-foreground-muted">
|
||||
Genereer per database nieuwe data en pulse de hele flow: bron -> Debezium -> Kafka -> S3
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex gap-2">
|
||||
<button type="button" onClick={() => { loadCounts(); SOURCES.forEach((s) => loadRuns(s.key)) }} className={cn('rounded-md px-3 py-1.5 text-[11px]', subTabIdle)}>
|
||||
<RefreshCw className="inline h-3 w-3" /> Refresh
|
||||
</button>
|
||||
<button type="button" onClick={onOpenPlatform} className={cn('rounded-md px-3 py-1.5 text-[11px]', subTabActive)}>
|
||||
Bekijk topology pulse
|
||||
</button>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<div className="flex shrink-0 flex-wrap gap-1 border-b border-border px-3 py-2">
|
||||
{SOURCES.map((s) => {
|
||||
const Icon = s.icon
|
||||
return (
|
||||
<button
|
||||
key={s.key}
|
||||
type="button"
|
||||
onClick={() => setActive(s.key)}
|
||||
className={cn('inline-flex items-center gap-1.5 rounded-md px-3 py-1.5 text-[11px] font-medium', active === s.key ? subTabActive : subTabIdle)}
|
||||
>
|
||||
<Icon className={cn('h-3.5 w-3.5', s.accent)} />
|
||||
{s.label}
|
||||
{busy[s.key] && <Loader2 className="h-3 w-3 animate-spin text-amber-400" />}
|
||||
</button>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
|
||||
<div className="scrollbar-thin min-h-0 flex-1 overflow-y-auto p-4">
|
||||
<div className="max-w-2xl space-y-4">
|
||||
<div className="rounded-lg border border-border bg-surface-overlay/40 p-4">
|
||||
<div className="mb-1 flex items-center gap-2">
|
||||
<meta.icon className={cn('h-5 w-5', meta.accent)} />
|
||||
<h3 className="text-sm font-semibold text-foreground">{meta.label}</h3>
|
||||
{meta.cdc ? (
|
||||
<span className="rounded bg-emerald-500/15 px-1.5 py-0.5 text-[9px] text-emerald-400">CDC actief</span>
|
||||
) : (
|
||||
<span className="rounded bg-white/10 px-1.5 py-0.5 text-[9px] text-foreground-muted">geen CDC-stream</span>
|
||||
)}
|
||||
{agentMap[active] && (
|
||||
<span className="inline-flex items-center gap-1 rounded bg-violet-500/15 px-1.5 py-0.5 text-[9px] text-violet-300">
|
||||
<Bot className="h-2.5 w-2.5" /> {agentMap[active].agent_name}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
<p className="mb-3 text-[11px] text-foreground-muted">{meta.desc} Doel: <span className="font-mono">{meta.target}</span></p>
|
||||
|
||||
<div className="flex flex-wrap items-end gap-3">
|
||||
<label className="flex flex-col gap-1 text-[10px] uppercase tracking-wide text-foreground-faint">
|
||||
Aantal rijen
|
||||
<input
|
||||
type="number"
|
||||
min={1}
|
||||
max={2000000}
|
||||
value={rows[active]}
|
||||
onChange={(e) => setRows((r) => ({ ...r, [active]: Number(e.target.value) }))}
|
||||
className="w-40 rounded border border-border bg-background px-2 py-1.5 text-[12px] text-foreground"
|
||||
/>
|
||||
</label>
|
||||
<button
|
||||
type="button"
|
||||
disabled={!!busy[active]}
|
||||
onClick={() => generate(active)}
|
||||
className="inline-flex items-center gap-2 rounded-md bg-violet-500/90 px-4 py-2 text-[12px] font-semibold text-black hover:bg-violet-400 disabled:opacity-40"
|
||||
>
|
||||
{busy[active] ? <Loader2 className="h-4 w-4 animate-spin" /> : <Play className="h-4 w-4" />}
|
||||
Genereer data
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
disabled={!!busy[active]}
|
||||
onClick={() => generate(active, true)}
|
||||
title="Laat de verantwoordelijke agent zelf data genereren (wordt gelogd)"
|
||||
className="inline-flex items-center gap-2 rounded-md border border-violet-400/50 px-3 py-2 text-[12px] font-medium text-violet-300 hover:bg-violet-500/10 disabled:opacity-40"
|
||||
>
|
||||
<Bot className="h-4 w-4" />
|
||||
Laat agent genereren
|
||||
</button>
|
||||
{active !== 'all' && COUNT_KEYS.includes(active) && (
|
||||
<div className="text-[11px] text-foreground-muted">
|
||||
Huidige rijen: <span className="font-mono text-foreground">{counts[active]?.toLocaleString() ?? '…'}</span>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
{msg && <p className="mt-3 text-[11px] text-foreground-muted">{msg}</p>}
|
||||
</div>
|
||||
|
||||
<div className="rounded-lg border border-border p-4">
|
||||
<h4 className="mb-2 text-[11px] font-semibold uppercase tracking-wide text-foreground-faint">Recente runs — {meta.label}</h4>
|
||||
{latest ? (
|
||||
<table className="w-full text-left text-[11px]">
|
||||
<thead>
|
||||
<tr className="border-b border-border text-[9px] uppercase text-foreground-faint">
|
||||
<th className="py-1 pr-2">State</th>
|
||||
<th className="py-1 pr-2">Rows</th>
|
||||
<th className="py-1 pr-2">Start</th>
|
||||
<th className="py-1 pr-2">Eind</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{(runs[active] || []).map((run) => (
|
||||
<tr key={run.run_id} className="border-b border-border/40">
|
||||
<td className="py-1 pr-2">{stateBadge(run.state)}</td>
|
||||
<td className="py-1 pr-2 font-mono text-foreground-muted">{run.conf?.rows ?? '—'}</td>
|
||||
<td className="py-1 pr-2 text-foreground-faint">{run.start?.slice(11, 19) || '—'}</td>
|
||||
<td className="py-1 pr-2 text-foreground-faint">{run.end?.slice(11, 19) || '—'}</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
) : (
|
||||
<p className="text-[11px] text-foreground-faint">Nog geen runs.</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="rounded-lg border border-border p-4">
|
||||
<h4 className="mb-2 text-[11px] font-semibold uppercase tracking-wide text-foreground-faint">Live tellingen (Trino)</h4>
|
||||
<div className="grid grid-cols-2 gap-2 sm:grid-cols-4">
|
||||
{COUNT_KEYS.map((k) => (
|
||||
<div key={k} className="rounded border border-border/60 px-2 py-1.5">
|
||||
<div className="text-[9px] uppercase text-foreground-faint">{k}</div>
|
||||
<div className="font-mono text-[12px] text-foreground">{counts[k]?.toLocaleString() ?? '…'}</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="rounded-lg border border-border p-4">
|
||||
<h4 className="mb-2 flex items-center gap-1.5 text-[11px] font-semibold uppercase tracking-wide text-foreground-faint">
|
||||
<ScrollText className="h-3.5 w-3.5" /> Agent-activiteit (wat de agents deden)
|
||||
</h4>
|
||||
{activity.length === 0 ? (
|
||||
<p className="text-[11px] text-foreground-faint">Nog geen agent-acties gelogd.</p>
|
||||
) : (
|
||||
<ul className="space-y-1">
|
||||
{activity.map((a) => (
|
||||
<li key={a.id} className="flex items-start gap-2 text-[11px]">
|
||||
<span className="mt-0.5 text-foreground-faint">{a.ts?.slice(11, 19) || ''}</span>
|
||||
<span className="inline-flex items-center gap-1 rounded bg-violet-500/15 px-1 text-[9px] text-violet-300">
|
||||
<Bot className="h-2.5 w-2.5" />{a.agent_name}
|
||||
</span>
|
||||
<span className={cn('flex-1', a.level === 'err' ? 'text-danger' : 'text-foreground-muted')}>{a.message}</span>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
return <DataGenPanel onPulse={onPulse} onOpenPlatform={onOpenPlatform} />
|
||||
}
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
import { useEffect, useState } from 'react'
|
||||
import { Database, Server } from 'lucide-react'
|
||||
import { DataSourcesView } from './DataSourcesView'
|
||||
import { HadoopSourcesView } from './HadoopSourcesView'
|
||||
import type { SourceEngine } from '../../lib/dataSourceCatalog'
|
||||
import { cn } from '../../lib/utils'
|
||||
import { subTabActive, subTabIdle } from '../../lib/tabActive'
|
||||
|
||||
type HubTab = 'sources' | 'hadoop'
|
||||
|
||||
type Props = {
|
||||
focusEngine?: SourceEngine | null
|
||||
initialTab?: HubTab
|
||||
onPulse?: () => void
|
||||
}
|
||||
|
||||
export function DataHubView({ focusEngine, initialTab = 'sources', onPulse }: Props) {
|
||||
const [tab, setTab] = useState<HubTab>(initialTab)
|
||||
|
||||
useEffect(() => {
|
||||
setTab(initialTab)
|
||||
}, [initialTab])
|
||||
|
||||
return (
|
||||
<div className="flex h-full min-h-0 flex-col overflow-hidden">
|
||||
<div className="panel mx-3 mt-3 flex shrink-0 items-center gap-1 px-2 py-1.5">
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setTab('sources')}
|
||||
className={cn(
|
||||
'inline-flex items-center gap-1.5 rounded-md px-3 py-1.5 text-[11px] font-medium transition-all',
|
||||
tab === 'sources' ? subTabActive : subTabIdle,
|
||||
)}
|
||||
>
|
||||
<Database className="h-3.5 w-3.5" /> Source Databases
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setTab('hadoop')}
|
||||
className={cn(
|
||||
'inline-flex items-center gap-1.5 rounded-md px-3 py-1.5 text-[11px] font-medium transition-all',
|
||||
tab === 'hadoop' ? subTabActive : subTabIdle,
|
||||
)}
|
||||
>
|
||||
<Server className="h-3.5 w-3.5" /> Hadoop
|
||||
</button>
|
||||
<span className="ml-auto text-[9px] text-foreground-muted">
|
||||
{tab === 'sources' ? 'PostgreSQL · MySQL · MongoDB · Cassandra · Neo4j' : 'HDFS · Hive · Iceberg · Spark · Kafka pipeline'}
|
||||
</span>
|
||||
</div>
|
||||
<div className="min-h-0 flex-1 overflow-hidden">
|
||||
{tab === 'sources' ? (
|
||||
<DataSourcesView focusEngine={focusEngine} onPulse={onPulse} />
|
||||
) : (
|
||||
<HadoopSourcesView onPulse={onPulse} />
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -6,15 +6,20 @@ import {
|
||||
FolderTree,
|
||||
Loader2,
|
||||
Network,
|
||||
Play,
|
||||
RefreshCw,
|
||||
Server,
|
||||
Table2,
|
||||
TerminalSquare,
|
||||
} from 'lucide-react'
|
||||
import { Badge } from '../ui/Badge'
|
||||
import { DbBrandIcon, dbBrandColor } from '../ui/DbBrandIcon'
|
||||
import { DataBrowserGrid } from './DataBrowserGrid'
|
||||
import { DataGenPanel } from './DataGenPanel'
|
||||
import { DbShell } from './DbShell'
|
||||
import { Neo4jGraphView } from './Neo4jGraphView'
|
||||
import { SqlWorkbench } from './SqlWorkbench'
|
||||
import { LakehouseWorkbench } from './SparkView'
|
||||
import {
|
||||
getSourceMeta,
|
||||
SOURCE_CATALOG,
|
||||
@@ -39,16 +44,33 @@ type SampleResponse = {
|
||||
row_count?: number
|
||||
elapsed_ms?: number
|
||||
error?: string
|
||||
primary_keys?: string[]
|
||||
cdc?: boolean
|
||||
editable?: boolean
|
||||
offset?: number
|
||||
limit?: number
|
||||
total_count?: number | null
|
||||
}
|
||||
|
||||
type Props = {
|
||||
focusEngine?: SourceEngine | null
|
||||
onPulse?: () => void
|
||||
}
|
||||
|
||||
const SUB_TABS: { id: SourceSubTab; label: string; icon: typeof FolderTree; neo4jOnly?: boolean }[] = [
|
||||
const ENGINE_CATALOG: Record<SourceEngine, string | undefined> = {
|
||||
postgres: 'postgres_sales',
|
||||
mysql: 'mysql_hr',
|
||||
mongodb: 'mongodb_supplychain',
|
||||
cassandra: 'cassandra_telemetry',
|
||||
neo4j: undefined,
|
||||
}
|
||||
|
||||
const SUB_TABS: { id: SourceSubTab; label: string; icon: typeof FolderTree; neo4jOnly?: boolean; hideNeo4j?: boolean }[] = [
|
||||
{ id: 'browser', label: 'Browser', icon: FolderTree },
|
||||
{ id: 'graph', label: 'Graph', icon: Network, neo4jOnly: true },
|
||||
{ id: 'console', label: 'Query Console', icon: Database },
|
||||
{ id: 'workbench', label: 'Workbench', icon: Activity, hideNeo4j: true },
|
||||
{ id: 'generate', label: 'Generate', icon: Play },
|
||||
{ id: 'shell', label: 'Shell', icon: TerminalSquare },
|
||||
]
|
||||
|
||||
@@ -59,7 +81,7 @@ function fmtCount(n?: number | null) {
|
||||
return String(n)
|
||||
}
|
||||
|
||||
export function DataSourcesView({ focusEngine }: Props) {
|
||||
export function DataSourcesView({ focusEngine, onPulse }: Props) {
|
||||
const [active, setActive] = useState<SourceEngine>(focusEngine || 'postgres')
|
||||
const [subTab, setSubTab] = useState<SourceSubTab>('browser')
|
||||
const [health, setHealth] = useState<HealthMap>({})
|
||||
@@ -68,6 +90,8 @@ export function DataSourcesView({ focusEngine }: Props) {
|
||||
const [selectedObject, setSelectedObject] = useState<CatalogObject | null>(null)
|
||||
const [sample, setSample] = useState<SampleResponse | null>(null)
|
||||
const [sampleLoading, setSampleLoading] = useState(false)
|
||||
const [page, setPage] = useState(1)
|
||||
const [pageSize, setPageSize] = useState(100)
|
||||
|
||||
const meta = getSourceMeta(active)
|
||||
|
||||
@@ -104,11 +128,14 @@ export function DataSourcesView({ focusEngine }: Props) {
|
||||
}
|
||||
}, [])
|
||||
|
||||
const loadSample = useCallback(async (engine: SourceEngine, obj: CatalogObject) => {
|
||||
const loadSample = useCallback(async (engine: SourceEngine, obj: CatalogObject, pg = page, ps = pageSize) => {
|
||||
setSampleLoading(true)
|
||||
setSample(null)
|
||||
const offset = (pg - 1) * ps
|
||||
try {
|
||||
const r = await fetch(`/api/sql/sample/${engine}?object=${encodeURIComponent(obj.fqn)}&limit=50`)
|
||||
const r = await fetch(
|
||||
`/api/sql/sample/${engine}?object=${encodeURIComponent(obj.fqn)}&limit=${ps}&offset=${offset}`,
|
||||
)
|
||||
const j = await r.json()
|
||||
setSample(j)
|
||||
} catch {
|
||||
@@ -116,7 +143,7 @@ export function DataSourcesView({ focusEngine }: Props) {
|
||||
} finally {
|
||||
setSampleLoading(false)
|
||||
}
|
||||
}, [])
|
||||
}, [page, pageSize])
|
||||
|
||||
useEffect(() => { loadHealth() }, [loadHealth])
|
||||
useEffect(() => {
|
||||
@@ -124,21 +151,26 @@ export function DataSourcesView({ focusEngine }: Props) {
|
||||
}, [active, subTab, loadCatalog])
|
||||
|
||||
useEffect(() => {
|
||||
if (selectedObject && subTab === 'browser') loadSample(active, selectedObject)
|
||||
}, [selectedObject, active, subTab, loadSample])
|
||||
setPage(1)
|
||||
}, [selectedObject?.fqn, active])
|
||||
|
||||
useEffect(() => {
|
||||
if (selectedObject && subTab === 'browser') loadSample(active, selectedObject, page, pageSize)
|
||||
}, [selectedObject, active, subTab, page, pageSize, loadSample])
|
||||
|
||||
const refreshAll = () => {
|
||||
loadHealth()
|
||||
if (subTab === 'browser') loadCatalog(active)
|
||||
else if (subTab === 'graph' && active === 'neo4j') { /* Neo4jGraphView reloads itself */ }
|
||||
else if (selectedObject) loadSample(active, selectedObject)
|
||||
if (subTab === 'browser') {
|
||||
loadCatalog(active)
|
||||
if (selectedObject) loadSample(active, selectedObject, page, pageSize)
|
||||
} else if (subTab === 'graph' && active === 'neo4j') { /* Neo4jGraphView reloads itself */ }
|
||||
else if (selectedObject) loadSample(active, selectedObject, page, pageSize)
|
||||
}
|
||||
|
||||
const visibleSubTabs = SUB_TABS.filter((t) => !t.neo4jOnly || active === 'neo4j')
|
||||
const visibleSubTabs = SUB_TABS.filter((t) => (!t.neo4jOnly || active === 'neo4j') && !(t.hideNeo4j && active === 'neo4j'))
|
||||
|
||||
return (
|
||||
<div className="flex h-full min-h-0 flex-col gap-2 p-3">
|
||||
{/* Header */}
|
||||
<header className="panel flex shrink-0 flex-wrap items-center justify-between gap-3 px-4 py-3">
|
||||
<div>
|
||||
<h1 className="flex items-center gap-2 text-base font-semibold text-foreground">
|
||||
@@ -146,7 +178,7 @@ export function DataSourcesView({ focusEngine }: Props) {
|
||||
Data Sources UI
|
||||
</h1>
|
||||
<p className="text-[11px] text-foreground-muted">
|
||||
Enterprise data browser — schema exploration, query console & interactive shells for all source databases
|
||||
Browse, filter, edit & query all source databases — edits on CDC sources flow to Live Changes
|
||||
</p>
|
||||
</div>
|
||||
<button type="button" onClick={refreshAll} className={cn('inline-flex items-center gap-1.5 rounded-md px-3 py-1.5 text-[11px]', subTabIdle)}>
|
||||
@@ -154,170 +186,159 @@ export function DataSourcesView({ focusEngine }: Props) {
|
||||
</button>
|
||||
</header>
|
||||
|
||||
<div className="flex min-h-0 flex-1 gap-2 overflow-hidden">
|
||||
{/* Left rail — database cards */}
|
||||
<aside className="panel flex w-[220px] shrink-0 flex-col overflow-hidden">
|
||||
<div className="shrink-0 border-b border-border px-3 py-2">
|
||||
<p className="text-[9px] font-semibold uppercase tracking-widest text-foreground-faint">Source Databases</p>
|
||||
</div>
|
||||
<div className="scrollbar-thin min-h-0 flex-1 space-y-1 overflow-y-auto p-2">
|
||||
{SOURCE_CATALOG.map((src) => {
|
||||
const Icon = src.icon
|
||||
const up = health[src.engine]?.ok
|
||||
const selected = active === src.engine
|
||||
return (
|
||||
<button
|
||||
key={src.engine}
|
||||
type="button"
|
||||
onClick={() => { setActive(src.engine); setSubTab('browser') }}
|
||||
className={cn(
|
||||
'flex w-full flex-col gap-1 rounded-lg border p-2.5 text-left transition-all',
|
||||
selected ? cn(src.border, src.accentBg, 'shadow-sm') : 'border-transparent hover:border-border hover:bg-surface-overlay',
|
||||
)}
|
||||
>
|
||||
<div className="flex items-center justify-between">
|
||||
<span className={cn('flex items-center gap-1.5 text-[12px] font-semibold', selected ? src.accent : 'text-foreground')}>
|
||||
<Icon className="h-4 w-4" />
|
||||
{src.label}
|
||||
</span>
|
||||
<span className={cn('h-2 w-2 rounded-full', up === true ? 'bg-emerald-400 shadow-[0_0_6px_rgba(52,211,153,0.6)]' : up === false ? 'bg-red-400' : 'bg-foreground-faint')} title={up ? 'Online' : up === false ? 'Offline' : 'Unknown'} />
|
||||
</div>
|
||||
<p className="text-[9px] leading-snug text-foreground-muted">{src.description}</p>
|
||||
<div className="flex flex-wrap gap-1">
|
||||
<Badge variant="default">{src.host}:{src.port}</Badge>
|
||||
{src.cdc && <Badge variant="accent">CDC</Badge>}
|
||||
</div>
|
||||
</button>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
</aside>
|
||||
|
||||
{/* Main panel */}
|
||||
<div className="panel flex min-h-0 min-w-0 flex-1 flex-col overflow-hidden">
|
||||
{/* Engine header */}
|
||||
<div className={cn('flex shrink-0 flex-wrap items-center justify-between gap-2 border-b border-border px-4 py-2.5', meta.accentBg)}>
|
||||
<div>
|
||||
<h2 className={cn('flex items-center gap-2 text-sm font-semibold', meta.accent)}>
|
||||
<meta.icon className="h-4 w-4" />
|
||||
{meta.label}
|
||||
{catalog?.version && (
|
||||
<span className="font-mono text-[10px] font-normal text-foreground-muted">v{catalog.version.split(' ')[0]?.slice(0, 20)}</span>
|
||||
{/* Horizontal database selector */}
|
||||
<div className="panel shrink-0 px-3 py-2">
|
||||
<p className="mb-2 text-[9px] font-semibold uppercase tracking-widest text-foreground-muted">Source Databases</p>
|
||||
<div className="scroll-x-stable scrollbar-thin flex gap-2 pb-1">
|
||||
{SOURCE_CATALOG.map((src) => {
|
||||
const up = health[src.engine]?.ok
|
||||
const selected = active === src.engine
|
||||
const brand = dbBrandColor(src.engine)
|
||||
return (
|
||||
<button
|
||||
key={src.engine}
|
||||
type="button"
|
||||
onClick={() => setActive(src.engine)}
|
||||
className={cn(
|
||||
'flex min-w-[148px] shrink-0 flex-col gap-1.5 rounded-lg border px-3 py-2.5 text-left transition-all',
|
||||
selected
|
||||
? 'shadow-md'
|
||||
: 'border-border/60 hover:border-border hover:bg-surface-overlay',
|
||||
)}
|
||||
</h2>
|
||||
<p className="font-mono text-[10px] text-foreground-muted">
|
||||
{meta.host}:{meta.port} · {meta.database} · container {meta.container}
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex gap-1">
|
||||
{visibleSubTabs.map(({ id, label, icon: Icon }) => (
|
||||
<button
|
||||
key={id}
|
||||
type="button"
|
||||
onClick={() => setSubTab(id)}
|
||||
className={cn('inline-flex items-center gap-1 rounded-md px-2.5 py-1 text-[10px] font-medium', subTab === id ? subTabActive : subTabIdle)}
|
||||
>
|
||||
<Icon className="h-3 w-3" /> {label}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Sub-tab content */}
|
||||
<div className="min-h-0 flex-1 overflow-hidden">
|
||||
{subTab === 'browser' && (
|
||||
<div className="flex h-full min-h-0">
|
||||
{/* Object tree */}
|
||||
<div className="flex w-[280px] shrink-0 flex-col border-r border-border/60">
|
||||
<div className="flex shrink-0 items-center justify-between border-b border-border/60 px-3 py-2">
|
||||
<span className="text-[10px] font-semibold uppercase tracking-wider text-foreground-faint">Objects</span>
|
||||
{catalogLoading && <Loader2 className="h-3 w-3 animate-spin text-foreground-muted" />}
|
||||
</div>
|
||||
<div className="scrollbar-thin min-h-0 flex-1 overflow-y-auto p-1.5">
|
||||
{catalog?.objects?.map((obj) => (
|
||||
<button
|
||||
key={obj.fqn}
|
||||
type="button"
|
||||
onClick={() => setSelectedObject(obj)}
|
||||
className={cn(
|
||||
'mb-0.5 flex w-full items-center gap-1.5 rounded px-2 py-1.5 text-left text-[10px] transition-colors',
|
||||
selectedObject?.fqn === obj.fqn ? subTabActive : 'hover:bg-surface-overlay',
|
||||
)}
|
||||
>
|
||||
{obj.type === 'node_label' ? <Activity className="h-3 w-3 shrink-0 text-pink-400" /> : <Table2 className="h-3 w-3 shrink-0 text-foreground-muted" />}
|
||||
<div className="min-w-0 flex-1">
|
||||
<p className="truncate font-medium text-foreground">{obj.name}</p>
|
||||
<p className="truncate font-mono text-[8px] text-foreground-faint">{obj.schema}{obj.type === 'relationship' ? ' · rel' : ''}</p>
|
||||
</div>
|
||||
<span className="shrink-0 font-mono text-[9px] text-foreground-muted">{fmtCount(obj.row_count)}</span>
|
||||
<ChevronRight className="h-3 w-3 shrink-0 text-foreground-faint" />
|
||||
</button>
|
||||
))}
|
||||
{!catalogLoading && !catalog?.objects?.length && (
|
||||
<p className="p-4 text-center text-[10px] text-foreground-faint">No objects found</p>
|
||||
style={selected ? { borderColor: brand, backgroundColor: `${brand}18`, boxShadow: `0 0 0 1px ${brand}40` } : undefined}
|
||||
>
|
||||
<div className="flex items-center justify-between gap-2">
|
||||
<span className="flex items-center gap-2 text-[12px] font-semibold text-foreground">
|
||||
<DbBrandIcon engine={src.engine} size={22} />
|
||||
{src.label}
|
||||
</span>
|
||||
<span
|
||||
className={cn(
|
||||
'h-2 w-2 shrink-0 rounded-full',
|
||||
up === true ? 'bg-emerald-400 shadow-[0_0_6px_rgba(52,211,153,0.6)]' : up === false ? 'bg-red-400' : 'bg-foreground-faint',
|
||||
)}
|
||||
</div>
|
||||
title={up ? 'Online' : up === false ? 'Offline' : 'Unknown'}
|
||||
/>
|
||||
</div>
|
||||
<p className="line-clamp-2 text-[9px] leading-snug text-foreground-muted">{src.description}</p>
|
||||
<div className="flex flex-wrap gap-1">
|
||||
<Badge variant="default">{src.host}:{src.port}</Badge>
|
||||
{src.cdc && <Badge variant="accent">CDC</Badge>}
|
||||
</div>
|
||||
</button>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Sample data grid */}
|
||||
<div className="flex min-h-0 min-w-0 flex-1 flex-col">
|
||||
<div className="flex shrink-0 items-center justify-between border-b border-border/60 px-3 py-2">
|
||||
<span className="text-[10px] font-semibold text-foreground">
|
||||
{selectedObject ? (
|
||||
<>Sample: <span className="font-mono text-docker">{selectedObject.fqn}</span></>
|
||||
) : 'Select an object'}
|
||||
</span>
|
||||
{sampleLoading && <Loader2 className="h-3 w-3 animate-spin" />}
|
||||
</div>
|
||||
<div className="scrollbar-thin min-h-0 flex-1 overflow-auto p-2">
|
||||
{sample?.ok && sample.columns && (
|
||||
<>
|
||||
<p className="mb-1 font-mono text-[9px] text-foreground-faint">
|
||||
{sample.row_count} rows · {sample.elapsed_ms}ms
|
||||
</p>
|
||||
<table className="w-full text-left font-mono text-[10px]">
|
||||
<thead>
|
||||
<tr className="sticky top-0 border-b border-border bg-surface-raised text-docker">
|
||||
{sample.columns.map((c) => <th key={c} className="px-2 py-1">{c}</th>)}
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{sample.rows?.map((row, i) => (
|
||||
<tr key={i} className="border-b border-border/30 hover:bg-white/5">
|
||||
{row.map((cell, j) => (
|
||||
<td key={j} className="max-w-[200px] truncate px-2 py-1 text-foreground-muted">
|
||||
{cell === null || cell === undefined ? 'NULL' : String(cell)}
|
||||
</td>
|
||||
))}
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</>
|
||||
)}
|
||||
{sample && !sample.ok && (
|
||||
<p className="p-4 text-[11px] text-danger">{sample.error || 'Failed to load sample'}</p>
|
||||
)}
|
||||
{!selectedObject && !sampleLoading && (
|
||||
<p className="py-8 text-center text-[11px] text-foreground-faint">Select a table, collection or label to preview data</p>
|
||||
)}
|
||||
</div>
|
||||
{/* Main panel — full width */}
|
||||
<div className="panel flex min-h-0 min-w-0 flex-1 flex-col overflow-hidden">
|
||||
<div
|
||||
className="flex shrink-0 flex-wrap items-center justify-between gap-2 border-b border-border px-4 py-2.5"
|
||||
style={{ backgroundColor: `${dbBrandColor(active)}12` }}
|
||||
>
|
||||
<div>
|
||||
<h2 className="flex items-center gap-2 text-sm font-semibold text-foreground">
|
||||
<DbBrandIcon engine={active} size={18} />
|
||||
{meta.label}
|
||||
{catalog?.version && (
|
||||
<span className="font-mono text-[10px] font-normal text-foreground-muted">v{catalog.version.split(' ')[0]?.slice(0, 20)}</span>
|
||||
)}
|
||||
</h2>
|
||||
<p className="font-mono text-[10px] text-foreground-muted">
|
||||
{meta.host}:{meta.port} · {meta.database} · container {meta.container}
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex flex-wrap gap-1">
|
||||
{visibleSubTabs.map(({ id, label, icon: Icon }) => (
|
||||
<button
|
||||
key={id}
|
||||
type="button"
|
||||
onClick={() => setSubTab(id)}
|
||||
className={cn('inline-flex items-center gap-1 rounded-md px-2.5 py-1 text-[10px] font-medium', subTab === id ? subTabActive : subTabIdle)}
|
||||
>
|
||||
<Icon className="h-3 w-3" /> {label}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="flex min-h-0 flex-1 flex-col overflow-hidden">
|
||||
{subTab === 'browser' && (
|
||||
<div className="flex h-full min-h-0 flex-1 overflow-hidden">
|
||||
<div className="flex w-[260px] shrink-0 flex-col border-r border-border/60">
|
||||
<div className="flex shrink-0 items-center justify-between border-b border-border/60 px-3 py-2">
|
||||
<span className="text-[10px] font-semibold uppercase tracking-wider text-foreground-muted">Objects</span>
|
||||
{catalogLoading && <Loader2 className="h-3 w-3 animate-spin text-foreground-muted" />}
|
||||
</div>
|
||||
<div className="scrollbar-thin min-h-0 flex-1 overflow-y-auto p-1.5">
|
||||
{catalog?.objects?.map((obj) => (
|
||||
<button
|
||||
key={obj.fqn}
|
||||
type="button"
|
||||
onClick={() => setSelectedObject(obj)}
|
||||
className={cn(
|
||||
'mb-0.5 flex w-full items-center gap-1.5 rounded px-2 py-1.5 text-left text-[10px] transition-colors',
|
||||
selectedObject?.fqn === obj.fqn ? subTabActive : 'hover:bg-surface-overlay',
|
||||
)}
|
||||
>
|
||||
{obj.type === 'node_label' ? (
|
||||
<Activity className="h-3 w-3 shrink-0" style={{ color: dbBrandColor('neo4j') }} />
|
||||
) : (
|
||||
<Table2 className="h-3 w-3 shrink-0 text-foreground-muted" />
|
||||
)}
|
||||
<div className="min-w-0 flex-1">
|
||||
<p className="truncate font-medium text-foreground">{obj.name}</p>
|
||||
<p className="truncate font-mono text-[8px] text-foreground-muted">{obj.schema}{obj.type === 'relationship' ? ' · rel' : ''}</p>
|
||||
</div>
|
||||
<span className="shrink-0 font-mono text-[9px] text-foreground-muted">{fmtCount(obj.row_count)}</span>
|
||||
<ChevronRight className="h-3 w-3 shrink-0 text-foreground-muted" />
|
||||
</button>
|
||||
))}
|
||||
{!catalogLoading && !catalog?.objects?.length && (
|
||||
<p className="p-4 text-center text-[10px] text-foreground-muted">No objects found</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{subTab === 'graph' && active === 'neo4j' && (
|
||||
<Neo4jGraphView />
|
||||
)}
|
||||
<div className="flex h-full min-h-0 min-w-0 flex-1 flex-col overflow-hidden">
|
||||
<div className="flex shrink-0 items-center border-b border-border/60 px-3 py-2">
|
||||
<span className="text-[10px] font-semibold text-foreground">
|
||||
{selectedObject ? (
|
||||
<>Data: <span className="font-mono text-docker">{selectedObject.fqn}</span></>
|
||||
) : 'Select an object'}
|
||||
</span>
|
||||
</div>
|
||||
<DataBrowserGrid
|
||||
engine={active}
|
||||
objectFqn={selectedObject?.fqn || ''}
|
||||
sample={selectedObject ? sample : null}
|
||||
loading={sampleLoading}
|
||||
page={page}
|
||||
pageSize={pageSize}
|
||||
onPageChange={setPage}
|
||||
onPageSizeChange={(ps) => { setPageSize(ps); setPage(1) }}
|
||||
onReload={() => selectedObject && loadSample(active, selectedObject, page, pageSize)}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{subTab === 'console' && (
|
||||
<SqlWorkbench engine={active} />
|
||||
)}
|
||||
{subTab === 'graph' && active === 'neo4j' && <Neo4jGraphView />}
|
||||
|
||||
{subTab === 'shell' && (
|
||||
<DbShell initialCommand={meta.shellCommand} />
|
||||
)}
|
||||
</div>
|
||||
{subTab === 'console' && <SqlWorkbench engine={active} />}
|
||||
|
||||
{subTab === 'workbench' && (
|
||||
<div className="flex min-h-0 flex-1 overflow-hidden">
|
||||
<LakehouseWorkbench lockedCatalog={ENGINE_CATALOG[active]} />
|
||||
</div>
|
||||
)}
|
||||
|
||||
{subTab === 'generate' && onPulse && (
|
||||
<DataGenPanel embedded focusSource={active} onPulse={onPulse} />
|
||||
)}
|
||||
|
||||
{subTab === 'shell' && <DbShell initialCommand={meta.shellCommand} />}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -0,0 +1,280 @@
|
||||
import { useCallback, useEffect, useState } from 'react'
|
||||
import {
|
||||
Activity,
|
||||
ChevronRight,
|
||||
FolderTree,
|
||||
GitBranch,
|
||||
Loader2,
|
||||
Play,
|
||||
RefreshCw,
|
||||
Server,
|
||||
Table2,
|
||||
TerminalSquare,
|
||||
Zap,
|
||||
} from 'lucide-react'
|
||||
import { Badge } from '../ui/Badge'
|
||||
import { DataBrowserGrid } from './DataBrowserGrid'
|
||||
import { HdfsView } from './HdfsView'
|
||||
import { SparkView } from './SparkView'
|
||||
import { SqlWorkbench } from './SqlWorkbench'
|
||||
import type { CatalogObject } from '../../lib/dataSourceCatalog'
|
||||
import { cn } from '../../lib/utils'
|
||||
import { subTabActive, subTabIdle } from '../../lib/tabActive'
|
||||
import { runDataflowMovement, triggerStreamingPipeline } from '../../lib/api'
|
||||
|
||||
type HadoopSubTab = 'browser' | 'files' | 'console' | 'spark' | 'pipeline'
|
||||
type CatalogResponse = { engine: string; version?: string; objects: CatalogObject[] }
|
||||
type SampleResponse = {
|
||||
ok: boolean
|
||||
columns?: string[]
|
||||
rows?: unknown[][]
|
||||
error?: string
|
||||
total_count?: number | null
|
||||
offset?: number
|
||||
limit?: number
|
||||
editable?: boolean
|
||||
}
|
||||
|
||||
const SUB_TABS: { id: HadoopSubTab; label: string; icon: typeof FolderTree }[] = [
|
||||
{ id: 'browser', label: 'Tables', icon: Table2 },
|
||||
{ id: 'files', label: 'HDFS Files', icon: FolderTree },
|
||||
{ id: 'console', label: 'Query Console', icon: TerminalSquare },
|
||||
{ id: 'spark', label: 'Spark', icon: Activity },
|
||||
{ id: 'pipeline', label: 'Pipeline', icon: GitBranch },
|
||||
]
|
||||
|
||||
export function HadoopSourcesView({ onPulse }: { onPulse?: () => void }) {
|
||||
const [subTab, setSubTab] = useState<HadoopSubTab>('browser')
|
||||
const [health, setHealth] = useState<{ ok?: boolean; error?: string } | null>(null)
|
||||
const [catalog, setCatalog] = useState<CatalogResponse | null>(null)
|
||||
const [catalogLoading, setCatalogLoading] = useState(false)
|
||||
const [selectedObject, setSelectedObject] = useState<CatalogObject | null>(null)
|
||||
const [sample, setSample] = useState<SampleResponse | null>(null)
|
||||
const [sampleLoading, setSampleLoading] = useState(false)
|
||||
const [page, setPage] = useState(1)
|
||||
const [pageSize, setPageSize] = useState(100)
|
||||
const [pipelineBusy, setPipelineBusy] = useState<string | null>(null)
|
||||
const [pipelineMsg, setPipelineMsg] = useState<string | null>(null)
|
||||
|
||||
const loadHealth = useCallback(async () => {
|
||||
try {
|
||||
const r = await fetch('/api/sql/health/hadoop')
|
||||
if (r.ok) setHealth(await r.json())
|
||||
} catch { /* */ }
|
||||
}, [])
|
||||
|
||||
const loadCatalog = useCallback(async () => {
|
||||
setCatalogLoading(true)
|
||||
try {
|
||||
const r = await fetch('/api/sql/catalog/hadoop')
|
||||
if (r.ok) {
|
||||
const j: CatalogResponse = await r.json()
|
||||
setCatalog(j)
|
||||
const first = j.objects?.[0]
|
||||
if (first) setSelectedObject(first)
|
||||
}
|
||||
} catch { /* */ } finally {
|
||||
setCatalogLoading(false)
|
||||
}
|
||||
}, [])
|
||||
|
||||
const loadSample = useCallback(async (obj: CatalogObject, pg = page, ps = pageSize) => {
|
||||
setSampleLoading(true)
|
||||
const offset = (pg - 1) * ps
|
||||
try {
|
||||
const r = await fetch(
|
||||
`/api/sql/sample/hadoop?object=${encodeURIComponent(obj.fqn)}&limit=${ps}&offset=${offset}`,
|
||||
)
|
||||
setSample(await r.json())
|
||||
} catch {
|
||||
setSample({ ok: false, error: 'Sample unavailable' })
|
||||
} finally {
|
||||
setSampleLoading(false)
|
||||
}
|
||||
}, [page, pageSize])
|
||||
|
||||
useEffect(() => { loadHealth() }, [loadHealth])
|
||||
useEffect(() => {
|
||||
if (subTab === 'browser') loadCatalog()
|
||||
}, [subTab, loadCatalog])
|
||||
useEffect(() => { setPage(1) }, [selectedObject?.fqn])
|
||||
useEffect(() => {
|
||||
if (selectedObject && subTab === 'browser') loadSample(selectedObject, page, pageSize)
|
||||
}, [selectedObject, subTab, page, pageSize, loadSample])
|
||||
|
||||
const refreshAll = () => {
|
||||
loadHealth()
|
||||
if (subTab === 'browser') {
|
||||
loadCatalog()
|
||||
if (selectedObject) loadSample(selectedObject, page, pageSize)
|
||||
}
|
||||
}
|
||||
|
||||
const runPipeline = async (kind: 'full' | 'hdfs_kafka' | 'spark_s3') => {
|
||||
setPipelineBusy(kind)
|
||||
setPipelineMsg(null)
|
||||
try {
|
||||
if (kind === 'full') {
|
||||
const r = await triggerStreamingPipeline('hadoop-lake')
|
||||
const j = await r.json()
|
||||
setPipelineMsg(j.ok ? `✓ Pipeline started: ${j.steps?.join(' → ') || 'ok'}` : j.error || 'Failed')
|
||||
} else if (kind === 'hdfs_kafka') {
|
||||
const r = await fetch('/api/pipeline/streaming/hdfs/to-kafka', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ source: 'trino', table: 'iceberg.hadoop.historical_sales_hdfs', topic: 'hdfs.historical.sales', limit: 2000 }),
|
||||
})
|
||||
const j = await r.json()
|
||||
setPipelineMsg(j.ok ? `✓ ${j.rows_sent} rows → Kafka topic ${j.topic}` : j.error || 'Failed')
|
||||
} else {
|
||||
await runDataflowMovement('spark_to_s3')
|
||||
setPipelineMsg('✓ Spark → S3 job triggered')
|
||||
}
|
||||
onPulse?.()
|
||||
} catch {
|
||||
setPipelineMsg('Pipeline failed')
|
||||
} finally {
|
||||
setPipelineBusy(null)
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="flex h-full min-h-0 flex-col gap-2 p-3 pt-1">
|
||||
<header className="panel flex shrink-0 flex-wrap items-center justify-between gap-3 px-4 py-3">
|
||||
<div>
|
||||
<h1 className="flex items-center gap-2 text-base font-semibold text-foreground">
|
||||
<Server className="h-5 w-5 text-emerald-400" />
|
||||
Hadoop Data Lake
|
||||
</h1>
|
||||
<p className="text-[11px] text-foreground-muted">
|
||||
HDFS · Hive · Iceberg tables · Spark transforms · Kafka bridge → S3
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<Badge variant={health?.ok ? 'success' : 'warning'}>
|
||||
{health?.ok ? 'Hadoop online' : health?.error?.slice(0, 40) || 'Checking…'}
|
||||
</Badge>
|
||||
<button type="button" onClick={refreshAll} className={cn('inline-flex items-center gap-1.5 rounded-md px-3 py-1.5 text-[11px]', subTabIdle)}>
|
||||
<RefreshCw className="h-3.5 w-3.5" /> Refresh
|
||||
</button>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<div className="panel flex shrink-0 gap-1 px-2 py-1.5">
|
||||
{SUB_TABS.map(({ id, label, icon: Icon }) => (
|
||||
<button
|
||||
key={id}
|
||||
type="button"
|
||||
onClick={() => setSubTab(id)}
|
||||
className={cn(
|
||||
'inline-flex items-center gap-1.5 rounded-md px-2.5 py-1.5 text-[10px] font-medium',
|
||||
subTab === id ? subTabActive : subTabIdle,
|
||||
)}
|
||||
>
|
||||
<Icon className="h-3.5 w-3.5" /> {label}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
|
||||
{subTab === 'browser' && (
|
||||
<div className="panel flex min-h-0 flex-1 gap-0 overflow-hidden">
|
||||
<aside className="scrollbar-thin w-52 shrink-0 overflow-y-auto border-r border-border p-2">
|
||||
<p className="mb-1 text-[9px] font-semibold uppercase tracking-wider text-foreground-muted">
|
||||
{catalogLoading ? 'Loading…' : `${catalog?.objects?.length ?? 0} objects`}
|
||||
</p>
|
||||
{catalog?.objects?.map((obj) => (
|
||||
<button
|
||||
key={obj.fqn}
|
||||
type="button"
|
||||
onClick={() => setSelectedObject(obj)}
|
||||
className={cn(
|
||||
'mb-0.5 flex w-full items-center gap-1 rounded px-2 py-1.5 text-left text-[10px] transition-colors',
|
||||
selectedObject?.fqn === obj.fqn ? 'bg-emerald-500/15 text-emerald-200' : 'hover:bg-surface-overlay text-foreground-muted',
|
||||
)}
|
||||
>
|
||||
<ChevronRight className="h-3 w-3 shrink-0 opacity-50" />
|
||||
<span className="min-w-0 truncate">
|
||||
<span className="block truncate font-medium text-foreground">{obj.name}</span>
|
||||
<span className="block truncate text-[8px] text-foreground-faint">{obj.schema}</span>
|
||||
</span>
|
||||
</button>
|
||||
))}
|
||||
</aside>
|
||||
<div className="min-h-0 min-w-0 flex-1 overflow-hidden p-2">
|
||||
{selectedObject ? (
|
||||
<DataBrowserGrid
|
||||
engine="hadoop"
|
||||
objectFqn={selectedObject.fqn}
|
||||
sample={sample}
|
||||
loading={sampleLoading}
|
||||
page={page}
|
||||
pageSize={pageSize}
|
||||
onPageChange={setPage}
|
||||
onPageSizeChange={(ps) => { setPageSize(ps); setPage(1) }}
|
||||
onReload={() => selectedObject && loadSample(selectedObject, page, pageSize)}
|
||||
/>
|
||||
) : (
|
||||
<p className="py-8 text-center text-sm text-foreground-muted">Select a table or HDFS path</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{subTab === 'files' && <HdfsView />}
|
||||
{subTab === 'console' && <SqlWorkbench engine="hadoop" />}
|
||||
{subTab === 'spark' && <SparkView embedded />}
|
||||
{subTab === 'pipeline' && (
|
||||
<div className="panel flex min-h-0 flex-1 flex-col gap-4 overflow-y-auto p-4">
|
||||
<div>
|
||||
<h3 className="mb-1 text-sm font-semibold text-foreground">Hadoop → Kafka → Spark → S3</h3>
|
||||
<p className="text-[11px] text-foreground-muted">
|
||||
Full lakehouse pipeline: export HDFS data to Kafka, Spark transforms into Iceberg/S3 curated layer.
|
||||
</p>
|
||||
</div>
|
||||
<div className="grid gap-3 sm:grid-cols-3">
|
||||
<PipelineCard
|
||||
title="1. HDFS → Kafka"
|
||||
desc="Export iceberg.hadoop.historical_sales_hdfs to Kafka topic hdfs.historical.sales"
|
||||
busy={pipelineBusy === 'hdfs_kafka'}
|
||||
onRun={() => runPipeline('hdfs_kafka')}
|
||||
/>
|
||||
<PipelineCard
|
||||
title="2. Spark transform"
|
||||
desc="Run mask_to_curated / hadoop_to_trino via Airflow on Spark cluster"
|
||||
busy={pipelineBusy === 'spark_s3'}
|
||||
onRun={() => runPipeline('spark_s3')}
|
||||
/>
|
||||
<PipelineCard
|
||||
title="Full pipeline"
|
||||
desc="HDFS → Kafka → Spark → Iceberg → S3 in one orchestrated run"
|
||||
busy={pipelineBusy === 'full'}
|
||||
onRun={() => runPipeline('full')}
|
||||
/>
|
||||
</div>
|
||||
{pipelineMsg && <p className="text-[11px] text-docker">{pipelineMsg}</p>}
|
||||
<div className="rounded-lg border border-border bg-surface-overlay/30 p-3 font-mono text-[10px] text-foreground-muted">
|
||||
hdfs:/data/historical/sales_orders → Kafka:hdfs.historical.sales → Spark → iceberg.hadoop → s3://data/hadoop/
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function PipelineCard({ title, desc, busy, onRun }: { title: string; desc: string; busy: boolean; onRun: () => void }) {
|
||||
return (
|
||||
<div className="rounded-lg border border-border bg-surface-overlay/20 p-3">
|
||||
<h4 className="mb-1 text-[11px] font-semibold text-foreground">{title}</h4>
|
||||
<p className="mb-2 min-h-[2.5rem] text-[10px] text-foreground-muted">{desc}</p>
|
||||
<button
|
||||
type="button"
|
||||
disabled={busy}
|
||||
onClick={onRun}
|
||||
className="inline-flex items-center gap-1 rounded-md border border-emerald-400/50 bg-emerald-500/15 px-3 py-1.5 text-[10px] font-medium text-emerald-200 hover:bg-emerald-500/25 disabled:opacity-50"
|
||||
>
|
||||
{busy ? <Loader2 className="h-3.5 w-3.5 animate-spin" /> : <Play className="h-3.5 w-3.5" />}
|
||||
Run
|
||||
</button>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
import { useState } from 'react'
|
||||
import { LayoutDashboard, Presentation } from 'lucide-react'
|
||||
import { PlatformTopology } from './PlatformTopology'
|
||||
import { PresentationView } from './PresentationView'
|
||||
import type { AgentAnim, WorkloadData } from '../../types'
|
||||
import { cn } from '../../lib/utils'
|
||||
import { subTabActive, subTabIdle } from '../../lib/tabActive'
|
||||
|
||||
type PlatformTab = 'topology' | 'presentation'
|
||||
|
||||
type Props = {
|
||||
workload: WorkloadData | null
|
||||
animations: Record<string, AgentAnim>
|
||||
selectedNodeId: string | null
|
||||
onNodeClick: (nodeId: string) => void
|
||||
pulse: boolean
|
||||
}
|
||||
|
||||
const TABS: { id: PlatformTab; label: string; icon: typeof LayoutDashboard }[] = [
|
||||
{ id: 'topology', label: 'Topology', icon: LayoutDashboard },
|
||||
{ id: 'presentation', label: 'Presentation', icon: Presentation },
|
||||
]
|
||||
|
||||
export function PlatformView({ workload, animations, selectedNodeId, onNodeClick, pulse }: Props) {
|
||||
const [tab, setTab] = useState<PlatformTab>('topology')
|
||||
|
||||
return (
|
||||
<div className="flex min-h-0 flex-1 flex-col overflow-hidden">
|
||||
<div className="panel flex shrink-0 items-center gap-1 px-2 py-1.5">
|
||||
{TABS.map(({ id, label, icon: Icon }) => (
|
||||
<button
|
||||
key={id}
|
||||
type="button"
|
||||
onClick={() => setTab(id)}
|
||||
className={cn(
|
||||
'inline-flex items-center gap-1.5 rounded-md px-3 py-1.5 text-[10px] font-medium transition-all',
|
||||
tab === id ? subTabActive : subTabIdle,
|
||||
)}
|
||||
>
|
||||
<Icon className="h-3.5 w-3.5" />
|
||||
{label}
|
||||
</button>
|
||||
))}
|
||||
{tab === 'presentation' && (
|
||||
<span className="ml-auto text-[9px] text-foreground-muted">
|
||||
Live cluster deck · all running services
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<div className="min-h-0 flex-1 overflow-hidden pt-1">
|
||||
{tab === 'topology' ? (
|
||||
<PlatformTopology
|
||||
workload={workload}
|
||||
animations={animations}
|
||||
selectedNodeId={selectedNodeId}
|
||||
onNodeClick={onNodeClick}
|
||||
pulse={pulse}
|
||||
/>
|
||||
) : (
|
||||
<PresentationView embedded />
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -1,8 +1,11 @@
|
||||
import { useCallback, useEffect, useRef, useState } from 'react'
|
||||
import { createPortal } from 'react-dom'
|
||||
import {
|
||||
Expand,
|
||||
ExternalLink,
|
||||
FileUp,
|
||||
ImagePlus,
|
||||
Maximize2,
|
||||
Monitor,
|
||||
Pencil,
|
||||
Plus,
|
||||
@@ -17,6 +20,136 @@ import { cn } from '../../lib/utils'
|
||||
import { subTabActive, subTabIdle } from '../../lib/tabActive'
|
||||
|
||||
type DeckSource = 'live' | 'data-maturity' | 'atc-platform' | string
|
||||
type SlideVariant = 'embedded' | 'normal' | 'present'
|
||||
type PresentMode = 'popup' | 'fullscreen' | null
|
||||
|
||||
function SlidePanel({
|
||||
slide,
|
||||
slideIdx,
|
||||
slideCount,
|
||||
variant,
|
||||
}: {
|
||||
slide: PresentationSlide
|
||||
slideIdx: number
|
||||
slideCount: number
|
||||
variant: SlideVariant
|
||||
}) {
|
||||
const isPresent = variant === 'present'
|
||||
const isEmbedded = variant === 'embedded'
|
||||
|
||||
return (
|
||||
<div className={cn('mx-auto w-full', isPresent ? 'max-w-6xl' : isEmbedded ? 'max-w-full' : 'max-w-5xl')}>
|
||||
<div className={cn('flex', isPresent ? 'gap-8 lg:gap-12' : isEmbedded ? 'gap-4' : 'gap-4 md:gap-6')}>
|
||||
<div className="min-w-0 flex-1">
|
||||
<p className={cn(
|
||||
'mb-1 font-medium uppercase tracking-widest text-docker/80',
|
||||
isPresent ? 'text-xs md:text-sm' : isEmbedded ? 'text-[9px]' : 'text-[10px]',
|
||||
)}>
|
||||
{slide.kind || 'slide'} · {slideIdx + 1}/{slideCount}
|
||||
</p>
|
||||
<h1 className={cn(
|
||||
'font-bold tracking-tight text-foreground',
|
||||
isPresent ? 'mb-3 text-3xl leading-tight md:text-5xl lg:text-6xl'
|
||||
: isEmbedded ? 'mb-1 text-lg leading-tight'
|
||||
: 'mb-2 text-2xl md:text-4xl',
|
||||
)}>
|
||||
{slide.title}
|
||||
</h1>
|
||||
{slide.subtitle && (
|
||||
<p className={cn(
|
||||
'text-foreground-muted',
|
||||
isPresent ? 'mb-6 text-lg md:text-2xl' : isEmbedded ? 'mb-2 text-xs' : 'mb-4 text-sm md:text-base',
|
||||
)}>
|
||||
{slide.subtitle}
|
||||
</p>
|
||||
)}
|
||||
{'animation' in slide && slide.animation && (
|
||||
<ArchitectureDiagram
|
||||
animation={String(slide.animation)}
|
||||
compact={isEmbedded}
|
||||
present={isPresent}
|
||||
/>
|
||||
)}
|
||||
<ul className={cn(
|
||||
'leading-relaxed text-foreground',
|
||||
isPresent ? 'space-y-3 text-lg md:text-xl lg:text-2xl'
|
||||
: isEmbedded ? 'space-y-1 text-xs'
|
||||
: 'space-y-2 text-sm md:text-base',
|
||||
)}>
|
||||
{(slide.bullets || []).map((b: string, bi: number) => (
|
||||
<li key={bi} className="flex gap-2"><span className="shrink-0 text-docker">▸</span><span>{b}</span></li>
|
||||
))}
|
||||
</ul>
|
||||
</div>
|
||||
{slide.image && !isEmbedded && (
|
||||
<div className={cn('hidden shrink-0 items-start', isPresent ? 'lg:flex' : 'md:flex')}>
|
||||
<img
|
||||
src={slide.image}
|
||||
alt=""
|
||||
className={cn(
|
||||
'rounded-lg border border-border object-contain shadow-lg',
|
||||
isPresent ? 'max-h-[55vh] max-w-[38vw]' : 'max-h-[46vh] max-w-[40vw]',
|
||||
)}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
{slide.image && isPresent && (
|
||||
<div className="mt-6 lg:hidden">
|
||||
<img src={slide.image} alt="" className="max-h-[35vh] w-full rounded-lg border border-border object-contain shadow-lg" />
|
||||
</div>
|
||||
)}
|
||||
{slide.image && !isPresent && (
|
||||
<div className={cn('mt-3', isEmbedded ? '' : 'md:hidden')}>
|
||||
<img
|
||||
src={slide.image}
|
||||
alt=""
|
||||
className={cn('rounded-lg border border-border object-contain', isEmbedded ? 'max-h-[22vh] w-full' : 'max-h-[30vh]')}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function SlideNav({
|
||||
slideIdx,
|
||||
slideCount,
|
||||
embedded,
|
||||
onPrev,
|
||||
onNext,
|
||||
onGo,
|
||||
className,
|
||||
}: {
|
||||
slideIdx: number
|
||||
slideCount: number
|
||||
embedded?: boolean
|
||||
onPrev: () => void
|
||||
onNext: () => void
|
||||
onGo?: (idx: number) => void
|
||||
className?: string
|
||||
}) {
|
||||
return (
|
||||
<div className={cn('flex shrink-0 items-center gap-2 border-t border-border bg-surface-raised/90 px-3 py-1.5', className)}>
|
||||
<button type="button" disabled={slideIdx === 0} onClick={onPrev} className="rounded border border-border px-2 py-1 text-[10px] disabled:opacity-40 md:text-xs">← Prev</button>
|
||||
{embedded ? (
|
||||
<span className="flex-1 text-center font-mono text-[10px] text-foreground-muted md:text-xs">{slideIdx + 1} / {slideCount}</span>
|
||||
) : (
|
||||
<div className="flex flex-1 flex-wrap justify-center gap-1">
|
||||
{Array.from({ length: slideCount }, (_, i) => (
|
||||
<button
|
||||
key={i}
|
||||
type="button"
|
||||
onClick={() => onGo?.(i)}
|
||||
className={cn('h-2 w-2 rounded-full', i === slideIdx ? 'scale-125 bg-docker' : 'bg-border')}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
<button type="button" disabled={slideIdx >= slideCount - 1} onClick={onNext} className="rounded border border-border px-2 py-1 text-[10px] disabled:opacity-40 md:text-xs">Next →</button>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
const KIND_STYLES: Record<string, string> = {
|
||||
hero: 'from-blue-600/25 via-violet-600/20 to-emerald-600/15',
|
||||
@@ -47,7 +180,7 @@ async function fetchDeck(id: DeckSource): Promise<PresentationData | null> {
|
||||
}
|
||||
}
|
||||
|
||||
export function PresentationView() {
|
||||
export function PresentationView({ embedded = false }: { embedded?: boolean }) {
|
||||
const [source, setSource] = useState<DeckSource>('live')
|
||||
const [data, setData] = useState<PresentationData | null>(null)
|
||||
const [slideIdx, setSlideIdx] = useState(0)
|
||||
@@ -63,8 +196,14 @@ export function PresentationView() {
|
||||
const [saving, setSaving] = useState(false)
|
||||
const [imgBusy, setImgBusy] = useState(false)
|
||||
const imgInputRef = useRef<HTMLInputElement>(null)
|
||||
const presentRef = useRef<HTMLDivElement>(null)
|
||||
const suppressFsPopup = useRef(false)
|
||||
|
||||
const [presentMode, setPresentMode] = useState<PresentMode>(null)
|
||||
|
||||
const isCustom = customDecks.some((d) => d.id === source)
|
||||
const canEditInPlace = source === 'live' || isCustom
|
||||
const liveEdited = source === 'live' && Boolean(data?.edited)
|
||||
|
||||
const refreshDeckList = useCallback(async () => {
|
||||
try {
|
||||
@@ -105,20 +244,94 @@ export function PresentationView() {
|
||||
refreshDeckList()
|
||||
}, [source, load, refreshDeckList])
|
||||
|
||||
const slides = data?.slides || []
|
||||
const slide: PresentationSlide | undefined = slides[slideIdx]
|
||||
|
||||
const closePresent = useCallback(() => {
|
||||
suppressFsPopup.current = true
|
||||
setPresentMode(null)
|
||||
if (document.fullscreenElement) {
|
||||
void document.exitFullscreen()
|
||||
}
|
||||
window.setTimeout(() => { suppressFsPopup.current = false }, 0)
|
||||
}, [])
|
||||
|
||||
const openPresent = useCallback((mode: 'popup' | 'fullscreen') => {
|
||||
if (!data?.slides?.length) return
|
||||
setPresentMode(mode)
|
||||
}, [data?.slides?.length])
|
||||
|
||||
const enterBrowserFullscreen = useCallback(async () => {
|
||||
const el = presentRef.current
|
||||
if (!el) return
|
||||
try {
|
||||
await el.requestFullscreen()
|
||||
setPresentMode('fullscreen')
|
||||
} catch {
|
||||
setPresentMode('popup')
|
||||
}
|
||||
}, [])
|
||||
|
||||
useEffect(() => {
|
||||
if (editing) return
|
||||
if (editing || presentMode) return
|
||||
const onKey = (e: KeyboardEvent) => {
|
||||
const n = data?.slides.length || 1
|
||||
if (e.key === 'ArrowRight' || e.key === ' ') { e.preventDefault(); setSlideIdx((i) => Math.min(n - 1, i + 1)) }
|
||||
if (e.key === 'ArrowLeft') setSlideIdx((i) => Math.max(0, i - 1))
|
||||
if (e.key === 'f' || e.key === 'F') document.documentElement.requestFullscreen?.()
|
||||
if ((e.key === 'f' || e.key === 'F') && data?.slides?.length) openPresent('fullscreen')
|
||||
}
|
||||
window.addEventListener('keydown', onKey)
|
||||
return () => window.removeEventListener('keydown', onKey)
|
||||
}, [data?.slides.length, editing])
|
||||
}, [data?.slides.length, editing, presentMode, openPresent])
|
||||
|
||||
const slides = data?.slides || []
|
||||
const slide: PresentationSlide | undefined = slides[slideIdx]
|
||||
useEffect(() => {
|
||||
if (presentMode !== 'fullscreen') return
|
||||
const id = window.requestAnimationFrame(() => { void enterBrowserFullscreen() })
|
||||
return () => window.cancelAnimationFrame(id)
|
||||
}, [presentMode, enterBrowserFullscreen])
|
||||
|
||||
useEffect(() => {
|
||||
const onFsChange = () => {
|
||||
if (!document.fullscreenElement && !suppressFsPopup.current) {
|
||||
setPresentMode((mode) => (mode === 'fullscreen' ? 'popup' : mode))
|
||||
}
|
||||
}
|
||||
document.addEventListener('fullscreenchange', onFsChange)
|
||||
return () => document.removeEventListener('fullscreenchange', onFsChange)
|
||||
}, [])
|
||||
|
||||
useEffect(() => {
|
||||
if (!presentMode) return
|
||||
const prev = document.body.style.overflow
|
||||
document.body.style.overflow = 'hidden'
|
||||
const onKey = (e: KeyboardEvent) => {
|
||||
const n = slides.length || 1
|
||||
if (e.key === 'Escape') {
|
||||
e.preventDefault()
|
||||
closePresent()
|
||||
return
|
||||
}
|
||||
if (e.key === 'ArrowRight' || e.key === ' ') {
|
||||
e.preventDefault()
|
||||
setSlideIdx((i) => Math.min(n - 1, i + 1))
|
||||
}
|
||||
if (e.key === 'ArrowLeft') setSlideIdx((i) => Math.max(0, i - 1))
|
||||
if (e.key === 'f' || e.key === 'F') {
|
||||
e.preventDefault()
|
||||
if (document.fullscreenElement) {
|
||||
void document.exitFullscreen()
|
||||
setPresentMode('popup')
|
||||
} else {
|
||||
void enterBrowserFullscreen()
|
||||
}
|
||||
}
|
||||
}
|
||||
window.addEventListener('keydown', onKey)
|
||||
return () => {
|
||||
document.body.style.overflow = prev
|
||||
window.removeEventListener('keydown', onKey)
|
||||
}
|
||||
}, [presentMode, slides.length, closePresent, enterBrowserFullscreen])
|
||||
|
||||
const editIdx = editing ? slideIdx : null
|
||||
const editSlides = draft?.slides || []
|
||||
@@ -221,6 +434,26 @@ export function PresentationView() {
|
||||
setDraft(null)
|
||||
}
|
||||
|
||||
const resetLive = async () => {
|
||||
if (!window.confirm('Reset Live Cluster to a fresh cluster snapshot? Your edits will be lost.')) return
|
||||
setUploadMsg(null)
|
||||
try {
|
||||
const r = await fetch('/api/presentation/live/reset', { method: 'POST' })
|
||||
const j = await r.json()
|
||||
if (j.ok && j.deck) {
|
||||
setData(j.deck)
|
||||
setSlideIdx(0)
|
||||
setEditing(false)
|
||||
setDraft(null)
|
||||
setUploadMsg('✓ Reset to cluster snapshot')
|
||||
} else {
|
||||
setUploadMsg(j.error || 'Reset failed')
|
||||
}
|
||||
} catch {
|
||||
setUploadMsg('Reset failed — check connection')
|
||||
}
|
||||
}
|
||||
|
||||
const patchSlide = (idx: number, patch: Partial<PresentationSlide>) => {
|
||||
setDraft((d) => {
|
||||
if (!d) return d
|
||||
@@ -279,74 +512,174 @@ export function PresentationView() {
|
||||
]
|
||||
|
||||
return (
|
||||
<div className="flex h-full min-h-[calc(100vh-140px)] flex-col overflow-hidden rounded-lg border border-border bg-surface-raised">
|
||||
<div className="flex shrink-0 flex-wrap items-center justify-between gap-2 border-b border-border bg-surface-raised/90 px-3 py-2">
|
||||
<div>
|
||||
<h2 className="text-[11px] font-semibold uppercase tracking-wider text-foreground">Presentation</h2>
|
||||
<p className="text-[9px] text-foreground-muted">
|
||||
Live cluster · HTML templates · PPT upload · editable decks with text & photos
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex flex-wrap gap-1">
|
||||
<>
|
||||
<div className={cn(
|
||||
'flex min-h-0 flex-col overflow-hidden bg-surface-raised',
|
||||
embedded ? 'h-full' : 'h-full min-h-[calc(100vh-140px)] rounded-lg border border-border',
|
||||
)}>
|
||||
{embedded ? (
|
||||
<div className="flex shrink-0 flex-col gap-1 border-b border-border bg-surface-raised/90 px-2 py-1.5">
|
||||
<div className="flex flex-wrap items-center justify-end gap-1">
|
||||
{!editing && (
|
||||
<>
|
||||
<button type="button" onClick={() => newDeck(false)} className="inline-flex items-center gap-1 rounded border border-border px-2 py-0.5 text-[9px] hover:bg-surface-overlay">
|
||||
<Plus className="h-3 w-3" /> New
|
||||
</button>
|
||||
{canEditInPlace ? (
|
||||
<button type="button" onClick={startEdit} className="inline-flex items-center gap-1 rounded-md px-2 py-1 text-[9px] text-docker hover:bg-surface-overlay">
|
||||
<Pencil className="h-3 w-3" /> Edit
|
||||
</button>
|
||||
) : (
|
||||
<button type="button" onClick={() => newDeck(true)} className="inline-flex items-center gap-1 rounded border border-border px-2 py-0.5 text-[9px] hover:bg-surface-overlay">
|
||||
<Pencil className="h-3 w-3" /> Edit a copy
|
||||
</button>
|
||||
)}
|
||||
{liveEdited && (
|
||||
<button type="button" onClick={resetLive} className="rounded border border-border px-2 py-0.5 text-[9px] text-foreground-muted hover:bg-surface-overlay">
|
||||
Reset snapshot
|
||||
</button>
|
||||
)}
|
||||
<a href="/dq/" target="_blank" rel="noreferrer" className={cn('inline-flex items-center gap-1 rounded-md px-2 py-1 text-[9px]', subTabIdle)}>
|
||||
<Monitor className="h-3 w-3" /> DQ Portal
|
||||
</a>
|
||||
<a href="/docling/ui/" target="_blank" rel="noreferrer" className="inline-flex items-center gap-1 rounded border border-border px-2 py-0.5 text-[9px] hover:bg-surface-overlay">
|
||||
<ExternalLink className="h-3 w-3" /> Docling
|
||||
</a>
|
||||
<button type="button" onClick={() => load(source)} className="rounded border border-border px-2 py-0.5 text-[9px] hover:bg-surface-overlay">Refresh</button>
|
||||
{!loading && slides.length > 0 && (
|
||||
<>
|
||||
<button type="button" onClick={() => openPresent('popup')} className="inline-flex items-center gap-1 rounded border border-border px-2 py-0.5 text-[9px] hover:bg-surface-overlay">
|
||||
<Maximize2 className="h-3 w-3" /> Popup
|
||||
</button>
|
||||
<button type="button" onClick={() => openPresent('fullscreen')} className={cn('inline-flex items-center gap-1 rounded-md px-2 py-1 text-[9px]', subTabIdle)}>
|
||||
<Expand className="h-3 w-3" /> Fullscreen
|
||||
</button>
|
||||
</>
|
||||
)}
|
||||
<button type="button" onClick={exportHtml} className={cn('rounded-md px-2 py-1 text-[9px]', subTabActive)}>Export HTML</button>
|
||||
</>
|
||||
)}
|
||||
{editing && (
|
||||
<>
|
||||
<span className="inline-flex items-center rounded bg-docker/15 px-2 py-0.5 text-[9px] font-medium text-docker">Editing</span>
|
||||
<button type="button" onClick={cancelEdit} className="inline-flex items-center gap-1 rounded border border-border px-2 py-0.5 text-[9px] hover:bg-surface-overlay">
|
||||
<X className="h-3 w-3" /> Cancel
|
||||
</button>
|
||||
<button type="button" onClick={saveDraft} disabled={saving} className={cn('inline-flex items-center gap-1 rounded-md px-2 py-1 text-[9px]', subTabActive, saving && 'opacity-50')}>
|
||||
<Save className="h-3 w-3" /> {saving ? 'Saving…' : 'Save'}
|
||||
</button>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
{!editing && (
|
||||
<>
|
||||
<button type="button" onClick={() => newDeck(false)} className="inline-flex items-center gap-1 rounded border border-border px-2 py-0.5 text-[9px] hover:bg-surface-overlay">
|
||||
<Plus className="h-3 w-3" /> New
|
||||
</button>
|
||||
{isCustom ? (
|
||||
<button type="button" onClick={startEdit} className="inline-flex items-center gap-1 rounded-md px-2 py-1 text-[9px] text-docker hover:bg-surface-overlay">
|
||||
<Pencil className="h-3 w-3" /> Edit
|
||||
<div className="scrollbar-thin flex shrink-0 gap-1 overflow-x-auto pb-0.5">
|
||||
{tabs.map((t) => (
|
||||
<button
|
||||
key={t.id}
|
||||
type="button"
|
||||
onClick={() => setSource(t.id)}
|
||||
className={cn(
|
||||
'shrink-0 rounded-md px-2 py-1 text-[10px] font-medium transition-all',
|
||||
source === t.id ? subTabActive : subTabIdle,
|
||||
)}
|
||||
>
|
||||
{t.label}
|
||||
</button>
|
||||
) : (
|
||||
<button type="button" onClick={() => newDeck(true)} className="inline-flex items-center gap-1 rounded border border-border px-2 py-0.5 text-[9px] hover:bg-surface-overlay">
|
||||
<Pencil className="h-3 w-3" /> Edit a copy
|
||||
</button>
|
||||
)}
|
||||
<a href="/dq/" target="_blank" rel="noreferrer" className={cn('inline-flex items-center gap-1 rounded-md px-2 py-1 text-[9px]', subTabIdle)}>
|
||||
<Monitor className="h-3 w-3" /> DQ Portal
|
||||
</a>
|
||||
<a href="/docling/ui/" target="_blank" rel="noreferrer" className="inline-flex items-center gap-1 rounded border border-border px-2 py-0.5 text-[9px] hover:bg-surface-overlay">
|
||||
<ExternalLink className="h-3 w-3" /> Docling
|
||||
</a>
|
||||
<button type="button" onClick={() => load(source)} className="rounded border border-border px-2 py-0.5 text-[9px] hover:bg-surface-overlay">Refresh</button>
|
||||
<button type="button" onClick={exportHtml} className={cn('rounded-md px-2 py-1 text-[9px]', subTabActive)}>Export HTML</button>
|
||||
</>
|
||||
)}
|
||||
{editing && (
|
||||
<>
|
||||
<span className="inline-flex items-center rounded bg-docker/15 px-2 py-0.5 text-[9px] font-medium text-docker">Editing</span>
|
||||
<button type="button" onClick={cancelEdit} className="inline-flex items-center gap-1 rounded border border-border px-2 py-0.5 text-[9px] hover:bg-surface-overlay">
|
||||
<X className="h-3 w-3" /> Cancel
|
||||
</button>
|
||||
<button type="button" onClick={saveDraft} disabled={saving} className={cn('inline-flex items-center gap-1 rounded-md px-2 py-1 text-[9px]', subTabActive, saving && 'opacity-50')}>
|
||||
<Save className="h-3 w-3" /> {saving ? 'Saving…' : 'Save'}
|
||||
</button>
|
||||
</>
|
||||
))}
|
||||
<label className={cn('ml-1 inline-flex shrink-0 cursor-pointer items-center gap-1 rounded-md border border-dashed border-border px-2 py-1 text-[10px]', uploading && 'opacity-50')}>
|
||||
<Upload className="h-3 w-3" />
|
||||
{uploading ? 'Uploading…' : 'PPT upload'}
|
||||
<input type="file" accept=".ppt,.pptx,.pdf,.docx" className="hidden" disabled={uploading} onChange={(e) => e.target.files?.[0] && onUpload(e.target.files[0])} />
|
||||
</label>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
) : (
|
||||
<>
|
||||
<div className="flex shrink-0 flex-wrap items-center justify-between gap-2 border-b border-border bg-surface-raised/90 px-3 py-2">
|
||||
<div>
|
||||
<h2 className="text-[11px] font-semibold uppercase tracking-wider text-foreground">Presentation</h2>
|
||||
<p className="text-[9px] text-foreground-muted">
|
||||
Live cluster · HTML templates · PPT upload · editable decks with text & photos
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex flex-wrap gap-1">
|
||||
{!editing && (
|
||||
<>
|
||||
<button type="button" onClick={() => newDeck(false)} className="inline-flex items-center gap-1 rounded border border-border px-2 py-0.5 text-[9px] hover:bg-surface-overlay">
|
||||
<Plus className="h-3 w-3" /> New
|
||||
</button>
|
||||
{canEditInPlace ? (
|
||||
<button type="button" onClick={startEdit} className="inline-flex items-center gap-1 rounded-md px-2 py-1 text-[9px] text-docker hover:bg-surface-overlay">
|
||||
<Pencil className="h-3 w-3" /> Edit
|
||||
</button>
|
||||
) : (
|
||||
<button type="button" onClick={() => newDeck(true)} className="inline-flex items-center gap-1 rounded border border-border px-2 py-0.5 text-[9px] hover:bg-surface-overlay">
|
||||
<Pencil className="h-3 w-3" /> Edit a copy
|
||||
</button>
|
||||
)}
|
||||
{liveEdited && (
|
||||
<button type="button" onClick={resetLive} className="rounded border border-border px-2 py-0.5 text-[9px] text-foreground-muted hover:bg-surface-overlay">
|
||||
Reset snapshot
|
||||
</button>
|
||||
)}
|
||||
<a href="/dq/" target="_blank" rel="noreferrer" className={cn('inline-flex items-center gap-1 rounded-md px-2 py-1 text-[9px]', subTabIdle)}>
|
||||
<Monitor className="h-3 w-3" /> DQ Portal
|
||||
</a>
|
||||
<a href="/docling/ui/" target="_blank" rel="noreferrer" className="inline-flex items-center gap-1 rounded border border-border px-2 py-0.5 text-[9px] hover:bg-surface-overlay">
|
||||
<ExternalLink className="h-3 w-3" /> Docling
|
||||
</a>
|
||||
<button type="button" onClick={() => load(source)} className="rounded border border-border px-2 py-0.5 text-[9px] hover:bg-surface-overlay">Refresh</button>
|
||||
{!loading && slides.length > 0 && (
|
||||
<>
|
||||
<button type="button" onClick={() => openPresent('popup')} className="inline-flex items-center gap-1 rounded border border-border px-2 py-0.5 text-[9px] hover:bg-surface-overlay">
|
||||
<Maximize2 className="h-3 w-3" /> Popup
|
||||
</button>
|
||||
<button type="button" onClick={() => openPresent('fullscreen')} className={cn('inline-flex items-center gap-1 rounded-md px-2 py-1 text-[9px]', subTabIdle)}>
|
||||
<Expand className="h-3 w-3" /> Fullscreen
|
||||
</button>
|
||||
</>
|
||||
)}
|
||||
<button type="button" onClick={exportHtml} className={cn('rounded-md px-2 py-1 text-[9px]', subTabActive)}>Export HTML</button>
|
||||
</>
|
||||
)}
|
||||
{editing && (
|
||||
<>
|
||||
<span className="inline-flex items-center rounded bg-docker/15 px-2 py-0.5 text-[9px] font-medium text-docker">Editing</span>
|
||||
<button type="button" onClick={cancelEdit} className="inline-flex items-center gap-1 rounded border border-border px-2 py-0.5 text-[9px] hover:bg-surface-overlay">
|
||||
<X className="h-3 w-3" /> Cancel
|
||||
</button>
|
||||
<button type="button" onClick={saveDraft} disabled={saving} className={cn('inline-flex items-center gap-1 rounded-md px-2 py-1 text-[9px]', subTabActive, saving && 'opacity-50')}>
|
||||
<Save className="h-3 w-3" /> {saving ? 'Saving…' : 'Save'}
|
||||
</button>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{!editing && (
|
||||
<div className="flex shrink-0 flex-wrap gap-1 border-b border-border bg-surface-overlay/40 px-2 py-1.5">
|
||||
{tabs.map((t) => (
|
||||
<button
|
||||
key={t.id}
|
||||
type="button"
|
||||
onClick={() => setSource(t.id)}
|
||||
className={cn(
|
||||
'rounded-md px-2.5 py-1.5 text-[10px] font-medium transition-all',
|
||||
source === t.id ? subTabActive : subTabIdle,
|
||||
)}
|
||||
>
|
||||
{t.label}
|
||||
</button>
|
||||
))}
|
||||
<label className={cn('ml-auto inline-flex cursor-pointer items-center gap-1 rounded-md border border-dashed border-border px-2 py-1 text-[10px]', uploading && 'opacity-50')}>
|
||||
<Upload className="h-3 w-3" />
|
||||
{uploading ? 'Uploading…' : 'PPT upload'}
|
||||
<input type="file" accept=".ppt,.pptx,.pdf,.docx" className="hidden" disabled={uploading} onChange={(e) => e.target.files?.[0] && onUpload(e.target.files[0])} />
|
||||
</label>
|
||||
</div>
|
||||
{!editing && (
|
||||
<div className="scrollbar-thin flex shrink-0 gap-1 overflow-x-auto border-b border-border bg-surface-overlay/40 px-2 py-1.5">
|
||||
{tabs.map((t) => (
|
||||
<button
|
||||
key={t.id}
|
||||
type="button"
|
||||
onClick={() => setSource(t.id)}
|
||||
className={cn(
|
||||
'shrink-0 rounded-md px-2.5 py-1.5 text-[10px] font-medium transition-all',
|
||||
source === t.id ? subTabActive : subTabIdle,
|
||||
)}
|
||||
>
|
||||
{t.label}
|
||||
</button>
|
||||
))}
|
||||
<label className={cn('ml-auto inline-flex shrink-0 cursor-pointer items-center gap-1 rounded-md border border-dashed border-border px-2 py-1 text-[10px]', uploading && 'opacity-50')}>
|
||||
<Upload className="h-3 w-3" />
|
||||
{uploading ? 'Uploading…' : 'PPT upload'}
|
||||
<input type="file" accept=".ppt,.pptx,.pdf,.docx" className="hidden" disabled={uploading} onChange={(e) => e.target.files?.[0] && onUpload(e.target.files[0])} />
|
||||
</label>
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
|
||||
{uploadMsg && <p className="shrink-0 px-3 py-1 text-[10px] text-docker">{uploadMsg}</p>}
|
||||
@@ -493,45 +826,103 @@ export function PresentationView() {
|
||||
</div>
|
||||
) : (
|
||||
/* ─────────── VIEW MODE ─────────── */
|
||||
<>
|
||||
<div className={cn('relative flex min-h-0 flex-1 flex-col justify-center bg-gradient-to-br p-6 md:p-10', KIND_STYLES[slide.kind || 'narrative'] || KIND_STYLES.narrative)}>
|
||||
<div className="flex max-w-5xl gap-6">
|
||||
<div className="min-w-0 flex-1">
|
||||
<p className="mb-1 text-[10px] font-medium uppercase tracking-widest text-docker/80">{slide.kind || 'slide'} · {slideIdx + 1}/{slides.length}</p>
|
||||
<h1 className="mb-2 text-2xl font-bold tracking-tight text-foreground md:text-4xl">{slide.title}</h1>
|
||||
{slide.subtitle && <p className="mb-4 text-sm text-foreground-muted md:text-base">{slide.subtitle}</p>}
|
||||
{'animation' in slide && slide.animation && (
|
||||
<ArchitectureDiagram animation={String(slide.animation)} />
|
||||
)}
|
||||
<ul className="space-y-2 text-sm leading-relaxed text-foreground md:text-base">
|
||||
{(slide.bullets || []).map((b: string, bi: number) => (
|
||||
<li key={bi} className="flex gap-2"><span className="shrink-0 text-docker">▸</span><span>{b}</span></li>
|
||||
))}
|
||||
</ul>
|
||||
</div>
|
||||
{slide.image && (
|
||||
<div className="hidden shrink-0 items-center md:flex">
|
||||
<img src={slide.image} alt="" className="max-h-[46vh] max-w-[40vw] rounded-lg border border-border object-contain shadow-lg" />
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
{slide.image && (
|
||||
<div className="mt-4 md:hidden">
|
||||
<img src={slide.image} alt="" className="max-h-[30vh] rounded-lg border border-border object-contain" />
|
||||
</div>
|
||||
)}
|
||||
<div className="flex min-h-0 flex-1 flex-col overflow-hidden">
|
||||
<div className={cn(
|
||||
'scrollbar-thin min-h-0 flex-1 overflow-y-auto bg-gradient-to-br',
|
||||
embedded ? 'p-3' : 'p-6 md:p-10',
|
||||
KIND_STYLES[slide.kind || 'narrative'] || KIND_STYLES.narrative,
|
||||
)}>
|
||||
<SlidePanel slide={slide} slideIdx={slideIdx} slideCount={slides.length} variant={embedded ? 'embedded' : 'normal'} />
|
||||
</div>
|
||||
<div className="flex shrink-0 items-center gap-2 border-t border-border bg-surface-raised/90 px-3 py-2">
|
||||
<button type="button" disabled={slideIdx === 0} onClick={() => setSlideIdx((i) => Math.max(0, i - 1))} className="rounded border border-border px-2 py-1 text-[10px] disabled:opacity-40">← Prev</button>
|
||||
<div className="flex flex-1 flex-wrap justify-center gap-1">
|
||||
{slides.map((_: PresentationSlide, i: number) => (
|
||||
<button key={i} type="button" onClick={() => setSlideIdx(i)} className={cn('h-2 w-2 rounded-full', i === slideIdx ? 'scale-125 bg-docker' : 'bg-border')} />
|
||||
))}
|
||||
</div>
|
||||
<button type="button" disabled={slideIdx >= slides.length - 1} onClick={() => setSlideIdx((i) => Math.min(slides.length - 1, i + 1))} className="rounded border border-border px-2 py-1 text-[10px] disabled:opacity-40">Next →</button>
|
||||
</div>
|
||||
</>
|
||||
<SlideNav
|
||||
slideIdx={slideIdx}
|
||||
slideCount={slides.length}
|
||||
embedded={embedded}
|
||||
onPrev={() => setSlideIdx((i) => Math.max(0, i - 1))}
|
||||
onNext={() => setSlideIdx((i) => Math.min(slides.length - 1, i + 1))}
|
||||
onGo={setSlideIdx}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
{presentMode && slide && createPortal(
|
||||
<>
|
||||
{presentMode === 'popup' && (
|
||||
<button
|
||||
type="button"
|
||||
aria-label="Close presentation"
|
||||
className="fixed inset-0 z-[199] bg-black/80 backdrop-blur-[2px]"
|
||||
onClick={closePresent}
|
||||
/>
|
||||
)}
|
||||
<div
|
||||
ref={presentRef}
|
||||
className={cn(
|
||||
'fixed z-[200] flex flex-col overflow-hidden bg-surface text-foreground shadow-2xl',
|
||||
presentMode === 'popup'
|
||||
? 'inset-2 rounded-xl border border-border ring-1 ring-white/10 sm:inset-4 md:inset-8 lg:inset-10'
|
||||
: 'inset-0',
|
||||
)}
|
||||
>
|
||||
<div className="flex shrink-0 items-center justify-between gap-2 border-b border-border bg-surface-raised/95 px-4 py-2">
|
||||
<div className="min-w-0 truncate text-xs text-foreground-muted md:text-sm">
|
||||
{data?.title || 'Presentation'} · slide {slideIdx + 1}/{slides.length}
|
||||
</div>
|
||||
<div className="flex shrink-0 items-center gap-1">
|
||||
{presentMode === 'popup' ? (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => void enterBrowserFullscreen()}
|
||||
className="inline-flex items-center gap-1 rounded border border-border px-2 py-1 text-[10px] hover:bg-surface-overlay md:text-xs"
|
||||
>
|
||||
<Expand className="h-3.5 w-3.5" /> Fullscreen
|
||||
</button>
|
||||
) : (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => {
|
||||
suppressFsPopup.current = true
|
||||
if (document.fullscreenElement) void document.exitFullscreen()
|
||||
setPresentMode('popup')
|
||||
window.setTimeout(() => { suppressFsPopup.current = false }, 0)
|
||||
}}
|
||||
className="inline-flex items-center gap-1 rounded border border-border px-2 py-1 text-[10px] hover:bg-surface-overlay md:text-xs"
|
||||
>
|
||||
<Maximize2 className="h-3.5 w-3.5" /> Popup
|
||||
</button>
|
||||
)}
|
||||
<button
|
||||
type="button"
|
||||
onClick={closePresent}
|
||||
className="inline-flex items-center gap-1 rounded border border-border px-2 py-1 text-[10px] hover:bg-surface-overlay md:text-xs"
|
||||
>
|
||||
<X className="h-3.5 w-3.5" /> Close
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
<div className={cn(
|
||||
'scrollbar-thin flex min-h-0 flex-1 flex-col overflow-hidden bg-gradient-to-br',
|
||||
KIND_STYLES[slide.kind || 'narrative'] || KIND_STYLES.narrative,
|
||||
)}>
|
||||
<div className="min-h-0 flex-1 overflow-y-auto p-8 md:p-12 lg:p-16">
|
||||
<SlidePanel slide={slide} slideIdx={slideIdx} slideCount={slides.length} variant="present" />
|
||||
</div>
|
||||
<SlideNav
|
||||
slideIdx={slideIdx}
|
||||
slideCount={slides.length}
|
||||
onPrev={() => setSlideIdx((i) => Math.max(0, i - 1))}
|
||||
onNext={() => setSlideIdx((i) => Math.min(slides.length - 1, i + 1))}
|
||||
onGo={setSlideIdx}
|
||||
className="border-t border-border/80 bg-surface-raised/95 px-4 py-2.5"
|
||||
/>
|
||||
</div>
|
||||
<p className="pointer-events-none absolute bottom-3 right-4 text-[10px] text-foreground-faint">
|
||||
← → navigate · F toggle fullscreen · Esc close
|
||||
</p>
|
||||
</div>
|
||||
</>,
|
||||
document.body,
|
||||
)}
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,346 @@
|
||||
import { useCallback, useEffect, useState } from 'react'
|
||||
import {
|
||||
Cpu,
|
||||
ExternalLink,
|
||||
Loader2,
|
||||
Pause,
|
||||
Play,
|
||||
RefreshCw,
|
||||
RotateCcw,
|
||||
Zap,
|
||||
} from 'lucide-react'
|
||||
import type { StreamingStatus } from '../../types'
|
||||
import {
|
||||
fetchStreamingStatus,
|
||||
restartKafkaConnector,
|
||||
pauseKafkaConnector,
|
||||
resumeKafkaConnector,
|
||||
triggerStreamingJob,
|
||||
} from '../../lib/api'
|
||||
import { cn } from '../../lib/utils'
|
||||
|
||||
type Tab = 'spark' | 'kafka' | 'jobs'
|
||||
|
||||
export function SparkKafkaPanel({
|
||||
embedded,
|
||||
selectedNodeId,
|
||||
streaming: initialStreaming,
|
||||
onRefreshGraph,
|
||||
}: {
|
||||
embedded?: boolean
|
||||
selectedNodeId?: string | null
|
||||
streaming?: StreamingStatus | null
|
||||
onRefreshGraph?: () => void
|
||||
}) {
|
||||
const [tab, setTab] = useState<Tab>('spark')
|
||||
const [streaming, setStreaming] = useState<StreamingStatus | null>(initialStreaming ?? null)
|
||||
const [loading, setLoading] = useState(!initialStreaming)
|
||||
const [busy, setBusy] = useState<string | null>(null)
|
||||
const [jobConf, setJobConf] = useState<Record<string, string>>({})
|
||||
const [selectedJob, setSelectedJob] = useState('spark_to_curated')
|
||||
const [showSparkUi, setShowSparkUi] = useState(false)
|
||||
|
||||
const load = useCallback(async (refresh = false) => {
|
||||
setLoading(true)
|
||||
const s = await fetchStreamingStatus(refresh)
|
||||
if (s) setStreaming(s)
|
||||
setLoading(false)
|
||||
}, [])
|
||||
|
||||
useEffect(() => {
|
||||
if (initialStreaming) setStreaming(initialStreaming)
|
||||
}, [initialStreaming])
|
||||
|
||||
useEffect(() => {
|
||||
if (selectedNodeId === 'spark') setTab('spark')
|
||||
if (selectedNodeId === 'kafka') setTab('kafka')
|
||||
}, [selectedNodeId])
|
||||
|
||||
useEffect(() => {
|
||||
const iv = setInterval(() => load(), 8000)
|
||||
return () => clearInterval(iv)
|
||||
}, [load])
|
||||
|
||||
const spark = streaming?.spark
|
||||
const kafka = streaming?.kafka
|
||||
const jobs = streaming?.jobs ?? []
|
||||
|
||||
const onJob = async (jobId: string) => {
|
||||
setBusy(`job:${jobId}`)
|
||||
try {
|
||||
let conf: Record<string, unknown> = {}
|
||||
const raw = jobConf[jobId]
|
||||
if (raw?.trim()) {
|
||||
conf = JSON.parse(raw)
|
||||
}
|
||||
await triggerStreamingJob(jobId, conf)
|
||||
onRefreshGraph?.()
|
||||
await load(true)
|
||||
} catch {
|
||||
/* ignore */
|
||||
} finally {
|
||||
setBusy(null)
|
||||
}
|
||||
}
|
||||
|
||||
const onConnector = async (name: string, action: 'restart' | 'pause' | 'resume') => {
|
||||
setBusy(`conn:${name}:${action}`)
|
||||
try {
|
||||
if (action === 'restart') await restartKafkaConnector(name)
|
||||
else if (action === 'pause') await pauseKafkaConnector(name)
|
||||
else await resumeKafkaConnector(name)
|
||||
await load(true)
|
||||
onRefreshGraph?.()
|
||||
} finally {
|
||||
setBusy(null)
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<div className={cn(
|
||||
'flex shrink-0 flex-col border-t border-border bg-surface/80',
|
||||
embedded ? 'max-h-[42vh]' : 'max-h-[48vh]',
|
||||
)}>
|
||||
<div className="flex shrink-0 items-center justify-between gap-2 border-b border-border px-3 py-1.5">
|
||||
<div className="flex items-center gap-1">
|
||||
{(['spark', 'kafka', 'jobs'] as Tab[]).map((t) => (
|
||||
<button
|
||||
key={t}
|
||||
type="button"
|
||||
onClick={() => setTab(t)}
|
||||
className={cn(
|
||||
'rounded-md px-2.5 py-1 text-[10px] font-medium capitalize transition-colors',
|
||||
tab === t
|
||||
? 'bg-docker/20 text-docker ring-1 ring-docker/40'
|
||||
: 'text-foreground-muted hover:bg-surface-overlay hover:text-foreground',
|
||||
)}
|
||||
>
|
||||
{t === 'jobs' ? 'Run jobs' : t === 'spark' ? 'Spark UI' : 'Kafka'}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
<div className="flex items-center gap-1">
|
||||
{spark?.ui_ok && (
|
||||
<span className="hidden text-[9px] text-emerald-300 sm:inline">
|
||||
Spark {spark.alive_workers}w · {spark.cores_used}/{spark.cores} cores
|
||||
</span>
|
||||
)}
|
||||
{kafka?.connect_ok && (
|
||||
<span className="hidden text-[9px] text-cyan-300 sm:inline">
|
||||
{kafka.connectors?.length ?? 0} connectors
|
||||
</span>
|
||||
)}
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => load(true)}
|
||||
className="inline-flex items-center gap-1 rounded border border-border px-1.5 py-0.5 text-[9px] text-foreground-muted hover:text-foreground"
|
||||
>
|
||||
<RefreshCw className={cn('h-3 w-3', loading && 'animate-spin')} /> Refresh
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="scrollbar-thin min-h-0 flex-1 overflow-y-auto p-3">
|
||||
{tab === 'spark' && (
|
||||
<div className="space-y-3">
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<Badge ok={spark?.ui_ok} label={spark?.ui_ok ? `Cluster ${spark.status}` : 'Spark UI offline'} />
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setShowSparkUi((v) => !v)}
|
||||
className="inline-flex items-center gap-1 rounded border border-border px-2 py-0.5 text-[9px] hover:bg-surface-overlay"
|
||||
>
|
||||
<Cpu className="h-3 w-3" /> {showSparkUi ? 'Hide' : 'Embed'} Spark UI
|
||||
</button>
|
||||
<a href="/spark-ui/" target="_blank" rel="noreferrer" className="inline-flex items-center gap-1 text-[9px] text-docker hover:underline">
|
||||
Open full Spark UI <ExternalLink className="h-3 w-3" />
|
||||
</a>
|
||||
</div>
|
||||
|
||||
{showSparkUi && (
|
||||
<iframe
|
||||
title="Spark Master UI"
|
||||
src="/spark-ui/"
|
||||
className="h-[280px] w-full rounded-lg border border-border bg-black"
|
||||
/>
|
||||
)}
|
||||
|
||||
<div className="grid gap-2 sm:grid-cols-3">
|
||||
<Stat label="Workers" value={String(spark?.alive_workers ?? 0)} />
|
||||
<Stat label="Cores used" value={`${spark?.cores_used ?? 0} / ${spark?.cores ?? 0}`} />
|
||||
<Stat label="Memory MB" value={`${spark?.memory_used_mb ?? 0} / ${spark?.memory_mb ?? 0}`} />
|
||||
</div>
|
||||
|
||||
{(spark?.workers?.length ?? 0) > 0 && (
|
||||
<div>
|
||||
<p className="mb-1 text-[9px] font-semibold uppercase tracking-wide text-foreground-muted">Workers</p>
|
||||
<div className="overflow-x-auto rounded border border-border">
|
||||
<table className="w-full text-left text-[9px]">
|
||||
<thead className="bg-surface-overlay/60 text-foreground-muted">
|
||||
<tr>
|
||||
<th className="px-2 py-1">Host</th>
|
||||
<th className="px-2 py-1">State</th>
|
||||
<th className="px-2 py-1">Cores</th>
|
||||
<th className="px-2 py-1">Memory</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{spark!.workers!.map((w) => (
|
||||
<tr key={w.id} className="border-t border-border/60">
|
||||
<td className="px-2 py-1 font-mono">{w.host}</td>
|
||||
<td className="px-2 py-1 text-emerald-300">{w.state}</td>
|
||||
<td className="px-2 py-1">{w.cores_used}/{w.cores}</td>
|
||||
<td className="px-2 py-1">{w.memory_mb} MB</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div>
|
||||
<p className="mb-1 text-[9px] font-semibold uppercase tracking-wide text-foreground-muted">Active applications</p>
|
||||
{(spark?.active_apps?.length ?? 0) === 0 ? (
|
||||
<p className="text-[10px] text-foreground-muted">No running Spark apps — trigger a job below or start streaming on lake01.</p>
|
||||
) : (
|
||||
<ul className="space-y-1">
|
||||
{spark!.active_apps!.map((a) => (
|
||||
<li key={a.id} className="rounded border border-border bg-surface-overlay/40 px-2 py-1 text-[10px]">
|
||||
<span className="font-semibold text-foreground">{a.name}</span>
|
||||
<span className="ml-2 font-mono text-foreground-muted">id={a.id} · {a.cores} cores</span>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{tab === 'kafka' && (
|
||||
<div className="space-y-3">
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<Badge ok={kafka?.ui_ok} label={kafka?.ui_ok ? `Cluster ${kafka.cluster?.name}` : 'Kafka UI offline'} />
|
||||
<Badge ok={kafka?.connect_ok} label={`${kafka?.connectors?.length ?? 0} Connectors`} />
|
||||
<a href="/kafka-ui/" target="_blank" rel="noreferrer" className="inline-flex items-center gap-1 text-[9px] text-docker hover:underline">
|
||||
Kafka UI <ExternalLink className="h-3 w-3" />
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<p className="mb-1 text-[9px] font-semibold uppercase tracking-wide text-foreground-muted">Debezium connectors</p>
|
||||
<div className="space-y-1">
|
||||
{(kafka?.connectors ?? []).map((c) => {
|
||||
const running = (c.state || '').toUpperCase() === 'RUNNING'
|
||||
const b = busy === `conn:${c.name}:restart` || busy === `conn:${c.name}:pause` || busy === `conn:${c.name}:resume`
|
||||
return (
|
||||
<div key={c.name} className="flex flex-wrap items-center gap-2 rounded border border-border bg-surface-overlay/30 px-2 py-1.5">
|
||||
<span className="min-w-0 flex-1 truncate font-mono text-[10px] text-foreground">{c.name}</span>
|
||||
<span className={cn('text-[9px] font-medium', running ? 'text-emerald-300' : 'text-amber-300')}>{c.state}</span>
|
||||
<button type="button" disabled={!!busy} onClick={() => onConnector(c.name, 'restart')} className="inline-flex items-center gap-0.5 rounded border border-border px-1.5 py-0.5 text-[8px] hover:bg-surface-overlay disabled:opacity-50">
|
||||
{b ? <Loader2 className="h-2.5 w-2.5 animate-spin" /> : <RotateCcw className="h-2.5 w-2.5" />} Restart
|
||||
</button>
|
||||
<button type="button" disabled={!!busy} onClick={() => onConnector(c.name, running ? 'pause' : 'resume')} className="inline-flex items-center gap-0.5 rounded border border-border px-1.5 py-0.5 text-[8px] hover:bg-surface-overlay disabled:opacity-50">
|
||||
{running ? <Pause className="h-2.5 w-2.5" /> : <Play className="h-2.5 w-2.5" />}
|
||||
{running ? 'Pause' : 'Resume'}
|
||||
</button>
|
||||
</div>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<p className="mb-1 text-[9px] font-semibold uppercase tracking-wide text-foreground-muted">Topics ({kafka?.topics?.length ?? 0})</p>
|
||||
<div className="max-h-40 overflow-y-auto rounded border border-border">
|
||||
<table className="w-full text-left text-[9px]">
|
||||
<thead className="sticky top-0 bg-surface-overlay/90 text-foreground-muted">
|
||||
<tr>
|
||||
<th className="px-2 py-1">Topic</th>
|
||||
<th className="px-2 py-1">Partitions</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{(kafka?.topics ?? []).slice(0, 30).map((t) => (
|
||||
<tr key={t.name} className="border-t border-border/50">
|
||||
<td className="max-w-[200px] truncate px-2 py-0.5 font-mono">{t.name}</td>
|
||||
<td className="px-2 py-0.5">{t.partitions ?? '—'}</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{tab === 'jobs' && (
|
||||
<div className="space-y-3">
|
||||
<p className="text-[10px] text-foreground-muted">
|
||||
Start lakehouse transforms manually via Airflow. Edit JSON config per job, then run.
|
||||
</p>
|
||||
<div className="flex flex-wrap gap-1">
|
||||
{jobs.map((j) => (
|
||||
<button
|
||||
key={j.id}
|
||||
type="button"
|
||||
onClick={() => setSelectedJob(j.id)}
|
||||
className={cn(
|
||||
'rounded border px-2 py-1 text-[9px] font-medium',
|
||||
selectedJob === j.id ? 'border-docker/50 bg-docker/15 text-docker' : 'border-border text-foreground-muted hover:text-foreground',
|
||||
)}
|
||||
>
|
||||
{j.label}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
{jobs.filter((j) => j.id === selectedJob).map((j) => (
|
||||
<div key={j.id} className="space-y-2 rounded-lg border border-border bg-surface-overlay/30 p-2">
|
||||
<p className="text-[10px] text-foreground">{j.description}</p>
|
||||
<p className="font-mono text-[9px] text-foreground-muted">DAG: {j.dag_id}</p>
|
||||
<label className="block">
|
||||
<span className="mb-0.5 block text-[8px] uppercase text-foreground-faint">Job config (JSON, editable)</span>
|
||||
<textarea
|
||||
value={jobConf[j.id] ?? JSON.stringify(j.default_conf ?? {}, null, 2)}
|
||||
onChange={(e) => setJobConf((prev) => ({ ...prev, [j.id]: e.target.value }))}
|
||||
rows={4}
|
||||
className="w-full rounded border border-border bg-surface px-2 py-1 font-mono text-[10px] text-foreground outline-none focus:border-docker"
|
||||
/>
|
||||
</label>
|
||||
<button
|
||||
type="button"
|
||||
disabled={busy === `job:${j.id}`}
|
||||
onClick={() => onJob(j.id)}
|
||||
className="inline-flex items-center gap-1 rounded-md border border-emerald-400/50 bg-emerald-500/15 px-3 py-1.5 text-[10px] font-medium text-emerald-200 hover:bg-emerald-500/25 disabled:opacity-50"
|
||||
>
|
||||
{busy === `job:${j.id}` ? <Loader2 className="h-3.5 w-3.5 animate-spin" /> : <Zap className="h-3.5 w-3.5" />}
|
||||
Start job
|
||||
</button>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function Badge({ ok, label }: { ok?: boolean; label: string }) {
|
||||
return (
|
||||
<span className={cn(
|
||||
'inline-flex items-center rounded border px-1.5 py-0.5 text-[9px] font-medium',
|
||||
ok ? 'border-emerald-400/40 bg-emerald-500/15 text-emerald-200' : 'border-amber-400/40 bg-amber-500/15 text-amber-200',
|
||||
)}>
|
||||
{label}
|
||||
</span>
|
||||
)
|
||||
}
|
||||
|
||||
function Stat({ label, value }: { label: string; value: string }) {
|
||||
return (
|
||||
<div className="rounded border border-border bg-surface-overlay/40 px-2 py-1.5">
|
||||
<p className="text-[8px] uppercase text-foreground-faint">{label}</p>
|
||||
<p className="font-mono text-sm font-semibold text-foreground">{value}</p>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,592 @@
|
||||
import { useCallback, useEffect, useRef, useState } from 'react'
|
||||
import {
|
||||
Activity,
|
||||
Cpu,
|
||||
Database,
|
||||
ExternalLink,
|
||||
Gauge,
|
||||
HardDrive,
|
||||
Layers,
|
||||
Loader2,
|
||||
Play,
|
||||
Plus,
|
||||
RefreshCw,
|
||||
Save,
|
||||
Server,
|
||||
Square,
|
||||
Table2,
|
||||
Trash2,
|
||||
Zap,
|
||||
} from 'lucide-react'
|
||||
import type { SparkLive, SparkRun, SparkRunStats } from '../../types'
|
||||
import {
|
||||
cancelSparkRun,
|
||||
createSparkRun,
|
||||
fetchSparkCatalogs,
|
||||
fetchSparkColumns,
|
||||
fetchSparkLive,
|
||||
fetchSparkRun,
|
||||
fetchSparkRuns,
|
||||
fetchSparkSchemas,
|
||||
fetchSparkTables,
|
||||
} from '../../lib/api'
|
||||
import { cn } from '../../lib/utils'
|
||||
import { subTabActive, subTabIdle } from '../../lib/tabActive'
|
||||
|
||||
type Tab = 'workbench' | 'cluster' | 'runs' | 'ui'
|
||||
type Operation = 'preview' | 'filter' | 'aggregate' | 'profile' | 'join' | 'sql'
|
||||
type Column = { name: string; type: string }
|
||||
type Metric = { fn: string; col: string; alias?: string }
|
||||
|
||||
const OPERATIONS: { id: Operation; label: string; desc: string }[] = [
|
||||
{ id: 'preview', label: 'Preview', desc: 'Sample rows from a table' },
|
||||
{ id: 'filter', label: 'Filter', desc: 'WHERE predicate on a table' },
|
||||
{ id: 'aggregate', label: 'Aggregate', desc: 'Group by + sum/avg/count/min/max' },
|
||||
{ id: 'profile', label: 'Profile', desc: 'Row count, distinct & non-null per column' },
|
||||
{ id: 'join', label: 'Join', desc: 'Join two tables on keys' },
|
||||
{ id: 'sql', label: 'SQL', desc: 'Run arbitrary distributed SQL' },
|
||||
]
|
||||
|
||||
const AGG_FNS = ['count', 'sum', 'avg', 'min', 'max', 'approx_distinct', 'count_distinct']
|
||||
|
||||
function fmtNum(n?: number | null) {
|
||||
if (n == null) return '—'
|
||||
if (n >= 1e9) return `${(n / 1e9).toFixed(2)}B`
|
||||
if (n >= 1e6) return `${(n / 1e6).toFixed(2)}M`
|
||||
if (n >= 1e3) return `${(n / 1e3).toFixed(1)}K`
|
||||
return String(n)
|
||||
}
|
||||
function fmtBytes(n?: number | null) {
|
||||
if (n == null) return '—'
|
||||
let v = n
|
||||
for (const u of ['B', 'KB', 'MB', 'GB', 'TB']) {
|
||||
if (v < 1024) return `${v.toFixed(u === 'B' ? 0 : 1)} ${u}`
|
||||
v /= 1024
|
||||
}
|
||||
return `${v.toFixed(1)} PB`
|
||||
}
|
||||
function fmtMs(n?: number | null) {
|
||||
if (n == null) return '—'
|
||||
if (n < 1000) return `${n} ms`
|
||||
if (n < 60000) return `${(n / 1000).toFixed(1)} s`
|
||||
return `${(n / 60000).toFixed(1)} m`
|
||||
}
|
||||
|
||||
const STATE_COLOR: Record<string, string> = {
|
||||
QUEUED: 'text-amber-300 bg-amber-500/15',
|
||||
RUNNING: 'text-sky-300 bg-sky-500/15',
|
||||
FINISHED: 'text-emerald-300 bg-emerald-500/15',
|
||||
FAILED: 'text-rose-300 bg-rose-500/15',
|
||||
CANCELED: 'text-foreground-muted bg-surface-overlay',
|
||||
}
|
||||
|
||||
export function SparkView({ embedded }: { embedded?: boolean }) {
|
||||
const [tab, setTab] = useState<Tab>('workbench')
|
||||
const [live, setLive] = useState<SparkLive | null>(null)
|
||||
|
||||
const loadLive = useCallback(async () => {
|
||||
const l = await fetchSparkLive()
|
||||
if (l) setLive(l)
|
||||
}, [])
|
||||
|
||||
useEffect(() => {
|
||||
loadLive()
|
||||
const iv = setInterval(loadLive, 4000)
|
||||
return () => clearInterval(iv)
|
||||
}, [loadLive])
|
||||
|
||||
const spark = live?.spark
|
||||
const alive = spark?.ui_ok && (spark?.status || '').toUpperCase() === 'ALIVE'
|
||||
const activeRuns = live?.active_runs ?? []
|
||||
|
||||
return (
|
||||
<div className={cn('flex min-h-0 flex-1 flex-col gap-2', embedded ? 'p-2' : 'p-3')}>
|
||||
<header className="panel flex shrink-0 flex-wrap items-center justify-between gap-3 px-4 py-3">
|
||||
<div>
|
||||
<h2 className="flex items-center gap-2 text-sm font-semibold text-foreground">
|
||||
<Zap className="h-4 w-4 text-amber-400" />
|
||||
Spark Lakehouse Workbench
|
||||
</h2>
|
||||
<p className="text-[10px] text-foreground-muted">
|
||||
Select data · transform on the distributed engine · materialize to Iceberg / S3 · live cluster metrics
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<LiveChip label="Cluster" value={alive ? 'ALIVE' : 'down'} ok={!!alive} />
|
||||
<LiveChip label="Cores" value={`${spark?.cores_used ?? 0}/${spark?.cores ?? 0}`} ok={(spark?.cores_used ?? 0) > 0} />
|
||||
<LiveChip label="Active jobs" value={String(activeRuns.length)} ok={activeRuns.length > 0} />
|
||||
<a href="/spark-ui/" target="_blank" rel="noreferrer" className="inline-flex items-center gap-1 rounded-md border border-border px-2 py-1 text-[10px] text-docker hover:bg-docker/10">
|
||||
<ExternalLink className="h-3 w-3" /> Native UI
|
||||
</a>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<div className="panel flex shrink-0 gap-1 px-2 py-1.5">
|
||||
{(['workbench', 'cluster', 'runs', 'ui'] as Tab[]).map((t) => (
|
||||
<button key={t} type="button" onClick={() => setTab(t)} className={cn('rounded-md px-2.5 py-1 text-[10px] font-medium capitalize', tab === t ? subTabActive : subTabIdle)}>
|
||||
{t === 'ui' ? 'Spark UI' : t}
|
||||
{t === 'runs' && activeRuns.length > 0 && <span className="ml-1 rounded-full bg-sky-500/30 px-1 text-[8px] text-sky-200">{activeRuns.length}</span>}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
|
||||
{tab === 'workbench' && <LakehouseWorkbench />}
|
||||
{tab === 'cluster' && <ClusterPanel live={live} />}
|
||||
{tab === 'runs' && <RunsPanel live={live} />}
|
||||
{tab === 'ui' && (
|
||||
<div className="panel min-h-0 flex-1 overflow-hidden p-1">
|
||||
<iframe title="Spark Master UI" src="/spark-ui/" className="h-full min-h-[420px] w-full rounded-md border-0 bg-black" />
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
/* ── Workbench: data selector + operation builder + live run ─────────────── */
|
||||
export function LakehouseWorkbench({ lockedCatalog }: { lockedCatalog?: string }) {
|
||||
const [catalogs, setCatalogs] = useState<string[]>([])
|
||||
const [catalog, setCatalog] = useState(lockedCatalog || 'iceberg')
|
||||
const [schemas, setSchemas] = useState<string[]>([])
|
||||
const [schema, setSchema] = useState('')
|
||||
const [tables, setTables] = useState<{ name: string; fqn: string }[]>([])
|
||||
const [table, setTable] = useState('')
|
||||
const [columns, setColumns] = useState<Column[]>([])
|
||||
|
||||
const [op, setOp] = useState<Operation>('preview')
|
||||
const [limit, setLimit] = useState(200)
|
||||
const [where, setWhere] = useState('')
|
||||
const [groupBy, setGroupBy] = useState<string[]>([])
|
||||
const [metrics, setMetrics] = useState<Metric[]>([{ fn: 'count', col: '*' }])
|
||||
const [profileCols, setProfileCols] = useState<string[]>([])
|
||||
const [sql, setSql] = useState('SELECT region, count(*) AS orders, sum(amount) AS revenue\nFROM iceberg.hadoop.historical_sales_hdfs\nGROUP BY region\nORDER BY revenue DESC')
|
||||
const [rightTable, setRightTable] = useState('')
|
||||
const [leftKey, setLeftKey] = useState('')
|
||||
const [rightKey, setRightKey] = useState('')
|
||||
const [joinType, setJoinType] = useState('INNER')
|
||||
|
||||
const [matEnabled, setMatEnabled] = useState(false)
|
||||
const [matSchema, setMatSchema] = useState('hadoop')
|
||||
const [matTable, setMatTable] = useState('')
|
||||
const [matMode, setMatMode] = useState('create')
|
||||
|
||||
const [run, setRun] = useState<SparkRun | null>(null)
|
||||
const [submitting, setSubmitting] = useState(false)
|
||||
const [err, setErr] = useState<string | null>(null)
|
||||
const pollRef = useRef<ReturnType<typeof setInterval> | null>(null)
|
||||
|
||||
useEffect(() => {
|
||||
if (lockedCatalog) { setCatalog(lockedCatalog); return }
|
||||
fetchSparkCatalogs().then((c) => {
|
||||
setCatalogs(c)
|
||||
if (c.length && !c.includes(catalog)) setCatalog(c[0])
|
||||
})
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [lockedCatalog])
|
||||
|
||||
useEffect(() => {
|
||||
if (!catalog) return
|
||||
setSchema(''); setTables([]); setTable(''); setColumns([])
|
||||
fetchSparkSchemas(catalog).then((s) => { setSchemas(s); if (s.length) setSchema(s[0]) })
|
||||
}, [catalog])
|
||||
|
||||
useEffect(() => {
|
||||
if (!catalog || !schema) return
|
||||
setTable(''); setColumns([])
|
||||
fetchSparkTables(catalog, schema).then((t) => { setTables(t); if (t.length) setTable(t[0].fqn) })
|
||||
}, [catalog, schema])
|
||||
|
||||
useEffect(() => {
|
||||
if (!table) { setColumns([]); return }
|
||||
fetchSparkColumns(table).then(setColumns)
|
||||
setGroupBy([]); setProfileCols([])
|
||||
}, [table])
|
||||
|
||||
useEffect(() => () => { if (pollRef.current) clearInterval(pollRef.current) }, [])
|
||||
|
||||
const startPolling = useCallback((runId: string) => {
|
||||
if (pollRef.current) clearInterval(pollRef.current)
|
||||
pollRef.current = setInterval(async () => {
|
||||
const j = await fetchSparkRun(runId)
|
||||
if (j?.run) {
|
||||
setRun(j.run)
|
||||
if (['FINISHED', 'FAILED', 'CANCELED'].includes(j.run.state)) {
|
||||
if (pollRef.current) clearInterval(pollRef.current)
|
||||
}
|
||||
}
|
||||
}, 700)
|
||||
}, [])
|
||||
|
||||
const onRun = async () => {
|
||||
setErr(null); setSubmitting(true); setRun(null)
|
||||
const body: Record<string, unknown> = { operation: op, limit }
|
||||
if (op !== 'sql' && op !== 'join') body.table = table
|
||||
if (op === 'filter') body.where = where
|
||||
if (op === 'aggregate') { body.table = table; body.group_by = groupBy; body.metrics = metrics }
|
||||
if (op === 'profile') { body.table = table; body.columns = profileCols.length ? profileCols : columns.slice(0, 8).map((c) => c.name) }
|
||||
if (op === 'sql') body.sql = sql
|
||||
if (op === 'join') {
|
||||
body.left = table; body.right = rightTable
|
||||
body.left_key = leftKey; body.right_key = rightKey; body.join_type = joinType
|
||||
}
|
||||
if (matEnabled && matTable) body.materialize = { enabled: true, schema: matSchema, table: matTable, mode: matMode }
|
||||
try {
|
||||
const r = await createSparkRun(body)
|
||||
if (!r.ok || !r.run_id) { setErr(r.error || 'Failed to submit'); setSubmitting(false); return }
|
||||
startPolling(r.run_id)
|
||||
} catch {
|
||||
setErr('Submit failed')
|
||||
} finally {
|
||||
setSubmitting(false)
|
||||
}
|
||||
}
|
||||
|
||||
const onCancel = async () => { if (run) await cancelSparkRun(run.id) }
|
||||
|
||||
const colNames = columns.map((c) => c.name)
|
||||
|
||||
return (
|
||||
<div className="flex min-h-0 flex-1 gap-2 overflow-hidden">
|
||||
{/* data selector */}
|
||||
<aside className="panel flex w-60 shrink-0 flex-col gap-2 overflow-y-auto p-3">
|
||||
<h3 className="flex items-center gap-1.5 text-[11px] font-semibold text-foreground">
|
||||
<Database className="h-3.5 w-3.5 text-emerald-400" /> Data
|
||||
</h3>
|
||||
{!lockedCatalog && (
|
||||
<Field label="Catalog">
|
||||
<select value={catalog} onChange={(e) => setCatalog(e.target.value)} className={selectCls}>
|
||||
{catalogs.map((c) => <option key={c} value={c}>{c}</option>)}
|
||||
</select>
|
||||
</Field>
|
||||
)}
|
||||
<Field label="Schema">
|
||||
<select value={schema} onChange={(e) => setSchema(e.target.value)} className={selectCls}>
|
||||
{schemas.map((s) => <option key={s} value={s}>{s}</option>)}
|
||||
</select>
|
||||
</Field>
|
||||
<Field label="Table">
|
||||
<select value={table} onChange={(e) => setTable(e.target.value)} className={selectCls}>
|
||||
{tables.map((t) => <option key={t.fqn} value={t.fqn}>{t.name}</option>)}
|
||||
</select>
|
||||
</Field>
|
||||
<div className="min-h-0 flex-1">
|
||||
<p className="mb-1 text-[9px] font-semibold uppercase tracking-wider text-foreground-muted">
|
||||
{columns.length} columns
|
||||
</p>
|
||||
<div className="scrollbar-thin space-y-0.5 overflow-y-auto">
|
||||
{columns.map((c) => (
|
||||
<div key={c.name} className="flex items-center justify-between gap-1 rounded px-1.5 py-0.5 text-[9px] hover:bg-surface-overlay">
|
||||
<span className="truncate font-mono text-foreground">{c.name}</span>
|
||||
<span className="shrink-0 text-[8px] text-foreground-faint">{c.type}</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</aside>
|
||||
|
||||
{/* builder + run */}
|
||||
<div className="flex min-h-0 flex-1 flex-col gap-2 overflow-hidden">
|
||||
<div className="panel shrink-0 space-y-2 p-3">
|
||||
<div className="flex flex-wrap gap-1">
|
||||
{OPERATIONS.map((o) => (
|
||||
<button key={o.id} type="button" onClick={() => setOp(o.id)} title={o.desc}
|
||||
className={cn('rounded-md px-2.5 py-1 text-[10px] font-medium', op === o.id ? subTabActive : subTabIdle)}>
|
||||
{o.label}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
|
||||
{op === 'sql' && (
|
||||
<textarea value={sql} onChange={(e) => setSql(e.target.value)} rows={4} spellCheck={false}
|
||||
className="w-full rounded-md border border-border bg-[#0d1117] p-2 font-mono text-[11px] text-emerald-100 outline-none" />
|
||||
)}
|
||||
|
||||
{op === 'filter' && (
|
||||
<Field label="WHERE predicate">
|
||||
<input value={where} onChange={(e) => setWhere(e.target.value)} placeholder="amount > 1000 AND region = 'EU'" className={inputCls} />
|
||||
</Field>
|
||||
)}
|
||||
|
||||
{op === 'aggregate' && (
|
||||
<div className="space-y-2">
|
||||
<Field label="Group by">
|
||||
<MultiChips options={colNames} selected={groupBy} onToggle={(c) =>
|
||||
setGroupBy((g) => g.includes(c) ? g.filter((x) => x !== c) : [...g, c])} />
|
||||
</Field>
|
||||
<div>
|
||||
<p className="mb-1 text-[9px] font-semibold uppercase tracking-wider text-foreground-muted">Metrics</p>
|
||||
{metrics.map((m, i) => (
|
||||
<div key={i} className="mb-1 flex items-center gap-1">
|
||||
<select value={m.fn} onChange={(e) => setMetrics((ms) => ms.map((x, j) => j === i ? { ...x, fn: e.target.value } : x))} className={cn(selectCls, 'w-32')}>
|
||||
{AGG_FNS.map((f) => <option key={f} value={f}>{f}</option>)}
|
||||
</select>
|
||||
<select value={m.col} onChange={(e) => setMetrics((ms) => ms.map((x, j) => j === i ? { ...x, col: e.target.value } : x))} className={cn(selectCls, 'flex-1')}>
|
||||
<option value="*">*</option>
|
||||
{colNames.map((c) => <option key={c} value={c}>{c}</option>)}
|
||||
</select>
|
||||
<button type="button" onClick={() => setMetrics((ms) => ms.filter((_, j) => j !== i))} className="rounded p-1 text-foreground-muted hover:text-rose-300">
|
||||
<Trash2 className="h-3 w-3" />
|
||||
</button>
|
||||
</div>
|
||||
))}
|
||||
<button type="button" onClick={() => setMetrics((ms) => [...ms, { fn: 'sum', col: colNames[0] || '*' }])}
|
||||
className="inline-flex items-center gap-1 rounded border border-border px-2 py-0.5 text-[9px] text-foreground-muted hover:text-foreground">
|
||||
<Plus className="h-3 w-3" /> Add metric
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{op === 'profile' && (
|
||||
<Field label="Columns (default: first 8)">
|
||||
<MultiChips options={colNames} selected={profileCols} onToggle={(c) =>
|
||||
setProfileCols((g) => g.includes(c) ? g.filter((x) => x !== c) : [...g, c])} />
|
||||
</Field>
|
||||
)}
|
||||
|
||||
{op === 'join' && (
|
||||
<div className="grid grid-cols-2 gap-2">
|
||||
<Field label="Right table (fqn)">
|
||||
<input value={rightTable} onChange={(e) => setRightTable(e.target.value)} placeholder="postgres_sales.public.customers" className={inputCls} />
|
||||
</Field>
|
||||
<Field label="Join type">
|
||||
<select value={joinType} onChange={(e) => setJoinType(e.target.value)} className={selectCls}>
|
||||
{['INNER', 'LEFT', 'RIGHT', 'FULL'].map((t) => <option key={t} value={t}>{t}</option>)}
|
||||
</select>
|
||||
</Field>
|
||||
<Field label="Left key">
|
||||
<select value={leftKey} onChange={(e) => setLeftKey(e.target.value)} className={selectCls}>
|
||||
<option value="">—</option>
|
||||
{colNames.map((c) => <option key={c} value={c}>{c}</option>)}
|
||||
</select>
|
||||
</Field>
|
||||
<Field label="Right key">
|
||||
<input value={rightKey} onChange={(e) => setRightKey(e.target.value)} placeholder="customer_id" className={inputCls} />
|
||||
</Field>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* materialize + run row */}
|
||||
<div className="flex flex-wrap items-end gap-2 border-t border-border pt-2">
|
||||
<label className="flex items-center gap-1.5 text-[10px] text-foreground">
|
||||
<input type="checkbox" checked={matEnabled} onChange={(e) => setMatEnabled(e.target.checked)} />
|
||||
<Save className="h-3 w-3 text-violet-300" /> Materialize → Iceberg/S3
|
||||
</label>
|
||||
{matEnabled && (
|
||||
<>
|
||||
<input value={matSchema} onChange={(e) => setMatSchema(e.target.value)} placeholder="schema" className={cn(inputCls, 'w-24')} />
|
||||
<input value={matTable} onChange={(e) => setMatTable(e.target.value)} placeholder="new_table" className={cn(inputCls, 'w-32')} />
|
||||
<select value={matMode} onChange={(e) => setMatMode(e.target.value)} className={cn(selectCls, 'w-28')}>
|
||||
<option value="create">create</option>
|
||||
<option value="replace">replace</option>
|
||||
<option value="insert">insert into</option>
|
||||
</select>
|
||||
</>
|
||||
)}
|
||||
{!matEnabled && (
|
||||
<Field label="Limit" inline>
|
||||
<input type="number" value={limit} onChange={(e) => setLimit(Math.max(1, Math.min(500, +e.target.value)))} className={cn(inputCls, 'w-20')} />
|
||||
</Field>
|
||||
)}
|
||||
<div className="ml-auto flex gap-2">
|
||||
{run && run.state === 'RUNNING' && (
|
||||
<button type="button" onClick={onCancel} className="inline-flex items-center gap-1 rounded-md border border-rose-400/50 bg-rose-500/15 px-3 py-1.5 text-[11px] text-rose-200 hover:bg-rose-500/25">
|
||||
<Square className="h-3.5 w-3.5" /> Cancel
|
||||
</button>
|
||||
)}
|
||||
<button type="button" onClick={onRun} disabled={submitting}
|
||||
className="inline-flex items-center gap-1.5 rounded-md border border-amber-400/50 bg-amber-500/15 px-4 py-1.5 text-[11px] font-medium text-amber-200 hover:bg-amber-500/25 disabled:opacity-50">
|
||||
{submitting ? <Loader2 className="h-4 w-4 animate-spin" /> : <Play className="h-4 w-4" />}
|
||||
Run on Spark
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
{err && <p className="text-[11px] text-rose-300">{err}</p>}
|
||||
</div>
|
||||
|
||||
{/* live run + results */}
|
||||
<div className="panel flex min-h-0 flex-1 flex-col overflow-hidden">
|
||||
{run ? <RunLive run={run} /> : (
|
||||
<div className="flex flex-1 items-center justify-center text-[11px] text-foreground-muted">
|
||||
Build an operation and Run to see the live execution matrix.
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function RunLive({ run }: { run: SparkRun }) {
|
||||
const s = run.stats || {}
|
||||
const pct = s.progress_pct ?? (run.state === 'FINISHED' ? 100 : 0)
|
||||
return (
|
||||
<div className="flex min-h-0 flex-1 flex-col">
|
||||
<div className="shrink-0 border-b border-border p-3">
|
||||
<div className="mb-2 flex items-center justify-between gap-2">
|
||||
<div className="min-w-0">
|
||||
<p className="truncate text-[11px] font-semibold text-foreground">{run.label}</p>
|
||||
{run.target && <p className="truncate font-mono text-[9px] text-violet-300">→ {run.target}</p>}
|
||||
</div>
|
||||
<span className={cn('rounded-full px-2 py-0.5 text-[10px] font-medium', STATE_COLOR[run.state] || STATE_COLOR.QUEUED)}>
|
||||
{run.state}
|
||||
</span>
|
||||
</div>
|
||||
<div className="h-1.5 w-full overflow-hidden rounded-full bg-surface-overlay">
|
||||
<div className="h-full rounded-full bg-gradient-to-r from-sky-400 to-emerald-400 transition-all" style={{ width: `${pct}%` }} />
|
||||
</div>
|
||||
<SparkMatrix stats={s} />
|
||||
{run.error && <p className="mt-2 rounded bg-rose-500/10 p-2 font-mono text-[10px] text-rose-300">{run.error}</p>}
|
||||
</div>
|
||||
<div className="min-h-0 flex-1 overflow-auto">
|
||||
{run.columns && run.columns.length > 0 ? (
|
||||
<table className="w-full border-collapse text-[10px]">
|
||||
<thead className="sticky top-0 bg-surface">
|
||||
<tr>{run.columns.map((c) => <th key={c} className="border-b border-border px-2 py-1 text-left font-semibold text-foreground-muted">{c}</th>)}</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{(run.rows || []).map((row, i) => (
|
||||
<tr key={i} className="hover:bg-surface-overlay/40">
|
||||
{row.map((cell, j) => <td key={j} className="border-b border-border/50 px-2 py-1 font-mono text-foreground">{cell == null ? '∅' : String(cell)}</td>)}
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
) : run.state === 'FINISHED' && run.target ? (
|
||||
<p className="p-3 text-[11px] text-emerald-300">✓ Materialized to {run.target}{run.update_type ? ` (${run.update_type})` : ''}</p>
|
||||
) : null}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function SparkMatrix({ stats }: { stats: SparkRunStats }) {
|
||||
const cells: { label: string; value: string; icon: typeof Cpu }[] = [
|
||||
{ label: 'Splits', value: `${fmtNum(stats.completed_splits)}/${fmtNum(stats.total_splits)}`, icon: Layers },
|
||||
{ label: 'Running', value: fmtNum(stats.running_splits), icon: Activity },
|
||||
{ label: 'Rows', value: fmtNum(stats.processed_rows), icon: Table2 },
|
||||
{ label: 'Input', value: fmtBytes(stats.processed_bytes), icon: HardDrive },
|
||||
{ label: 'CPU', value: fmtMs(stats.cpu_time_ms), icon: Cpu },
|
||||
{ label: 'Wall', value: fmtMs(stats.elapsed_ms ?? stats.wall_time_ms), icon: Gauge },
|
||||
{ label: 'Peak mem', value: fmtBytes(stats.peak_memory_bytes), icon: Server },
|
||||
{ label: 'Nodes', value: fmtNum(stats.nodes), icon: Cpu },
|
||||
]
|
||||
return (
|
||||
<div className="mt-2 grid grid-cols-4 gap-1.5 lg:grid-cols-8">
|
||||
{cells.map((c) => (
|
||||
<div key={c.label} className="rounded-md border border-border bg-surface-overlay/30 px-2 py-1.5">
|
||||
<div className="flex items-center gap-1 text-[8px] uppercase tracking-wider text-foreground-muted">
|
||||
<c.icon className="h-2.5 w-2.5" /> {c.label}
|
||||
</div>
|
||||
<p className="font-mono text-[11px] font-semibold tabular-nums text-foreground">{c.value}</p>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
/* ── Cluster panel ───────────────────────────────────────────────────────── */
|
||||
function ClusterPanel({ live }: { live: SparkLive | null }) {
|
||||
const spark = live?.spark
|
||||
return (
|
||||
<div className="panel grid min-h-0 flex-1 gap-3 overflow-y-auto p-4 sm:grid-cols-2 lg:grid-cols-4">
|
||||
<Metric label="Workers" value={String(spark?.alive_workers ?? 0)} icon={Server} />
|
||||
<Metric label="Cores" value={`${spark?.cores_used ?? 0} / ${spark?.cores ?? 0}`} icon={Cpu} />
|
||||
<Metric label="Memory" value={`${fmtNum(spark?.memory_used_mb)} / ${fmtNum(spark?.memory_mb)} MB`} icon={Activity} />
|
||||
<Metric label="Active jobs" value={String(live?.active_runs?.length ?? 0)} icon={Zap} />
|
||||
<div className="col-span-full">
|
||||
<h3 className="mb-2 text-[11px] font-semibold uppercase tracking-wider text-foreground-muted">Workers</h3>
|
||||
<div className="grid gap-2 sm:grid-cols-2 lg:grid-cols-3">
|
||||
{(spark?.workers || []).map((w) => (
|
||||
<div key={w.id} className="rounded-lg border border-border bg-surface-overlay/30 p-3">
|
||||
<p className="truncate font-mono text-[10px] text-foreground">{w.host || w.id}</p>
|
||||
<p className="text-[9px] text-foreground-muted">{w.cores_used}/{w.cores} cores · {w.memory_mb} MB · {w.state}</p>
|
||||
</div>
|
||||
))}
|
||||
{!spark?.workers?.length && <p className="text-[10px] text-foreground-muted">No worker details</p>}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
/* ── Runs history ────────────────────────────────────────────────────────── */
|
||||
function RunsPanel({ live }: { live: SparkLive | null }) {
|
||||
const [runs, setRuns] = useState<SparkRun[]>([])
|
||||
useEffect(() => {
|
||||
fetchSparkRuns().then(setRuns)
|
||||
const iv = setInterval(() => fetchSparkRuns().then(setRuns), 3000)
|
||||
return () => clearInterval(iv)
|
||||
}, [])
|
||||
const list = runs.length ? runs : (live?.recent_runs ?? [])
|
||||
return (
|
||||
<div className="panel min-h-0 flex-1 overflow-y-auto p-3">
|
||||
{list.length === 0 ? (
|
||||
<p className="p-4 text-center text-[11px] text-foreground-muted">No runs yet. Submit an operation from the Workbench.</p>
|
||||
) : (
|
||||
<div className="space-y-1.5">
|
||||
{list.map((r) => (
|
||||
<div key={r.id} className="flex items-center justify-between gap-3 rounded-lg border border-border bg-surface-overlay/20 px-3 py-2">
|
||||
<div className="min-w-0">
|
||||
<p className="truncate text-[11px] font-medium text-foreground">{r.label}</p>
|
||||
<p className="truncate font-mono text-[9px] text-foreground-muted">{r.sql.slice(0, 90)}</p>
|
||||
</div>
|
||||
<div className="flex shrink-0 items-center gap-3 text-[9px] text-foreground-muted">
|
||||
<span>{fmtNum(r.stats?.processed_rows)} rows</span>
|
||||
<span>{fmtMs(r.stats?.elapsed_ms)}</span>
|
||||
<span className={cn('rounded-full px-2 py-0.5 font-medium', STATE_COLOR[r.state] || STATE_COLOR.QUEUED)}>{r.state}</span>
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
/* ── small UI helpers ────────────────────────────────────────────────────── */
|
||||
const selectCls = 'w-full rounded-md border border-border bg-surface px-2 py-1 text-[10px] text-foreground outline-none'
|
||||
const inputCls = 'rounded-md border border-border bg-surface px-2 py-1 text-[10px] text-foreground outline-none'
|
||||
|
||||
function Field({ label, children, inline }: { label: string; children: React.ReactNode; inline?: boolean }) {
|
||||
return (
|
||||
<label className={cn('text-[9px] font-semibold uppercase tracking-wider text-foreground-muted', inline ? 'flex items-center gap-1.5' : 'block')}>
|
||||
{label}
|
||||
<div className={inline ? '' : 'mt-1'}>{children}</div>
|
||||
</label>
|
||||
)
|
||||
}
|
||||
|
||||
function MultiChips({ options, selected, onToggle }: { options: string[]; selected: string[]; onToggle: (c: string) => void }) {
|
||||
return (
|
||||
<div className="flex max-h-24 flex-wrap gap-1 overflow-y-auto">
|
||||
{options.map((o) => (
|
||||
<button key={o} type="button" onClick={() => onToggle(o)}
|
||||
className={cn('rounded border px-1.5 py-0.5 font-mono text-[9px]', selected.includes(o) ? 'border-emerald-400/50 bg-emerald-500/15 text-emerald-200' : 'border-border text-foreground-muted hover:bg-surface-overlay')}>
|
||||
{o}
|
||||
</button>
|
||||
))}
|
||||
{!options.length && <span className="text-[9px] text-foreground-faint">select a table</span>}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function Metric({ label, value, icon: Icon }: { label: string; value: string; icon: typeof Cpu }) {
|
||||
return (
|
||||
<div className="rounded-lg border border-border bg-surface-overlay/25 p-3">
|
||||
<div className="mb-1 flex items-center gap-1.5 text-[9px] uppercase tracking-wider text-foreground-muted">
|
||||
<Icon className="h-3 w-3" /> {label}
|
||||
</div>
|
||||
<p className="text-lg font-semibold tabular-nums text-foreground">{value}</p>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function LiveChip({ label, value, ok }: { label: string; value: string; ok: boolean }) {
|
||||
return (
|
||||
<span className="inline-flex items-center gap-1 rounded-full border border-border px-2 py-0.5 text-[9px]">
|
||||
<span className={cn('h-1.5 w-1.5 rounded-full', ok ? 'bg-emerald-400' : 'bg-amber-400')} />
|
||||
<span className="text-foreground-muted">{label}</span>
|
||||
<span className="font-mono font-semibold text-foreground">{value}</span>
|
||||
</span>
|
||||
)
|
||||
}
|
||||
@@ -15,7 +15,7 @@ type SqlResult = {
|
||||
sql?: string
|
||||
}
|
||||
|
||||
type Engine = SourceEngine | 'trino'
|
||||
type Engine = SourceEngine | 'trino' | 'hadoop'
|
||||
|
||||
type Props = {
|
||||
engine: Engine
|
||||
@@ -53,6 +53,12 @@ const ENGINE_META: Record<Engine, { title: string; sub: string; accent: string;
|
||||
accent: 'text-pink-400',
|
||||
queryLabel: 'Cypher',
|
||||
},
|
||||
hadoop: {
|
||||
title: 'Hadoop / Trino Console',
|
||||
sub: 'Lakehouse · Trino 10.0.21.50:8089 · iceberg.hadoop · hive',
|
||||
accent: 'text-emerald-400',
|
||||
queryLabel: 'SQL',
|
||||
},
|
||||
trino: {
|
||||
title: 'Trino SQL Console',
|
||||
sub: 'Lakehouse · 10.0.21.50:8089 · federated queries',
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { Database, DatabaseZap, HardDrive, Search, LayoutDashboard, MessageSquare, Presentation, Server, TerminalSquare, Cpu, Activity, GitBranch } from 'lucide-react'
|
||||
import { Database, DatabaseZap, HardDrive, Search, LayoutDashboard, MessageSquare, Server, TerminalSquare, Activity, GitBranch } from 'lucide-react'
|
||||
import type { GpuStatus, WorkloadData } from '../../types'
|
||||
import type { GpuLiveMetrics } from '../../hooks/useLiveMetrics'
|
||||
import { cn } from '../../lib/utils'
|
||||
@@ -6,7 +6,7 @@ import { viewTabActive, viewTabIdle } from '../../lib/tabActive'
|
||||
import { GpuMatrixPanel } from '../features/GpuMatrixPanel'
|
||||
import { LabHealthPanel } from '../features/LabHealthPanel'
|
||||
|
||||
type MainView = 'platform' | 'presentation' | 'dataquality' | 'knowledge' | 'storage' | 'hdfs' | 'search' | 'datagen' | 'approvals' | 'changes' | 'dataflow' | 'datasources'
|
||||
type MainView = 'platform' | 'dataquality' | 'knowledge' | 'storage' | 'hdfs' | 'search' | 'approvals' | 'changes' | 'dataflow' | 'datasources'
|
||||
|
||||
type Props = {
|
||||
workload: WorkloadData | null
|
||||
@@ -24,16 +24,13 @@ type Props = {
|
||||
|
||||
const VIEWS: { id: MainView; label: string; icon: typeof LayoutDashboard }[] = [
|
||||
{ id: 'platform', label: 'Data Platform', icon: LayoutDashboard },
|
||||
{ id: 'datasources', label: 'Data Sources UI', icon: Database },
|
||||
{ id: 'datagen', label: 'Data Generation', icon: Cpu },
|
||||
{ id: 'datasources', label: 'Data Hub', icon: Database },
|
||||
{ id: 'changes', label: 'Live Changes', icon: Activity },
|
||||
{ id: 'dataflow', label: 'Data Flow', icon: GitBranch },
|
||||
{ id: 'presentation', label: 'Presentation', icon: Presentation },
|
||||
{ id: 'dataquality', label: 'Data Quality', icon: DatabaseZap },
|
||||
{ id: 'knowledge', label: 'Knowledge Chat', icon: MessageSquare },
|
||||
{ id: 'storage', label: 'Object Storage', icon: HardDrive },
|
||||
{ id: 'hdfs', label: 'Hadoop HDFS', icon: Server },
|
||||
{ id: 'search', label: 'Elasticsearch', icon: Search },
|
||||
{ id: 'search', label: 'Elasticsearch', icon: Search },
|
||||
]
|
||||
|
||||
export function SideNav({
|
||||
@@ -52,10 +49,10 @@ export function SideNav({
|
||||
const matrixBoost = gpuBoost || mainView === 'knowledge'
|
||||
|
||||
return (
|
||||
<nav className="flex w-[240px] shrink-0 flex-col border-r border-border bg-surface-raised">
|
||||
<section className="shrink-0 border-b border-border p-3">
|
||||
<h2 className="mb-2 text-[9px] font-semibold uppercase tracking-widest text-foreground-faint">Views</h2>
|
||||
<div className="space-y-1">
|
||||
<nav className="flex h-full min-h-0 w-[240px] shrink-0 flex-col border-r border-border bg-surface-raised">
|
||||
<section className="flex max-h-[42%] min-h-0 shrink-0 flex-col border-b border-border p-3">
|
||||
<h2 className="mb-2 shrink-0 text-[9px] font-semibold uppercase tracking-widest text-foreground-muted">Views</h2>
|
||||
<div className="scrollbar-thin min-h-0 flex-1 space-y-1 overflow-y-auto">
|
||||
{VIEWS.map(({ id, label, icon: Icon }) => (
|
||||
<button
|
||||
key={id}
|
||||
@@ -81,21 +78,25 @@ export function SideNav({
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<GpuMatrixPanel
|
||||
gpu={gpu}
|
||||
live={gpuLive}
|
||||
boost={matrixBoost}
|
||||
onSelectGpu={() => onSelectZone('gpu')}
|
||||
/>
|
||||
<div className="flex min-h-0 flex-1 flex-col overflow-hidden">
|
||||
<GpuMatrixPanel
|
||||
gpu={gpu}
|
||||
live={gpuLive}
|
||||
boost={matrixBoost}
|
||||
onSelectGpu={() => onSelectZone('gpu')}
|
||||
/>
|
||||
|
||||
<LabHealthPanel
|
||||
workload={workload}
|
||||
gpu={gpu}
|
||||
selectedNodeId={selectedNodeId}
|
||||
approvalCount={approvalCount}
|
||||
onSelectZone={onSelectZone}
|
||||
onOpenApprovals={onOpenApprovals}
|
||||
/>
|
||||
<div className="scrollbar-thin min-h-0 flex-1 overflow-y-auto">
|
||||
<LabHealthPanel
|
||||
workload={workload}
|
||||
gpu={gpu}
|
||||
selectedNodeId={selectedNodeId}
|
||||
approvalCount={approvalCount}
|
||||
onSelectZone={onSelectZone}
|
||||
onOpenApprovals={onOpenApprovals}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</nav>
|
||||
)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
import { cn } from '../../lib/utils'
|
||||
import type { SourceEngine } from '../../lib/dataSourceCatalog'
|
||||
|
||||
const BRAND: Record<SourceEngine, { color: string; label: string }> = {
|
||||
postgres: { color: '#336791', label: 'PG' },
|
||||
mysql: { color: '#00758F', label: 'MY' },
|
||||
mongodb: { color: '#47A248', label: 'MG' },
|
||||
cassandra: { color: '#1287B1', label: 'CS' },
|
||||
neo4j: { color: '#018BFF', label: 'NJ' },
|
||||
}
|
||||
|
||||
type Props = {
|
||||
engine: SourceEngine
|
||||
size?: number
|
||||
className?: string
|
||||
}
|
||||
|
||||
export function DbBrandIcon({ engine, size = 20, className }: Props) {
|
||||
const b = BRAND[engine]
|
||||
const s = size
|
||||
return (
|
||||
<svg
|
||||
width={s}
|
||||
height={s}
|
||||
viewBox="0 0 24 24"
|
||||
className={cn('shrink-0', className)}
|
||||
aria-hidden
|
||||
>
|
||||
<circle cx="12" cy="12" r="11" fill={b.color} />
|
||||
{engine === 'postgres' && (
|
||||
<path fill="#fff" d="M7 8h10v1.5H7V8zm0 3.5h10V13H7v-1.5zm0 3.5h7V16H7v-1z" opacity="0.95" />
|
||||
)}
|
||||
{engine === 'mysql' && (
|
||||
<path fill="#F29111" d="M12 5c-3 0-5 2-5 4.5 0 2 1.5 3.5 3.5 4.5-1 .5-1.5 1.5-1.5 2.5 0 2 2 3.5 4.5 3.5s4.5-1.5 4.5-3.5c0-1-.5-2-1.5-2.5 2-1 3.5-2.5 3.5-4.5C17 7 15 5 12 5z" />
|
||||
)}
|
||||
{engine === 'mongodb' && (
|
||||
<path fill="#fff" d="M12 6c-2.5 2-4 5-4 8.5 0 2 .5 3.5 1.5 4.5.5-2 1.5-3.5 2.5-4.5 1 1 2 2.5 2.5 4.5 1-1 1.5-2.5 1.5-4.5C16 11 14.5 8 12 6z" />
|
||||
)}
|
||||
{engine === 'cassandra' && (
|
||||
<path fill="#fff" d="M12 5l6 3.5v7L12 19l-6-3.5v-7L12 5zm0 2.2L8.5 9v4L12 14.8l3.5-1.8V9L12 7.2z" opacity="0.95" />
|
||||
)}
|
||||
{engine === 'neo4j' && (
|
||||
<>
|
||||
<circle cx="8" cy="10" r="2.2" fill="#fff" />
|
||||
<circle cx="16" cy="10" r="2.2" fill="#fff" />
|
||||
<circle cx="12" cy="16" r="2.2" fill="#fff" />
|
||||
<path stroke="#fff" strokeWidth="1.2" d="M9.5 10.8 11 14M14.5 10.8 13 14M9.8 10 14.2 10" />
|
||||
</>
|
||||
)}
|
||||
</svg>
|
||||
)
|
||||
}
|
||||
|
||||
export function dbBrandColor(engine: SourceEngine): string {
|
||||
return BRAND[engine].color
|
||||
}
|
||||
Reference in New Issue
Block a user