feat: Spark Workbench everywhere, autonomous Hadoop offload & LLM masking-aware
- Data Hub with Hadoop tab (HDFS/Iceberg browser, Spark, pipeline) - Databricks-style Lakehouse Workbench (Trino engine, live exec matrix, materialize to Iceberg/S3); reused & embedded in every source-DB UI - HDFS -> Kafka -> Spark -> Iceberg/S3 pipeline; WebHDFS hostname resolver - Data Flow master pulse switch (Run/Pause/Stop) gating animated edges - Data Custodian autonomous Hadoop offload loop (batch counterpart to CDC), pulsing source -> HDFS edges; toggle in Data Flow - LLM now autonomously aware of all latest platform changes (live platform context) and enforces masking policy: never reveals masked PII, still answers helpfully with aggregates/explanations
This commit is contained in:
mo
@@ -0,0 +1,346 @@
|
||||
import { useCallback, useEffect, useState } from 'react'
|
||||
import {
|
||||
Cpu,
|
||||
ExternalLink,
|
||||
Loader2,
|
||||
Pause,
|
||||
Play,
|
||||
RefreshCw,
|
||||
RotateCcw,
|
||||
Zap,
|
||||
} from 'lucide-react'
|
||||
import type { StreamingStatus } from '../../types'
|
||||
import {
|
||||
fetchStreamingStatus,
|
||||
restartKafkaConnector,
|
||||
pauseKafkaConnector,
|
||||
resumeKafkaConnector,
|
||||
triggerStreamingJob,
|
||||
} from '../../lib/api'
|
||||
import { cn } from '../../lib/utils'
|
||||
|
||||
type Tab = 'spark' | 'kafka' | 'jobs'
|
||||
|
||||
export function SparkKafkaPanel({
|
||||
embedded,
|
||||
selectedNodeId,
|
||||
streaming: initialStreaming,
|
||||
onRefreshGraph,
|
||||
}: {
|
||||
embedded?: boolean
|
||||
selectedNodeId?: string | null
|
||||
streaming?: StreamingStatus | null
|
||||
onRefreshGraph?: () => void
|
||||
}) {
|
||||
const [tab, setTab] = useState<Tab>('spark')
|
||||
const [streaming, setStreaming] = useState<StreamingStatus | null>(initialStreaming ?? null)
|
||||
const [loading, setLoading] = useState(!initialStreaming)
|
||||
const [busy, setBusy] = useState<string | null>(null)
|
||||
const [jobConf, setJobConf] = useState<Record<string, string>>({})
|
||||
const [selectedJob, setSelectedJob] = useState('spark_to_curated')
|
||||
const [showSparkUi, setShowSparkUi] = useState(false)
|
||||
|
||||
const load = useCallback(async (refresh = false) => {
|
||||
setLoading(true)
|
||||
const s = await fetchStreamingStatus(refresh)
|
||||
if (s) setStreaming(s)
|
||||
setLoading(false)
|
||||
}, [])
|
||||
|
||||
useEffect(() => {
|
||||
if (initialStreaming) setStreaming(initialStreaming)
|
||||
}, [initialStreaming])
|
||||
|
||||
useEffect(() => {
|
||||
if (selectedNodeId === 'spark') setTab('spark')
|
||||
if (selectedNodeId === 'kafka') setTab('kafka')
|
||||
}, [selectedNodeId])
|
||||
|
||||
useEffect(() => {
|
||||
const iv = setInterval(() => load(), 8000)
|
||||
return () => clearInterval(iv)
|
||||
}, [load])
|
||||
|
||||
const spark = streaming?.spark
|
||||
const kafka = streaming?.kafka
|
||||
const jobs = streaming?.jobs ?? []
|
||||
|
||||
const onJob = async (jobId: string) => {
|
||||
setBusy(`job:${jobId}`)
|
||||
try {
|
||||
let conf: Record<string, unknown> = {}
|
||||
const raw = jobConf[jobId]
|
||||
if (raw?.trim()) {
|
||||
conf = JSON.parse(raw)
|
||||
}
|
||||
await triggerStreamingJob(jobId, conf)
|
||||
onRefreshGraph?.()
|
||||
await load(true)
|
||||
} catch {
|
||||
/* ignore */
|
||||
} finally {
|
||||
setBusy(null)
|
||||
}
|
||||
}
|
||||
|
||||
const onConnector = async (name: string, action: 'restart' | 'pause' | 'resume') => {
|
||||
setBusy(`conn:${name}:${action}`)
|
||||
try {
|
||||
if (action === 'restart') await restartKafkaConnector(name)
|
||||
else if (action === 'pause') await pauseKafkaConnector(name)
|
||||
else await resumeKafkaConnector(name)
|
||||
await load(true)
|
||||
onRefreshGraph?.()
|
||||
} finally {
|
||||
setBusy(null)
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<div className={cn(
|
||||
'flex shrink-0 flex-col border-t border-border bg-surface/80',
|
||||
embedded ? 'max-h-[42vh]' : 'max-h-[48vh]',
|
||||
)}>
|
||||
<div className="flex shrink-0 items-center justify-between gap-2 border-b border-border px-3 py-1.5">
|
||||
<div className="flex items-center gap-1">
|
||||
{(['spark', 'kafka', 'jobs'] as Tab[]).map((t) => (
|
||||
<button
|
||||
key={t}
|
||||
type="button"
|
||||
onClick={() => setTab(t)}
|
||||
className={cn(
|
||||
'rounded-md px-2.5 py-1 text-[10px] font-medium capitalize transition-colors',
|
||||
tab === t
|
||||
? 'bg-docker/20 text-docker ring-1 ring-docker/40'
|
||||
: 'text-foreground-muted hover:bg-surface-overlay hover:text-foreground',
|
||||
)}
|
||||
>
|
||||
{t === 'jobs' ? 'Run jobs' : t === 'spark' ? 'Spark UI' : 'Kafka'}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
<div className="flex items-center gap-1">
|
||||
{spark?.ui_ok && (
|
||||
<span className="hidden text-[9px] text-emerald-300 sm:inline">
|
||||
Spark {spark.alive_workers}w · {spark.cores_used}/{spark.cores} cores
|
||||
</span>
|
||||
)}
|
||||
{kafka?.connect_ok && (
|
||||
<span className="hidden text-[9px] text-cyan-300 sm:inline">
|
||||
{kafka.connectors?.length ?? 0} connectors
|
||||
</span>
|
||||
)}
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => load(true)}
|
||||
className="inline-flex items-center gap-1 rounded border border-border px-1.5 py-0.5 text-[9px] text-foreground-muted hover:text-foreground"
|
||||
>
|
||||
<RefreshCw className={cn('h-3 w-3', loading && 'animate-spin')} /> Refresh
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="scrollbar-thin min-h-0 flex-1 overflow-y-auto p-3">
|
||||
{tab === 'spark' && (
|
||||
<div className="space-y-3">
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<Badge ok={spark?.ui_ok} label={spark?.ui_ok ? `Cluster ${spark.status}` : 'Spark UI offline'} />
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setShowSparkUi((v) => !v)}
|
||||
className="inline-flex items-center gap-1 rounded border border-border px-2 py-0.5 text-[9px] hover:bg-surface-overlay"
|
||||
>
|
||||
<Cpu className="h-3 w-3" /> {showSparkUi ? 'Hide' : 'Embed'} Spark UI
|
||||
</button>
|
||||
<a href="/spark-ui/" target="_blank" rel="noreferrer" className="inline-flex items-center gap-1 text-[9px] text-docker hover:underline">
|
||||
Open full Spark UI <ExternalLink className="h-3 w-3" />
|
||||
</a>
|
||||
</div>
|
||||
|
||||
{showSparkUi && (
|
||||
<iframe
|
||||
title="Spark Master UI"
|
||||
src="/spark-ui/"
|
||||
className="h-[280px] w-full rounded-lg border border-border bg-black"
|
||||
/>
|
||||
)}
|
||||
|
||||
<div className="grid gap-2 sm:grid-cols-3">
|
||||
<Stat label="Workers" value={String(spark?.alive_workers ?? 0)} />
|
||||
<Stat label="Cores used" value={`${spark?.cores_used ?? 0} / ${spark?.cores ?? 0}`} />
|
||||
<Stat label="Memory MB" value={`${spark?.memory_used_mb ?? 0} / ${spark?.memory_mb ?? 0}`} />
|
||||
</div>
|
||||
|
||||
{(spark?.workers?.length ?? 0) > 0 && (
|
||||
<div>
|
||||
<p className="mb-1 text-[9px] font-semibold uppercase tracking-wide text-foreground-muted">Workers</p>
|
||||
<div className="overflow-x-auto rounded border border-border">
|
||||
<table className="w-full text-left text-[9px]">
|
||||
<thead className="bg-surface-overlay/60 text-foreground-muted">
|
||||
<tr>
|
||||
<th className="px-2 py-1">Host</th>
|
||||
<th className="px-2 py-1">State</th>
|
||||
<th className="px-2 py-1">Cores</th>
|
||||
<th className="px-2 py-1">Memory</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{spark!.workers!.map((w) => (
|
||||
<tr key={w.id} className="border-t border-border/60">
|
||||
<td className="px-2 py-1 font-mono">{w.host}</td>
|
||||
<td className="px-2 py-1 text-emerald-300">{w.state}</td>
|
||||
<td className="px-2 py-1">{w.cores_used}/{w.cores}</td>
|
||||
<td className="px-2 py-1">{w.memory_mb} MB</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div>
|
||||
<p className="mb-1 text-[9px] font-semibold uppercase tracking-wide text-foreground-muted">Active applications</p>
|
||||
{(spark?.active_apps?.length ?? 0) === 0 ? (
|
||||
<p className="text-[10px] text-foreground-muted">No running Spark apps — trigger a job below or start streaming on lake01.</p>
|
||||
) : (
|
||||
<ul className="space-y-1">
|
||||
{spark!.active_apps!.map((a) => (
|
||||
<li key={a.id} className="rounded border border-border bg-surface-overlay/40 px-2 py-1 text-[10px]">
|
||||
<span className="font-semibold text-foreground">{a.name}</span>
|
||||
<span className="ml-2 font-mono text-foreground-muted">id={a.id} · {a.cores} cores</span>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{tab === 'kafka' && (
|
||||
<div className="space-y-3">
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<Badge ok={kafka?.ui_ok} label={kafka?.ui_ok ? `Cluster ${kafka.cluster?.name}` : 'Kafka UI offline'} />
|
||||
<Badge ok={kafka?.connect_ok} label={`${kafka?.connectors?.length ?? 0} Connectors`} />
|
||||
<a href="/kafka-ui/" target="_blank" rel="noreferrer" className="inline-flex items-center gap-1 text-[9px] text-docker hover:underline">
|
||||
Kafka UI <ExternalLink className="h-3 w-3" />
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<p className="mb-1 text-[9px] font-semibold uppercase tracking-wide text-foreground-muted">Debezium connectors</p>
|
||||
<div className="space-y-1">
|
||||
{(kafka?.connectors ?? []).map((c) => {
|
||||
const running = (c.state || '').toUpperCase() === 'RUNNING'
|
||||
const b = busy === `conn:${c.name}:restart` || busy === `conn:${c.name}:pause` || busy === `conn:${c.name}:resume`
|
||||
return (
|
||||
<div key={c.name} className="flex flex-wrap items-center gap-2 rounded border border-border bg-surface-overlay/30 px-2 py-1.5">
|
||||
<span className="min-w-0 flex-1 truncate font-mono text-[10px] text-foreground">{c.name}</span>
|
||||
<span className={cn('text-[9px] font-medium', running ? 'text-emerald-300' : 'text-amber-300')}>{c.state}</span>
|
||||
<button type="button" disabled={!!busy} onClick={() => onConnector(c.name, 'restart')} className="inline-flex items-center gap-0.5 rounded border border-border px-1.5 py-0.5 text-[8px] hover:bg-surface-overlay disabled:opacity-50">
|
||||
{b ? <Loader2 className="h-2.5 w-2.5 animate-spin" /> : <RotateCcw className="h-2.5 w-2.5" />} Restart
|
||||
</button>
|
||||
<button type="button" disabled={!!busy} onClick={() => onConnector(c.name, running ? 'pause' : 'resume')} className="inline-flex items-center gap-0.5 rounded border border-border px-1.5 py-0.5 text-[8px] hover:bg-surface-overlay disabled:opacity-50">
|
||||
{running ? <Pause className="h-2.5 w-2.5" /> : <Play className="h-2.5 w-2.5" />}
|
||||
{running ? 'Pause' : 'Resume'}
|
||||
</button>
|
||||
</div>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<p className="mb-1 text-[9px] font-semibold uppercase tracking-wide text-foreground-muted">Topics ({kafka?.topics?.length ?? 0})</p>
|
||||
<div className="max-h-40 overflow-y-auto rounded border border-border">
|
||||
<table className="w-full text-left text-[9px]">
|
||||
<thead className="sticky top-0 bg-surface-overlay/90 text-foreground-muted">
|
||||
<tr>
|
||||
<th className="px-2 py-1">Topic</th>
|
||||
<th className="px-2 py-1">Partitions</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{(kafka?.topics ?? []).slice(0, 30).map((t) => (
|
||||
<tr key={t.name} className="border-t border-border/50">
|
||||
<td className="max-w-[200px] truncate px-2 py-0.5 font-mono">{t.name}</td>
|
||||
<td className="px-2 py-0.5">{t.partitions ?? '—'}</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{tab === 'jobs' && (
|
||||
<div className="space-y-3">
|
||||
<p className="text-[10px] text-foreground-muted">
|
||||
Start lakehouse transforms manually via Airflow. Edit JSON config per job, then run.
|
||||
</p>
|
||||
<div className="flex flex-wrap gap-1">
|
||||
{jobs.map((j) => (
|
||||
<button
|
||||
key={j.id}
|
||||
type="button"
|
||||
onClick={() => setSelectedJob(j.id)}
|
||||
className={cn(
|
||||
'rounded border px-2 py-1 text-[9px] font-medium',
|
||||
selectedJob === j.id ? 'border-docker/50 bg-docker/15 text-docker' : 'border-border text-foreground-muted hover:text-foreground',
|
||||
)}
|
||||
>
|
||||
{j.label}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
{jobs.filter((j) => j.id === selectedJob).map((j) => (
|
||||
<div key={j.id} className="space-y-2 rounded-lg border border-border bg-surface-overlay/30 p-2">
|
||||
<p className="text-[10px] text-foreground">{j.description}</p>
|
||||
<p className="font-mono text-[9px] text-foreground-muted">DAG: {j.dag_id}</p>
|
||||
<label className="block">
|
||||
<span className="mb-0.5 block text-[8px] uppercase text-foreground-faint">Job config (JSON, editable)</span>
|
||||
<textarea
|
||||
value={jobConf[j.id] ?? JSON.stringify(j.default_conf ?? {}, null, 2)}
|
||||
onChange={(e) => setJobConf((prev) => ({ ...prev, [j.id]: e.target.value }))}
|
||||
rows={4}
|
||||
className="w-full rounded border border-border bg-surface px-2 py-1 font-mono text-[10px] text-foreground outline-none focus:border-docker"
|
||||
/>
|
||||
</label>
|
||||
<button
|
||||
type="button"
|
||||
disabled={busy === `job:${j.id}`}
|
||||
onClick={() => onJob(j.id)}
|
||||
className="inline-flex items-center gap-1 rounded-md border border-emerald-400/50 bg-emerald-500/15 px-3 py-1.5 text-[10px] font-medium text-emerald-200 hover:bg-emerald-500/25 disabled:opacity-50"
|
||||
>
|
||||
{busy === `job:${j.id}` ? <Loader2 className="h-3.5 w-3.5 animate-spin" /> : <Zap className="h-3.5 w-3.5" />}
|
||||
Start job
|
||||
</button>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
function Badge({ ok, label }: { ok?: boolean; label: string }) {
|
||||
return (
|
||||
<span className={cn(
|
||||
'inline-flex items-center rounded border px-1.5 py-0.5 text-[9px] font-medium',
|
||||
ok ? 'border-emerald-400/40 bg-emerald-500/15 text-emerald-200' : 'border-amber-400/40 bg-amber-500/15 text-amber-200',
|
||||
)}>
|
||||
{label}
|
||||
</span>
|
||||
)
|
||||
}
|
||||
|
||||
function Stat({ label, value }: { label: string; value: string }) {
|
||||
return (
|
||||
<div className="rounded border border-border bg-surface-overlay/40 px-2 py-1.5">
|
||||
<p className="text-[8px] uppercase text-foreground-faint">{label}</p>
|
||||
<p className="font-mono text-sm font-semibold text-foreground">{value}</p>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
Reference in New Issue
Block a user