mo
46b9c50e73
- Data Hub with Hadoop tab (HDFS/Iceberg browser, Spark, pipeline) - Databricks-style Lakehouse Workbench (Trino engine, live exec matrix, materialize to Iceberg/S3); reused & embedded in every source-DB UI - HDFS -> Kafka -> Spark -> Iceberg/S3 pipeline; WebHDFS hostname resolver - Data Flow master pulse switch (Run/Pause/Stop) gating animated edges - Data Custodian autonomous Hadoop offload loop (batch counterpart to CDC), pulsing source -> HDFS edges; toggle in Data Flow - LLM now autonomously aware of all latest platform changes (live platform context) and enforces masking policy: never reveals masked PII, still answers helpfully with aggregates/explanations
347 lines
15 KiB
TypeScript
347 lines
15 KiB
TypeScript
import { useCallback, useEffect, useState } from 'react'
|
|
import {
|
|
Cpu,
|
|
ExternalLink,
|
|
Loader2,
|
|
Pause,
|
|
Play,
|
|
RefreshCw,
|
|
RotateCcw,
|
|
Zap,
|
|
} from 'lucide-react'
|
|
import type { StreamingStatus } from '../../types'
|
|
import {
|
|
fetchStreamingStatus,
|
|
restartKafkaConnector,
|
|
pauseKafkaConnector,
|
|
resumeKafkaConnector,
|
|
triggerStreamingJob,
|
|
} from '../../lib/api'
|
|
import { cn } from '../../lib/utils'
|
|
|
|
type Tab = 'spark' | 'kafka' | 'jobs'
|
|
|
|
export function SparkKafkaPanel({
|
|
embedded,
|
|
selectedNodeId,
|
|
streaming: initialStreaming,
|
|
onRefreshGraph,
|
|
}: {
|
|
embedded?: boolean
|
|
selectedNodeId?: string | null
|
|
streaming?: StreamingStatus | null
|
|
onRefreshGraph?: () => void
|
|
}) {
|
|
const [tab, setTab] = useState<Tab>('spark')
|
|
const [streaming, setStreaming] = useState<StreamingStatus | null>(initialStreaming ?? null)
|
|
const [loading, setLoading] = useState(!initialStreaming)
|
|
const [busy, setBusy] = useState<string | null>(null)
|
|
const [jobConf, setJobConf] = useState<Record<string, string>>({})
|
|
const [selectedJob, setSelectedJob] = useState('spark_to_curated')
|
|
const [showSparkUi, setShowSparkUi] = useState(false)
|
|
|
|
const load = useCallback(async (refresh = false) => {
|
|
setLoading(true)
|
|
const s = await fetchStreamingStatus(refresh)
|
|
if (s) setStreaming(s)
|
|
setLoading(false)
|
|
}, [])
|
|
|
|
useEffect(() => {
|
|
if (initialStreaming) setStreaming(initialStreaming)
|
|
}, [initialStreaming])
|
|
|
|
useEffect(() => {
|
|
if (selectedNodeId === 'spark') setTab('spark')
|
|
if (selectedNodeId === 'kafka') setTab('kafka')
|
|
}, [selectedNodeId])
|
|
|
|
useEffect(() => {
|
|
const iv = setInterval(() => load(), 8000)
|
|
return () => clearInterval(iv)
|
|
}, [load])
|
|
|
|
const spark = streaming?.spark
|
|
const kafka = streaming?.kafka
|
|
const jobs = streaming?.jobs ?? []
|
|
|
|
const onJob = async (jobId: string) => {
|
|
setBusy(`job:${jobId}`)
|
|
try {
|
|
let conf: Record<string, unknown> = {}
|
|
const raw = jobConf[jobId]
|
|
if (raw?.trim()) {
|
|
conf = JSON.parse(raw)
|
|
}
|
|
await triggerStreamingJob(jobId, conf)
|
|
onRefreshGraph?.()
|
|
await load(true)
|
|
} catch {
|
|
/* ignore */
|
|
} finally {
|
|
setBusy(null)
|
|
}
|
|
}
|
|
|
|
const onConnector = async (name: string, action: 'restart' | 'pause' | 'resume') => {
|
|
setBusy(`conn:${name}:${action}`)
|
|
try {
|
|
if (action === 'restart') await restartKafkaConnector(name)
|
|
else if (action === 'pause') await pauseKafkaConnector(name)
|
|
else await resumeKafkaConnector(name)
|
|
await load(true)
|
|
onRefreshGraph?.()
|
|
} finally {
|
|
setBusy(null)
|
|
}
|
|
}
|
|
|
|
return (
|
|
<div className={cn(
|
|
'flex shrink-0 flex-col border-t border-border bg-surface/80',
|
|
embedded ? 'max-h-[42vh]' : 'max-h-[48vh]',
|
|
)}>
|
|
<div className="flex shrink-0 items-center justify-between gap-2 border-b border-border px-3 py-1.5">
|
|
<div className="flex items-center gap-1">
|
|
{(['spark', 'kafka', 'jobs'] as Tab[]).map((t) => (
|
|
<button
|
|
key={t}
|
|
type="button"
|
|
onClick={() => setTab(t)}
|
|
className={cn(
|
|
'rounded-md px-2.5 py-1 text-[10px] font-medium capitalize transition-colors',
|
|
tab === t
|
|
? 'bg-docker/20 text-docker ring-1 ring-docker/40'
|
|
: 'text-foreground-muted hover:bg-surface-overlay hover:text-foreground',
|
|
)}
|
|
>
|
|
{t === 'jobs' ? 'Run jobs' : t === 'spark' ? 'Spark UI' : 'Kafka'}
|
|
</button>
|
|
))}
|
|
</div>
|
|
<div className="flex items-center gap-1">
|
|
{spark?.ui_ok && (
|
|
<span className="hidden text-[9px] text-emerald-300 sm:inline">
|
|
Spark {spark.alive_workers}w · {spark.cores_used}/{spark.cores} cores
|
|
</span>
|
|
)}
|
|
{kafka?.connect_ok && (
|
|
<span className="hidden text-[9px] text-cyan-300 sm:inline">
|
|
{kafka.connectors?.length ?? 0} connectors
|
|
</span>
|
|
)}
|
|
<button
|
|
type="button"
|
|
onClick={() => load(true)}
|
|
className="inline-flex items-center gap-1 rounded border border-border px-1.5 py-0.5 text-[9px] text-foreground-muted hover:text-foreground"
|
|
>
|
|
<RefreshCw className={cn('h-3 w-3', loading && 'animate-spin')} /> Refresh
|
|
</button>
|
|
</div>
|
|
</div>
|
|
|
|
<div className="scrollbar-thin min-h-0 flex-1 overflow-y-auto p-3">
|
|
{tab === 'spark' && (
|
|
<div className="space-y-3">
|
|
<div className="flex flex-wrap items-center gap-2">
|
|
<Badge ok={spark?.ui_ok} label={spark?.ui_ok ? `Cluster ${spark.status}` : 'Spark UI offline'} />
|
|
<button
|
|
type="button"
|
|
onClick={() => setShowSparkUi((v) => !v)}
|
|
className="inline-flex items-center gap-1 rounded border border-border px-2 py-0.5 text-[9px] hover:bg-surface-overlay"
|
|
>
|
|
<Cpu className="h-3 w-3" /> {showSparkUi ? 'Hide' : 'Embed'} Spark UI
|
|
</button>
|
|
<a href="/spark-ui/" target="_blank" rel="noreferrer" className="inline-flex items-center gap-1 text-[9px] text-docker hover:underline">
|
|
Open full Spark UI <ExternalLink className="h-3 w-3" />
|
|
</a>
|
|
</div>
|
|
|
|
{showSparkUi && (
|
|
<iframe
|
|
title="Spark Master UI"
|
|
src="/spark-ui/"
|
|
className="h-[280px] w-full rounded-lg border border-border bg-black"
|
|
/>
|
|
)}
|
|
|
|
<div className="grid gap-2 sm:grid-cols-3">
|
|
<Stat label="Workers" value={String(spark?.alive_workers ?? 0)} />
|
|
<Stat label="Cores used" value={`${spark?.cores_used ?? 0} / ${spark?.cores ?? 0}`} />
|
|
<Stat label="Memory MB" value={`${spark?.memory_used_mb ?? 0} / ${spark?.memory_mb ?? 0}`} />
|
|
</div>
|
|
|
|
{(spark?.workers?.length ?? 0) > 0 && (
|
|
<div>
|
|
<p className="mb-1 text-[9px] font-semibold uppercase tracking-wide text-foreground-muted">Workers</p>
|
|
<div className="overflow-x-auto rounded border border-border">
|
|
<table className="w-full text-left text-[9px]">
|
|
<thead className="bg-surface-overlay/60 text-foreground-muted">
|
|
<tr>
|
|
<th className="px-2 py-1">Host</th>
|
|
<th className="px-2 py-1">State</th>
|
|
<th className="px-2 py-1">Cores</th>
|
|
<th className="px-2 py-1">Memory</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
{spark!.workers!.map((w) => (
|
|
<tr key={w.id} className="border-t border-border/60">
|
|
<td className="px-2 py-1 font-mono">{w.host}</td>
|
|
<td className="px-2 py-1 text-emerald-300">{w.state}</td>
|
|
<td className="px-2 py-1">{w.cores_used}/{w.cores}</td>
|
|
<td className="px-2 py-1">{w.memory_mb} MB</td>
|
|
</tr>
|
|
))}
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
</div>
|
|
)}
|
|
|
|
<div>
|
|
<p className="mb-1 text-[9px] font-semibold uppercase tracking-wide text-foreground-muted">Active applications</p>
|
|
{(spark?.active_apps?.length ?? 0) === 0 ? (
|
|
<p className="text-[10px] text-foreground-muted">No running Spark apps — trigger a job below or start streaming on lake01.</p>
|
|
) : (
|
|
<ul className="space-y-1">
|
|
{spark!.active_apps!.map((a) => (
|
|
<li key={a.id} className="rounded border border-border bg-surface-overlay/40 px-2 py-1 text-[10px]">
|
|
<span className="font-semibold text-foreground">{a.name}</span>
|
|
<span className="ml-2 font-mono text-foreground-muted">id={a.id} · {a.cores} cores</span>
|
|
</li>
|
|
))}
|
|
</ul>
|
|
)}
|
|
</div>
|
|
</div>
|
|
)}
|
|
|
|
{tab === 'kafka' && (
|
|
<div className="space-y-3">
|
|
<div className="flex flex-wrap items-center gap-2">
|
|
<Badge ok={kafka?.ui_ok} label={kafka?.ui_ok ? `Cluster ${kafka.cluster?.name}` : 'Kafka UI offline'} />
|
|
<Badge ok={kafka?.connect_ok} label={`${kafka?.connectors?.length ?? 0} Connectors`} />
|
|
<a href="/kafka-ui/" target="_blank" rel="noreferrer" className="inline-flex items-center gap-1 text-[9px] text-docker hover:underline">
|
|
Kafka UI <ExternalLink className="h-3 w-3" />
|
|
</a>
|
|
</div>
|
|
|
|
<div>
|
|
<p className="mb-1 text-[9px] font-semibold uppercase tracking-wide text-foreground-muted">Debezium connectors</p>
|
|
<div className="space-y-1">
|
|
{(kafka?.connectors ?? []).map((c) => {
|
|
const running = (c.state || '').toUpperCase() === 'RUNNING'
|
|
const b = busy === `conn:${c.name}:restart` || busy === `conn:${c.name}:pause` || busy === `conn:${c.name}:resume`
|
|
return (
|
|
<div key={c.name} className="flex flex-wrap items-center gap-2 rounded border border-border bg-surface-overlay/30 px-2 py-1.5">
|
|
<span className="min-w-0 flex-1 truncate font-mono text-[10px] text-foreground">{c.name}</span>
|
|
<span className={cn('text-[9px] font-medium', running ? 'text-emerald-300' : 'text-amber-300')}>{c.state}</span>
|
|
<button type="button" disabled={!!busy} onClick={() => onConnector(c.name, 'restart')} className="inline-flex items-center gap-0.5 rounded border border-border px-1.5 py-0.5 text-[8px] hover:bg-surface-overlay disabled:opacity-50">
|
|
{b ? <Loader2 className="h-2.5 w-2.5 animate-spin" /> : <RotateCcw className="h-2.5 w-2.5" />} Restart
|
|
</button>
|
|
<button type="button" disabled={!!busy} onClick={() => onConnector(c.name, running ? 'pause' : 'resume')} className="inline-flex items-center gap-0.5 rounded border border-border px-1.5 py-0.5 text-[8px] hover:bg-surface-overlay disabled:opacity-50">
|
|
{running ? <Pause className="h-2.5 w-2.5" /> : <Play className="h-2.5 w-2.5" />}
|
|
{running ? 'Pause' : 'Resume'}
|
|
</button>
|
|
</div>
|
|
)
|
|
})}
|
|
</div>
|
|
</div>
|
|
|
|
<div>
|
|
<p className="mb-1 text-[9px] font-semibold uppercase tracking-wide text-foreground-muted">Topics ({kafka?.topics?.length ?? 0})</p>
|
|
<div className="max-h-40 overflow-y-auto rounded border border-border">
|
|
<table className="w-full text-left text-[9px]">
|
|
<thead className="sticky top-0 bg-surface-overlay/90 text-foreground-muted">
|
|
<tr>
|
|
<th className="px-2 py-1">Topic</th>
|
|
<th className="px-2 py-1">Partitions</th>
|
|
</tr>
|
|
</thead>
|
|
<tbody>
|
|
{(kafka?.topics ?? []).slice(0, 30).map((t) => (
|
|
<tr key={t.name} className="border-t border-border/50">
|
|
<td className="max-w-[200px] truncate px-2 py-0.5 font-mono">{t.name}</td>
|
|
<td className="px-2 py-0.5">{t.partitions ?? '—'}</td>
|
|
</tr>
|
|
))}
|
|
</tbody>
|
|
</table>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
)}
|
|
|
|
{tab === 'jobs' && (
|
|
<div className="space-y-3">
|
|
<p className="text-[10px] text-foreground-muted">
|
|
Start lakehouse transforms manually via Airflow. Edit JSON config per job, then run.
|
|
</p>
|
|
<div className="flex flex-wrap gap-1">
|
|
{jobs.map((j) => (
|
|
<button
|
|
key={j.id}
|
|
type="button"
|
|
onClick={() => setSelectedJob(j.id)}
|
|
className={cn(
|
|
'rounded border px-2 py-1 text-[9px] font-medium',
|
|
selectedJob === j.id ? 'border-docker/50 bg-docker/15 text-docker' : 'border-border text-foreground-muted hover:text-foreground',
|
|
)}
|
|
>
|
|
{j.label}
|
|
</button>
|
|
))}
|
|
</div>
|
|
{jobs.filter((j) => j.id === selectedJob).map((j) => (
|
|
<div key={j.id} className="space-y-2 rounded-lg border border-border bg-surface-overlay/30 p-2">
|
|
<p className="text-[10px] text-foreground">{j.description}</p>
|
|
<p className="font-mono text-[9px] text-foreground-muted">DAG: {j.dag_id}</p>
|
|
<label className="block">
|
|
<span className="mb-0.5 block text-[8px] uppercase text-foreground-faint">Job config (JSON, editable)</span>
|
|
<textarea
|
|
value={jobConf[j.id] ?? JSON.stringify(j.default_conf ?? {}, null, 2)}
|
|
onChange={(e) => setJobConf((prev) => ({ ...prev, [j.id]: e.target.value }))}
|
|
rows={4}
|
|
className="w-full rounded border border-border bg-surface px-2 py-1 font-mono text-[10px] text-foreground outline-none focus:border-docker"
|
|
/>
|
|
</label>
|
|
<button
|
|
type="button"
|
|
disabled={busy === `job:${j.id}`}
|
|
onClick={() => onJob(j.id)}
|
|
className="inline-flex items-center gap-1 rounded-md border border-emerald-400/50 bg-emerald-500/15 px-3 py-1.5 text-[10px] font-medium text-emerald-200 hover:bg-emerald-500/25 disabled:opacity-50"
|
|
>
|
|
{busy === `job:${j.id}` ? <Loader2 className="h-3.5 w-3.5 animate-spin" /> : <Zap className="h-3.5 w-3.5" />}
|
|
Start job
|
|
</button>
|
|
</div>
|
|
))}
|
|
</div>
|
|
)}
|
|
</div>
|
|
</div>
|
|
)
|
|
}
|
|
|
|
function Badge({ ok, label }: { ok?: boolean; label: string }) {
|
|
return (
|
|
<span className={cn(
|
|
'inline-flex items-center rounded border px-1.5 py-0.5 text-[9px] font-medium',
|
|
ok ? 'border-emerald-400/40 bg-emerald-500/15 text-emerald-200' : 'border-amber-400/40 bg-amber-500/15 text-amber-200',
|
|
)}>
|
|
{label}
|
|
</span>
|
|
)
|
|
}
|
|
|
|
function Stat({ label, value }: { label: string; value: string }) {
|
|
return (
|
|
<div className="rounded border border-border bg-surface-overlay/40 px-2 py-1.5">
|
|
<p className="text-[8px] uppercase text-foreground-faint">{label}</p>
|
|
<p className="font-mono text-sm font-semibold text-foreground">{value}</p>
|
|
</div>
|
|
)
|
|
}
|