feat: Spark Workbench everywhere, autonomous Hadoop offload & LLM masking-aware

- Data Hub with Hadoop tab (HDFS/Iceberg browser, Spark, pipeline)
- Databricks-style Lakehouse Workbench (Trino engine, live exec matrix,
  materialize to Iceberg/S3); reused & embedded in every source-DB UI
- HDFS -> Kafka -> Spark -> Iceberg/S3 pipeline; WebHDFS hostname resolver
- Data Flow master pulse switch (Run/Pause/Stop) gating animated edges
- Data Custodian autonomous Hadoop offload loop (batch counterpart to CDC),
  pulsing source -> HDFS edges; toggle in Data Flow
- LLM now autonomously aware of all latest platform changes (live platform
  context) and enforces masking policy: never reveals masked PII, still
  answers helpfully with aggregates/explanations

ui/src/lib/api.ts

@@ -9,6 +9,9 @@ import type {
   GpuStatus,
   Movement,
   PiiDataset,
+  SparkRun,
+  SparkLive,
+  StreamingStatus,
   StatusData,
   TerminalLine,
   WorkloadData,
@@ -178,3 +181,100 @@ export async function decideApproval(id: string, approved: boolean, decidedBy: s
     body: JSON.stringify({ approved, decided_by: decidedBy, note }),
   })
 }
+
+export async function fetchStreamingStatus(refresh = false): Promise<StreamingStatus | null> {
+  return fetchJson<StreamingStatus>(`/api/pipeline/streaming/status${refresh ? '?refresh=true' : ''}`, 20000)
+}
+
+export function triggerStreamingJob(jobId: string, conf?: Record<string, unknown>) {
+  return fetch(`/api/pipeline/streaming/jobs/${jobId}/trigger`, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({ conf: conf ?? {} }),
+  })
+}
+
+export function restartKafkaConnector(name: string) {
+  return fetch(`/api/pipeline/streaming/kafka/connectors/${encodeURIComponent(name)}/restart`, { method: 'POST' })
+}
+
+export function pauseKafkaConnector(name: string) {
+  return fetch(`/api/pipeline/streaming/kafka/connectors/${encodeURIComponent(name)}/pause`, { method: 'POST' })
+}
+
+export function resumeKafkaConnector(name: string) {
+  return fetch(`/api/pipeline/streaming/kafka/connectors/${encodeURIComponent(name)}/resume`, { method: 'POST' })
+}
+
+export function triggerStreamingPipeline(pipelineId: string) {
+  return fetch(`/api/pipeline/streaming/pipeline/${encodeURIComponent(pipelineId)}`, { method: 'POST' })
+}
+
+export function exportHdfsToKafka(body?: { path?: string; topic?: string; limit?: number }) {
+  return fetch('/api/pipeline/streaming/hdfs/to-kafka', {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify(body ?? {}),
+  })
+}
+
+export function setStreamingFlow(action: 'pause' | 'resume' | 'stop') {
+  return fetch(`/api/pipeline/streaming/flow/${action}`, { method: 'POST' })
+}
+
+// ── Spark Workbench ──────────────────────────────────────────────
+export async function fetchSparkCatalogs(): Promise<string[]> {
+  const j = await fetchJson<{ catalogs?: string[] }>('/api/spark/catalogs', 15000)
+  return j?.catalogs ?? []
+}
+
+export async function fetchSparkSchemas(catalog: string): Promise<string[]> {
+  const j = await fetchJson<{ schemas?: string[] }>(`/api/spark/schemas?catalog=${encodeURIComponent(catalog)}`, 15000)
+  return j?.schemas ?? []
+}
+
+export async function fetchSparkTables(catalog: string, schema: string): Promise<{ name: string; fqn: string }[]> {
+  const j = await fetchJson<{ tables?: { name: string; fqn: string }[] }>(
+    `/api/spark/tables?catalog=${encodeURIComponent(catalog)}&schema=${encodeURIComponent(schema)}`, 15000)
+  return j?.tables ?? []
+}
+
+export async function fetchSparkColumns(table: string): Promise<{ name: string; type: string }[]> {
+  const j = await fetchJson<{ columns?: { name: string; type: string }[] }>(
+    `/api/spark/columns?table=${encodeURIComponent(table)}`, 15000)
+  return j?.columns ?? []
+}
+
+export async function createSparkRun(body: Record<string, unknown>): Promise<{ ok: boolean; run_id?: string; sql?: string; error?: string; target?: string | null }> {
+  const r = await fetch('/api/spark/run', {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify(body),
+  })
+  return r.json()
+}
+
+export async function fetchSparkRun(runId: string) {
+  return fetchJson<{ ok: boolean; run?: SparkRun }>(`/api/spark/run/${runId}`, 15000)
+}
+
+export function cancelSparkRun(runId: string) {
+  return fetch(`/api/spark/run/${runId}/cancel`, { method: 'POST' })
+}
+
+export async function fetchSparkLive() {
+  return fetchJson<SparkLive>('/api/spark/live', 15000)
+}
+
+export async function fetchSparkRuns(): Promise<SparkRun[]> {
+  const j = await fetchJson<{ runs?: SparkRun[] }>('/api/spark/runs', 15000)
+  return j?.runs ?? []
+}
+
+export function toggleCustodianOffload(enabled?: boolean) {
+  return fetch('/api/agent-ops/custodian/toggle', {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify(enabled === undefined ? {} : { enabled }),
+  })
+}