feat: Spark Workbench everywhere, autonomous Hadoop offload & LLM masking-aware

- Data Hub with Hadoop tab (HDFS/Iceberg browser, Spark, pipeline)
- Databricks-style Lakehouse Workbench (Trino engine, live exec matrix,
  materialize to Iceberg/S3); reused & embedded in every source-DB UI
- HDFS -> Kafka -> Spark -> Iceberg/S3 pipeline; WebHDFS hostname resolver
- Data Flow master pulse switch (Run/Pause/Stop) gating animated edges
- Data Custodian autonomous Hadoop offload loop (batch counterpart to CDC),
  pulsing source -> HDFS edges; toggle in Data Flow
- LLM now autonomously aware of all latest platform changes (live platform
  context) and enforces masking policy: never reveals masked PII, still
  answers helpfully with aggregates/explanations

ui/src/types.ts

@@ -108,11 +108,61 @@ export type DataflowGraph = {
   ok: boolean
   nodes: DataflowNode[]
   edges: DataflowEdge[]
+  flow?: string
   pii_summary: { datasets?: number; pii_columns?: number; masked_columns?: number; unmasked_columns?: number }
   cdc: { connected?: boolean; consumed?: number; window_total?: number }
+  streaming?: StreamingStatus | null
   ts: number
 }
 
+export type StreamingSparkWorker = {
+  id?: string
+  host?: string
+  cores?: number
+  cores_used?: number
+  memory_mb?: number
+  state?: string
+  webui?: string
+}
+
+export type StreamingSparkApp = {
+  id?: string
+  name?: string
+  cores?: number
+  memory_mb?: number
+  submitdate?: string
+  duration_ms?: number
+  user?: string
+}
+
+export type StreamingStatus = {
+  ok?: boolean
+  spark?: {
+    ui_url?: string
+    ui_ok?: boolean
+    status?: string
+    workers?: StreamingSparkWorker[]
+    alive_workers?: number
+    cores?: number
+    cores_used?: number
+    memory_mb?: number
+    memory_used_mb?: number
+    active_apps?: StreamingSparkApp[]
+    completed_apps?: StreamingSparkApp[]
+  }
+  kafka?: {
+    ui_url?: string
+    ui_ok?: boolean
+    connect_ok?: boolean
+    cluster?: { name?: string; status?: string; broker_count?: number; topic_count?: number }
+    topics?: { name?: string; partitions?: number; replicas?: number; messages?: number }[]
+    connectors?: { name: string; state?: string; worker?: string; tasks?: { id?: number; state?: string }[]; type?: string }[]
+  }
+  edges?: Record<string, boolean>
+  jobs?: { id: string; label: string; dag_id: string; description?: string; default_conf?: Record<string, unknown> }[]
+  ts?: number
+}
+
 export type Movement = {
   id: string
   label: string
@@ -334,6 +384,53 @@ export type PresentationData = {
   pipeline_active?: boolean
   slides: PresentationSlide[]
   slide_count: number
+  edited?: boolean
+  override_ts?: string
+  id?: string
+  source?: string
   workload?: WorkloadData
   topologies?: Record<string, TopologyViewData>
 }
+
+export type SparkRunStats = {
+  state?: string
+  nodes?: number
+  total_splits?: number
+  queued_splits?: number
+  running_splits?: number
+  completed_splits?: number
+  processed_rows?: number
+  processed_bytes?: number
+  physical_input_bytes?: number
+  peak_memory_bytes?: number
+  cpu_time_ms?: number
+  wall_time_ms?: number
+  elapsed_ms?: number
+  progress_pct?: number
+}
+
+export type SparkRun = {
+  id: string
+  operation: string
+  label: string
+  sql: string
+  target?: string | null
+  state: string
+  engine_state?: string
+  stats?: SparkRunStats
+  columns?: string[]
+  rows?: unknown[][]
+  row_count?: number | null
+  error?: string | null
+  started_at?: number
+  ended_at?: number | null
+  update_type?: string
+}
+
+export type SparkLive = {
+  ok?: boolean
+  spark?: StreamingStatus['spark']
+  active_runs?: SparkRun[]
+  recent_runs?: SparkRun[]
+  ts?: number
+}