feat: Spark Workbench everywhere, autonomous Hadoop offload & LLM masking-aware

- Data Hub with Hadoop tab (HDFS/Iceberg browser, Spark, pipeline)
- Databricks-style Lakehouse Workbench (Trino engine, live exec matrix,
  materialize to Iceberg/S3); reused & embedded in every source-DB UI
- HDFS -> Kafka -> Spark -> Iceberg/S3 pipeline; WebHDFS hostname resolver
- Data Flow master pulse switch (Run/Pause/Stop) gating animated edges
- Data Custodian autonomous Hadoop offload loop (batch counterpart to CDC),
  pulsing source -> HDFS edges; toggle in Data Flow
- LLM now autonomously aware of all latest platform changes (live platform
  context) and enforces masking policy: never reveals masked PII, still
  answers helpfully with aggregates/explanations

api/main.py

@@ -26,13 +26,16 @@ from agent_terminal import (
 from lab_context import collect_full_lab_context, format_context_for_agent
 from presentation import build_presentation_payload, render_presentation_html
 from presentation_upload import (
+    clear_live_override,
     create_deck,
     delete_deck,
     get_asset_path,
     get_deck,
+    get_live_override,
     list_decks,
     save_deck,
     save_image,
+    save_live_override,
     save_upload,
 )
 from presentation_static import get_static_deck, list_static_decks
@@ -42,11 +45,13 @@ from pipeline_ops import router as pipeline_router
 from hadoop_analytics import router as hadoop_router
 from elasticsearch_api import router as elasticsearch_router
 from sql_console import router as sql_router
-from agent_ops import router as agent_ops_router, agent_dml_loop, etl_agent_loop
+from agent_ops import router as agent_ops_router, agent_dml_loop, etl_agent_loop, custodian_offload_loop
 from cdc_consumer import router as cdc_router, cdc_consumer_loop
 from movements import router as movements_router
 from movements import MOVEMENT_BY_ID, trigger_and_watch
 from dataflow import router as dataflow_router
+from streaming_ops import router as streaming_router
+from spark_workbench import router as spark_workbench_router
 from pii_catalog import router as pii_router
 from ssh_terminal import ssh_session
 from node_registry import NODE_IDS, NODE_AGENT, NODE_REGISTRY, is_node_id
@@ -339,6 +344,11 @@ async def gather_agent_context(
         sup = " [supervisor]" if a.get("supervisor") else ""
         agent_lines.append(f"  - {a['name']} ({a['id']}){sup}: {a['role']}")
     ctx = ctx + "\n".join(agent_lines)
+    try:
+        from platform_context import build_llm_addendum
+        ctx = ctx + "\n\n" + build_llm_addendum()
+    except Exception:
+        pass
     if log:
         await log("info", "fetch", f"▸ Context assembled: {len(ctx)} chars for LLM")
     return ctx
@@ -363,6 +373,8 @@ Rules:
 - Use ONLY the live data below — do not invent hosts, ports, numbers or connector names.
 - Use exact container/connector names from the data (e.g. mysql-hr-connector, not "Debezium").
 - If something is DOWN or 0 GB, say so honestly.
+- Respect the data masking policy: NEVER reveal, guess or reconstruct raw values of MASKED columns (they arrive as the token 🔒 MASKED). You MUST still answer helpfully — confirm the column is masked for privacy/governance, explain why, and you may use non-sensitive aggregates/counts over it.
+- You are fully aware of all latest platform changes via the section PLATFORM CAPABILITIES & RECENT CHANGES below; use it to answer questions about recent changes, the Spark Workbench, the Hadoop pipeline, the Data Flow pulse switch and the autonomous agents (DML, ETL, Custodian Hadoop offload).
 - Be concise and helpful (max ~10 sentences); bullet lists are fine when they aid clarity.
 
 --- LIVE LAB DATA (primary domain first, then full stack) ---
@@ -720,6 +732,7 @@ async def lifespan(app: FastAPI):
     dml_task = asyncio.create_task(agent_dml_loop())
     cdc_task = asyncio.create_task(cdc_consumer_loop())
     etl_task = asyncio.create_task(etl_agent_loop())
+    cust_task = asyncio.create_task(custodian_offload_loop())
     add_feed("infra-sentinel", "ATC Command Center API online", "info")
     yield
     task.cancel()
@@ -741,6 +754,8 @@ app.include_router(agent_ops_router)
 app.include_router(cdc_router)
 app.include_router(movements_router)
 app.include_router(dataflow_router)
+app.include_router(streaming_router)
+app.include_router(spark_workbench_router)
 app.include_router(pii_router)
 app.add_middleware(
     CORSMiddleware,
@@ -794,7 +809,18 @@ async def get_presentation_data(*, use_cache: bool = True) -> dict[str, Any]:
     gpu = await collect_gpu()
     snap = await collect_full_lab_context(gpu_data=gpu, include_inventory=False)
     data = build_presentation_payload(snap)
+    override = get_live_override()
+    if override and override.get("slides"):
+        data["title"] = override.get("title") or data.get("title")
+        data["subtitle"] = override.get("subtitle") or data.get("subtitle", "")
+        data["slides"] = override["slides"]
+        data["slide_count"] = len(override["slides"])
+        data["edited"] = True
+        data["override_ts"] = override.get("ts")
+    else:
+        data["edited"] = False
     data["source"] = "live"
+    data["id"] = "live"
     _presentation_cache["ts"] = now
     _presentation_cache["data"] = data
     return data
@@ -867,6 +893,7 @@ async def create_presentation_deck(body: dict[str, Any] | None = Body(default=No
                     "bullets": list(s.get("bullets") or []),
                     "image": s.get("image") or "",
                     "kind": s.get("kind") or "narrative",
+                    **({"animation": s["animation"]} if s.get("animation") else {}),
                 }
                 for i, s in enumerate(src["slides"], start=1)
             ]
@@ -878,12 +905,27 @@ async def create_presentation_deck(body: dict[str, Any] | None = Body(default=No
 
 @app.put("/api/presentation/decks/{deck_id}")
 async def update_presentation_deck(deck_id: str, body: dict[str, Any] = Body(...)):
+    if deck_id == "live":
+        save_live_override(body)
+        _presentation_cache["ts"] = 0
+        _presentation_cache["data"] = None
+        payload = await get_presentation_data(use_cache=False)
+        return {"ok": True, "deck": payload}
     deck = save_deck(deck_id, body)
     if not deck:
         return {"error": "deck not found or not editable"}
     return {"ok": True, "deck": deck}
 
 
+@app.post("/api/presentation/live/reset")
+async def reset_live_presentation():
+    clear_live_override()
+    _presentation_cache["ts"] = 0
+    _presentation_cache["data"] = None
+    payload = await get_presentation_data(use_cache=False)
+    return {"ok": True, "deck": payload}
+
+
 @app.delete("/api/presentation/decks/{deck_id}")
 async def remove_presentation_deck(deck_id: str):
     return {"ok": delete_deck(deck_id)}