feat: Spark Workbench everywhere, autonomous Hadoop offload & LLM masking-aware

- Data Hub with Hadoop tab (HDFS/Iceberg browser, Spark, pipeline)
- Databricks-style Lakehouse Workbench (Trino engine, live exec matrix,
  materialize to Iceberg/S3); reused & embedded in every source-DB UI
- HDFS -> Kafka -> Spark -> Iceberg/S3 pipeline; WebHDFS hostname resolver
- Data Flow master pulse switch (Run/Pause/Stop) gating animated edges
- Data Custodian autonomous Hadoop offload loop (batch counterpart to CDC),
  pulsing source -> HDFS edges; toggle in Data Flow
- LLM now autonomously aware of all latest platform changes (live platform
  context) and enforces masking policy: never reveals masked PII, still
  answers helpfully with aggregates/explanations

api/presentation_upload.py

@@ -13,6 +13,7 @@ from typing import Any
 import httpx
 
 PRESENTATIONS_DIR = Path(os.getenv("PRESENTATIONS_DIR", "/data/presentations"))
+LIVE_OVERRIDE_DIR = PRESENTATIONS_DIR / "live-override"
 DOCLING_URL = os.getenv("DOCLING_URL", "http://docling-serve:5001").rstrip("/")
 
 
@@ -165,6 +166,65 @@ def get_deck(deck_id: str) -> dict[str, Any] | None:
     return json.loads(path.read_text())
 
 
+def get_live_override() -> dict[str, Any] | None:
+    path = LIVE_OVERRIDE_DIR / "meta.json"
+    if not path.exists():
+        return None
+    try:
+        return json.loads(path.read_text())
+    except Exception:
+        return None
+
+
+def save_live_override(body: dict[str, Any]) -> dict[str, Any]:
+    """Persist user edits for the Live Cluster deck."""
+    LIVE_OVERRIDE_DIR.mkdir(parents=True, exist_ok=True)
+    (LIVE_OVERRIDE_DIR / "assets").mkdir(parents=True, exist_ok=True)
+    existing = get_live_override() or {"id": "live", "source": "live-override", "editable": True}
+    slides = _clean_slides(body.get("slides"))
+    payload = {
+        **existing,
+        "id": "live",
+        "source": "live-override",
+        "editable": True,
+        "title": str(body.get("title") or existing.get("title") or "Live Cluster")[:120],
+        "subtitle": str(body.get("subtitle") or existing.get("subtitle") or "")[:300],
+        "ts": datetime.now(timezone.utc).isoformat(),
+        "slides": slides,
+        "slide_count": len(slides),
+    }
+    (LIVE_OVERRIDE_DIR / "meta.json").write_text(json.dumps(payload, indent=2, default=str))
+    return payload
+
+
+def clear_live_override() -> bool:
+    import shutil
+    if not LIVE_OVERRIDE_DIR.exists():
+        return True
+    shutil.rmtree(LIVE_OVERRIDE_DIR, ignore_errors=True)
+    return True
+
+
+def _clean_slides(incoming_slides: Any) -> list[dict[str, Any]]:
+    clean_slides: list[dict[str, Any]] = []
+    for i, s in enumerate(incoming_slides or [], start=1):
+        if not isinstance(s, dict):
+            continue
+        bullets = [str(b).strip()[:400] for b in (s.get("bullets") or []) if str(b).strip()]
+        slide: dict[str, Any] = {
+            "id": str(s.get("id") or f"slide-{i}"),
+            "title": str(s.get("title") or f"Slide {i}")[:200],
+            "subtitle": str(s.get("subtitle") or "")[:300],
+            "bullets": bullets,
+            "image": str(s.get("image") or "")[:300],
+            "kind": str(s.get("kind") or "narrative")[:40],
+        }
+        if s.get("animation"):
+            slide["animation"] = str(s.get("animation"))[:40]
+        clean_slides.append(slide)
+    return clean_slides or [_blank_slide(1)]
+
+
 def _blank_slide(idx: int = 1) -> dict[str, Any]:
     return {
         "id": f"slide-{idx}",
@@ -208,22 +268,7 @@ def save_deck(deck_id: str, body: dict[str, Any]) -> dict[str, Any] | None:
         return None
     existing = json.loads(meta_path.read_text())
 
-    incoming_slides = body.get("slides")
-    clean_slides: list[dict[str, Any]] = []
-    for i, s in enumerate(incoming_slides or [], start=1):
-        if not isinstance(s, dict):
-            continue
-        bullets = [str(b).strip()[:400] for b in (s.get("bullets") or []) if str(b).strip()]
-        clean_slides.append({
-            "id": str(s.get("id") or f"slide-{i}"),
-            "title": str(s.get("title") or f"Slide {i}")[:200],
-            "subtitle": str(s.get("subtitle") or "")[:300],
-            "bullets": bullets,
-            "image": str(s.get("image") or "")[:300],
-            "kind": str(s.get("kind") or "narrative")[:40],
-        })
-    if not clean_slides:
-        clean_slides = [_blank_slide(1)]
+    clean_slides = _clean_slides(body.get("slides"))
 
     existing.update({
         "title": str(body.get("title") or existing.get("title") or "Untitled deck")[:120],
@@ -250,9 +295,12 @@ def delete_deck(deck_id: str) -> bool:
 
 def save_image(deck_id: str, filename: str, content: bytes) -> dict[str, Any] | None:
     """Store an image in the deck's assets folder; return its served URL."""
-    deck_dir = PRESENTATIONS_DIR / deck_id
-    if not deck_dir.exists():
-        return None
+    if deck_id == "live":
+        deck_dir = LIVE_OVERRIDE_DIR
+    else:
+        deck_dir = PRESENTATIONS_DIR / deck_id
+        if not deck_dir.exists():
+            return None
     assets = deck_dir / "assets"
     assets.mkdir(parents=True, exist_ok=True)
     ext = ""
@@ -265,7 +313,8 @@ def save_image(deck_id: str, filename: str, content: bytes) -> dict[str, Any] |
 
 def get_asset_path(deck_id: str, name: str) -> Path | None:
     safe = _safe_name(name)
-    path = PRESENTATIONS_DIR / deck_id / "assets" / safe
+    base = LIVE_OVERRIDE_DIR if deck_id == "live" else PRESENTATIONS_DIR / deck_id
+    path = base / "assets" / safe
     if not path.exists() or not path.is_file():
         return None
     return path