feat: Spark Workbench everywhere, autonomous Hadoop offload & LLM masking-aware

- Data Hub with Hadoop tab (HDFS/Iceberg browser, Spark, pipeline)
- Databricks-style Lakehouse Workbench (Trino engine, live exec matrix,
  materialize to Iceberg/S3); reused & embedded in every source-DB UI
- HDFS -> Kafka -> Spark -> Iceberg/S3 pipeline; WebHDFS hostname resolver
- Data Flow master pulse switch (Run/Pause/Stop) gating animated edges
- Data Custodian autonomous Hadoop offload loop (batch counterpart to CDC),
  pulsing source -> HDFS edges; toggle in Data Flow
- LLM now autonomously aware of all latest platform changes (live platform
  context) and enforces masking policy: never reveals masked PII, still
  answers helpfully with aggregates/explanations

api/movements.py

@@ -39,6 +39,10 @@ MOVEMENTS: list[dict[str, Any]] = [
     {"id": "gen_mongodb", "label": "Generate → MongoDB", "kind": "generate",
      "dag_id": "gen_mongodb", "agent": "data-custodian", "from": "generator", "to": "mongodb",
      "default_conf": {"rows": 3000}},
+    {"id": "hdfs_to_kafka", "label": "HDFS → Kafka export", "kind": "stream",
+     "dag_id": None, "agent": "hadoop-ranger", "from": "hdfs", "to": "kafka",
+     "api": "/api/pipeline/streaming/hdfs/to-kafka",
+     "default_conf": {"source": "trino", "table": "iceberg.hadoop.historical_sales_hdfs", "topic": "hdfs.historical.sales"}},
     {"id": "hadoop_to_trino", "label": "HDFS → Iceberg (Trino)", "kind": "movement",
      "dag_id": "hadoop_to_trino", "agent": "hadoop-ranger", "from": "hdfs", "to": "iceberg_hadoop",
      "default_conf": {"mode": "refresh"},
@@ -47,6 +51,14 @@ MOVEMENTS: list[dict[str, Any]] = [
      "dag_id": "mask_to_curated", "agent": "lakehouse-ops", "from": "sources", "to": "iceberg_curated",
      "default_conf": {},
      "count_sql": "SELECT count(*) FROM iceberg.curated_masked.sales_orders_masked"},
+    {"id": "spark_to_s3", "label": "Spark → S3 curated", "kind": "movement",
+     "dag_id": "mask_to_curated", "agent": "lakehouse-ops", "from": "spark", "to": "s3_cdc",
+     "default_conf": {"target": "s3"},
+     "count_sql": "SELECT count(*) FROM iceberg.curated_masked.sales_orders_masked"},
+    {"id": "spark_to_curated", "label": "Spark → Iceberg curated", "kind": "movement",
+     "dag_id": "mask_to_curated", "agent": "lakehouse-ops", "from": "spark", "to": "iceberg_curated",
+     "default_conf": {},
+     "count_sql": "SELECT count(*) FROM iceberg.curated_masked.sales_orders_masked"},
 ]
 MOVEMENT_BY_ID = {m["id"]: m for m in MOVEMENTS}
 
@@ -116,6 +128,30 @@ async def trigger_and_watch(mid: str, conf: dict[str, Any] | None = None, *, aut
     mv = MOVEMENT_BY_ID.get(mid)
     if not mv:
         return {"ok": False, "error": f"unknown movement {mid}"}
+    if mv.get("api"):
+        try:
+            payload = {**(mv.get("default_conf") or {}), **(conf or {})}
+            t0 = time.time()
+            async with httpx.AsyncClient(timeout=120.0) as client:
+                r = await client.post(f"http://127.0.0.1:8000{mv['api']}", json=payload)
+            dur = round(time.time() - t0, 1)
+            body = r.json() if r.headers.get("content-type", "").startswith("application/json") else {}
+            state = "success" if r.status_code < 400 and body.get("ok", True) else "failed"
+            rows = body.get("rows_sent") or body.get("rows")
+            run = {
+                "movement_id": mid, "state": state, "duration_s": dur, "rows": rows,
+                "ended_at": datetime.now(timezone.utc).isoformat(), "conf": payload,
+            }
+            _last_runs[mid] = run
+            await _publish({"type": "movement", **run})
+            lvl = "info" if state == "success" else "err"
+            _feed(agent, f"[etl] {mv['label']}: {state} in {dur}s", lvl)
+            return {"ok": state == "success", **run}
+        except Exception as exc:
+            _last_runs[mid] = {**_last_runs.get(mid, {}), "state": "failed", "error": str(exc)}
+            _feed(agent, f"[etl] {mv['label']}: error {str(exc)[:120]}", "err")
+            return {"ok": False, "error": str(exc)}
+
     conf = {**(mv.get("default_conf") or {}), **(conf or {})}
     agent = mv["agent"]
     count_sql = mv.get("count_sql")