feat: Spark Workbench everywhere, autonomous Hadoop offload & LLM masking-aware

- Data Hub with Hadoop tab (HDFS/Iceberg browser, Spark, pipeline)
- Databricks-style Lakehouse Workbench (Trino engine, live exec matrix,
  materialize to Iceberg/S3); reused & embedded in every source-DB UI
- HDFS -> Kafka -> Spark -> Iceberg/S3 pipeline; WebHDFS hostname resolver
- Data Flow master pulse switch (Run/Pause/Stop) gating animated edges
- Data Custodian autonomous Hadoop offload loop (batch counterpart to CDC),
  pulsing source -> HDFS edges; toggle in Data Flow
- LLM now autonomously aware of all latest platform changes (live platform
  context) and enforces masking policy: never reveals masked PII, still
  answers helpfully with aggregates/explanations

api/Dockerfile

@@ -4,7 +4,7 @@ WORKDIR /app
 RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/*
 COPY requirements.txt .
 RUN pip install --no-cache-dir -r requirements.txt
-COPY main.py lab_context.py agent_terminal.py workload.py node_registry.py node_ops.py topology_views.py supervisor.py approval_service.py db.py dockhand_envs.py presentation.py database_inventory.py presentation_upload.py presentation_static.py storage_s3.py elasticsearch_api.py sql_console.py hdfs_api.py ssh_terminal.py pipeline_ops.py hadoop_analytics.py agent_ops.py cdc_consumer.py movements.py dataflow.py pii_catalog.py hive_bench_seed.json .
+COPY main.py lab_context.py agent_terminal.py workload.py node_registry.py node_ops.py topology_views.py supervisor.py approval_service.py db.py dockhand_envs.py presentation.py database_inventory.py presentation_upload.py presentation_static.py storage_s3.py elasticsearch_api.py sql_console.py hdfs_api.py ssh_terminal.py pipeline_ops.py hadoop_analytics.py agent_ops.py cdc_consumer.py movements.py dataflow.py streaming_ops.py spark_workbench.py hadoop_sql.py hdfs_kafka.py webhdfs_util.py pii_catalog.py platform_context.py hive_bench_seed.json .
 RUN mkdir -p /data
 ENV DATABASE_URL=sqlite:////data/atc-agents.db
 EXPOSE 3201

api/agent_ops.py

@@ -328,11 +328,98 @@ async def etl_agent_loop() -> None:
         await asyncio.sleep(max(30.0, float(_etl_state["interval"])))
 
 
+
+# ── Custodian Hadoop offload (batch counterpart to CDC) ─────────────────────
+_CUST_INTERVAL = float(os.getenv("CUSTODIAN_OFFLOAD_INTERVAL_SECONDS", "120"))
+_CUST_BATCH = int(os.getenv("CUSTODIAN_OFFLOAD_BATCH", "200"))
+_CUST_TARGETS = [
+    {"label": "postgres sales_orders", "src": "postgres_sales.public.sales_orders",
+     "target": "iceberg.hadoop.sales_orders_offload"},
+    {"label": "mysql employee_events", "src": "mysql_hr.hr.employee_events",
+     "target": "iceberg.hadoop.employee_events_offload"},
+]
+_custodian_state: dict[str, Any] = {
+    "enabled": os.getenv("CUSTODIAN_OFFLOAD_ENABLED", "1") not in ("0", "false", "False", ""),
+    "interval": _CUST_INTERVAL,
+    "targets": [c["target"] for c in _CUST_TARGETS],
+    "idx": 0,
+    "runs_total": 0,
+    "last": None,
+    "started": False,
+}
+
+
+async def _custodian_offload_once(idx: int | None = None) -> dict[str, Any]:
+    """Offload a batch of source rows into the Hadoop Iceberg lake via Trino."""
+    from spark_workbench import _trino_collect
+    i = _custodian_state["idx"] if idx is None else idx
+    tgt = _CUST_TARGETS[i % len(_CUST_TARGETS)]
+    _custodian_state["idx"] = i + 1
+    await _trino_collect(
+        f"CREATE TABLE IF NOT EXISTS {tgt['target']} AS SELECT * FROM {tgt['src']} WHERE 1=0", 1)
+    ins = await _trino_collect(
+        f"INSERT INTO {tgt['target']} SELECT * FROM {tgt['src']} LIMIT {_CUST_BATCH}", 1)
+    ok = bool(ins.get("ok"))
+    _custodian_state["runs_total"] += 1
+    _custodian_state["last"] = {
+        "target": tgt["target"], "src": tgt["src"], "ok": ok,
+        "rows": _CUST_BATCH if ok else 0,
+        "ts": datetime.now(timezone.utc).isoformat(), "error": ins.get("error"),
+    }
+    if ok:
+        await _emit(f"[custodian-offload] {tgt['label']} → {tgt['target']}: offloaded ~{_CUST_BATCH} rows to Hadoop", "info")
+    else:
+        await _emit(f"[custodian-offload] {tgt['label']} failed: {str(ins.get('error'))[:120]}", "err")
+    return _custodian_state["last"]
+
+
+async def custodian_offload_loop() -> None:
+    _custodian_state["started"] = True
+    await asyncio.sleep(45)
+    await _emit("[custodian-offload] Autonomous Hadoop offload online — batching source data into the Iceberg lake", "info")
+    while True:
+        try:
+            if _custodian_state["enabled"]:
+                await _custodian_offload_once()
+        except Exception as exc:
+            _custodian_state["last"] = {"error": str(exc), "ts": datetime.now(timezone.utc).isoformat()}
+        await asyncio.sleep(max(30.0, float(_custodian_state["interval"])))
+
+
+def custodian_recent() -> bool:
+    last = _custodian_state.get("last") or {}
+    ts = last.get("ts")
+    if not ts or not last.get("ok"):
+        return False
+    try:
+        from datetime import datetime as _dt
+        t = _dt.fromisoformat(str(ts).replace("Z", "+00:00"))
+        window = max(60.0, float(_custodian_state["interval"]) * 1.5)
+        return (datetime.now(timezone.utc) - t).total_seconds() < window
+    except Exception:
+        return False
+
+
 # ── Endpoints ────────────────────────────────────────────────────────────────
 @router.get("/status")
 async def status() -> JSONResponse:
     pools = {k: len(v) for k, v in _pools.items()}
-    return JSONResponse({"ok": True, "pools": pools, "etl": _etl_state, **_state})
+    return JSONResponse({"ok": True, "pools": pools, "etl": _etl_state, "custodian": _custodian_state, **_state})
+
+
+@router.post("/custodian/toggle")
+async def custodian_toggle(body: dict[str, Any] = Body(default={})) -> JSONResponse:
+    if "enabled" in body:
+        _custodian_state["enabled"] = bool(body["enabled"])
+    else:
+        _custodian_state["enabled"] = not _custodian_state["enabled"]
+    if "interval" in body:
+        try:
+            _custodian_state["interval"] = max(30.0, float(body["interval"]))
+        except (TypeError, ValueError):
+            pass
+    await _emit(f"[custodian-offload] Hadoop offload {'ENABLED' if _custodian_state['enabled'] else 'PAUSED'} by operator", "warn")
+    return JSONResponse({"ok": True, "enabled": _custodian_state["enabled"], "interval": _custodian_state["interval"]})
 
 
 @router.post("/etl/toggle")

api/dataflow.py

@@ -38,7 +38,9 @@ NODES: list[dict[str, Any]] = [
     {"id": "mysql", "label": "MySQL", "sub": "employee_events", "kind": "source", "x": 28, "y": 38},
     {"id": "mongodb", "label": "MongoDB", "sub": "events", "kind": "source", "x": 28, "y": 60},
     # col 2 — change data capture
-    {"id": "kafka", "label": "Kafka · Debezium", "sub": "CDC topics", "kind": "stream", "x": 47, "y": 34},
+    {"id": "kafka", "label": "Kafka · Debezium", "sub": "CDC topics", "kind": "stream", "x": 47, "y": 24},
+    {"id": "spark", "label": "Apache Spark", "sub": "Streaming · batch", "kind": "compute", "x": 62, "y": 38,
+     "url": "/spark-ui/"},
     # col 3 — storage / lakehouse
     {"id": "s3_cdc", "label": "S3 CDC Archive", "sub": "object store", "kind": "sink", "x": 67, "y": 13},
     {"id": "iceberg_curated", "label": "Iceberg · curated_masked", "sub": "masked PII", "kind": "lakehouse", "x": 67, "y": 45},
@@ -58,7 +60,13 @@ EDGES: list[dict[str, Any]] = [
     {"from": "postgres", "to": "kafka", "kind": "cdc"},
     {"from": "mysql", "to": "kafka", "kind": "cdc"},
     {"from": "mongodb", "to": "kafka", "kind": "cdc"},
+    {"from": "kafka", "to": "spark", "kind": "stream"},
+    {"from": "spark", "to": "iceberg_curated", "kind": "movement", "movement_id": "spark_to_curated"},
+    {"from": "spark", "to": "s3_cdc", "kind": "movement", "movement_id": "spark_to_s3"},
     {"from": "kafka", "to": "s3_cdc", "kind": "archive"},
+    {"from": "postgres", "to": "hdfs", "kind": "archive", "offload": True},
+    {"from": "mysql", "to": "hdfs", "kind": "archive", "offload": True},
+    {"from": "hdfs", "to": "kafka", "kind": "stream", "movement_id": "hdfs_to_kafka"},
     {"from": "hdfs", "to": "iceberg_hadoop", "kind": "movement", "movement_id": "hadoop_to_trino"},
     {"from": "postgres", "to": "iceberg_curated", "kind": "mask", "movement_id": "mask_to_curated"},
     {"from": "mysql", "to": "iceberg_curated", "kind": "mask", "movement_id": "mask_to_curated"},
@@ -112,7 +120,7 @@ def _iceberg_hadoop_count() -> int | None:
     return v
 
 
-def _build() -> dict[str, Any]:
+async def _build() -> dict[str, Any]:
     # Live signals
     try:
         from movements import last_runs
@@ -129,6 +137,14 @@ def _build() -> dict[str, Any]:
         pii = get_pii()
     except Exception:
         pii = {"datasets": []}
+    try:
+        from streaming_ops import build_streaming_status
+        streaming = await build_streaming_status()
+    except Exception:
+        streaming = {}
+    spark = streaming.get("spark") or {}
+    kafka = streaming.get("kafka") or {}
+    edge_live = streaming.get("edges") or {}
     pii_by_node = {d["node_id"]: d for d in pii.get("datasets", [])}
 
     nodes = []
@@ -138,9 +154,17 @@ def _build() -> dict[str, Any]:
         metric = None
         if n["id"] == "openmetadata":
             metric = f"{pii.get('summary', {}).get('pii_columns', 0)} PII cols cataloged"
-        if n["id"] == "kafka":
-            metric = f"{cdc.get('window_total', 0)} chg/15m · {cdc.get('consumed', 0)} total"
-            node["level"] = "ok" if cdc.get("connected") else "warn"
+        elif n["id"] == "kafka":
+            topics = len(kafka.get("topics") or [])
+            conn_n = len(kafka.get("connectors") or [])
+            metric = f"{cdc.get('window_total', 0)} chg/15m · {topics} topics · {conn_n} connectors"
+            node["level"] = "ok" if cdc.get("connected") and kafka.get("ui_ok") else "warn"
+        elif n["id"] == "spark":
+            apps = len(spark.get("active_apps") or [])
+            cores = spark.get("cores") or 0
+            used = spark.get("cores_used") or 0
+            metric = f"{spark.get('alive_workers', 0)} workers · {used}/{cores} cores · {apps} apps"
+            node["level"] = "ok" if spark.get("ui_ok") and (spark.get("status") or "").upper() == "ALIVE" else "warn"
         elif n["id"] in ("postgres", "mysql", "mongodb"):
             metric = f"{cdc.get('by_source', {}).get(n['id'], 0)} CDC/15m"
         elif n["id"] == "iceberg_hadoop":
@@ -161,6 +185,12 @@ def _build() -> dict[str, Any]:
         node["metric"] = metric
         nodes.append(node)
 
+    try:
+        from streaming_ops import flow_mode
+        _flow = flow_mode()
+    except Exception:
+        _flow = "running"
+
     edges = []
     for e in EDGES:
         edge = dict(e)
@@ -173,6 +203,22 @@ def _build() -> dict[str, Any]:
             edge["active"] = lr.get("state") == "running"
         if e["kind"] == "cdc":
             edge["active"] = cdc.get("by_source", {}).get(e["from"], 0) > 0
+        elif e.get("from") == "hdfs" and e.get("to") == "kafka":
+            edge["active"] = bool(edge_live.get("hdfs→kafka"))
+        elif e.get("from") == "kafka" and e.get("to") == "spark":
+            edge["active"] = bool(edge_live.get("kafka→spark"))
+        elif e.get("from") == "spark" and e.get("to") == "iceberg_curated":
+            edge["active"] = bool(edge_live.get("spark→iceberg")) or edge.get("active")
+        elif e.get("from") == "spark" and e.get("to") == "s3_cdc":
+            edge["active"] = bool(edge_live.get("spark→s3")) or edge.get("active")
+        if e.get("offload"):
+            try:
+                from agent_ops import custodian_recent
+                edge["active"] = custodian_recent()
+            except Exception:
+                pass
+        if _flow != "running":
+            edge["active"] = False
         edges.append(edge)
 
     return {
@@ -181,6 +227,8 @@ def _build() -> dict[str, Any]:
         "edges": edges,
         "pii_summary": pii.get("summary", {}),
         "cdc": {"connected": cdc.get("connected"), "consumed": cdc.get("consumed"), "window_total": cdc.get("window_total")},
+        "streaming": streaming,
+        "flow": _flow,
         "ts": time.time(),
     }
 
@@ -190,7 +238,7 @@ async def get_dataflow(refresh: bool = False) -> JSONResponse:
     now = time.time()
     if not refresh and _cache["data"] and now - _cache["ts"] < _TTL:
         return JSONResponse(_cache["data"])
-    data = _build()
+    data = await _build()
     _cache["data"] = data
     _cache["ts"] = now
     return JSONResponse(data)

api/hadoop_sql.py

@@ -0,0 +1,152 @@
+"""Hadoop / HDFS / Iceberg catalog and sampling for the Data Hub."""
+
+from __future__ import annotations
+
+import csv
+import io
+import os
+from typing import Any, Callable
+
+from webhdfs_util import open_bytes
+
+HDFS_PATHS = [
+    "/data/historical/sales_orders",
+    "/data/historical/sales_orders/year=2020/part-0.csv",
+]
+
+ICEBERG_SCHEMAS = ["hadoop", "curated_masked", "curated"]
+
+HADOOP_SAMPLES: list[dict[str, str]] = [
+    {"id": "hd1", "label": "Iceberg historical sales", "sql": "SELECT * FROM iceberg.hadoop.historical_sales_hdfs LIMIT 20"},
+    {"id": "hd2", "label": "Row count historical", "sql": "SELECT count(*) FROM iceberg.hadoop.historical_sales_hdfs"},
+    {"id": "hd3", "label": "Curated masked sample", "sql": "SELECT * FROM iceberg.curated_masked.sales_orders_masked LIMIT 15"},
+    {"id": "hd4", "label": "Hive lake schemas", "sql": "SHOW SCHEMAS FROM hive"},
+    {"id": "hd5", "label": "Iceberg hadoop tables", "sql": "SHOW TABLES FROM iceberg.hadoop"},
+]
+
+
+def catalog_hadoop(run_trino: Callable[[str, int], dict[str, Any]]) -> dict[str, Any]:
+    objects: list[dict[str, Any]] = []
+
+    for schema in ICEBERG_SCHEMAS:
+        show = run_trino(f"SHOW TABLES FROM iceberg.{schema}", 500)
+        if not show.get("ok"):
+            continue
+        for row in show.get("rows") or []:
+            name = str(row[0])
+            fqn = f"iceberg.{schema}.{name}"
+            cnt = run_trino(f"SELECT count(*) FROM iceberg.{schema}.{name}", 1)
+            count = None
+            if cnt.get("ok") and cnt.get("rows"):
+                try:
+                    count = int(cnt["rows"][0][0])
+                except (TypeError, ValueError, IndexError):
+                    pass
+            objects.append({
+                "type": "table",
+                "schema": f"iceberg.{schema}",
+                "name": name,
+                "fqn": fqn,
+                "row_count": count,
+            })
+
+    for path in HDFS_PATHS:
+        label = path.rstrip("/").rsplit("/", 1)[-1]
+        objects.append({
+            "type": "file",
+            "schema": "hdfs",
+            "name": label,
+            "fqn": f"hdfs:{path}",
+            "row_count": None,
+        })
+
+    return {"engine": "hadoop", "version": "HDFS + Iceberg", "objects": objects}
+
+
+def sample_hdfs_csv(path: str, limit: int, offset: int, tabular: Callable[..., dict[str, Any]]) -> dict[str, Any]:
+    try:
+        text = open_bytes(path).decode("utf-8", errors="replace")
+        reader = csv.reader(io.StringIO(text))
+        all_rows = list(reader)
+        if not all_rows:
+            return tabular([], [], 0, row_count=0)
+        columns = [c.strip() for c in all_rows[0]]
+        data = all_rows[1 + offset: 1 + offset + limit]
+        rows = [[cell.strip() for cell in row] for row in data]
+        return tabular(columns, rows, 0, row_count=max(0, len(all_rows) - 1))
+    except Exception as exc:
+        return {"ok": False, "error": str(exc)[:500]}
+
+
+def sample_hadoop(
+    object_name: str,
+    limit: int,
+    offset: int,
+    run_trino: Callable[[str, int], dict[str, Any]],
+    tabular: Callable[..., dict[str, Any]],
+) -> dict[str, Any]:
+    if object_name.startswith("hdfs:"):
+        return sample_hdfs_csv(object_name[5:], limit, offset, tabular)
+    sql = f"SELECT * FROM {object_name} OFFSET {offset} LIMIT {limit}"
+    return run_trino(sql, limit)
+
+
+def table_row_count_hadoop(object_name: str, run_trino: Callable[[str, int], dict[str, Any]]) -> int | None:
+    if object_name.startswith("hdfs:"):
+        try:
+            text = open_bytes(object_name[5:]).decode("utf-8", errors="replace")
+            return max(0, sum(1 for _ in csv.reader(io.StringIO(text))) - 1)
+        except Exception:
+            return None
+    cnt = run_trino(f"SELECT count(*) FROM {object_name}", 1)
+    if cnt.get("ok") and cnt.get("rows"):
+        try:
+            return int(cnt["rows"][0][0])
+        except (TypeError, ValueError, IndexError):
+            pass
+    return None
+
+
+def health_hadoop(run_trino: Callable[[str, int], dict[str, Any]]) -> dict[str, Any]:
+    from webhdfs_util import HDFS_NN_URL
+    import httpx
+
+    nn_ok = False
+    live_dn = None
+    err_parts: list[str] = []
+    try:
+        with httpx.Client(timeout=6.0) as client:
+            r = client.get(f"{HDFS_NN_URL}/jmx?qry=Hadoop:service=NameNode,name=FSNamesystemState")
+            beans = (r.json().get("beans") or [{}])[0]
+            nn_ok = r.status_code < 400
+            live_dn = beans.get("NumLiveDataNodes")
+    except Exception as exc:
+        err_parts.append(f"NN: {exc}")
+
+    trino_ok = False
+    try:
+        tr = run_trino("SELECT 1", 1)
+        trino_ok = bool(tr.get("ok"))
+        if not trino_ok and tr.get("error"):
+            err_parts.append(f"Trino: {tr['error'][:80]}")
+    except Exception as exc:
+        err_parts.append(f"Trino: {exc}")
+
+    ok = nn_ok and trino_ok
+    return {
+        "ok": ok,
+        "namenode": HDFS_NN_URL,
+        "live_datanodes": live_dn,
+        "trino_ok": trino_ok,
+        "error": "; ".join(err_parts) if err_parts else None,
+    }
+
+
+def connection_info() -> dict[str, str]:
+    from webhdfs_util import HDFS_NN_URL, HDFS_USER
+    return {
+        "namenode": HDFS_NN_URL,
+        "user": HDFS_USER,
+        "trino": os.getenv("TRINO_URL", "http://10.0.21.50:8089"),
+        "spark": os.getenv("SPARK_UI_URL", "http://10.0.21.50:8080"),
+    }

api/hdfs_kafka.py

@@ -0,0 +1,108 @@
+"""HDFS → Kafka export and full hadoop-lake pipeline orchestration."""
+
+from __future__ import annotations
+
+import csv
+import io
+import json
+import os
+import time
+from typing import Any
+
+from aiokafka import AIOKafkaProducer
+from webhdfs_util import open_bytes
+
+KAFKA_BOOTSTRAP = os.getenv("KAFKA_BOOTSTRAP", "10.0.21.36:9092")
+DEFAULT_TRINO_TABLE = os.getenv("HADOOP_EXPORT_TABLE", "iceberg.hadoop.historical_sales_hdfs")
+DEFAULT_HDFS_FILE = os.getenv("HADOOP_EXPORT_HDFS", "/data/historical/sales_orders/year=2020/part-0.csv")
+
+_last_hdfs_export: dict[str, Any] = {"ts": 0.0, "rows": 0, "topic": None}
+
+
+def hdfs_export_snapshot() -> dict[str, Any]:
+    age = time.time() - float(_last_hdfs_export.get("ts") or 0)
+    return {
+        **(_last_hdfs_export or {}),
+        "recent": age < 120,
+        "age_s": round(age, 1) if _last_hdfs_export.get("ts") else None,
+    }
+
+
+def _read_hdfs_csv(path: str, limit: int) -> tuple[list[str], list[list[str]]]:
+    raw = open_bytes(path).decode("utf-8", errors="replace")
+    rows = list(csv.reader(io.StringIO(raw)))
+    if not rows:
+        return [], []
+    header = [c.strip() for c in rows[0]]
+    data = [[cell.strip() for cell in row] for row in rows[1: 1 + limit]]
+    return header, data
+
+
+def _read_trino_table(table: str, limit: int) -> tuple[list[str], list[list[Any]]]:
+    from sql_console import _run_trino
+
+    result = _run_trino(f"SELECT * FROM {table} LIMIT {limit}", limit)
+    if not result.get("ok"):
+        raise RuntimeError(result.get("error") or "Trino query failed")
+    return list(result.get("columns") or []), list(result.get("rows") or [])
+
+
+async def export_hdfs_to_kafka(
+    path: str | None = None,
+    topic: str = "hdfs.historical.sales",
+    limit: int = 2000,
+    source: str = "trino",
+    table: str | None = None,
+    feed: Any = None,
+) -> dict[str, Any]:
+    global _last_hdfs_export
+    header: list[str] = []
+    data: list[list[Any]] = []
+    src_label = source
+
+    try:
+        if source == "trino":
+            tbl = table or DEFAULT_TRINO_TABLE
+            header, data = _read_trino_table(tbl, limit)
+            src_label = tbl
+        else:
+            p = path or DEFAULT_HDFS_FILE
+            header, data = _read_hdfs_csv(p, limit)
+            src_label = p
+    except Exception as exc:
+        if source == "trino" and path:
+            try:
+                header, data = _read_hdfs_csv(path, limit)
+                src_label = path
+            except Exception:
+                return {"ok": False, "error": str(exc)[:400]}
+        else:
+            return {"ok": False, "error": str(exc)[:400]}
+
+    if not data:
+        return {"ok": False, "error": "No data rows to export"}
+
+    producer = AIOKafkaProducer(
+        bootstrap_servers=KAFKA_BOOTSTRAP,
+        value_serializer=lambda v: json.dumps(v, default=str).encode("utf-8"),
+    )
+    await producer.start()
+    rows_sent = 0
+    try:
+        for i, row in enumerate(data):
+            payload = {
+                header[j] if j < len(header) else f"col_{j}": row[j] if j < len(row) else None
+                for j in range(max(len(header), len(row)))
+            }
+            payload["_source"] = "hadoop"
+            payload["_origin"] = src_label
+            payload["_row"] = i + 1
+            await producer.send_and_wait(topic, payload)
+            rows_sent += 1
+    finally:
+        await producer.stop()
+
+    _last_hdfs_export = {"ts": time.time(), "rows": rows_sent, "topic": topic, "source": src_label}
+    if feed:
+        feed("hadoop-ranger", f"[hadoop→kafka] {rows_sent} rows from {src_label} → {topic}", "info")
+    return {"ok": True, "rows_sent": rows_sent, "topic": topic, "source": src_label, "columns": header}

api/main.py

@@ -26,13 +26,16 @@ from agent_terminal import (
 from lab_context import collect_full_lab_context, format_context_for_agent
 from presentation import build_presentation_payload, render_presentation_html
 from presentation_upload import (
+    clear_live_override,
     create_deck,
     delete_deck,
     get_asset_path,
     get_deck,
+    get_live_override,
     list_decks,
     save_deck,
     save_image,
+    save_live_override,
     save_upload,
 )
 from presentation_static import get_static_deck, list_static_decks
@@ -42,11 +45,13 @@ from pipeline_ops import router as pipeline_router
 from hadoop_analytics import router as hadoop_router
 from elasticsearch_api import router as elasticsearch_router
 from sql_console import router as sql_router
-from agent_ops import router as agent_ops_router, agent_dml_loop, etl_agent_loop
+from agent_ops import router as agent_ops_router, agent_dml_loop, etl_agent_loop, custodian_offload_loop
 from cdc_consumer import router as cdc_router, cdc_consumer_loop
 from movements import router as movements_router
 from movements import MOVEMENT_BY_ID, trigger_and_watch
 from dataflow import router as dataflow_router
+from streaming_ops import router as streaming_router
+from spark_workbench import router as spark_workbench_router
 from pii_catalog import router as pii_router
 from ssh_terminal import ssh_session
 from node_registry import NODE_IDS, NODE_AGENT, NODE_REGISTRY, is_node_id
@@ -339,6 +344,11 @@ async def gather_agent_context(
         sup = " [supervisor]" if a.get("supervisor") else ""
         agent_lines.append(f"  - {a['name']} ({a['id']}){sup}: {a['role']}")
     ctx = ctx + "\n".join(agent_lines)
+    try:
+        from platform_context import build_llm_addendum
+        ctx = ctx + "\n\n" + build_llm_addendum()
+    except Exception:
+        pass
     if log:
         await log("info", "fetch", f"▸ Context assembled: {len(ctx)} chars for LLM")
     return ctx
@@ -363,6 +373,8 @@ Rules:
 - Use ONLY the live data below — do not invent hosts, ports, numbers or connector names.
 - Use exact container/connector names from the data (e.g. mysql-hr-connector, not "Debezium").
 - If something is DOWN or 0 GB, say so honestly.
+- Respect the data masking policy: NEVER reveal, guess or reconstruct raw values of MASKED columns (they arrive as the token 🔒 MASKED). You MUST still answer helpfully — confirm the column is masked for privacy/governance, explain why, and you may use non-sensitive aggregates/counts over it.
+- You are fully aware of all latest platform changes via the section PLATFORM CAPABILITIES & RECENT CHANGES below; use it to answer questions about recent changes, the Spark Workbench, the Hadoop pipeline, the Data Flow pulse switch and the autonomous agents (DML, ETL, Custodian Hadoop offload).
 - Be concise and helpful (max ~10 sentences); bullet lists are fine when they aid clarity.
 
 --- LIVE LAB DATA (primary domain first, then full stack) ---
@@ -720,6 +732,7 @@ async def lifespan(app: FastAPI):
     dml_task = asyncio.create_task(agent_dml_loop())
     cdc_task = asyncio.create_task(cdc_consumer_loop())
     etl_task = asyncio.create_task(etl_agent_loop())
+    cust_task = asyncio.create_task(custodian_offload_loop())
     add_feed("infra-sentinel", "ATC Command Center API online", "info")
     yield
     task.cancel()
@@ -741,6 +754,8 @@ app.include_router(agent_ops_router)
 app.include_router(cdc_router)
 app.include_router(movements_router)
 app.include_router(dataflow_router)
+app.include_router(streaming_router)
+app.include_router(spark_workbench_router)
 app.include_router(pii_router)
 app.add_middleware(
     CORSMiddleware,
@@ -794,7 +809,18 @@ async def get_presentation_data(*, use_cache: bool = True) -> dict[str, Any]:
     gpu = await collect_gpu()
     snap = await collect_full_lab_context(gpu_data=gpu, include_inventory=False)
     data = build_presentation_payload(snap)
+    override = get_live_override()
+    if override and override.get("slides"):
+        data["title"] = override.get("title") or data.get("title")
+        data["subtitle"] = override.get("subtitle") or data.get("subtitle", "")
+        data["slides"] = override["slides"]
+        data["slide_count"] = len(override["slides"])
+        data["edited"] = True
+        data["override_ts"] = override.get("ts")
+    else:
+        data["edited"] = False
     data["source"] = "live"
+    data["id"] = "live"
     _presentation_cache["ts"] = now
     _presentation_cache["data"] = data
     return data
@@ -867,6 +893,7 @@ async def create_presentation_deck(body: dict[str, Any] | None = Body(default=No
                     "bullets": list(s.get("bullets") or []),
                     "image": s.get("image") or "",
                     "kind": s.get("kind") or "narrative",
+                    **({"animation": s["animation"]} if s.get("animation") else {}),
                 }
                 for i, s in enumerate(src["slides"], start=1)
             ]
@@ -878,12 +905,27 @@ async def create_presentation_deck(body: dict[str, Any] | None = Body(default=No
 
 @app.put("/api/presentation/decks/{deck_id}")
 async def update_presentation_deck(deck_id: str, body: dict[str, Any] = Body(...)):
+    if deck_id == "live":
+        save_live_override(body)
+        _presentation_cache["ts"] = 0
+        _presentation_cache["data"] = None
+        payload = await get_presentation_data(use_cache=False)
+        return {"ok": True, "deck": payload}
     deck = save_deck(deck_id, body)
     if not deck:
         return {"error": "deck not found or not editable"}
     return {"ok": True, "deck": deck}
 
 
+@app.post("/api/presentation/live/reset")
+async def reset_live_presentation():
+    clear_live_override()
+    _presentation_cache["ts"] = 0
+    _presentation_cache["data"] = None
+    payload = await get_presentation_data(use_cache=False)
+    return {"ok": True, "deck": payload}
+
+
 @app.delete("/api/presentation/decks/{deck_id}")
 async def remove_presentation_deck(deck_id: str):
     return {"ok": delete_deck(deck_id)}

api/movements.py

@@ -39,6 +39,10 @@ MOVEMENTS: list[dict[str, Any]] = [
     {"id": "gen_mongodb", "label": "Generate → MongoDB", "kind": "generate",
      "dag_id": "gen_mongodb", "agent": "data-custodian", "from": "generator", "to": "mongodb",
      "default_conf": {"rows": 3000}},
+    {"id": "hdfs_to_kafka", "label": "HDFS → Kafka export", "kind": "stream",
+     "dag_id": None, "agent": "hadoop-ranger", "from": "hdfs", "to": "kafka",
+     "api": "/api/pipeline/streaming/hdfs/to-kafka",
+     "default_conf": {"source": "trino", "table": "iceberg.hadoop.historical_sales_hdfs", "topic": "hdfs.historical.sales"}},
     {"id": "hadoop_to_trino", "label": "HDFS → Iceberg (Trino)", "kind": "movement",
      "dag_id": "hadoop_to_trino", "agent": "hadoop-ranger", "from": "hdfs", "to": "iceberg_hadoop",
      "default_conf": {"mode": "refresh"},
@@ -47,6 +51,14 @@ MOVEMENTS: list[dict[str, Any]] = [
      "dag_id": "mask_to_curated", "agent": "lakehouse-ops", "from": "sources", "to": "iceberg_curated",
      "default_conf": {},
      "count_sql": "SELECT count(*) FROM iceberg.curated_masked.sales_orders_masked"},
+    {"id": "spark_to_s3", "label": "Spark → S3 curated", "kind": "movement",
+     "dag_id": "mask_to_curated", "agent": "lakehouse-ops", "from": "spark", "to": "s3_cdc",
+     "default_conf": {"target": "s3"},
+     "count_sql": "SELECT count(*) FROM iceberg.curated_masked.sales_orders_masked"},
+    {"id": "spark_to_curated", "label": "Spark → Iceberg curated", "kind": "movement",
+     "dag_id": "mask_to_curated", "agent": "lakehouse-ops", "from": "spark", "to": "iceberg_curated",
+     "default_conf": {},
+     "count_sql": "SELECT count(*) FROM iceberg.curated_masked.sales_orders_masked"},
 ]
 MOVEMENT_BY_ID = {m["id"]: m for m in MOVEMENTS}
 
@@ -116,6 +128,30 @@ async def trigger_and_watch(mid: str, conf: dict[str, Any] | None = None, *, aut
     mv = MOVEMENT_BY_ID.get(mid)
     if not mv:
         return {"ok": False, "error": f"unknown movement {mid}"}
+    if mv.get("api"):
+        try:
+            payload = {**(mv.get("default_conf") or {}), **(conf or {})}
+            t0 = time.time()
+            async with httpx.AsyncClient(timeout=120.0) as client:
+                r = await client.post(f"http://127.0.0.1:8000{mv['api']}", json=payload)
+            dur = round(time.time() - t0, 1)
+            body = r.json() if r.headers.get("content-type", "").startswith("application/json") else {}
+            state = "success" if r.status_code < 400 and body.get("ok", True) else "failed"
+            rows = body.get("rows_sent") or body.get("rows")
+            run = {
+                "movement_id": mid, "state": state, "duration_s": dur, "rows": rows,
+                "ended_at": datetime.now(timezone.utc).isoformat(), "conf": payload,
+            }
+            _last_runs[mid] = run
+            await _publish({"type": "movement", **run})
+            lvl = "info" if state == "success" else "err"
+            _feed(agent, f"[etl] {mv['label']}: {state} in {dur}s", lvl)
+            return {"ok": state == "success", **run}
+        except Exception as exc:
+            _last_runs[mid] = {**_last_runs.get(mid, {}), "state": "failed", "error": str(exc)}
+            _feed(agent, f"[etl] {mv['label']}: error {str(exc)[:120]}", "err")
+            return {"ok": False, "error": str(exc)}
+
     conf = {**(mv.get("default_conf") or {}), **(conf or {})}
     agent = mv["agent"]
     count_sql = mv.get("count_sql")

api/pipeline_ops.py

@@ -114,7 +114,7 @@ def _feed(agent_id: str, message: str, level: str = "info") -> None:
 async def _watch_run(source: str, dag_id: str, run_id: str, agent_id: str, rows: int | None) -> None:
     """Poll an Airflow run to completion and log the outcome to the feed."""
     name = AGENT_NAME.get(agent_id, agent_id)
-    label = f"{rows} rijen" if rows else "data"
+    label = f"{rows} rows" if rows else "data"
     try:
         async with httpx.AsyncClient() as client:
             tok = await _airflow_token(client)
@@ -130,10 +130,10 @@ async def _watch_run(source: str, dag_id: str, run_id: str, agent_id: str, rows:
                 except Exception:
                     continue
                 if state == "success":
-                    _feed(agent_id, f"[datagen] {name} genereerde {label} in {source} — klaar, data stroomt via CDC naar Kafka/S3", "info")
+                    _feed(agent_id, f"[datagen] {name} generated {label} in {source} — complete, data flowing via CDC to Kafka/S3", "info")
                     return
                 if state == "failed":
-                    _feed(agent_id, f"[datagen] {name}: generatie voor {source} is mislukt (zie Airflow logs)", "err")
+                    _feed(agent_id, f"[datagen] {name}: generation for {source} failed (see Airflow logs)", "err")
                     return
     except Exception:
         pass
@@ -205,12 +205,12 @@ async def generate(source: str, body: dict[str, Any] = Body(default={})) -> JSON
                 timeout=15,
             )
         if r.status_code >= 400:
-            _feed(agent_id, f"[datagen] {name}: kon generatie voor {source} niet starten (Airflow {r.status_code})", "err")
+            _feed(agent_id, f"[datagen] {name}: could not start generation for {source} (Airflow {r.status_code})", "err")
             return JSONResponse({"ok": False, "error": f"Airflow {r.status_code}: {r.text[:300]}"}, status_code=200)
         j = r.json()
         run_id = j.get("dag_run_id")
-        verb = "genereert zelf" if autonomous else "startte generatie:"
-        rows_txt = f"{conf['rows']} rijen" if conf.get("rows") else "data"
+        verb = "generating autonomously" if autonomous else "started generation of"
+        rows_txt = f"{conf['rows']} rows" if conf.get("rows") else "data"
         _feed(agent_id, f"[datagen] {name} {verb} {rows_txt} in {source}", "info")
         if run_id:
             asyncio.create_task(_watch_run(source, dag_id, run_id, agent_id, conf.get("rows")))

api/platform_context.py

@@ -0,0 +1,144 @@
+"""Live 'platform capabilities + recent changes + masking guidance' block for the LLM.
+
+This is rebuilt on every question from live in-process state, so the assistant is
+always autonomously aware of the latest things running in the lab (Data Hub,
+Spark Workbench, HDFS→Kafka→Spark→S3 pipeline, autonomous agents) and of the
+exact masking policy currently in force.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+def _fmt_ts(ts: Any) -> str:
+    try:
+        return str(ts)[:19]
+    except Exception:
+        return "?"
+
+
+def build_platform_section() -> str:
+    lines: list[str] = ["=== PLATFORM CAPABILITIES & RECENT CHANGES (live) ==="]
+
+    lines += [
+        "Command Center features currently deployed:",
+        "  - Data Hub: tabbed UI with 'Source Databases' (PostgreSQL, MySQL, MongoDB, Cassandra, Neo4j) and 'Hadoop' (HDFS files, Hive/Iceberg tables, Spark, pipeline).",
+        "  - Spark Lakehouse Workbench (Databricks-style): pick any federated table, run preview/filter/aggregate/profile/join/SQL on the distributed engine, and materialize results to Iceberg (S3/HDFS-backed). Live execution matrix: splits, rows, bytes, CPU, wall-time, peak memory, nodes.",
+        "  - The same workbench is embedded in each source-database UI (scoped to that source's Trino catalog).",
+        "  - Data Flow: live lineage graph with a master pulse switch (Run / Pause / Stop) that starts/stops the animated flow.",
+        "  - Pipeline: HDFS (Iceberg historical_sales) → Kafka topic hdfs.historical.sales → Spark transform → Iceberg curated → S3.",
+        "Autonomous agents (run continuously, toggleable):",
+        "  - Data Custodian (DML): generates live INSERT/UPDATE/DELETE on the source DBs so Debezium CDC streams to Kafka.",
+        "  - Data Custodian (Hadoop offload): periodically offloads recent source rows into the Hadoop Iceberg lake (iceberg.hadoop.*_offload), the batch counterpart to CDC.",
+        "  - ETL agent: autonomously triggers data movements (HDFS→Iceberg, mask→curated, generators).",
+    ]
+
+    # live streaming + flow
+    try:
+        from streaming_ops import build_streaming_status, flow_snapshot  # type: ignore
+        flow = flow_snapshot()
+        lines.append(f"Data Flow pulse: {flow.get('mode')}")
+    except Exception:
+        pass
+
+    # autonomous agent state
+    try:
+        from agent_ops import _state, _etl_state, _custodian_state  # type: ignore
+        lines.append(
+            f"DML agent: {'on' if _state.get('enabled') else 'off'} "
+            f"(ops_total={_state.get('ops_total')}, last={_state.get('last_op')})"
+        )
+        lines.append(
+            f"ETL agent: {'on' if _etl_state.get('enabled') else 'off'} "
+            f"(runs={_etl_state.get('runs_total')}, last={_etl_state.get('last')})"
+        )
+        cust = _custodian_state
+        lines.append(
+            f"Custodian Hadoop offload: {'on' if cust.get('enabled') else 'off'} "
+            f"(offloads={cust.get('runs_total')}, last={cust.get('last')})"
+        )
+    except Exception:
+        pass
+
+    # recent movements
+    try:
+        from movements import last_runs  # type: ignore
+        runs = last_runs()
+        if runs:
+            lines.append("Recent data-movement runs:")
+            for mid, r in list(runs.items())[-6:]:
+                lines.append(f"  - {mid}: {r.get('state')} rows={r.get('rows')} {_fmt_ts(r.get('ended_at'))}")
+    except Exception:
+        pass
+
+    # recent spark workbench runs
+    try:
+        from spark_workbench import _runs as wb_runs, _run_order  # type: ignore
+        recent = [wb_runs[r] for r in _run_order[-6:] if r in wb_runs]
+        if recent:
+            lines.append("Recent Spark Workbench runs:")
+            for r in recent:
+                st = (r.get("stats") or {})
+                tgt = f" → {r.get('target')}" if r.get("target") else ""
+                lines.append(
+                    f"  - {r.get('label')}: {r.get('state')} rows={st.get('processed_rows')}{tgt}"
+                )
+    except Exception:
+        pass
+
+    return "\n".join(lines)
+
+
+def build_masking_section() -> str:
+    """Exact masking policy + strict guidance so the LLM can answer about masked
+    data without ever revealing masked raw values."""
+    lines: list[str] = ["=== DATA MASKING POLICY (enforced) ==="]
+    masked: list[str] = []
+    unmasked: list[str] = []
+    try:
+        from pii_catalog import get_pii  # type: ignore
+        data = get_pii()
+        for d in data.get("datasets", []):
+            for c in d.get("pii_columns", []):
+                tag = f"{d.get('label')}.{c.get('name')} [{c.get('category')}]"
+                (masked if c.get("masked") else unmasked).append(tag)
+        summ = data.get("summary", {})
+        lines.append(
+            f"PII columns: {summ.get('pii_columns', 0)} total — "
+            f"{summ.get('masked_columns', 0)} masked, {summ.get('unmasked_columns', 0)} visible."
+        )
+    except Exception as exc:
+        lines.append(f"(masking catalog unavailable: {exc})")
+
+    if masked:
+        lines.append("MASKED columns (raw values are withheld — token 🔒 MASKED):")
+        for m in masked[:40]:
+            lines.append(f"  - {m}")
+    if unmasked:
+        lines.append("Visible PII columns (operator opted out of masking):")
+        for u in unmasked[:40]:
+            lines.append(f"  - {u}")
+
+    lines += [
+        "",
+        "How to handle masked data when answering:",
+        "  1. NEVER reveal, guess, reconstruct or print the raw value of a MASKED column. If a value comes in as '🔒 MASKED', keep it masked.",
+        "  2. DO still answer helpfully: confirm the column exists and is masked for privacy/governance, and explain why (PII protection policy).",
+        "  3. You MAY use and report non-sensitive aggregates, counts, distributions and derived metrics over masked columns (e.g. 'there are N distinct customers') as long as no individual raw value is exposed.",
+        "  4. Tell the operator they can unmask a specific column from the Data Flow PII overlay if they have the authority, and that the curated/masked Iceberg layer is physically masked and cannot be unmasked.",
+        "  5. Unmasked PII columns may be shown, but flag that they are sensitive.",
+    ]
+    return "\n".join(lines)
+
+
+def build_llm_addendum() -> str:
+    try:
+        platform = build_platform_section()
+    except Exception as exc:
+        platform = f"(platform section error: {exc})"
+    try:
+        masking = build_masking_section()
+    except Exception as exc:
+        masking = f"(masking section error: {exc})"
+    return "\n\n".join([platform, masking])

api/presentation.py

@@ -18,6 +18,30 @@ def _slide(slide_id: str, title: str, subtitle: str, bullets: list[str], **extra
     return {"id": slide_id, "title": title, "subtitle": subtitle, "bullets": bullets, **extra}
 
 
+def _node_slide(nid: str, extra_bullets: list[str] | None = None) -> dict[str, Any]:
+    reg = NODE_REGISTRY.get(nid, {})
+    bullets: list[str] = []
+    if reg.get("description"):
+        bullets.append(reg["description"])
+    bullets.append(f"VM: {reg.get('vm', '?')} (VMID {reg.get('vmid', '?')}) @ {reg.get('ip', '?')}")
+    for link in reg.get("links", [])[:5]:
+        bullets.append(f"{link.get('label', 'Link')}: {link.get('url', '')}")
+    for ep in reg.get("endpoints", [])[:6]:
+        bullets.append(f"{ep.get('name', '?')}: {ep.get('host', '?')}:{ep.get('port', '?')}")
+    if extra_bullets:
+        bullets.extend(extra_bullets)
+    kind = "gpu" if nid == "gpu" else "command" if nid == "command" else "zone"
+    if nid in ("openmetadata", "elastic"):
+        kind = "architecture"
+    return _slide(
+        f"node-{nid}",
+        reg.get("label", nid),
+        f"{reg.get('role', 'service').upper()} · {reg.get('vm', '')}",
+        bullets[:14],
+        kind=kind,
+    )
+
+
 def build_presentation_payload(snap: dict[str, Any]) -> dict[str, Any]:
     workload = build_workload_payload(snap)
     topologies = workload.get("topologies") or build_all_topologies(snap)
@@ -28,20 +52,24 @@ def build_presentation_payload(snap: dict[str, Any]) -> dict[str, Any]:
     hadoop = snap.get("hadoop", {})
     objectscale = snap.get("objectscale", {})
     command = snap.get("command_center", {})
+    databases = snap.get("databases", {})
+    docker = snap.get("docker", {})
+    lakehouse = snap.get("lakehouse", {})
+    governance = snap.get("governance") or {}
 
     slides: list[dict[str, Any]] = []
 
     slides.append(_slide(
         "title",
         "Dell ATC Data Lab",
-        "Live demo & presentation — Command Center",
+        "Live infrastructure presentation — Command Center",
         [
             f"Snapshot: {snap.get('ts', 'now')}",
             f"Pipeline: {'ACTIVE' if totals.get('pipeline_active') else 'INACTIVE'}",
             f"Apps running: {totals.get('apps_running', 0)}/{totals.get('apps_total', 0)}",
-            f"CDC connectors: {totals.get('connectors', 0)}",
+            f"CDC connectors: {totals.get('connectors', 0)} · Source DBs: 5 engines on db02",
             f"LLM: {gpu.get('model') or 'offline'} ({gpu.get('gpu_count', 0)}× V100)",
-            "Command Center → http://10.0.21.33/",
+            "Command Center → http://10.0.21.33/ · Data Platform tab → Presentation",
         ],
         kind="hero",
     ))
@@ -51,53 +79,151 @@ def build_presentation_payload(snap: dict[str, Any]) -> dict[str, Any]:
         "Mission",
         "End-to-end modern data platform on Dell infrastructure",
         [
-            "Ingest change data from operational databases (PostgreSQL, MySQL, MongoDB, Cassandra)",
-            "Stream via Kafka & Debezium into the lakehouse (Spark, Trino, Iceberg)",
-            "Land curated data on ObjectScale S3 — query with Trino & visualize in Superset",
-            "Parallel HDFS cluster for batch / legacy workloads",
-            "GPU lab powers autonomous ops agents with local vLLM inference",
-            "This dashboard orchestrates agents, approvals, and live cluster visibility",
+            "Ingest CDC from PostgreSQL, MySQL, MongoDB (+ Cassandra & Neo4j for analytics)",
+            "Stream via Kafka & Debezium into Spark, Trino & Iceberg on the lakehouse",
+            "Land curated data on ObjectScale S3 — federated SQL with Trino",
+            "Govern with OpenMetadata — catalog, lineage, PII classification",
+            "Search & observe via Elasticsearch/Kibana · BI via Superset",
+            "Parallel 9-node Hadoop HDFS cluster for batch workloads",
+            "GPU lab (4× V100) powers autonomous agents with local vLLM inference",
+            "This Command Center orchestrates agents, approvals & live visibility",
         ],
         kind="narrative",
     ))
 
+    slides.append(_slide(
+        "command-center-ui",
+        "Command Center UI",
+        "Everything you operate from this dashboard",
+        [
+            "Data Platform — interactive topology + live presentation deck",
+            "Data Sources UI — browse, filter & edit all 5 source databases",
+            "Live Changes — real-time Debezium CDC event stream",
+            "Data Flow — OpenMetadata catalog, lineage & pipeline map",
+            "Data Quality — Docling document QA + RAG ingest",
+            "Knowledge Chat — GPU-backed RAG over lab documentation",
+            "Object Storage · HDFS · Elasticsearch · SSH terminal",
+            "Agent fleet · Approval inbox · Live GPU matrix",
+        ],
+        kind="command",
+    ))
+
     arch = topologies.get("architecture") or workload.get("topology") or {}
     arch_nodes = arch.get("nodes", [])
     slides.append(_slide(
         "architecture",
         "Data Platform Architecture",
         arch.get("subtitle", "Sources → Ingestion → Compute → Storage → Consumers"),
-        [f"{n.get('label', n.get('id'))}: {n.get('subtitle', n.get('role', ''))}" for n in arch_nodes[:14]],
+        [f"{n.get('label', n.get('id'))}: {n.get('subtitle', n.get('role', ''))}" for n in arch_nodes[:16]],
         kind="topology",
         topology=arch,
     ))
 
+    db_lines = [
+        f"Host atc-db02 (10.0.21.51) — {databases.get('running', 0)}/{databases.get('total', 0)} containers up",
+        "postgres_sales — PostgreSQL sales_orders (CDC → Debezium)",
+        "mysql_hr — MySQL employee_events (CDC → Debezium)",
+        "mongodb_supplychain — MongoDB supplychain.events (CDC → Debezium)",
+        "cassandra_telemetry — Cassandra device_metrics (Trino federated)",
+        "neo4j_graph — Product/Supplier graph · 4.5M nodes",
+    ]
+    for c in (databases.get("containers") or [])[:8]:
+        db_lines.append(f"  · {c.get('name', '?')}: {c.get('state', '?')}")
+    slides.append(_slide(
+        "source-databases",
+        "Source Databases",
+        "DB Vault · atc-db02 · 10.0.21.51",
+        db_lines,
+        kind="topology",
+    ))
+
     pipeline = topologies.get("pipeline", {})
     connector_lines = [
         f"  · {cs['name']}: {cs.get('state', '?')}"
-        for cs in (etl.get("connector_status") or [])[:6]
+        for cs in (etl.get("connector_status") or [])[:8]
     ]
     slides.append(_slide(
         "pipeline",
         "CDC Pipeline",
         pipeline.get("subtitle", "Airflow → DB → Debezium → Kafka → Lakehouse → S3"),
         [
-            f"Airflow: {'healthy' if etl.get('airflow_healthy') else 'degraded'} ({etl.get('airflow_url', '')})",
-            f"Kafka UI: {'UP' if etl.get('kafka_ui_ok') else 'DOWN'}",
+            f"Airflow: {'healthy' if etl.get('airflow_healthy') else 'degraded'} — {etl.get('airflow_url', 'http://10.0.21.55:8080')}",
+            f"Kafka UI: {'UP' if etl.get('kafka_ui_ok') else 'DOWN'} — http://10.0.21.36:9000",
+            f"Debezium Connect: http://10.0.21.50:8083",
             f"Connectors: {', '.join(etl.get('connectors') or []) or 'none'}",
             *connector_lines,
-            f"Spark UI: {'UP' if etl.get('spark_ui_ok') else 'DOWN'}",
-            f"ObjectScale: {'reachable' if objectscale.get('reachable') else 'down'} bucket={objectscale.get('bucket', 'data')}",
+            f"Spark UI: {'UP' if etl.get('spark_ui_ok') else 'DOWN'} — http://10.0.21.50:8080",
+            f"ObjectScale S3: {'reachable' if objectscale.get('reachable') else 'down'} — bucket={objectscale.get('bucket', 'data')}",
         ],
         kind="topology",
         topology=pipeline,
     ))
 
+    lake_lines = [
+        f"atc-lake01 @ {lakehouse.get('host', '10.0.21.50')} — {lakehouse.get('running', 0)}/{lakehouse.get('total', 0)} containers",
+        f"Trino: {'UP' if lakehouse.get('trino_ok') else 'DOWN'} — http://10.0.21.50:8089",
+        "Spark — batch & streaming compute",
+        "Kafka Connect + Debezium — CDC ingestion",
+        "s3-kafka-consumer — events → ObjectScale S3",
+        "Iceberg catalog — bronze → silver → gold tables",
+    ]
+    for c in (lakehouse.get("containers") or [])[:8]:
+        lake_lines.append(f"  · {c.get('name', '?')}: {c.get('state', '?')}")
+    slides.append(_slide(
+        "lakehouse",
+        "Lakehouse Hub",
+        "Spark · Trino · Iceberg · Kafka Connect",
+        lake_lines,
+        kind="topology",
+    ))
+
+    docker_lines = [
+        f"atc-docker01 @ 10.0.21.45 — {docker.get('running', 0)}/{docker.get('total', 0)} containers",
+        "Homepage — http://10.0.21.45",
+        "Dockhand — container management http://10.0.21.45:8082",
+        "Apache Superset — BI dashboards http://10.0.21.45:8088",
+        "Forgejo / Gitea · monitoring · nginx · redis",
+    ]
+    for c in (docker.get("containers") or [])[:10]:
+        docker_lines.append(f"  · {c.get('name', '?')}: {c.get('state', '?')}")
+    slides.append(_slide(
+        "docker-rack",
+        "Docker Rack & Analytics",
+        "Platform services on atc-docker01",
+        docker_lines,
+        kind="zone",
+    ))
+
+    cdc = governance.get("cdc") or {}
+    gov_lines = [
+        f"OpenMetadata UI: {governance.get('openmetadata_url', 'http://10.0.21.47:8585')}",
+        "Ingestion Airflow: http://10.0.21.47:8080",
+        "Catalog · lineage · data quality · PII auto-classification (Presidio NER)",
+    ]
+    if "error" not in cdc:
+        by_src = ", ".join(f"{k}={v}" for k, v in (cdc.get("by_source") or {}).items()) or "none"
+        gov_lines.append(f"CDC stream: connected={cdc.get('connected')} · {cdc.get('window_total', 0)} changes/15m ({by_src})")
+    ps = governance.get("pii_summary") or {}
+    if "error" not in ps:
+        gov_lines.append(f"PII: {ps.get('pii_columns', 0)} columns · {ps.get('masked_columns', 0)} masked")
+    for ln in (governance.get("lineage") or [])[:4]:
+        gov_lines.append(f"Lineage: {ln}")
+    slides.append(_slide(
+        "governance",
+        "Governance & Metadata",
+        "OpenMetadata · CDC · PII · Lineage",
+        gov_lines,
+        kind="architecture",
+    ))
+
+    slides.append(_node_slide("openmetadata"))
+    slides.append(_node_slide("elastic"))
+
     for zone in zones:
         apps = zone.get("apps") or []
         app_lines = [
             f"{a['name']}: {a['state']}" + (f" ({a.get('host', '')})" if a.get("host") else "")
-            for a in apps[:10]
+            for a in apps[:12]
         ]
         slides.append(_slide(
             f"zone-{zone['id']}",
@@ -111,32 +237,36 @@ def build_presentation_payload(snap: dict[str, Any]) -> dict[str, Any]:
             zone=zone,
         ))
 
-    infra_nodes = [
-        nid for nid in NODE_REGISTRY
-        if nid not in ("mo-commander", "bart-commander", "network-watcher", "mcp-coordinator")
+    PRESENTATION_NODES = [
+        "airflow", "db", "debezium", "kafka", "lakehouse", "s3",
+        "docker", "hadoop", "gpu", "command",
     ]
     slides.append(_slide(
-        "infrastructure",
+        "infrastructure-map",
         "Infrastructure Map",
         "Proxmox VMs & services across VLAN 20/21",
         [
             f"{NODE_REGISTRY[nid]['label']} — {NODE_REGISTRY[nid].get('vm')} "
             f"(VMID {NODE_REGISTRY[nid].get('vmid', '?')}) @ {NODE_REGISTRY[nid].get('ip')}"
-            for nid in infra_nodes
+            for nid in PRESENTATION_NODES if nid in NODE_REGISTRY
         ],
         kind="registry",
     ))
 
+    for nid in PRESENTATION_NODES:
+        if nid in NODE_REGISTRY and nid not in ("docker", "db", "lakehouse"):
+            slides.append(_node_slide(nid))
+
     dn_lines = [
         f"  · {dn['host']}: {dn.get('used_gb', 0)} GB — {dn.get('state', '')}"
-        for dn in (hadoop.get("datanodes") or [])[:5]
+        for dn in (hadoop.get("datanodes") or [])[:6]
     ]
     slides.append(_slide(
         "hadoop",
         "Hadoop HDFS",
         "9-node parallel storage cluster",
         [
-            f"NameNode: {'UP' if hadoop.get('reachable') else 'DOWN'} — {hadoop.get('namenode', '')}",
+            f"NameNode: {'UP' if hadoop.get('reachable') else 'DOWN'} — {hadoop.get('namenode', 'http://10.0.21.61:9870')}",
             f"Capacity: {hadoop.get('capacity_used_gb', '?')} / {hadoop.get('capacity_total_gb', '?')} GB",
             f"DataNodes: {hadoop.get('live_datanodes', 0)} live, {hadoop.get('dead_datanodes', 0)} dead",
             f"Files: {hadoop.get('files_total', 0)}, Blocks: {hadoop.get('blocks_total', 0)}",
@@ -158,7 +288,8 @@ def build_presentation_payload(snap: dict[str, Any]) -> dict[str, Any]:
         [
             f"Inference: {'ON' if gpu.get('inference_active') else 'OFF'}",
             f"API: {snap.get('gpu', {}).get('vllm_url') or 'http://10.0.20.106:8001/v1'}",
-            "Manager: http://10.0.20.106:9000",
+            "GPU Lab UI: http://10.0.20.106:9000",
+            "Kibana/Elastic: http://10.0.21.46:5601",
             *gpu_lines,
         ],
         kind="gpu",
@@ -171,7 +302,7 @@ def build_presentation_payload(snap: dict[str, Any]) -> dict[str, Any]:
         [
             "ETL Guardian — Airflow, Kafka, Debezium, connectors",
             "Data Custodian — PostgreSQL, MySQL, MongoDB, Cassandra, Neo4j",
-            "Lakehouse Ops — Spark, Trino, Iceberg, ObjectScale S3",
+            "Lakehouse Ops — Spark, Trino, Iceberg, ObjectScale S3, OpenMetadata",
             "Hadoop Ranger — HDFS NameNode, DataNodes, block health",
             "Infra Sentinel — Docker rack, GPU lab, Command Center",
             "All agents receive LIVE cluster snapshot in every LLM prompt",
@@ -182,13 +313,13 @@ def build_presentation_payload(snap: dict[str, Any]) -> dict[str, Any]:
     cc_apps = [f"{c['name']}: {c['state']}" for c in (command.get("containers") or [])]
     slides.append(_slide(
         "command",
-        "Command Center",
-        "VM 304 — this presentation runs here",
+        "Command Center Stack",
+        "VM 304 — this dashboard runs here",
         [
             f"Host: {command.get('host', '10.0.21.33')} (VMID {command.get('vmid', 304)})",
-            f"Stack: {command.get('running', 0)}/{command.get('total', 0)} containers",
+            f"Stack: {command.get('running', 0)}/{command.get('total', 0)} containers (api, ui, caddy, redis, postgres)",
             *cc_apps,
-            "WebSocket ops feed · Approval inbox · Agent terminals",
+            "WebSocket ops feed · Approval inbox · Agent terminals · Data Sources UI",
         ],
         kind="command",
     ))
@@ -198,12 +329,12 @@ def build_presentation_payload(snap: dict[str, Any]) -> dict[str, Any]:
         "Live Demo Tips",
         "Use this deck during customer presentations",
         [
-            "Press ← → or click dots to navigate slides",
-            "F = fullscreen presentation mode",
+            "Press ← → or click dots to navigate slides · F = fullscreen",
+            "Data Platform tab → Presentation sub-tab (this deck)",
+            "Data Platform → Topology for interactive pipeline map",
             "Export HTML opens a standalone deck for projectors / offline",
             "Ask agents in the Command Bar — they see full cluster context",
-            "Switch to Data Platform tab for interactive topology",
-            "GPU Lab chat: http://10.0.20.106:9000/chat",
+            "Trigger data generation in Data Sources UI → Generate tab",
         ],
         kind="cta",
     ))

api/presentation_upload.py

@@ -13,6 +13,7 @@ from typing import Any
 import httpx
 
 PRESENTATIONS_DIR = Path(os.getenv("PRESENTATIONS_DIR", "/data/presentations"))
+LIVE_OVERRIDE_DIR = PRESENTATIONS_DIR / "live-override"
 DOCLING_URL = os.getenv("DOCLING_URL", "http://docling-serve:5001").rstrip("/")
 
 
@@ -165,6 +166,65 @@ def get_deck(deck_id: str) -> dict[str, Any] | None:
     return json.loads(path.read_text())
 
 
+def get_live_override() -> dict[str, Any] | None:
+    path = LIVE_OVERRIDE_DIR / "meta.json"
+    if not path.exists():
+        return None
+    try:
+        return json.loads(path.read_text())
+    except Exception:
+        return None
+
+
+def save_live_override(body: dict[str, Any]) -> dict[str, Any]:
+    """Persist user edits for the Live Cluster deck."""
+    LIVE_OVERRIDE_DIR.mkdir(parents=True, exist_ok=True)
+    (LIVE_OVERRIDE_DIR / "assets").mkdir(parents=True, exist_ok=True)
+    existing = get_live_override() or {"id": "live", "source": "live-override", "editable": True}
+    slides = _clean_slides(body.get("slides"))
+    payload = {
+        **existing,
+        "id": "live",
+        "source": "live-override",
+        "editable": True,
+        "title": str(body.get("title") or existing.get("title") or "Live Cluster")[:120],
+        "subtitle": str(body.get("subtitle") or existing.get("subtitle") or "")[:300],
+        "ts": datetime.now(timezone.utc).isoformat(),
+        "slides": slides,
+        "slide_count": len(slides),
+    }
+    (LIVE_OVERRIDE_DIR / "meta.json").write_text(json.dumps(payload, indent=2, default=str))
+    return payload
+
+
+def clear_live_override() -> bool:
+    import shutil
+    if not LIVE_OVERRIDE_DIR.exists():
+        return True
+    shutil.rmtree(LIVE_OVERRIDE_DIR, ignore_errors=True)
+    return True
+
+
+def _clean_slides(incoming_slides: Any) -> list[dict[str, Any]]:
+    clean_slides: list[dict[str, Any]] = []
+    for i, s in enumerate(incoming_slides or [], start=1):
+        if not isinstance(s, dict):
+            continue
+        bullets = [str(b).strip()[:400] for b in (s.get("bullets") or []) if str(b).strip()]
+        slide: dict[str, Any] = {
+            "id": str(s.get("id") or f"slide-{i}"),
+            "title": str(s.get("title") or f"Slide {i}")[:200],
+            "subtitle": str(s.get("subtitle") or "")[:300],
+            "bullets": bullets,
+            "image": str(s.get("image") or "")[:300],
+            "kind": str(s.get("kind") or "narrative")[:40],
+        }
+        if s.get("animation"):
+            slide["animation"] = str(s.get("animation"))[:40]
+        clean_slides.append(slide)
+    return clean_slides or [_blank_slide(1)]
+
+
 def _blank_slide(idx: int = 1) -> dict[str, Any]:
     return {
         "id": f"slide-{idx}",
@@ -208,22 +268,7 @@ def save_deck(deck_id: str, body: dict[str, Any]) -> dict[str, Any] | None:
         return None
     existing = json.loads(meta_path.read_text())
 
-    incoming_slides = body.get("slides")
-    clean_slides: list[dict[str, Any]] = []
-    for i, s in enumerate(incoming_slides or [], start=1):
-        if not isinstance(s, dict):
-            continue
-        bullets = [str(b).strip()[:400] for b in (s.get("bullets") or []) if str(b).strip()]
-        clean_slides.append({
-            "id": str(s.get("id") or f"slide-{i}"),
-            "title": str(s.get("title") or f"Slide {i}")[:200],
-            "subtitle": str(s.get("subtitle") or "")[:300],
-            "bullets": bullets,
-            "image": str(s.get("image") or "")[:300],
-            "kind": str(s.get("kind") or "narrative")[:40],
-        })
-    if not clean_slides:
-        clean_slides = [_blank_slide(1)]
+    clean_slides = _clean_slides(body.get("slides"))
 
     existing.update({
         "title": str(body.get("title") or existing.get("title") or "Untitled deck")[:120],
@@ -250,9 +295,12 @@ def delete_deck(deck_id: str) -> bool:
 
 def save_image(deck_id: str, filename: str, content: bytes) -> dict[str, Any] | None:
     """Store an image in the deck's assets folder; return its served URL."""
-    deck_dir = PRESENTATIONS_DIR / deck_id
-    if not deck_dir.exists():
-        return None
+    if deck_id == "live":
+        deck_dir = LIVE_OVERRIDE_DIR
+    else:
+        deck_dir = PRESENTATIONS_DIR / deck_id
+        if not deck_dir.exists():
+            return None
     assets = deck_dir / "assets"
     assets.mkdir(parents=True, exist_ok=True)
     ext = ""
@@ -265,7 +313,8 @@ def save_image(deck_id: str, filename: str, content: bytes) -> dict[str, Any] |
 
 def get_asset_path(deck_id: str, name: str) -> Path | None:
     safe = _safe_name(name)
-    path = PRESENTATIONS_DIR / deck_id / "assets" / safe
+    base = LIVE_OVERRIDE_DIR if deck_id == "live" else PRESENTATIONS_DIR / deck_id
+    path = base / "assets" / safe
     if not path.exists() or not path.is_file():
         return None
     return path

api/spark_workbench.py

@@ -0,0 +1,435 @@
+"""Databricks-style lakehouse workbench for the Command Center.
+
+Lets the user interactively select data from any federated source (Iceberg,
+Hive/HDFS, Postgres, MySQL, Mongo, Cassandra, Kafka) and run distributed
+transformations on the lakehouse compute layer:
+
+    explore -> transform (aggregate / filter / join / profile) -> materialize to Iceberg (S3/HDFS)
+
+Execution streams live engine metrics (state, splits, rows, bytes, CPU, wall,
+peak memory, nodes) so the UI can show a live "what the cluster is doing" matrix
+exactly like Databricks' Spark UI. Compute runs on the lakehouse engine (Trino
+coordinator on the Spark cluster host) which distributes work across the
+workers; Spark batch DAGs remain available via the jobs API.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import re
+import time
+import uuid
+from typing import Any
+
+import httpx
+from fastapi import APIRouter, Body, Query
+from fastapi.responses import JSONResponse
+
+TRINO_URL = os.getenv("TRINO_URL", "http://10.0.21.50:8089").rstrip("/")
+TRINO_USER = os.getenv("TRINO_USER", "mo")
+SPARK_UI_URL = os.getenv("SPARK_UI_URL", "http://10.0.21.50:8080").rstrip("/")
+
+router = APIRouter(prefix="/api/spark", tags=["spark-workbench"])
+
+_runs: dict[str, dict[str, Any]] = {}
+_run_order: list[str] = []
+_MAX_RUNS = 40
+_MAX_RESULT_ROWS = 500
+
+_IDENT_RE = re.compile(r"^[A-Za-z0-9_.\" ]+$")
+_TABLE_RE = re.compile(r"^[A-Za-z0-9_.\"]+$")
+
+AGG_FUNCS = {"count", "sum", "avg", "min", "max", "approx_distinct", "count_distinct"}
+JOIN_TYPES = {"INNER", "LEFT", "RIGHT", "FULL"}
+
+
+def _feed(agent_id: str, message: str, level: str = "info") -> None:
+    try:
+        from main import add_feed
+        add_feed(agent_id, message, level)
+    except Exception:
+        pass
+
+
+def _qident(name: str) -> str:
+    name = name.strip().strip('"')
+    if not re.match(r"^[A-Za-z0-9_]+$", name):
+        raise ValueError(f"invalid identifier: {name}")
+    return f'"{name}"'
+
+
+def _safe_table(name: str) -> str:
+    name = (name or "").strip()
+    if not name or not _TABLE_RE.match(name):
+        raise ValueError(f"invalid table reference: {name}")
+    return name
+
+
+# ── Trino helpers ────────────────────────────────────────────────────────────
+def _trino_headers() -> dict[str, str]:
+    return {"X-Trino-User": TRINO_USER, "Content-Type": "text/plain"}
+
+
+async def _trino_collect(sql: str, limit: int = 200, timeout: float = 60.0) -> dict[str, Any]:
+    """Synchronous-style helper: run a query and return all rows (for catalog ops)."""
+    columns: list[str] = []
+    rows: list[list[Any]] = []
+    async with httpx.AsyncClient(timeout=timeout) as client:
+        r = await client.post(f"{TRINO_URL}/v1/statement", content=sql.encode(), headers=_trino_headers())
+        if r.status_code >= 400:
+            return {"ok": False, "error": r.text[:400]}
+        data = r.json()
+        while True:
+            if data.get("error"):
+                return {"ok": False, "error": str(data["error"])[:400]}
+            if data.get("columns") and not columns:
+                columns = [c["name"] for c in data["columns"]]
+            for row in data.get("data") or []:
+                rows.append(row)
+                if len(rows) >= limit:
+                    break
+            nxt = data.get("nextUri")
+            if not nxt or len(rows) >= limit:
+                if nxt:
+                    try:
+                        await client.delete(nxt, headers=_trino_headers())
+                    except Exception:
+                        pass
+                break
+            data = (await client.get(nxt, headers=_trino_headers())).json()
+    return {"ok": True, "columns": columns, "rows": rows}
+
+
+def _norm_stats(stats: dict[str, Any]) -> dict[str, Any]:
+    return {
+        "state": stats.get("state"),
+        "nodes": stats.get("nodes"),
+        "total_splits": stats.get("totalSplits"),
+        "queued_splits": stats.get("queuedSplits"),
+        "running_splits": stats.get("runningSplits"),
+        "completed_splits": stats.get("completedSplits"),
+        "processed_rows": stats.get("processedRows"),
+        "processed_bytes": stats.get("processedBytes"),
+        "physical_input_bytes": stats.get("physicalInputBytes"),
+        "peak_memory_bytes": stats.get("peakMemoryBytes"),
+        "cpu_time_ms": stats.get("cpuTimeMillis"),
+        "wall_time_ms": stats.get("wallTimeMillis"),
+        "elapsed_ms": stats.get("elapsedTimeMillis"),
+        "progress_pct": round(
+            (stats.get("completedSplits") or 0) / stats["totalSplits"] * 100, 1
+        ) if stats.get("totalSplits") else (100.0 if stats.get("state") == "FINISHED" else 0.0),
+    }
+
+
+async def _execute_run(run_id: str, sql: str, returns_rows: bool, pre_sql: str | None = None) -> None:
+    run = _runs[run_id]
+    run["state"] = "RUNNING"
+    columns: list[str] = []
+    rows: list[list[Any]] = []
+    try:
+        if pre_sql:
+            pre = await _trino_collect(pre_sql, 1)
+            if not pre.get("ok"):
+                run.update(state="FAILED", error=pre.get("error"), ended_at=time.time())
+                _feed("lakehouse-ops", f"[workbench] {run['label']}: pre-step failed", "err")
+                return
+        async with httpx.AsyncClient(timeout=None) as client:
+            r = await client.post(f"{TRINO_URL}/v1/statement", content=sql.encode(), headers=_trino_headers())
+            if r.status_code >= 400:
+                run.update(state="FAILED", error=r.text[:500], ended_at=time.time())
+                _feed("lakehouse-ops", f"[workbench] {run['label']}: failed ({r.status_code})", "err")
+                return
+            data = r.json()
+            run["query_id"] = data.get("id")
+            while True:
+                if run.get("cancel_requested"):
+                    nxt = data.get("nextUri")
+                    if nxt:
+                        try:
+                            await client.delete(nxt, headers=_trino_headers())
+                        except Exception:
+                            pass
+                    run.update(state="CANCELED", ended_at=time.time())
+                    _feed("lakehouse-ops", f"[workbench] {run['label']}: canceled", "warn")
+                    return
+                if data.get("stats"):
+                    run["stats"] = _norm_stats(data["stats"])
+                    st = data["stats"].get("state")
+                    if st:
+                        run["engine_state"] = st
+                if data.get("error"):
+                    run.update(state="FAILED", error=str(data["error"])[:500], ended_at=time.time())
+                    _feed("lakehouse-ops", f"[workbench] {run['label']}: {str(data['error'])[:120]}", "err")
+                    return
+                if data.get("columns") and not columns:
+                    columns = [c["name"] for c in data["columns"]]
+                    run["columns"] = columns
+                for row in data.get("data") or []:
+                    if returns_rows and len(rows) < _MAX_RESULT_ROWS:
+                        rows.append(row)
+                if data.get("updateType"):
+                    run["update_type"] = data.get("updateType")
+                nxt = data.get("nextUri")
+                if not nxt:
+                    break
+                data = (await client.get(nxt, headers=_trino_headers())).json()
+                run["next_uri"] = nxt
+        run["rows"] = rows
+        run["row_count"] = len(rows)
+        run.update(state="FINISHED", ended_at=time.time())
+        if run.get("stats"):
+            run["stats"]["state"] = "FINISHED"
+            run["stats"]["progress_pct"] = 100.0
+        msg = f"[workbench] {run['label']}: finished"
+        if run.get("target"):
+            msg = f"[workbench] {run['label']}: materialized → {run['target']}"
+        _feed("lakehouse-ops", msg, "info")
+    except Exception as exc:
+        run.update(state="FAILED", error=str(exc)[:500], ended_at=time.time())
+        _feed("lakehouse-ops", f"[workbench] {run['label']}: error {str(exc)[:120]}", "err")
+
+
+# ── SQL builders ─────────────────────────────────────────────────────────────
+def _build_sql(body: dict[str, Any]) -> tuple[str, bool, str, str | None, str | None]:
+    """Return (sql, returns_rows, label, materialize_target, pre_sql)."""
+    op = body.get("operation", "preview")
+    limit = max(1, min(int(body.get("limit", 200)), _MAX_RESULT_ROWS))
+    materialize = body.get("materialize") or {}
+    target = None
+
+    if op == "sql":
+        sql = (body.get("sql") or "").strip().rstrip(";")
+        if not sql:
+            raise ValueError("empty SQL")
+        select_sql = sql
+        label = "Custom SQL"
+        returns_rows = sql.lower().lstrip().startswith(("select", "show", "describe", "with", "explain"))
+
+    elif op == "preview":
+        table = _safe_table(body.get("table"))
+        select_sql = f"SELECT * FROM {table} LIMIT {limit}"
+        label = f"Preview {table}"
+        returns_rows = True
+
+    elif op == "filter":
+        table = _safe_table(body.get("table"))
+        where = (body.get("where") or "").strip()
+        clause = f" WHERE {where}" if where else ""
+        select_sql = f"SELECT * FROM {table}{clause} LIMIT {limit}"
+        label = f"Filter {table}"
+        returns_rows = True
+
+    elif op == "aggregate":
+        table = _safe_table(body.get("table"))
+        group_by = [c for c in (body.get("group_by") or []) if c]
+        metrics = body.get("metrics") or []
+        select_parts: list[str] = [_qident(c) for c in group_by]
+        for m in metrics:
+            fn = (m.get("fn") or "count").lower()
+            if fn not in AGG_FUNCS:
+                raise ValueError(f"unsupported function {fn}")
+            col = m.get("col")
+            alias = m.get("alias") or (f"{fn}_{col}" if col else fn)
+            if fn == "count" and (not col or col == "*"):
+                expr = "count(*)"
+            elif fn == "count_distinct":
+                expr = f"count(DISTINCT {_qident(col)})"
+            else:
+                expr = f"{fn}({_qident(col)})"
+            select_parts.append(f"{expr} AS {_qident(alias)}")
+        if not select_parts:
+            select_parts = ["count(*) AS cnt"]
+        gb = f" GROUP BY {', '.join(_qident(c) for c in group_by)}" if group_by else ""
+        order = ""
+        if group_by:
+            order = f" ORDER BY {', '.join(_qident(c) for c in group_by)}"
+        select_sql = f"SELECT {', '.join(select_parts)} FROM {table}{gb}{order} LIMIT {limit}"
+        label = f"Aggregate {table}"
+        returns_rows = True
+
+    elif op == "join":
+        left = _safe_table(body.get("left"))
+        right = _safe_table(body.get("right"))
+        jt = (body.get("join_type") or "INNER").upper()
+        if jt not in JOIN_TYPES:
+            raise ValueError(f"invalid join type {jt}")
+        lk = _qident(body.get("left_key"))
+        rk = _qident(body.get("right_key"))
+        select_sql = (
+            f"SELECT l.*, r.* FROM {left} l {jt} JOIN {right} r "
+            f"ON l.{lk} = r.{rk} LIMIT {limit}"
+        )
+        label = f"Join {left} ⋈ {right}"
+        returns_rows = True
+
+    elif op == "profile":
+        table = _safe_table(body.get("table"))
+        cols = [c for c in (body.get("columns") or []) if c][:12]
+        parts = ["count(*) AS row_count"]
+        for c in cols:
+            qc = _qident(c)
+            parts.append(f"approx_distinct({qc}) AS {_qident(c + '_distinct')}")
+            parts.append(f"count({qc}) AS {_qident(c + '_nonnull')}")
+        select_sql = f"SELECT {', '.join(parts)} FROM {table}"
+        label = f"Profile {table}"
+        returns_rows = True
+
+    else:
+        raise ValueError(f"unknown operation {op}")
+
+    if materialize.get("enabled"):
+        schema = materialize.get("schema", "hadoop")
+        name = materialize.get("table")
+        if not name:
+            raise ValueError("materialize target table required")
+        target = f"iceberg.{_qident(schema).strip(chr(34))}.{_qident(name).strip(chr(34))}"
+        mode = (materialize.get("mode") or "create").lower()
+        pre = None
+        if mode == "replace":
+            pre = f"DROP TABLE IF EXISTS {target}"
+            ddl = f"CREATE TABLE {target} AS {select_sql}"
+        elif mode == "insert":
+            ddl = f"INSERT INTO {target} {select_sql}"
+        else:
+            ddl = f"CREATE TABLE {target} AS {select_sql}"
+        return ddl, False, f"Materialize → {target}", target, pre
+
+    return select_sql, returns_rows, label, None, None
+
+
+# ── catalog endpoints ────────────────────────────────────────────────────────
+@router.get("/catalogs")
+async def list_catalogs() -> JSONResponse:
+    res = await _trino_collect("SHOW CATALOGS", 100)
+    if not res.get("ok"):
+        return JSONResponse(res, status_code=502)
+    cats = [r[0] for r in res["rows"] if r[0] not in ("system",)]
+    return JSONResponse({"ok": True, "catalogs": cats})
+
+
+@router.get("/schemas")
+async def list_schemas(catalog: str = Query(...)) -> JSONResponse:
+    cat = _safe_table(catalog)
+    res = await _trino_collect(f"SHOW SCHEMAS FROM {cat}", 200)
+    if not res.get("ok"):
+        return JSONResponse(res, status_code=502)
+    skip = {"information_schema"}
+    schemas = [r[0] for r in res["rows"] if r[0] not in skip]
+    return JSONResponse({"ok": True, "catalog": catalog, "schemas": schemas})
+
+
+@router.get("/tables")
+async def list_tables(catalog: str = Query(...), schema: str = Query(...)) -> JSONResponse:
+    cat = _safe_table(catalog)
+    sch = _safe_table(schema)
+    res = await _trino_collect(f"SHOW TABLES FROM {cat}.{sch}", 500)
+    if not res.get("ok"):
+        return JSONResponse(res, status_code=502)
+    tables = [{"name": r[0], "fqn": f"{catalog}.{schema}.{r[0]}"} for r in res["rows"]]
+    return JSONResponse({"ok": True, "catalog": catalog, "schema": schema, "tables": tables})
+
+
+@router.get("/columns")
+async def list_columns(table: str = Query(...)) -> JSONResponse:
+    tbl = _safe_table(table)
+    res = await _trino_collect(f"DESCRIBE {tbl}", 500)
+    if not res.get("ok"):
+        return JSONResponse(res, status_code=502)
+    cols = [{"name": r[0], "type": r[1] if len(r) > 1 else ""} for r in res["rows"]]
+    return JSONResponse({"ok": True, "table": table, "columns": cols})
+
+
+# ── run endpoints ────────────────────────────────────────────────────────────
+@router.post("/run")
+async def create_run(body: dict[str, Any] = Body(...)) -> JSONResponse:
+    try:
+        sql, returns_rows, label, target, pre_sql = _build_sql(body)
+    except ValueError as exc:
+        return JSONResponse({"ok": False, "error": str(exc)}, status_code=400)
+
+    run_id = uuid.uuid4().hex[:12]
+    _runs[run_id] = {
+        "id": run_id,
+        "operation": body.get("operation", "preview"),
+        "label": label,
+        "sql": sql,
+        "target": target,
+        "state": "QUEUED",
+        "engine_state": "QUEUED",
+        "stats": {},
+        "columns": [],
+        "rows": [],
+        "row_count": None,
+        "error": None,
+        "started_at": time.time(),
+        "ended_at": None,
+        "cancel_requested": False,
+    }
+    _run_order.append(run_id)
+    while len(_run_order) > _MAX_RUNS:
+        old = _run_order.pop(0)
+        _runs.pop(old, None)
+
+    _feed("lakehouse-ops", f"[workbench] {label}: submitted", "info")
+    asyncio.create_task(_execute_run(run_id, sql, returns_rows, pre_sql))
+    return JSONResponse({"ok": True, "run_id": run_id, "sql": sql, "label": label, "target": target})
+
+
+def _run_public(run: dict[str, Any], include_rows: bool = True) -> dict[str, Any]:
+    out = {k: v for k, v in run.items() if k not in ("next_uri", "cancel_requested")}
+    if not include_rows:
+        out.pop("rows", None)
+    return out
+
+
+@router.get("/run/{run_id}")
+async def get_run(run_id: str) -> JSONResponse:
+    run = _runs.get(run_id)
+    if not run:
+        return JSONResponse({"ok": False, "error": "unknown run"}, status_code=404)
+    return JSONResponse({"ok": True, "run": _run_public(run)})
+
+
+@router.post("/run/{run_id}/cancel")
+async def cancel_run(run_id: str) -> JSONResponse:
+    run = _runs.get(run_id)
+    if not run:
+        return JSONResponse({"ok": False, "error": "unknown run"}, status_code=404)
+    run["cancel_requested"] = True
+    return JSONResponse({"ok": True, "run_id": run_id, "state": "canceling"})
+
+
+@router.get("/runs")
+async def list_runs() -> JSONResponse:
+    out = [_run_public(_runs[r], include_rows=False) for r in reversed(_run_order) if r in _runs]
+    return JSONResponse({"ok": True, "runs": out})
+
+
+@router.get("/live")
+async def spark_live() -> JSONResponse:
+    """Live cluster + active-run matrix for the workbench dashboard."""
+    spark: dict[str, Any] = {}
+    try:
+        from streaming_ops import collect_spark
+        spark = await collect_spark()
+    except Exception as exc:
+        spark = {"error": str(exc)[:200]}
+
+    active = [
+        _run_public(_runs[r], include_rows=False)
+        for r in reversed(_run_order)
+        if r in _runs and _runs[r]["state"] in ("QUEUED", "RUNNING")
+    ]
+    recent = [
+        _run_public(_runs[r], include_rows=False)
+        for r in reversed(_run_order[-8:])
+        if r in _runs
+    ]
+    return JSONResponse({
+        "ok": True,
+        "spark": spark,
+        "active_runs": active,
+        "recent_runs": recent,
+        "ts": time.time(),
+    })

api/sql_console.py

@@ -48,6 +48,17 @@ TRINO_USER = os.getenv("TRINO_USER", "atc")
 SOURCE_ENGINES = ("postgres", "mysql", "mongodb", "cassandra", "neo4j")
 ENGINES = SOURCE_ENGINES + ("trino",)
 
+from hadoop_sql import (
+    HADOOP_SAMPLES,
+    catalog_hadoop,
+    connection_info as hadoop_connection_info,
+    health_hadoop,
+    sample_hadoop,
+    table_row_count_hadoop,
+)
+
+LAKE_ENGINES = ("hadoop",)
+
 router = APIRouter(prefix="/api/sql", tags=["sql"])
 
 SAMPLES: dict[str, list[dict[str, str]]] = {
@@ -111,6 +122,7 @@ SAMPLES: dict[str, list[dict[str, str]]] = {
         {"id": "nj9", "label": "Schema visualization", "sql": "CALL db.schema.visualization()"},
         {"id": "nj10", "label": "Constraint info", "sql": "SHOW CONSTRAINTS"},
     ],
+    "hadoop": HADOOP_SAMPLES,
     "trino": [
         {"id": "tq1", "label": "Trino version", "sql": "SELECT version()"},
         {"id": "tq2", "label": "Catalogs", "sql": "SHOW CATALOGS"},
@@ -332,10 +344,17 @@ def _run_trino(sql: str, limit: int = 200) -> dict[str, Any]:
 
 
 class SqlRequest(BaseModel):
-    engine: str = Field(..., pattern="^(postgres|mysql|mongodb|trino|cassandra|neo4j)$")
+    engine: str = Field(..., pattern="^(postgres|mysql|mongodb|trino|cassandra|neo4j|hadoop)$")
     sql: str = Field(..., min_length=1, max_length=8000)
 
 
+class RowUpdateRequest(BaseModel):
+    engine: str = Field(..., pattern="^(postgres|mysql|mongodb|cassandra|neo4j)$")
+    object: str = Field(..., min_length=1, max_length=256)
+    pk: dict[str, Any] = Field(..., min_length=1)
+    changes: dict[str, Any] = Field(..., min_length=1)
+
+
 def _connection_info(engine: str) -> dict[str, str]:
     if engine == "postgres":
         return {"host": DB_HOST, "port": str(PG_PORT), "database": PG_DB, "user": PG_USER}
@@ -361,6 +380,8 @@ def _dispatch(engine: str, sql: str, limit: int = 200) -> dict[str, Any]:
         return _run_cassandra(sql, limit)
     if engine == "neo4j":
         return _run_neo4j(sql, limit)
+    if engine == "hadoop":
+        return _run_trino(sql, limit)
     return _run_trino(sql, limit)
 
 
@@ -458,6 +479,15 @@ def _catalog_neo4j() -> dict[str, Any]:
         driver.close()
 
 
+
+def _catalog_hadoop() -> dict[str, Any]:
+    return catalog_hadoop(_run_trino)
+
+
+def _sample_hadoop(object_name: str, limit: int, offset: int = 0) -> dict[str, Any]:
+    return sample_hadoop(object_name, limit, offset, _run_trino, _tabular)
+
+
 def _catalog(engine: str) -> dict[str, Any]:
     if engine == "postgres":
         return _catalog_postgres()
@@ -469,37 +499,74 @@ def _catalog(engine: str) -> dict[str, Any]:
         return _catalog_cassandra()
     if engine == "neo4j":
         return _catalog_neo4j()
+    if engine == "hadoop":
+        return _catalog_hadoop()
     raise ValueError(f"Catalog not supported for {engine}")
 
 
-def _sample_postgres(object_name: str, limit: int) -> dict[str, Any]:
+def _sample_postgres(object_name: str, limit: int, offset: int = 0) -> dict[str, Any]:
     if "." in object_name:
         schema, table = object_name.split(".", 1)
-        sql = f'SELECT * FROM "{schema}"."{table}" LIMIT {limit}'
+        sql = f'SELECT * FROM "{schema}"."{table}" OFFSET {int(offset)} LIMIT {int(limit)}'
     else:
-        sql = f'SELECT * FROM public."{object_name}" LIMIT {limit}'
+        sql = f'SELECT * FROM public."{object_name}" OFFSET {int(offset)} LIMIT {int(limit)}'
     return _run_postgres(sql, limit)
 
 
-def _sample_mysql(object_name: str, limit: int) -> dict[str, Any]:
+def _sample_mysql(object_name: str, limit: int, offset: int = 0) -> dict[str, Any]:
     table = object_name.split(".")[-1]
-    return _run_mysql(f"SELECT * FROM `{table}` LIMIT {limit}", limit)
+    return _run_mysql(f"SELECT * FROM `{table}` LIMIT {int(limit)} OFFSET {int(offset)}", limit)
 
 
-def _sample_mongodb(object_name: str, limit: int) -> dict[str, Any]:
+def _sample_mongodb(object_name: str, limit: int, offset: int = 0) -> dict[str, Any]:
     if "." in object_name:
         db_name, coll = object_name.split(".", 1)
     else:
         db_name, coll = MONGO_DB, object_name
-    return _run_mongo(f"FIND {db_name}.{coll} LIMIT {limit}", limit)
+    t0 = time.perf_counter()
+    client = _mongo_client()
+    try:
+        docs = list(client[db_name][coll].find({}).skip(int(offset)).limit(int(limit)))
+        elapsed_ms = int((time.perf_counter() - t0) * 1000)
+        if not docs:
+            return _tabular(["result"], [["(empty)"]], elapsed_ms)
+        columns = sorted({k for d in docs for k in d})
+        rows = [[_fmt(d.get(c)) for c in columns] for d in docs]
+        return _tabular(columns, rows, elapsed_ms, object=object_name)
+    finally:
+        client.close()
 
 
-def _sample_cassandra(object_name: str, limit: int) -> dict[str, Any]:
+def _sample_cassandra(object_name: str, limit: int, offset: int = 0) -> dict[str, Any]:
     if "." in object_name:
         ks, table = object_name.split(".", 1)
     else:
         ks, table = CASS_KS, object_name
-    return _run_cassandra(f"SELECT * FROM {ks}.{table} LIMIT {limit}", limit)
+    t0 = time.perf_counter()
+    cluster = _cass_cluster()
+    session = cluster.connect()
+    try:
+        stmt = f"SELECT * FROM {ks}.{table}"
+        result = session.execute(stmt, timeout=60)
+        skipped = 0
+        picked: list[Any] = []
+        columns: list[str] = []
+        for row in result:
+            if skipped < offset:
+                skipped += 1
+                continue
+            if not columns:
+                columns = list(row._fields)
+            picked.append(row)
+            if len(picked) >= limit:
+                break
+        elapsed_ms = int((time.perf_counter() - t0) * 1000)
+        if not picked:
+            return _tabular(["result"], [["(empty)"]], elapsed_ms)
+        rows = [[_fmt(getattr(r, c)) for c in columns] for r in picked]
+        return _tabular(columns, rows, elapsed_ms, truncated=len(picked) >= limit)
+    finally:
+        cluster.shutdown()
 
 
 def _graph_neo4j(edge_limit: int = 60, rel_type: str | None = None) -> dict[str, Any]:
@@ -582,9 +649,9 @@ def _graph_neo4j(edge_limit: int = 60, rel_type: str | None = None) -> dict[str,
         driver.close()
 
 
-def _sample_neo4j(object_name: str, limit: int) -> dict[str, Any]:
+def _sample_neo4j(object_name: str, limit: int, offset: int = 0) -> dict[str, Any]:
     label = object_name.split(".")[-1]
-    cypher = f"MATCH (n:`{label}`) RETURN n LIMIT {limit}"
+    cypher = f"MATCH (n:`{label}`) RETURN n SKIP {int(offset)} LIMIT {int(limit)}"
     t0 = time.perf_counter()
     driver = _neo4j_driver()
     try:
@@ -601,24 +668,319 @@ def _sample_neo4j(object_name: str, limit: int) -> dict[str, Any]:
         driver.close()
 
 
-def _sample(engine: str, object_name: str, limit: int) -> dict[str, Any]:
+def _sample(engine: str, object_name: str, limit: int, offset: int = 0) -> dict[str, Any]:
     if engine == "postgres":
-        return _sample_postgres(object_name, limit)
+        return _sample_postgres(object_name, limit, offset)
     if engine == "mysql":
-        return _sample_mysql(object_name, limit)
+        return _sample_mysql(object_name, limit, offset)
     if engine == "mongodb":
-        return _sample_mongodb(object_name, limit)
+        return _sample_mongodb(object_name, limit, offset)
     if engine == "cassandra":
-        return _sample_cassandra(object_name, limit)
+        return _sample_cassandra(object_name, limit, offset)
     if engine == "neo4j":
-        return _sample_neo4j(object_name, limit)
+        return _sample_neo4j(object_name, limit, offset)
+    if engine == "hadoop":
+        return _sample_hadoop(object_name, limit, offset)
     raise ValueError(f"Sample not supported for {engine}")
 
 
+def _table_row_count(engine: str, object_name: str) -> int | None:
+    try:
+        if engine == "postgres":
+            schema, table = _parse_fqn(engine, object_name)
+            conn = psycopg2.connect(
+                host=DB_HOST, port=PG_PORT, user=PG_USER, password=PG_PASS, dbname=PG_DB, connect_timeout=8,
+            )
+            try:
+                cur = conn.cursor()
+                cur.execute(f'SELECT count(*) FROM "{schema}"."{table}"')
+                return int(cur.fetchone()[0])
+            finally:
+                conn.close()
+        if engine == "mysql":
+            schema, table = _parse_fqn(engine, object_name)
+            conn = pymysql.connect(
+                host=DB_HOST, port=MYSQL_PORT, user=MYSQL_USER, password=MYSQL_PASS,
+                database=MYSQL_DB, connect_timeout=8,
+            )
+            try:
+                cur = conn.cursor()
+                cur.execute(f"SELECT count(*) FROM `{table}`")
+                return int(cur.fetchone()[0])
+            finally:
+                conn.close()
+        if engine == "mongodb":
+            db_name, coll = _parse_fqn(engine, object_name)
+            client = _mongo_client()
+            try:
+                return int(client[db_name][coll].estimated_document_count())
+            finally:
+                client.close()
+        if engine == "neo4j":
+            label = object_name.split(".")[-1]
+            driver = _neo4j_driver()
+            try:
+                with driver.session() as session:
+                    return int(session.run(f"MATCH (n:`{label}`) RETURN count(n) AS c").single()["c"])
+            finally:
+                driver.close()
+    except Exception:
+        pass
+    return None
+
+
+def _parse_fqn(engine: str, object_name: str) -> tuple[str, str]:
+    if engine == "postgres":
+        if "." in object_name:
+            return object_name.split(".", 1)
+        return "public", object_name
+    if engine == "mysql":
+        if "." in object_name:
+            return object_name.split(".", 1)
+        return MYSQL_DB, object_name
+    if engine == "mongodb":
+        if "." in object_name:
+            return object_name.split(".", 1)
+        return MONGO_DB, object_name
+    if engine == "cassandra":
+        if "." in object_name:
+            return object_name.split(".", 1)
+        return CASS_KS, object_name
+    return "graph", object_name.split(".")[-1]
+
+
+def _object_primary_keys(engine: str, object_name: str) -> list[str]:
+    try:
+        if engine == "postgres":
+            schema, table = _parse_fqn(engine, object_name)
+            conn = psycopg2.connect(
+                host=DB_HOST, port=PG_PORT, user=PG_USER, password=PG_PASS, dbname=PG_DB, connect_timeout=8,
+            )
+            try:
+                cur = conn.cursor()
+                cur.execute(
+                    """
+                    SELECT a.attname
+                    FROM pg_constraint c
+                    JOIN pg_class t ON t.oid = c.conrelid
+                    JOIN pg_namespace n ON n.oid = t.relnamespace
+                    JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = ANY(c.conkey)
+                    WHERE c.contype = 'p' AND n.nspname = %s AND t.relname = %s
+                    ORDER BY array_position(c.conkey, a.attnum)
+                    """,
+                    (schema, table),
+                )
+                return [r[0] for r in cur.fetchall()]
+            finally:
+                conn.close()
+        if engine == "mysql":
+            schema, table = _parse_fqn(engine, object_name)
+            conn = pymysql.connect(
+                host=DB_HOST, port=MYSQL_PORT, user=MYSQL_USER, password=MYSQL_PASS,
+                database=MYSQL_DB, connect_timeout=8,
+            )
+            try:
+                cur = conn.cursor()
+                cur.execute(
+                    """
+                    SELECT COLUMN_NAME FROM information_schema.KEY_COLUMN_USAGE
+                    WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s AND CONSTRAINT_NAME = 'PRIMARY'
+                    ORDER BY ORDINAL_POSITION
+                    """,
+                    (schema, table),
+                )
+                return [r[0] for r in cur.fetchall()]
+            finally:
+                conn.close()
+        if engine == "mongodb":
+            return ["_id"]
+        if engine == "cassandra":
+            ks, table = _parse_fqn(engine, object_name)
+            cluster = _cass_cluster()
+            session = cluster.connect()
+            try:
+                rows = session.execute(
+                    "SELECT column_name FROM system_schema.columns "
+                    "WHERE keyspace_name=%s AND table_name=%s AND kind IN ('partition_key','clustering') "
+                    "ORDER BY position",
+                    (ks, table),
+                )
+                return [r.column_name for r in rows]
+            finally:
+                cluster.shutdown()
+        if engine == "neo4j":
+            label = object_name.split(".")[-1]
+            if label == "Product":
+                return ["product_id"]
+            if label == "Supplier":
+                return ["supplier_id"]
+    except Exception:
+        pass
+    return []
+
+
+def _coerce_value(val: Any) -> Any:
+    if val is None or val == "":
+        return None
+    if isinstance(val, (int, float, bool)):
+        return val
+    s = str(val)
+    if s.lower() == "null":
+        return None
+    if re.fullmatch(r"-?\d+", s):
+        try:
+            return int(s)
+        except ValueError:
+            pass
+    if re.fullmatch(r"-?\d+\.\d+", s):
+        try:
+            return float(s)
+        except ValueError:
+            pass
+    return s
+
+
+def _safe_ident(name: str) -> bool:
+    return bool(re.fullmatch(r"[a-zA-Z_][a-zA-Z0-9_]*", name))
+
+
+def _update_postgres(object_name: str, pk: dict[str, Any], changes: dict[str, Any]) -> str:
+    schema, table = _parse_fqn("postgres", object_name)
+    if not changes or not pk:
+        raise ValueError("Primary key and changes required")
+    for c in list(changes) + list(pk):
+        if not _safe_ident(c):
+            raise ValueError(f"Invalid column name: {c}")
+    set_sql = ", ".join(f'"{c}"=%s' for c in changes)
+    where_sql = " AND ".join(f'"{c}"=%s' for c in pk)
+    sql = f'UPDATE "{schema}"."{table}" SET {set_sql} WHERE {where_sql}'
+    params = list(changes.values()) + list(pk.values())
+    conn = psycopg2.connect(
+        host=DB_HOST, port=PG_PORT, user=PG_USER, password=PG_PASS, dbname=PG_DB, connect_timeout=8,
+    )
+    try:
+        conn.autocommit = True
+        cur = conn.cursor()
+        cur.execute(sql, params)
+        if cur.rowcount == 0:
+            raise ValueError("No row matched primary key")
+        return sql
+    finally:
+        conn.close()
+
+
+def _update_mysql(object_name: str, pk: dict[str, Any], changes: dict[str, Any]) -> str:
+    _schema, table = _parse_fqn("mysql", object_name)
+    if not changes or not pk:
+        raise ValueError("Primary key and changes required")
+    for c in list(changes) + list(pk):
+        if not _safe_ident(c):
+            raise ValueError(f"Invalid column name: {c}")
+    set_sql = ", ".join(f"`{c}`=%s" for c in changes)
+    where_sql = " AND ".join(f"`{c}`=%s" for c in pk)
+    sql = f"UPDATE `{table}` SET {set_sql} WHERE {where_sql}"
+    params = list(changes.values()) + list(pk.values())
+    conn = pymysql.connect(
+        host=DB_HOST, port=MYSQL_PORT, user=MYSQL_USER, password=MYSQL_PASS,
+        database=MYSQL_DB, connect_timeout=8,
+    )
+    try:
+        conn.autocommit = True
+        cur = conn.cursor()
+        cur.execute(sql, params)
+        if cur.rowcount == 0:
+            raise ValueError("No row matched primary key")
+        return sql
+    finally:
+        conn.close()
+
+
+def _update_mongodb(object_name: str, pk: dict[str, Any], changes: dict[str, Any]) -> str:
+    db_name, coll = _parse_fqn("mongodb", object_name)
+    if not pk or not changes:
+        raise ValueError("Primary key and changes required")
+    client = _mongo_client()
+    try:
+        from bson import ObjectId
+        filt = dict(pk)
+        if "_id" in filt and isinstance(filt["_id"], str) and len(filt["_id"]) == 24:
+            try:
+                filt["_id"] = ObjectId(filt["_id"])
+            except Exception:
+                pass
+        res = client[db_name][coll].update_one(filt, {"$set": changes})
+        if res.matched_count == 0:
+            raise ValueError("No document matched primary key")
+        return f"UPDATE {db_name}.{coll} {filt}"
+    finally:
+        client.close()
+
+
+def _update_cassandra(object_name: str, pk: dict[str, Any], changes: dict[str, Any]) -> str:
+    ks, table = _parse_fqn("cassandra", object_name)
+    if not pk or not changes:
+        raise ValueError("Primary key and changes required")
+    set_cql = ", ".join(f"{c}=%s" for c in changes)
+    where_cql = " AND ".join(f"{c}=%s" for c in pk)
+    cql = f"UPDATE {ks}.{table} SET {set_cql} WHERE {where_cql}"
+    cluster = _cass_cluster()
+    session = cluster.connect()
+    try:
+        session.execute(cql, list(changes.values()) + list(pk.values()))
+        return cql
+    finally:
+        cluster.shutdown()
+
+
+def _update_neo4j(object_name: str, pk: dict[str, Any], changes: dict[str, Any]) -> str:
+    label = object_name.split(".")[-1]
+    if not pk or not changes:
+        raise ValueError("Primary key and changes required")
+    pk_col, pk_val = next(iter(pk.items()))
+    if not _safe_ident(pk_col):
+        raise ValueError(f"Invalid property name: {pk_col}")
+    for c in changes:
+        if not _safe_ident(c):
+            raise ValueError(f"Invalid property name: {c}")
+    set_frag = ", ".join(f"n.{c} = ${c}" for c in changes)
+    cypher = f"MATCH (n:`{label}` {{{pk_col}: $pk}}) SET {set_frag} RETURN n"
+    params: dict[str, Any] = {"pk": _coerce_value(pk_val)}
+    params.update({c: _coerce_value(v) for c, v in changes.items()})
+    driver = _neo4j_driver()
+    try:
+        with driver.session() as session:
+            rec = session.run(cypher, **params).single()
+            if not rec:
+                raise ValueError("No node matched primary key")
+        return cypher
+    finally:
+        driver.close()
+
+
+def _update_row(engine: str, object_name: str, pk: dict[str, Any], changes: dict[str, Any]) -> dict[str, Any]:
+    pk = {k: _coerce_value(v) for k, v in pk.items()}
+    changes = {k: _coerce_value(v) for k, v in changes.items()}
+    if engine == "postgres":
+        sql = _update_postgres(object_name, pk, changes)
+    elif engine == "mysql":
+        sql = _update_mysql(object_name, pk, changes)
+    elif engine == "mongodb":
+        sql = _update_mongodb(object_name, pk, changes)
+    elif engine == "cassandra":
+        sql = _update_cassandra(object_name, pk, changes)
+    elif engine == "neo4j":
+        sql = _update_neo4j(object_name, pk, changes)
+    else:
+        raise ValueError(f"Update not supported for {engine}")
+    return {"ok": True, "engine": engine, "object": object_name, "statement": sql, "cdc": engine in ("postgres", "mysql", "mongodb")}
+
+
 @router.get("/samples/{engine}")
 async def get_samples(engine: str):
-    if engine not in SAMPLES:
+    if engine not in SAMPLES and engine not in LAKE_ENGINES:
         return JSONResponse({"error": "unknown engine"}, status_code=404)
+    if engine == "hadoop":
+        return {"engine": engine, "samples": SAMPLES[engine], "connection": hadoop_connection_info()}
     return {"engine": engine, "samples": SAMPLES[engine], "connection": _connection_info(engine)}
 
 
@@ -647,9 +1009,20 @@ async def sql_health():
     return out
 
 
+
+
+@router.get("/health/{engine}")
+async def sql_health_engine(engine: str):
+    if engine == "hadoop":
+        return health_hadoop(_run_trino)
+    if engine not in SOURCE_ENGINES:
+        return JSONResponse({"error": "unknown engine"}, status_code=404)
+    all_h = await sql_health()
+    return all_h.get(engine) or {"ok": False, "error": "not found"}
+
 @router.get("/catalog/{engine}")
 async def get_catalog(engine: str):
-    if engine not in SOURCE_ENGINES:
+    if engine not in SOURCE_ENGINES and engine not in LAKE_ENGINES:
         return JSONResponse({"error": "unknown engine"}, status_code=404)
     try:
         return _catalog(engine)
@@ -658,14 +1031,31 @@ async def get_catalog(engine: str):
 
 
 @router.get("/sample/{engine}")
-async def get_sample(engine: str, object: str = Query(..., min_length=1), limit: int = Query(50, ge=1, le=200)):
-    if engine not in SOURCE_ENGINES:
+async def get_sample(
+    engine: str,
+    object: str = Query(..., min_length=1),
+    limit: int = Query(100, ge=1, le=500),
+    offset: int = Query(0, ge=0),
+):
+    if engine not in SOURCE_ENGINES and engine not in LAKE_ENGINES:
         return JSONResponse({"error": "unknown engine"}, status_code=404)
     try:
-        result = _sample(engine, object, limit)
+        result = _sample(engine, object, limit, offset)
         if not result.get("ok"):
             return JSONResponse(result, status_code=422)
-        return {**result, "engine": engine, "object": object}
+        pks = _object_primary_keys(engine, object)
+        total = _table_row_count(engine, object)
+        return {
+            **result,
+            "engine": engine,
+            "object": object,
+            "offset": offset,
+            "limit": limit,
+            "total_count": total,
+            "primary_keys": pks,
+            "cdc": engine in ("postgres", "mysql", "mongodb"),
+            "editable": bool(pks),
+        }
     except Exception as exc:
         return JSONResponse({"ok": False, "error": str(exc)[:500]}, status_code=502)
 
@@ -698,6 +1088,22 @@ async def execute_sql(body: SqlRequest):
         return JSONResponse({"ok": False, "error": str(exc)[:500]}, status_code=502)
 
 
+@router.post("/row/update")
+async def update_row(body: RowUpdateRequest):
+    allowed_pks = _object_primary_keys(body.engine, body.object)
+    if allowed_pks:
+        for k in body.pk:
+            if k not in allowed_pks:
+                return JSONResponse({"ok": False, "error": f"Invalid primary key column '{k}'"}, status_code=400)
+    for k in body.changes:
+        if k in body.pk:
+            return JSONResponse({"ok": False, "error": "Cannot change primary key columns"}, status_code=400)
+    try:
+        return _update_row(body.engine, body.object, body.pk, body.changes)
+    except Exception as exc:
+        return JSONResponse({"ok": False, "error": str(exc)[:500]}, status_code=422)
+
+
 @router.post("/benchmark")
 async def benchmark():
     """Parallel analytics: PostgreSQL OLTP vs Trino distributed engine (5M sin() rows)."""

api/streaming_ops.py

@@ -0,0 +1,435 @@
+"""Live Spark + Kafka visibility and manual job control for the Command Center."""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import time
+from typing import Any
+
+import httpx
+from fastapi import APIRouter, Body
+from fastapi.responses import JSONResponse, Response
+
+SPARK_UI_URL = os.getenv("SPARK_UI_URL", "http://10.0.21.50:8080").rstrip("/")
+KAFKA_UI_URL = os.getenv("KAFKA_UI_URL", "http://10.0.21.36:9000").rstrip("/")
+KAFKA_CONNECT_URL = os.getenv("KAFKA_CONNECT_URL", "http://10.0.21.50:8083").rstrip("/")
+AIRFLOW_URL = os.getenv("AIRFLOW_URL", "http://10.0.21.55:8080").rstrip("/")
+AIRFLOW_USER = os.getenv("AIRFLOW_USER", "admin")
+AIRFLOW_PASSWORD = os.getenv("AIRFLOW_PASSWORD", "")
+
+from hdfs_kafka import export_hdfs_to_kafka, hdfs_export_snapshot
+
+router = APIRouter(prefix="/api/pipeline/streaming", tags=["streaming"])
+
+# Manual Spark / lakehouse jobs (Airflow DAGs on the lab)
+SPARK_JOBS: dict[str, dict[str, Any]] = {
+    "spark_to_curated": {
+        "label": "Spark → Iceberg curated (mask PII)",
+        "dag_id": "mask_to_curated",
+        "agent": "lakehouse-ops",
+        "description": "Runs the mask_to_curated DAG — Spark/SQL transform into iceberg.curated_masked",
+        "default_conf": {},
+        "editable_fields": ["batch_size", "sources"],
+    },
+    "hadoop_to_trino": {
+        "label": "HDFS → Iceberg (historical)",
+        "dag_id": "hadoop_to_trino",
+        "agent": "hadoop-ranger",
+        "description": "Load historical_sales from HDFS into Iceberg via Trino/Spark pipeline",
+        "default_conf": {"mode": "refresh"},
+        "editable_fields": ["mode"],
+    },
+    "spark_to_s3": {
+        "label": "Spark → S3 curated layer",
+        "dag_id": "mask_to_curated",
+        "agent": "lakehouse-ops",
+        "description": "Spark transform into Iceberg curated tables mirrored to S3",
+        "default_conf": {"target": "s3"},
+        "editable_fields": ["target", "batch_size"],
+    },
+    "generate_all": {
+        "label": "Seed all source DBs",
+        "dag_id": "generate_data_all_databases",
+        "agent": "data-custodian",
+        "description": "Airflow DAG — generates rows into PostgreSQL, MySQL, MongoDB, Cassandra, Neo4j",
+        "default_conf": {"rows": 3000},
+        "editable_fields": ["rows"],
+    },
+}
+
+# Master pipeline pulse switch: running | paused | stopped
+_flow_state: dict[str, Any] = {"mode": "running", "since": time.time()}
+
+def flow_mode() -> str:
+    return _flow_state.get("mode", "running")
+
+def flow_snapshot() -> dict[str, Any]:
+    return {"mode": _flow_state.get("mode", "running"), "since": _flow_state.get("since")}
+
+_cache: dict[str, Any] = {"ts": 0.0, "data": None}
+_TTL = 8.0
+_token_cache: dict[str, Any] = {"token": None, "exp": 0.0}
+
+
+def _feed(agent_id: str, message: str, level: str = "info") -> None:
+    try:
+        from main import add_feed
+        add_feed(agent_id, message, level)
+    except Exception:
+        pass
+
+
+async def _airflow_token(client: httpx.AsyncClient) -> str:
+    now = time.time()
+    if _token_cache["token"] and _token_cache["exp"] > now + 30:
+        return _token_cache["token"]
+    r = await client.post(
+        f"{AIRFLOW_URL}/auth/token",
+        json={"username": AIRFLOW_USER, "password": AIRFLOW_PASSWORD},
+        timeout=10,
+    )
+    r.raise_for_status()
+    tok = r.json()["access_token"]
+    _token_cache["token"] = tok
+    _token_cache["exp"] = now + 20 * 60
+    return tok
+
+
+async def collect_spark() -> dict[str, Any]:
+    out: dict[str, Any] = {
+        "ui_url": SPARK_UI_URL,
+        "ui_ok": False,
+        "status": "UNKNOWN",
+        "workers": [],
+        "alive_workers": 0,
+        "cores": 0,
+        "cores_used": 0,
+        "memory_mb": 0,
+        "memory_used_mb": 0,
+        "active_apps": [],
+        "completed_apps": [],
+    }
+    try:
+        async with httpx.AsyncClient(timeout=6.0) as client:
+            r = await client.get(f"{SPARK_UI_URL}/json/")
+            if r.status_code >= 400:
+                return out
+            d = r.json()
+            out["ui_ok"] = True
+            out["status"] = d.get("status") or "ALIVE"
+            out["alive_workers"] = int(d.get("aliveworkers") or 0)
+            out["cores"] = int(d.get("cores") or 0)
+            out["cores_used"] = int(d.get("coresused") or 0)
+            out["memory_mb"] = int(d.get("memory") or 0)
+            out["memory_used_mb"] = int(d.get("memoryused") or 0)
+            workers = d.get("workers") or []
+            out["workers"] = [
+                {
+                    "id": w.get("id"),
+                    "host": w.get("host"),
+                    "cores": w.get("cores"),
+                    "cores_used": w.get("coresused"),
+                    "memory_mb": w.get("memory"),
+                    "state": w.get("state"),
+                    "webui": w.get("webuiaddress"),
+                }
+                for w in workers
+            ]
+            for app in (d.get("activeapps") or [])[:20]:
+                out["active_apps"].append({
+                    "id": app.get("id"),
+                    "name": app.get("name"),
+                    "cores": app.get("cores"),
+                    "memory_mb": app.get("memory"),
+                    "submitdate": app.get("submitdate"),
+                    "duration_ms": app.get("duration"),
+                    "user": app.get("user"),
+                })
+            for app in (d.get("completedapps") or [])[:10]:
+                out["completed_apps"].append({
+                    "id": app.get("id"),
+                    "name": app.get("name"),
+                    "duration_ms": app.get("duration"),
+                })
+    except Exception as exc:
+        out["error"] = str(exc)[:200]
+    return out
+
+
+async def collect_kafka() -> dict[str, Any]:
+    out: dict[str, Any] = {
+        "ui_url": KAFKA_UI_URL,
+        "ui_ok": False,
+        "connect_url": KAFKA_CONNECT_URL,
+        "connect_ok": False,
+        "cluster": {},
+        "topics": [],
+        "connectors": [],
+    }
+    try:
+        async with httpx.AsyncClient(timeout=8.0) as client:
+            cr = await client.get(f"{KAFKA_UI_URL}/api/clusters")
+            if cr.status_code < 400:
+                clusters = cr.json()
+                out["ui_ok"] = True
+                if clusters:
+                    name = clusters[0].get("name", "local")
+                    out["cluster"] = {
+                        "name": name,
+                        "status": clusters[0].get("status"),
+                        "broker_count": clusters[0].get("brokerCount"),
+                        "topic_count": clusters[0].get("topicCount"),
+                        "online_partitions": clusters[0].get("onlinePartitionCount"),
+                    }
+                    tr = await client.get(
+                        f"{KAFKA_UI_URL}/api/clusters/{name}/topics",
+                        params={"page": 1, "perPage": 50, "showInternal": False},
+                    )
+                    if tr.status_code < 400:
+                        topics = tr.json().get("topics") or []
+                        out["topics"] = [
+                            {
+                                "name": t.get("name"),
+                                "partitions": t.get("partitionCount"),
+                                "replicas": t.get("replicationFactor"),
+                                "messages": t.get("messagesCount"),
+                            }
+                            for t in topics[:50]
+                        ]
+            lr = await client.get(f"{KAFKA_CONNECT_URL}/connectors")
+            if lr.status_code < 400:
+                out["connect_ok"] = True
+                names_raw = lr.json()
+                names = names_raw if isinstance(names_raw, list) else []
+                for cn in names[:20]:
+                    try:
+                        sr = await client.get(f"{KAFKA_CONNECT_URL}/connectors/{cn}/status")
+                        st = sr.json() if sr.status_code < 400 else {}
+                        conn = st.get("connector") or {}
+                        tasks = st.get("tasks") or []
+                        out["connectors"].append({
+                            "name": cn,
+                            "state": conn.get("state"),
+                            "worker": conn.get("worker_id"),
+                            "tasks": [{"id": t.get("id"), "state": t.get("state")} for t in tasks],
+                            "type": st.get("type"),
+                        })
+                    except Exception:
+                        out["connectors"].append({"name": cn, "state": "UNKNOWN"})
+    except Exception as exc:
+        out["error"] = str(exc)[:200]
+    return out
+
+
+async def build_streaming_status() -> dict[str, Any]:
+    spark, kafka = await asyncio.gather(collect_spark(), collect_kafka())
+    cdc_active = False
+    try:
+        from cdc_consumer import snapshot as cdc_snapshot
+        cdc = cdc_snapshot(15)
+        cdc_active = bool(cdc.get("connected")) and int(cdc.get("window_total") or 0) > 0
+    except Exception:
+        pass
+
+    connectors_running = sum(
+        1 for c in kafka.get("connectors", [])
+        if (c.get("state") or "").upper() == "RUNNING"
+    )
+    spark_alive = spark.get("ui_ok") and (spark.get("status") or "").upper() == "ALIVE"
+    apps_running = len(spark.get("active_apps") or [])
+
+    hdfs_recent = hdfs_export_snapshot().get("recent")
+
+    edges = {
+        "hdfs→kafka": bool(hdfs_recent),
+        "kafka→spark": cdc_active and spark_alive or hdfs_recent,
+        "spark→iceberg": apps_running > 0 or spark_alive,
+        "spark→s3": apps_running > 0 or spark_alive,
+        "sources→kafka": cdc_active,
+        "connectors": connectors_running > 0,
+    }
+
+    mode = flow_mode()
+    if mode != "running":
+        edges = {k: False for k in edges}
+
+    return {
+        "ok": True,
+        "flow": mode,
+        "spark": spark,
+        "kafka": kafka,
+        "edges": edges,
+        "jobs": [
+            {**{"id": k}, **{kk: vv for kk, vv in v.items() if kk != "editable_fields"}}
+            for k, v in SPARK_JOBS.items()
+        ],
+        "ts": time.time(),
+    }
+
+
+@router.get("/status")
+async def streaming_status(refresh: bool = False) -> JSONResponse:
+    now = time.time()
+    if not refresh and _cache["data"] and now - _cache["ts"] < _TTL:
+        return JSONResponse(_cache["data"])
+    data = await build_streaming_status()
+    _cache["data"] = data
+    _cache["ts"] = now
+    return JSONResponse(data)
+
+
+@router.get("/jobs")
+async def list_spark_jobs() -> JSONResponse:
+    return JSONResponse({"ok": True, "jobs": SPARK_JOBS})
+
+
+
+
+@router.post("/flow/{action}")
+async def set_flow(action: str) -> JSONResponse:
+    mapping = {
+        "pause": "paused",
+        "stop": "stopped",
+        "resume": "running",
+        "start": "running",
+        "run": "running",
+    }
+    if action not in mapping:
+        return JSONResponse({"ok": False, "error": f"unknown action {action}"}, status_code=400)
+    _flow_state["mode"] = mapping[action]
+    _flow_state["since"] = time.time()
+    _cache["ts"] = 0
+    label = {"running": "resumed", "paused": "paused", "stopped": "stopped"}[mapping[action]]
+    _feed("lakehouse-ops", f"[pipeline] data flow {label}", "info" if mapping[action] == "running" else "warn")
+    return JSONResponse({"ok": True, "flow": flow_snapshot()})
+
+
+@router.get("/flow")
+async def get_flow() -> JSONResponse:
+    return JSONResponse({"ok": True, "flow": flow_snapshot()})
+
+@router.post("/jobs/{job_id}/trigger")
+async def trigger_spark_job(job_id: str, body: dict[str, Any] = Body(default={})) -> JSONResponse:
+    job = SPARK_JOBS.get(job_id)
+    if not job:
+        return JSONResponse({"ok": False, "error": f"unknown job {job_id}"}, status_code=400)
+    conf = {**(job.get("default_conf") or {}), **(body.get("conf") or {})}
+    agent = job.get("agent", "lakehouse-ops")
+    dag_id = job["dag_id"]
+    _feed(agent, f"[spark] Triggering {job['label']} conf={conf}", "info")
+
+    try:
+        async with httpx.AsyncClient() as client:
+            tok = await _airflow_token(client)
+            h = {"Authorization": f"Bearer {tok}"}
+            r = await client.post(
+                f"{AIRFLOW_URL}/api/v2/dags/{dag_id}/dagRuns",
+                headers=h,
+                json={"logical_date": None, "conf": conf},
+                timeout=20,
+            )
+            if r.status_code >= 400:
+                _feed(agent, f"[spark] {job['label']}: Airflow {r.status_code}", "err")
+                return JSONResponse({"ok": False, "error": f"airflow {r.status_code}: {r.text[:300]}"}, status_code=502)
+            run = r.json()
+            run_id = run.get("dag_run_id")
+            _feed(agent, f"[spark] {job['label']}: started run {run_id}", "info")
+            return JSONResponse({"ok": True, "job_id": job_id, "dag_id": dag_id, "run_id": run_id, "conf": conf})
+    except Exception as exc:
+        _feed(agent, f"[spark] {job['label']}: {str(exc)[:120]}", "err")
+        return JSONResponse({"ok": False, "error": str(exc)}, status_code=500)
+
+
+
+
+@router.post("/hdfs/to-kafka")
+async def hdfs_to_kafka(body: dict[str, Any] = Body(default={})) -> JSONResponse:
+    path = body.get("path", "/data/historical/sales_orders")
+    topic = body.get("topic", "hdfs.historical.sales")
+    limit = int(body.get("limit", 2000))
+    try:
+        result = await export_hdfs_to_kafka(path=path, topic=topic, limit=limit, feed=_feed)
+        if not result.get("ok"):
+            return JSONResponse(result, status_code=502)
+        _cache["ts"] = 0
+        return JSONResponse(result)
+    except Exception as exc:
+        _feed("hadoop-ranger", f"[hdfs→kafka] failed: {str(exc)[:120]}", "err")
+        return JSONResponse({"ok": False, "error": str(exc)}, status_code=500)
+
+
+@router.post("/pipeline/{pipeline_id}")
+async def run_streaming_pipeline(pipeline_id: str) -> JSONResponse:
+    if pipeline_id != "hadoop-lake":
+        return JSONResponse({"ok": False, "error": f"unknown pipeline {pipeline_id}"}, status_code=400)
+    steps: list[str] = []
+    try:
+        exp = await export_hdfs_to_kafka(feed=_feed)
+        if not exp.get("ok"):
+            return JSONResponse({"ok": False, "error": exp.get("error"), "steps": steps}, status_code=502)
+        steps.append(f"hdfs→kafka ({exp.get('rows_sent')} rows)")
+
+        async with httpx.AsyncClient() as client:
+            tok = await _airflow_token(client)
+            h = {"Authorization": f"Bearer {tok}"}
+            for job_id, dag_id, agent in [
+                ("hadoop_to_trino", "hadoop_to_trino", "hadoop-ranger"),
+                ("spark_to_curated", "mask_to_curated", "lakehouse-ops"),
+            ]:
+                r = await client.post(
+                    f"{AIRFLOW_URL}/api/v2/dags/{dag_id}/dagRuns",
+                    headers=h,
+                    json={"logical_date": None, "conf": {}},
+                    timeout=20,
+                )
+                if r.status_code >= 400:
+                    _feed(agent, f"[pipeline] {dag_id} failed {r.status_code}", "err")
+                    return JSONResponse({"ok": False, "error": f"{dag_id}: {r.text[:200]}", "steps": steps}, status_code=502)
+                steps.append(dag_id)
+                _feed(agent, f"[pipeline] started {dag_id}", "info")
+
+        _cache["ts"] = 0
+        return JSONResponse({"ok": True, "pipeline": pipeline_id, "steps": steps, "export": exp})
+    except Exception as exc:
+        return JSONResponse({"ok": False, "error": str(exc), "steps": steps}, status_code=500)
+
+@router.post("/kafka/connectors/{name}/restart")
+async def restart_kafka_connector(name: str) -> JSONResponse:
+    _feed("etl-guardian", f"[kafka] Restarting connector {name}", "info")
+    try:
+        async with httpx.AsyncClient(timeout=15.0) as client:
+            r = await client.post(f"{KAFKA_CONNECT_URL}/connectors/{name}/restart")
+            if r.status_code >= 400:
+                return JSONResponse({"ok": False, "error": r.text[:300]}, status_code=502)
+        _feed("etl-guardian", f"[kafka] Connector {name} restart requested", "info")
+        _cache["ts"] = 0
+        return JSONResponse({"ok": True, "connector": name, "action": "restart"})
+    except Exception as exc:
+        return JSONResponse({"ok": False, "error": str(exc)}, status_code=500)
+
+
+@router.post("/kafka/connectors/{name}/pause")
+async def pause_kafka_connector(name: str) -> JSONResponse:
+    try:
+        async with httpx.AsyncClient(timeout=10.0) as client:
+            r = await client.put(f"{KAFKA_CONNECT_URL}/connectors/{name}/pause")
+            if r.status_code >= 400:
+                return JSONResponse({"ok": False, "error": r.text[:300]}, status_code=502)
+        _cache["ts"] = 0
+        return JSONResponse({"ok": True, "connector": name, "action": "pause"})
+    except Exception as exc:
+        return JSONResponse({"ok": False, "error": str(exc)}, status_code=500)
+
+
+@router.post("/kafka/connectors/{name}/resume")
+async def resume_kafka_connector(name: str) -> JSONResponse:
+    try:
+        async with httpx.AsyncClient(timeout=10.0) as client:
+            r = await client.put(f"{KAFKA_CONNECT_URL}/connectors/{name}/resume")
+            if r.status_code >= 400:
+                return JSONResponse({"ok": False, "error": r.text[:300]}, status_code=502)
+        _cache["ts"] = 0
+        return JSONResponse({"ok": True, "connector": name, "action": "resume"})
+    except Exception as exc:
+        return JSONResponse({"ok": False, "error": str(exc)}, status_code=500)

api/webhdfs_util.py

@@ -0,0 +1,77 @@
+"""Shared WebHDFS helpers — resolve datanode redirects via NameNode JMX."""
+
+from __future__ import annotations
+
+import os
+import time
+from typing import Any
+from urllib.parse import quote, urlparse, urlunparse
+
+import httpx
+
+HDFS_NN_URL = os.getenv("HDFS_NN_URL", "http://10.0.21.61:9870").rstrip("/")
+HDFS_USER = os.getenv("HDFS_USER", "hdfs")
+WEBHDFS = f"{HDFS_NN_URL}/webhdfs/v1"
+
+_dn_cache: dict[str, Any] = {"ts": 0.0, "map": {}}
+
+
+def encode_path(path: str) -> str:
+    p = path if path.startswith("/") else f"/{path}"
+    return "/".join(quote(seg, safe="") for seg in p.split("/"))
+
+
+def datanode_host_map() -> dict[str, str]:
+    now = time.time()
+    if _dn_cache["map"] and now - _dn_cache["ts"] < 120:
+        return _dn_cache["map"]
+    mapping: dict[str, str] = {}
+    try:
+        with httpx.Client(timeout=8.0) as client:
+            r = client.get(f"{HDFS_NN_URL}/jmx?qry=Hadoop:service=NameNode,name=NameNodeInfo")
+            raw = (r.json().get("beans") or [{}])[0].get("LiveNodes") or "{}"
+            if isinstance(raw, str):
+                import json
+                nodes = json.loads(raw)
+            else:
+                nodes = raw
+            for key, info in nodes.items():
+                host = key.split(":")[0]
+                info_addr = (info or {}).get("infoAddr") or ""
+                if info_addr:
+                    mapping[host] = info_addr  # ip:9864
+    except Exception:
+        pass
+    _dn_cache["map"] = mapping
+    _dn_cache["ts"] = now
+    return mapping
+
+
+def resolve_redirect(url: str) -> str:
+    parsed = urlparse(url)
+    host = parsed.hostname or ""
+    if not host or host.replace(".", "").isdigit():
+        return url
+    dn = datanode_host_map().get(host)
+    if not dn:
+        return url
+    ip, _, port = dn.partition(":")
+    port = parsed.port or port or "9864"
+    return urlunparse(parsed._replace(netloc=f"{ip}:{port}"))
+
+
+def open_bytes(path: str, max_bytes: int = 8_000_000) -> bytes:
+    url = f"{WEBHDFS}{encode_path(path)}"
+    with httpx.Client(timeout=90.0, follow_redirects=False) as client:
+        r = client.get(url, params={"op": "OPEN", "user.name": HDFS_USER})
+        if r.status_code in (301, 302, 307, 308):
+            loc = r.headers.get("location") or ""
+            if loc:
+                r = client.get(resolve_redirect(loc), follow_redirects=True)
+        elif r.status_code == 200 and r.headers.get("content-type", "").startswith("application/json"):
+            loc = r.json().get("Location") or r.json().get("location") or ""
+            if loc:
+                r = client.get(resolve_redirect(loc), follow_redirects=True)
+        if r.status_code >= 400:
+            raise RuntimeError(r.text[:300])
+        return r.content[:max_bytes]

ui/index.html

@@ -1,5 +1,5 @@
 <!DOCTYPE html>
-<html lang="en" class="light">
+<html lang="en" class="dark">
   <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />

ui/nginx.conf

@@ -14,6 +14,31 @@ server {
         proxy_buffering off;
     }
 
+    location /spark-ui/ {
+        proxy_pass http://10.0.21.50:8080/;
+        proxy_http_version 1.1;
+        proxy_set_header Host 10.0.21.50:8080;
+        proxy_hide_header X-Frame-Options;
+        proxy_redirect http://10.0.21.50:8080/ /spark-ui/;
+        sub_filter_once off;
+        sub_filter 'href="/' 'href="/spark-ui/';
+        sub_filter 'src="/' 'src="/spark-ui/';
+        sub_filter "href='/" "href='/spark-ui/";
+        sub_filter_types text/css application/javascript text/html;
+    }
+
+    location /kafka-ui/ {
+        proxy_pass http://10.0.21.36:9000/;
+        proxy_http_version 1.1;
+        proxy_set_header Host 10.0.21.36:9000;
+        proxy_hide_header X-Frame-Options;
+        proxy_redirect http://10.0.21.36:9000/ /kafka-ui/;
+        sub_filter_once off;
+        sub_filter 'href="/' 'href="/kafka-ui/';
+        sub_filter 'src="/' 'src="/kafka-ui/';
+        sub_filter_types text/html application/javascript text/css;
+    }
+
     location /assets/ {
         add_header Cache-Control "public, max-age=31536000, immutable";
         try_files $uri =404;

ui/src/App.tsx

@@ -11,17 +11,14 @@ import { ChatDrawer } from './components/features/ChatDrawer'
 import { GpuMonitor } from './components/features/GpuMonitor'
 import { InfraQuickAccess } from './components/features/InfraQuickAccess'
 import { InspectorPanel } from './components/features/InspectorPanel'
-import { PlatformTopology } from './components/features/PlatformTopology'
-import { PresentationView } from './components/features/PresentationView'
+import { PlatformView } from './components/features/PlatformView'
 import { DataQualityView } from './components/features/DataQualityView'
 import { KnowledgeChatView } from './components/features/KnowledgeChatView'
 import { StorageView } from './components/features/StorageView'
-import { HdfsView } from './components/features/HdfsView'
-import { DataGenView } from './components/features/DataGenView'
 import { ChangesView } from './components/features/ChangesView'
 import { DataFlowView } from './components/features/DataFlowView'
 import { SearchView } from './components/features/SearchView'
-import { DataSourcesView } from './components/features/DataSourcesView'
+import { DataHubView } from './components/features/DataHubView'
 import { SshTerminal } from './components/features/SshTerminal'
 import { TerminalDock } from './components/features/TerminalDock'
 import { WorkbenchPanel } from './components/features/WorkbenchPanel'
@@ -39,7 +36,7 @@ export default function App() {
   const mainScrollRef = useRef<HTMLDivElement>(null)
 
   const isPlatform = cc.mainView === 'platform'
-  const isDataSources = cc.mainView === 'datasources'
+  const isDataSources = cc.mainView === 'datasources' || cc.mainView === 'hdfs'
 
   const openApprovals = () => {
     cc.setMainView('approvals')
@@ -123,31 +120,25 @@ export default function App() {
 
               <div className={cn('flex min-h-0 flex-col', (isPlatform || isDataSources) ? 'min-h-0 flex-1 overflow-hidden' : 'min-h-0 flex-1')}>
                 {cc.mainView === 'platform' ? (
-                  <PlatformTopology
+                  <PlatformView
                     workload={cc.workload}
                     animations={cc.anims}
                     selectedNodeId={cc.selectedNodeId}
                     onNodeClick={cc.selectNode}
                     pulse={cc.genPulse}
                   />
-                ) : cc.mainView === 'datasources' ? (
-                  <DataSourcesView focusEngine={cc.dataSourceFocus} />
-                ) : cc.mainView === 'datagen' ? (
-                  <DataGenView onPulse={cc.pulseFlow} onOpenPlatform={() => cc.setMainView('platform')} />
+                ) : cc.mainView === 'datasources' || cc.mainView === 'hdfs' ? (
+                  <DataHubView focusEngine={cc.dataSourceFocus} initialTab={cc.mainView === 'hdfs' ? 'hadoop' : 'sources'} onPulse={cc.pulseFlow} />
                 ) : cc.mainView === 'changes' ? (
                   <ChangesView liveChanges={cc.changes} />
                 ) : cc.mainView === 'dataflow' ? (
                   <DataFlowView />
-                ) : cc.mainView === 'presentation' ? (
-                  <PresentationView />
                 ) : cc.mainView === 'dataquality' ? (
                   <DataQualityView />
                 ) : cc.mainView === 'knowledge' ? (
                   <KnowledgeChatView onGpuActivity={setGpuChatActive} />
                 ) : cc.mainView === 'storage' ? (
                   <StorageView />
-                ) : cc.mainView === 'hdfs' ? (
-                  <HdfsView />
                 ) : cc.mainView === 'search' ? (
                   <SearchView />
                 ) : (

ui/src/components/features/ArchitectureDiagram.tsx

@@ -146,9 +146,56 @@ const POSITIONS: Record<string, Record<string, { x: number; y: number }>> = {
   },
 }
 
-export function ArchitectureDiagram({ animation }: { animation: string }) {
+const COMPACT_POSITIONS: Record<string, Record<string, { x: number; y: number }>> = {
+  'full-stack': {
+    user: { x: 14, y: 18 },
+    caddy: { x: 32, y: 18 },
+    ui: { x: 50, y: 18 },
+    api: { x: 12, y: 48 },
+    dq: { x: 34, y: 48 },
+    rag: { x: 56, y: 48 },
+    docling: { x: 34, y: 78 },
+    chroma: { x: 72, y: 48 },
+    llm: { x: 88, y: 78 },
+    lake: { x: 12, y: 78 },
+  },
+  'rag-flow': {
+    upload: { x: 10, y: 22 },
+    store: { x: 10, y: 72 },
+    docling: { x: 28, y: 22 },
+    chunk: { x: 46, y: 22 },
+    embed: { x: 46, y: 47 },
+    chroma: { x: 46, y: 72 },
+    query: { x: 72, y: 22 },
+    retrieve: { x: 72, y: 47 },
+    llm: { x: 90, y: 72 },
+  },
+  'dq-flow': {
+    data: { x: 8, y: 50 },
+    docling: { x: 24, y: 22 },
+    pandas: { x: 24, y: 78 },
+    ge: { x: 46, y: 32 },
+    soda: { x: 46, y: 68 },
+    maturity: { x: 68, y: 50 },
+    report: { x: 88, y: 50 },
+  },
+  'lakehouse': {
+    pg: { x: 6, y: 22 },
+    mysql: { x: 6, y: 42 },
+    mongo: { x: 6, y: 62 },
+    debezium: { x: 24, y: 42 },
+    kafka: { x: 38, y: 42 },
+    spark: { x: 52, y: 42 },
+    iceberg: { x: 66, y: 42 },
+    trino: { x: 80, y: 42 },
+    bi: { x: 92, y: 62 },
+  },
+}
+
+export function ArchitectureDiagram({ animation, compact = false, present = false }: { animation: string; compact?: boolean; present?: boolean }) {
   const flow = FLOWS[animation] || FLOWS['full-stack']
-  const positions = POSITIONS[animation] || POSITIONS['full-stack']
+  const positions = (compact ? COMPACT_POSITIONS[animation] : POSITIONS[animation])
+    || (compact ? COMPACT_POSITIONS['full-stack'] : POSITIONS['full-stack'])
   const [tick, setTick] = useState(0)
 
   useEffect(() => {
@@ -159,7 +206,12 @@ export function ArchitectureDiagram({ animation }: { animation: string }) {
   const activeEdge = tick % flow.edges.length
 
   return (
-    <div className="relative mx-auto mb-6 h-[280px] w-full max-w-4xl rounded-xl border border-docker/30 bg-surface-overlay/60 p-2 md:h-[320px]">
+    <div className={cn(
+      'relative mx-auto w-full rounded-xl border border-docker/30 bg-surface-overlay/60 p-2',
+      present ? 'mb-8 h-[340px] max-w-5xl md:h-[400px]'
+        : compact ? 'mb-3 h-[200px] max-w-none'
+        : 'mb-6 h-[280px] max-w-4xl md:h-[320px]',
+    )}>
       <svg className="absolute inset-0 h-full w-full" viewBox="0 0 100 100" preserveAspectRatio="none">
         {flow.edges.map((edge, i) => {
           const from = positions[edge.from]
@@ -195,15 +247,19 @@ export function ArchitectureDiagram({ animation }: { animation: string }) {
           <div
             key={node.id}
             className={cn(
-              'absolute -translate-x-1/2 -translate-y-1/2 rounded-lg border px-2 py-1 text-center transition-all duration-500',
+              'absolute -translate-x-1/2 -translate-y-1/2 rounded-lg border text-center transition-all duration-500',
+              compact ? 'px-1 py-0.5' : present ? 'px-3 py-2' : 'px-2 py-1',
               lit ? 'scale-105 border-docker shadow-docker bg-docker/20' : 'border-border bg-surface-raised/90',
             )}
-            style={{ left: `${pos.x}%`, top: `${pos.y}%`, minWidth: '72px' }}
+            style={{ left: `${pos.x}%`, top: `${pos.y}%`, minWidth: compact ? '52px' : present ? '88px' : '72px', maxWidth: compact ? '64px' : present ? '120px' : '96px' }}
           >
-            <p className="text-[9px] font-semibold leading-tight text-foreground md:text-[10px]" style={{ color: lit ? node.color : undefined }}>
+            <p className={cn(
+              'font-semibold leading-tight text-foreground',
+              compact ? 'text-[6px]' : present ? 'text-[11px] md:text-xs' : 'text-[9px] md:text-[10px]',
+            )} style={{ color: lit ? node.color : undefined }}>
               {node.label}
             </p>
-            {node.sub && <p className="text-[7px] text-foreground-faint md:text-[8px]">{node.sub}</p>}
+            {node.sub && <p className={cn('text-foreground-faint', compact ? 'text-[5px] leading-none' : present ? 'text-[9px] md:text-[10px]' : 'text-[7px] md:text-[8px]')}>{node.sub}</p>}
           </div>
         )
       })}

ui/src/components/features/DataBrowserGrid.tsx

@@ -0,0 +1,376 @@
+import { useCallback, useMemo, useState } from 'react'
+import { Check, Filter, Loader2, Pencil, RotateCcw, Save, Search, X } from 'lucide-react'
+import type { SourceEngine } from '../../lib/dataSourceCatalog'
+import { cn } from '../../lib/utils'
+import { subTabActive, subTabIdle } from '../../lib/tabActive'
+
+type SampleData = {
+  ok: boolean
+  columns?: string[]
+  rows?: unknown[][]
+  row_count?: number
+  elapsed_ms?: number
+  error?: string
+  primary_keys?: string[]
+  cdc?: boolean
+  editable?: boolean
+  offset?: number
+  limit?: number
+  total_count?: number | null
+}
+
+type DataEngine = SourceEngine | 'hadoop'
+
+type Props = {
+  engine: DataEngine
+  objectFqn: string
+  sample: SampleData | null
+  loading: boolean
+  page: number
+  pageSize: number
+  onPageChange: (page: number) => void
+  onPageSizeChange: (size: number) => void
+  onReload: () => void
+}
+
+const PAGE_SIZES = [100, 250, 500]
+
+function cellStr(v: unknown) {
+  if (v === null || v === undefined) return ''
+  return String(v)
+}
+
+function rowMatches(row: unknown[], columns: string[], filters: Record<string, string>, global: string) {
+  if (global) {
+    const hay = row.map(cellStr).join(' ').toLowerCase()
+    if (!hay.includes(global.toLowerCase())) return false
+  }
+  for (const col of columns) {
+    const f = filters[col]?.trim()
+    if (!f) continue
+    const idx = columns.indexOf(col)
+    const val = cellStr(row[idx]).toLowerCase()
+    if (!val.includes(f.toLowerCase())) return false
+  }
+  return true
+}
+
+export function DataBrowserGrid({
+  engine,
+  objectFqn,
+  sample,
+  loading,
+  page,
+  pageSize,
+  onPageChange,
+  onPageSizeChange,
+  onReload,
+}: Props) {
+  const [globalFilter, setGlobalFilter] = useState('')
+  const [colFilters, setColFilters] = useState<Record<string, string>>({})
+  const [editMode, setEditMode] = useState(false)
+  const [drafts, setDrafts] = useState<Record<number, Record<string, string>>>({})
+  const [savingRow, setSavingRow] = useState<number | null>(null)
+  const [msg, setMsg] = useState<{ text: string; ok: boolean } | null>(null)
+
+  const columns = sample?.columns || []
+  const rows = sample?.rows || []
+  const pks = sample?.primary_keys || []
+  const editable = sample?.editable && pks.length > 0
+  const totalCount = sample?.total_count ?? null
+  const offset = sample?.offset ?? (page - 1) * pageSize
+  const totalPages = totalCount != null ? Math.max(1, Math.ceil(totalCount / pageSize)) : null
+  const rowFrom = rows.length ? offset + 1 : 0
+  const rowTo = offset + rows.length
+
+  const filteredRows = useMemo(() => {
+    return rows
+      .map((row, idx) => ({ row, idx }))
+      .filter(({ row }) => rowMatches(row, columns, colFilters, globalFilter))
+  }, [rows, columns, colFilters, globalFilter])
+
+  const pkValuesForRow = useCallback(
+    (row: unknown[]) => {
+      const pk: Record<string, unknown> = {}
+      for (const k of pks) {
+        const i = columns.indexOf(k)
+        if (i >= 0) pk[k] = row[i]
+      }
+      return pk
+    },
+    [columns, pks],
+  )
+
+  const getDraft = (rowIdx: number, col: string, original: unknown) => {
+    if (drafts[rowIdx]?.[col] !== undefined) return drafts[rowIdx][col]
+    return cellStr(original)
+  }
+
+  const setDraft = (rowIdx: number, col: string, val: string) => {
+    setDrafts((d) => ({ ...d, [rowIdx]: { ...d[rowIdx], [col]: val } }))
+  }
+
+  const rowDirty = (rowIdx: number, row: unknown[]) => {
+    const d = drafts[rowIdx]
+    if (!d) return false
+    return columns.some((col, j) => {
+      if (pks.includes(col)) return false
+      return d[col] !== undefined && d[col] !== cellStr(row[j])
+    })
+  }
+
+  const saveRow = async (rowIdx: number, row: unknown[]) => {
+    const pk = pkValuesForRow(row)
+    if (!Object.keys(pk).length) {
+      setMsg({ text: 'No primary key — cannot save', ok: false })
+      return
+    }
+    const changes: Record<string, unknown> = {}
+    const d = drafts[rowIdx] || {}
+    for (const col of columns) {
+      if (pks.includes(col)) continue
+      if (d[col] !== undefined && d[col] !== cellStr(row[columns.indexOf(col)])) {
+        changes[col] = d[col] === '' ? null : d[col]
+      }
+    }
+    if (!Object.keys(changes).length) return
+
+    setSavingRow(rowIdx)
+    setMsg(null)
+    try {
+      const r = await fetch('/api/sql/row/update', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ engine, object: objectFqn, pk, changes }),
+      })
+      const j = await r.json()
+      if (!j.ok) {
+        setMsg({ text: j.error || 'Save failed', ok: false })
+        return
+      }
+      setMsg({
+        text: j.cdc
+          ? 'Saved — change will appear in Live Changes via CDC'
+          : 'Saved (this source has no CDC stream)',
+        ok: true,
+      })
+      setDrafts((d) => {
+        const next = { ...d }
+        delete next[rowIdx]
+        return next
+      })
+      setTimeout(onReload, 600)
+    } catch {
+      setMsg({ text: 'API unreachable', ok: false })
+    } finally {
+      setSavingRow(null)
+    }
+  }
+
+  const clearFilters = () => {
+    setGlobalFilter('')
+    setColFilters({})
+  }
+
+  if (!sample?.ok && !loading) {
+    return (
+      <p className="p-4 text-[11px] text-danger">{sample?.error || 'Failed to load data'}</p>
+    )
+  }
+
+  return (
+    <div className="flex h-full min-h-0 flex-1 flex-col overflow-hidden">
+      {/* Toolbar */}
+      <div className="flex shrink-0 flex-wrap items-center gap-2 border-b border-border/60 px-3 py-2">
+        <div className="relative min-w-[160px] flex-1">
+          <Search className="absolute left-2 top-1/2 h-3 w-3 -translate-y-1/2 text-foreground-muted" />
+          <input
+            type="text"
+            placeholder="Search all columns…"
+            value={globalFilter}
+            onChange={(e) => setGlobalFilter(e.target.value)}
+            className="w-full rounded border border-border bg-surface-overlay py-1 pl-7 pr-2 text-[10px] text-foreground"
+          />
+        </div>
+        {(globalFilter || Object.values(colFilters).some(Boolean)) && (
+          <button type="button" onClick={clearFilters} className={cn('inline-flex items-center gap-1 rounded-md px-2 py-1 text-[10px]', subTabIdle)}>
+            <X className="h-3 w-3" /> Clear filters
+          </button>
+        )}
+        {editable && (
+          <button
+            type="button"
+            onClick={() => { setEditMode((v) => !v); setDrafts({}) }}
+            className={cn('inline-flex items-center gap-1 rounded-md px-2 py-1 text-[10px]', editMode ? subTabActive : subTabIdle)}
+          >
+            <Pencil className="h-3 w-3" /> {editMode ? 'Editing' : 'Edit rows'}
+          </button>
+        )}
+        {sample?.cdc && (
+          <span className="rounded bg-emerald-500/15 px-2 py-0.5 text-[9px] text-emerald-300">CDC → Live Changes</span>
+        )}
+        {loading && <Loader2 className="h-3.5 w-3.5 animate-spin text-foreground-muted" />}
+      </div>
+
+      {msg && (
+        <p className={cn('shrink-0 px-3 py-1.5 text-[10px]', msg.ok ? 'text-emerald-300' : 'text-danger')}>{msg.text}</p>
+      )}
+
+      <div className="scrollbar-thin min-h-0 flex-1 overflow-x-auto overflow-y-auto overscroll-contain p-2">
+        {columns.length > 0 && (
+          <>
+            <p className="mb-1 font-mono text-[9px] text-foreground-muted">
+              {filteredRows.length} shown
+              {totalCount != null
+                ? ` · rows ${rowFrom.toLocaleString()}–${rowTo.toLocaleString()} of ${totalCount.toLocaleString()}`
+                : ` · page ${page}${totalPages ? ` of ${totalPages.toLocaleString()}` : ''}`}
+              {sample?.elapsed_ms != null && ` · ${sample.elapsed_ms}ms`}
+              {pks.length > 0 && ` · PK: ${pks.join(', ')}`}
+            </p>
+            <table className="w-full text-left font-mono text-[10px]">
+              <thead>
+                <tr className="sticky top-0 z-10 border-b border-border bg-surface-raised text-docker">
+                  {editMode && editable && <th className="w-8 px-1 py-1" />}
+                  {columns.map((c) => (
+                    <th key={c} className="px-2 py-1">
+                      <div className="flex flex-col gap-0.5">
+                        <span className={cn(pks.includes(c) && 'text-amber-300')}>{c}{pks.includes(c) ? ' (PK)' : ''}</span>
+                        <div className="relative">
+                          <Filter className="pointer-events-none absolute left-1 top-1/2 h-2.5 w-2.5 -translate-y-1/2 text-foreground-faint" />
+                          <input
+                            type="text"
+                            placeholder="filter"
+                            value={colFilters[c] || ''}
+                            onChange={(e) => setColFilters((f) => ({ ...f, [c]: e.target.value }))}
+                            className="w-full min-w-[60px] rounded border border-border/60 bg-surface py-0.5 pl-5 pr-1 text-[9px] font-normal text-foreground"
+                          />
+                        </div>
+                      </div>
+                    </th>
+                  ))}
+                </tr>
+              </thead>
+              <tbody>
+                {filteredRows.map(({ row, idx }) => {
+                  const dirty = rowDirty(idx, row)
+                  return (
+                    <tr key={idx} className={cn('border-b border-border/30', dirty && 'bg-violet-500/10')}>
+                      {editMode && editable && (
+                        <td className="px-1 py-1">
+                          {dirty && (
+                            <button
+                              type="button"
+                              disabled={savingRow === idx}
+                              onClick={() => saveRow(idx, row)}
+                              title="Save row"
+                              className="rounded p-0.5 text-emerald-400 hover:bg-emerald-500/20 disabled:opacity-40"
+                            >
+                              {savingRow === idx ? <Loader2 className="h-3.5 w-3.5 animate-spin" /> : <Save className="h-3.5 w-3.5" />}
+                            </button>
+                          )}
+                        </td>
+                      )}
+                      {row.map((cell, j) => {
+                        const col = columns[j]
+                        const isPk = pks.includes(col)
+                        if (editMode && editable && !isPk) {
+                          return (
+                            <td key={j} className="max-w-[180px] px-1 py-0.5">
+                              <input
+                                type="text"
+                                value={getDraft(idx, col, cell)}
+                                onChange={(e) => setDraft(idx, col, e.target.value)}
+                                className="w-full rounded border border-border/60 bg-surface-overlay px-1.5 py-0.5 text-[10px] text-foreground"
+                              />
+                            </td>
+                          )
+                        }
+                        return (
+                          <td key={j} className={cn('max-w-[200px] truncate px-2 py-1', isPk ? 'text-amber-200' : 'text-foreground')}>
+                            {cell === null || cell === undefined ? 'NULL' : String(cell)}
+                          </td>
+                        )
+                      })}
+                    </tr>
+                  )
+                })}
+              </tbody>
+            </table>
+            {filteredRows.length === 0 && (
+              <p className="py-6 text-center text-[11px] text-foreground-muted">No rows match filters</p>
+            )}
+          </>
+        )}
+        {!columns.length && !loading && (
+          <p className="py-8 text-center text-[11px] text-foreground-muted">Select a table, collection or label to preview data</p>
+        )}
+      </div>
+
+      {/* Pagination */}
+      {columns.length > 0 && (
+        <div className="flex shrink-0 flex-wrap items-center justify-between gap-2 border-t border-border/60 bg-surface-raised px-3 py-2">
+          <div className="flex items-center gap-2 text-[10px] text-foreground-muted">
+            <span>Rows per page</span>
+            <select
+              value={pageSize}
+              onChange={(e) => onPageSizeChange(Number(e.target.value))}
+              className="rounded border border-border bg-surface-overlay px-2 py-0.5 text-[10px] text-foreground"
+            >
+              {PAGE_SIZES.map((s) => (
+                <option key={s} value={s}>{s}</option>
+              ))}
+            </select>
+          </div>
+          <div className="flex flex-wrap items-center gap-1">
+            <button
+              type="button"
+              disabled={page <= 1 || loading}
+              onClick={() => onPageChange(1)}
+              className={cn('rounded px-2 py-1 text-[10px]', subTabIdle, 'disabled:opacity-30')}
+            >
+              First
+            </button>
+            <button
+              type="button"
+              disabled={page <= 1 || loading}
+              onClick={() => onPageChange(page - 1)}
+              className={cn('rounded px-2 py-1 text-[10px]', subTabIdle, 'disabled:opacity-30')}
+            >
+              Prev
+            </button>
+            <span className="px-2 font-mono text-[10px] text-foreground">
+              Page {page.toLocaleString()}{totalPages != null ? ` / ${totalPages.toLocaleString()}` : ''}
+            </span>
+            <button
+              type="button"
+              disabled={loading || (totalPages != null ? page >= totalPages : rows.length < pageSize)}
+              onClick={() => onPageChange(page + 1)}
+              className={cn('rounded px-2 py-1 text-[10px]', subTabIdle, 'disabled:opacity-30')}
+            >
+              Next
+            </button>
+            <button
+              type="button"
+              disabled={totalPages == null || page >= totalPages || loading}
+              onClick={() => totalPages && onPageChange(totalPages)}
+              className={cn('rounded px-2 py-1 text-[10px]', subTabIdle, 'disabled:opacity-30')}
+            >
+              Last
+            </button>
+          </div>
+        </div>
+      )}
+
+      {editMode && editable && (
+        <div className="flex shrink-0 items-center gap-2 border-t border-border/60 px-3 py-1.5 text-[9px] text-foreground-muted">
+          <Check className="h-3 w-3 text-emerald-400" />
+          Edit cells, then click <Save className="inline h-3 w-3" /> on a row to save.
+          {sample?.cdc && ' Changes on CDC sources appear in Live Changes within seconds.'}
+          <button type="button" onClick={() => { setDrafts({}); onReload() }} className="ml-auto inline-flex items-center gap-1 text-docker hover:underline">
+            <RotateCcw className="h-3 w-3" /> Reset
+          </button>
+        </div>
+      )}
+    </div>
+  )
+}

ui/src/components/features/DataFlowView.tsx

@@ -1,7 +1,8 @@
 import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from 'react'
-import { GitBranch, Lock, LockOpen, Play, RefreshCw, ShieldAlert, ShieldCheck, Loader2, Bot } from 'lucide-react'
+import { GitBranch, Lock, LockOpen, Play, Pause, Square, RefreshCw, ShieldAlert, ShieldCheck, Loader2, Bot } from 'lucide-react'
 import type { DataflowEdge, DataflowGraph, DataflowNode } from '../../types'
-import { fetchDataflow, runDataflowMovement, toggleEtlAgent, fetchAgentOpsStatus, setPiiMask } from '../../lib/api'
+import { fetchDataflow, runDataflowMovement, toggleEtlAgent, toggleCustodianOffload, fetchAgentOpsStatus, setPiiMask, setStreamingFlow } from '../../lib/api'
+import { SparkKafkaPanel } from './SparkKafkaPanel'
 import { Badge } from '../ui/Badge'
 import { cn } from '../../lib/utils'
 
@@ -14,6 +15,7 @@ const NODE_KIND: Record<string, { ring: string; chip: string; dot: string }> = {
   stream: { ring: 'border-cyan-400/60', chip: 'bg-cyan-500/15 text-cyan-300 border-cyan-400/40', dot: '#22d3ee' },
   sink: { ring: 'border-sky-400/60', chip: 'bg-sky-500/15 text-sky-300 border-sky-400/40', dot: '#38bdf8' },
   lakehouse: { ring: 'border-blue-400/60', chip: 'bg-blue-500/15 text-blue-300 border-blue-400/40', dot: '#60a5fa' },
+  compute: { ring: 'border-violet-400/60', chip: 'bg-violet-500/15 text-violet-300 border-violet-400/40', dot: '#a78bfa' },
   engine: { ring: 'border-violet-400/60', chip: 'bg-violet-500/15 text-violet-300 border-violet-400/40', dot: '#a78bfa' },
   governance: { ring: 'border-fuchsia-400/60', chip: 'bg-fuchsia-500/15 text-fuchsia-300 border-fuchsia-400/40', dot: '#d946ef' },
 }
@@ -70,6 +72,7 @@ export function DataFlowView() {
   const [selected, setSelected] = useState<string | null>(null)
   const [triggering, setTriggering] = useState<string | null>(null)
   const [etlEnabled, setEtlEnabled] = useState<boolean | null>(null)
+  const [custEnabled, setCustEnabled] = useState<boolean | null>(null)
 
   const canvasRef = useRef<HTMLDivElement>(null)
   const nodeRefs = useRef<Record<string, HTMLButtonElement | null>>({})
@@ -91,6 +94,8 @@ export function DataFlowView() {
     fetchAgentOpsStatus().then((s) => {
       const etl = (s as { etl?: { enabled?: boolean } })?.etl
       if (etl) setEtlEnabled(!!etl.enabled)
+      const cust = (s as { custodian?: { enabled?: boolean } })?.custodian
+      if (cust) setCustEnabled(!!cust.enabled)
     })
     const iv = setInterval(() => load(), 6000)
     return () => clearInterval(iv)
@@ -152,12 +157,24 @@ export function DataFlowView() {
     await toggleEtlAgent(next)
   }, [etlEnabled])
 
+  const onToggleCust = useCallback(async () => {
+    const next = !custEnabled
+    setCustEnabled(next)
+    await toggleCustodianOffload(next)
+  }, [custEnabled])
+
   const onRefresh = useCallback(async () => {
     setRefreshing(true)
     await load(true)
     setTimeout(() => setRefreshing(false), 400)
   }, [load])
 
+  const flowMode = (graph as unknown as { flow?: string })?.flow ?? 'running'
+  const onFlow = useCallback(async (action: 'pause' | 'resume' | 'stop') => {
+    await setStreamingFlow(action)
+    setTimeout(() => load(true), 300)
+  }, [load])
+
   const [maskBusy, setMaskBusy] = useState<string | null>(null)
   const onToggleMask = useCallback(async (key: string, column: string, masked: boolean) => {
     setMaskBusy(`${key}.${column}`)
@@ -202,7 +219,7 @@ export function DataFlowView() {
             <div className="min-w-0">
               <h2 className="truncate text-xs font-semibold text-foreground">Data Flow · live lineage</h2>
               <p className="truncate text-[9px] text-foreground-muted">
-                Generators → sources → CDC/Kafka → lakehouse · click a node for PII detail · trigger movements below
+                Generators → sources → CDC/Kafka → Spark → lakehouse · pulses show live flow · Spark/Kafka panel below
               </p>
             </div>
           </div>
@@ -240,6 +257,33 @@ export function DataFlowView() {
             >
               <Bot className="h-3 w-3" /> ETL agent {etlEnabled == null ? '' : etlEnabled ? 'on' : 'off'}
             </button>
+            <button
+              type="button"
+              onClick={onToggleCust}
+              title="Data Custodian: autonomous batch offload of source data into the Hadoop Iceberg lake"
+              className={cn(
+                'inline-flex items-center gap-1 rounded border px-1.5 py-0.5 text-[9px] font-medium transition-colors',
+                custEnabled
+                  ? 'border-orange-400/50 bg-orange-500/20 text-orange-200'
+                  : 'border-border bg-transparent text-foreground-muted hover:text-foreground',
+              )}
+            >
+              <Bot className="h-3 w-3" /> Hadoop offload {custEnabled == null ? '' : custEnabled ? 'on' : 'off'}
+            </button>
+            <div className="inline-flex items-center gap-0.5 rounded border border-border p-0.5" title="Master pulse control">
+              <button type="button" onClick={() => onFlow('resume')}
+                className={cn('inline-flex items-center gap-1 rounded px-1.5 py-0.5 text-[9px] font-medium transition-colors', flowMode === 'running' ? 'bg-emerald-500/25 text-emerald-200' : 'text-foreground-muted hover:text-foreground')}>
+                <Play className="h-3 w-3" /> Run
+              </button>
+              <button type="button" onClick={() => onFlow('pause')}
+                className={cn('inline-flex items-center gap-1 rounded px-1.5 py-0.5 text-[9px] font-medium transition-colors', flowMode === 'paused' ? 'bg-amber-500/25 text-amber-200' : 'text-foreground-muted hover:text-foreground')}>
+                <Pause className="h-3 w-3" /> Pause
+              </button>
+              <button type="button" onClick={() => onFlow('stop')}
+                className={cn('inline-flex items-center gap-1 rounded px-1.5 py-0.5 text-[9px] font-medium transition-colors', flowMode === 'stopped' ? 'bg-rose-500/25 text-rose-200' : 'text-foreground-muted hover:text-foreground')}>
+                <Square className="h-3 w-3" /> Stop
+              </button>
+            </div>
             <button
               type="button"
               onClick={onRefresh}
@@ -333,11 +377,19 @@ export function DataFlowView() {
                 href={selNode.url}
                 target="_blank"
                 rel="noreferrer"
-                className="mt-1 inline-block rounded border border-fuchsia-400/40 bg-fuchsia-500/15 px-1.5 py-0.5 text-[8px] font-medium text-fuchsia-200 hover:bg-fuchsia-500/25"
+                className={cn(
+                  'mt-1 inline-block rounded border px-1.5 py-0.5 text-[8px] font-medium hover:opacity-90',
+                  selNode.id === 'openmetadata'
+                    ? 'border-fuchsia-400/40 bg-fuchsia-500/15 text-fuchsia-200'
+                    : 'border-docker/40 bg-docker/15 text-docker',
+                )}
               >
-                Open in OpenMetadata ↗
+                Open {selNode.label} ↗
               </a>
             )}
+            {(selNode.id === 'spark' || selNode.id === 'kafka') && (
+              <p className="mt-1 text-[8px] text-docker/80">See Spark/Kafka panel below for full UI + job control.</p>
+            )}
             {selNode.pii?.has_pii ? (
               <div className="mt-1.5 border-t border-border pt-1.5">
                 <div className="mb-1 flex items-center justify-between gap-1 text-[9px] font-medium text-rose-300">
@@ -400,6 +452,13 @@ export function DataFlowView() {
         )}
       </div>
 
+      <SparkKafkaPanel
+        embedded
+        selectedNodeId={selected}
+        streaming={graph?.streaming ?? null}
+        onRefreshGraph={() => load(true)}
+      />
+
       {/* movement control strip */}
       <div className="shrink-0 border-t border-border bg-surface/60 px-3 py-1.5">
         <div className="mb-1 flex items-center gap-1 text-[8px] font-semibold uppercase tracking-wide text-foreground-muted">

ui/src/components/features/DataGenPanel.tsx

@@ -0,0 +1,370 @@
+import { useCallback, useEffect, useRef, useState } from 'react'
+import { Database, Boxes, Activity, Network, Layers, Play, Loader2, CheckCircle2, XCircle, RefreshCw, Bot, ScrollText } from 'lucide-react'
+import { cn } from '../../lib/utils'
+import { subTabActive, subTabIdle } from '../../lib/tabActive'
+
+export type SourceKey = 'all' | 'postgres' | 'mysql' | 'mongodb' | 'cassandra' | 'neo4j'
+
+type SourceMeta = {
+  key: SourceKey
+  label: string
+  icon: typeof Database
+  accent: string
+  target: string
+  cdc: boolean
+  desc: string
+  defaultRows: number
+}
+
+export const DATA_GEN_SOURCES: SourceMeta[] = [
+  { key: 'all', label: 'All sources', icon: Layers, accent: 'text-violet-400', target: 'all 5 databases', cdc: true, desc: 'Generate across all databases at once.', defaultRows: 5000 },
+  { key: 'postgres', label: 'PostgreSQL', icon: Database, accent: 'text-sky-400', target: 'sales_orders', cdc: true, desc: 'Sales orders. CDC via Debezium -> Kafka -> S3.', defaultRows: 5000 },
+  { key: 'mysql', label: 'MySQL', icon: Database, accent: 'text-amber-400', target: 'employee_events', cdc: true, desc: 'HR employee events. CDC via Debezium -> Kafka -> S3.', defaultRows: 5000 },
+  { key: 'mongodb', label: 'MongoDB', icon: Boxes, accent: 'text-emerald-400', target: 'supplychain.events', cdc: true, desc: 'Supply chain events. CDC via Debezium -> Kafka -> S3.', defaultRows: 5000 },
+  { key: 'cassandra', label: 'Cassandra', icon: Activity, accent: 'text-cyan-400', target: 'device_metrics', cdc: false, desc: 'Telemetry metrics. Queryable via Trino.', defaultRows: 5000 },
+  { key: 'neo4j', label: 'Neo4j', icon: Network, accent: 'text-pink-400', target: 'Product/Supplier graph', cdc: false, desc: 'Graph data (products, suppliers, relationships).', defaultRows: 2000 },
+]
+
+type RunInfo = { run_id?: string; state?: string; start?: string; end?: string; conf?: { rows?: number } }
+type AgentInfo = { agent_id: string; agent_name: string }
+type ActivityItem = { id: string; ts?: string; agent_id: string; agent_name: string; message: string; level: string }
+
+type Props = {
+  onPulse: () => void
+  /** Lock to one source (embedded in Data Sources UI) */
+  focusSource?: Exclude<SourceKey, 'all'>
+  embedded?: boolean
+  onOpenPlatform?: () => void
+}
+
+const COUNT_KEYS: SourceKey[] = ['postgres', 'mysql', 'mongodb', 'cassandra']
+
+export function DataGenPanel({ onPulse, focusSource, embedded = false, onOpenPlatform }: Props) {
+  const [active, setActive] = useState<SourceKey>(focusSource || 'all')
+  const [rows, setRows] = useState<Record<SourceKey, number>>(
+    Object.fromEntries(DATA_GEN_SOURCES.map((s) => [s.key, s.defaultRows])) as Record<SourceKey, number>,
+  )
+  const [busy, setBusy] = useState<Record<string, boolean>>({})
+  const [runs, setRuns] = useState<Record<string, RunInfo[]>>({})
+  const [counts, setCounts] = useState<Record<string, number | null>>({})
+  const [msg, setMsg] = useState<string | null>(null)
+  const [agentMap, setAgentMap] = useState<Record<string, AgentInfo>>({})
+  const [activity, setActivity] = useState<ActivityItem[]>([])
+  const pollRef = useRef<Record<string, ReturnType<typeof setInterval>>>({})
+
+  useEffect(() => {
+    if (focusSource) setActive(focusSource)
+  }, [focusSource])
+
+  const meta = DATA_GEN_SOURCES.find((s) => s.key === active)!
+
+  const loadCounts = useCallback(async () => {
+    try {
+      const r = await fetch('/api/pipeline/sync')
+      const j = await r.json()
+      if (j.ok) setCounts(j.counts || {})
+    } catch { /* */ }
+  }, [])
+
+  const loadActivity = useCallback(async () => {
+    try {
+      const r = await fetch('/api/pipeline/activity?limit=25')
+      const j = await r.json()
+      if (j.ok) setActivity(j.activity || [])
+    } catch { /* */ }
+  }, [])
+
+  const loadRuns = useCallback(async (source: SourceKey) => {
+    try {
+      const r = await fetch(`/api/pipeline/runs/${source}?limit=5`)
+      const j = await r.json()
+      if (j.ok) setRuns((prev) => ({ ...prev, [source]: j.runs || [] }))
+      return (j.runs || [])[0] as RunInfo | undefined
+    } catch {
+      return undefined
+    }
+  }, [])
+
+  useEffect(() => {
+    loadCounts()
+    loadActivity()
+    const keys: SourceKey[] = focusSource ? [focusSource, 'all'] : DATA_GEN_SOURCES.map((s) => s.key)
+    keys.forEach((k) => loadRuns(k))
+    fetch('/api/pipeline/agents').then((r) => r.json()).then((j) => { if (j.ok) setAgentMap(j.agents || {}) }).catch(() => {})
+    const act = setInterval(loadActivity, 8000)
+    const poll = pollRef.current
+    return () => { Object.values(poll).forEach(clearInterval); clearInterval(act) }
+  }, [loadCounts, loadRuns, loadActivity, focusSource])
+
+  const startPolling = useCallback((source: SourceKey) => {
+    if (pollRef.current[source]) clearInterval(pollRef.current[source])
+    pollRef.current[source] = setInterval(async () => {
+      const latest = await loadRuns(source)
+      if (latest && (latest.state === 'success' || latest.state === 'failed')) {
+        clearInterval(pollRef.current[source])
+        delete pollRef.current[source]
+        setBusy((b) => ({ ...b, [source]: false }))
+        loadCounts()
+        if (latest.state === 'success') setMsg(`${source}: complete — new data generated`)
+        else setMsg(`${source}: run failed — check Airflow logs`)
+      }
+    }, 3000)
+  }, [loadRuns, loadCounts])
+
+  const generate = useCallback(async (source: SourceKey, autonomous = false) => {
+    setMsg(null)
+    setBusy((b) => ({ ...b, [source]: true }))
+    onPulse()
+    try {
+      const r = await fetch(`/api/pipeline/generate/${source}`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ rows: rows[source], autonomous }),
+      })
+      const j = await r.json()
+      if (!j.ok) {
+        setMsg(`Error: ${j.error || 'could not trigger run'}`)
+        setBusy((b) => ({ ...b, [source]: false }))
+        return
+      }
+      const who = autonomous ? `${j.agent_name || 'Agent'} generating autonomously` : 'started'
+      setMsg(`${source}: ${who} (run ${String(j.run_id).slice(-8)})`)
+      startPolling(source)
+      setTimeout(loadActivity, 800)
+    } catch {
+      setMsg('API unreachable')
+      setBusy((b) => ({ ...b, [source]: false }))
+    }
+  }, [rows, onPulse, startPolling, loadActivity])
+
+  const stateBadge = (state?: string) => {
+    if (state === 'success') return <span className="inline-flex items-center gap-1 text-emerald-400"><CheckCircle2 className="h-3 w-3" /> success</span>
+    if (state === 'failed') return <span className="inline-flex items-center gap-1 text-danger"><XCircle className="h-3 w-3" /> failed</span>
+    if (state === 'running' || state === 'queued') return <span className="inline-flex items-center gap-1 text-amber-400"><Loader2 className="h-3 w-3 animate-spin" /> {state}</span>
+    return <span className="text-foreground-faint">{state || '—'}</span>
+  }
+
+  const latest = (runs[active] || [])[0]
+  const showSourceTabs = !embedded && !focusSource
+
+  return (
+    <div className={cn('flex h-full min-h-0 flex-col', embedded ? '' : 'panel flex-1 overflow-hidden')}>
+      {!embedded && (
+        <header className="flex shrink-0 flex-wrap items-center justify-between gap-2 border-b border-border px-4 py-3">
+          <div>
+            <h2 className="text-sm font-semibold text-foreground">Data Generation</h2>
+            <p className="text-[10px] text-foreground-muted">
+              Generate new data per database — source → Debezium → Kafka → S3
+            </p>
+          </div>
+          <div className="flex gap-2">
+            <button type="button" onClick={() => { loadCounts(); DATA_GEN_SOURCES.forEach((s) => loadRuns(s.key)) }} className={cn('rounded-md px-3 py-1.5 text-[11px]', subTabIdle)}>
+              <RefreshCw className="inline h-3 w-3" /> Refresh
+            </button>
+            {onOpenPlatform && (
+              <button type="button" onClick={onOpenPlatform} className={cn('rounded-md px-3 py-1.5 text-[11px]', subTabActive)}>
+                View topology pulse
+              </button>
+            )}
+          </div>
+        </header>
+      )}
+
+      {showSourceTabs && (
+        <div className="flex shrink-0 flex-wrap gap-1 border-b border-border px-3 py-2">
+          {DATA_GEN_SOURCES.map((s) => {
+            const Icon = s.icon
+            return (
+              <button
+                key={s.key}
+                type="button"
+                onClick={() => setActive(s.key)}
+                className={cn('inline-flex items-center gap-1.5 rounded-md px-3 py-1.5 text-[11px] font-medium', active === s.key ? subTabActive : subTabIdle)}
+              >
+                <Icon className={cn('h-3.5 w-3.5', s.accent)} />
+                {s.label}
+                {busy[s.key] && <Loader2 className="h-3 w-3 animate-spin text-amber-400" />}
+              </button>
+            )
+          })}
+        </div>
+      )}
+
+      <div className="scrollbar-thin min-h-0 flex-1 overflow-y-auto p-4">
+        <div className={cn('space-y-4', embedded ? 'max-w-none' : 'max-w-2xl')}>
+          {embedded && (
+            <div className="flex flex-wrap items-center gap-2">
+              <button
+                type="button"
+                disabled={!!busy.all}
+                onClick={() => generate('all')}
+                className="inline-flex items-center gap-1.5 rounded-md border border-violet-400/40 bg-violet-500/10 px-3 py-1.5 text-[10px] font-medium text-violet-300 hover:bg-violet-500/20 disabled:opacity-40"
+              >
+                {busy.all ? <Loader2 className="h-3 w-3 animate-spin" /> : <Layers className="h-3 w-3" />}
+                Generate all databases
+              </button>
+              <button type="button" onClick={() => { loadCounts(); loadRuns(active); loadActivity() }} className={cn('rounded-md px-2 py-1 text-[10px]', subTabIdle)}>
+                <RefreshCw className="inline h-3 w-3" /> Refresh
+              </button>
+            </div>
+          )}
+
+          <div className="rounded-lg border border-border bg-surface-overlay/40 p-4">
+            <div className="mb-1 flex flex-wrap items-center gap-2">
+              <meta.icon className={cn('h-5 w-5', meta.accent)} />
+              <h3 className="text-sm font-semibold text-foreground">{meta.label}</h3>
+              {meta.cdc ? (
+                <span className="rounded bg-emerald-500/15 px-1.5 py-0.5 text-[9px] text-emerald-400">CDC active</span>
+              ) : (
+                <span className="rounded bg-white/10 px-1.5 py-0.5 text-[9px] text-foreground-muted">no CDC stream</span>
+              )}
+              {agentMap[active] && (
+                <span className="inline-flex items-center gap-1 rounded bg-violet-500/15 px-1.5 py-0.5 text-[9px] text-violet-300">
+                  <Bot className="h-2.5 w-2.5" /> {agentMap[active].agent_name}
+                </span>
+              )}
+            </div>
+            <p className="mb-3 text-[11px] text-foreground-muted">{meta.desc} Target: <span className="font-mono text-foreground">{meta.target}</span></p>
+
+            <div className="flex flex-wrap items-end gap-3">
+              <label className="flex flex-col gap-1 text-[10px] uppercase tracking-wide text-foreground-muted">
+                Row count
+                <input
+                  type="number"
+                  min={1}
+                  max={2000000}
+                  value={rows[active]}
+                  onChange={(e) => setRows((r) => ({ ...r, [active]: Number(e.target.value) }))}
+                  className="w-40 rounded border border-border bg-background px-2 py-1.5 text-[12px] text-foreground"
+                />
+              </label>
+              <button
+                type="button"
+                disabled={!!busy[active]}
+                onClick={() => generate(active)}
+                className="inline-flex items-center gap-2 rounded-md bg-violet-500/90 px-4 py-2 text-[12px] font-semibold text-black hover:bg-violet-400 disabled:opacity-40"
+              >
+                {busy[active] ? <Loader2 className="h-4 w-4 animate-spin" /> : <Play className="h-4 w-4" />}
+                Generate data
+              </button>
+              <button
+                type="button"
+                disabled={!!busy[active]}
+                onClick={() => generate(active, true)}
+                title="Let the assigned agent generate data autonomously"
+                className="inline-flex items-center gap-2 rounded-md border border-violet-400/50 px-3 py-2 text-[12px] font-medium text-violet-300 hover:bg-violet-500/10 disabled:opacity-40"
+              >
+                <Bot className="h-4 w-4" />
+                Let agent generate
+              </button>
+              {active !== 'all' && COUNT_KEYS.includes(active) && (
+                <div className="text-[11px] text-foreground-muted">
+                  Current rows: <span className="font-mono text-foreground">{counts[active]?.toLocaleString() ?? '…'}</span>
+                </div>
+              )}
+            </div>
+            {msg && <p className="mt-3 text-[11px] text-foreground-muted">{msg}</p>}
+          </div>
+
+          <div className="rounded-lg border border-border p-4">
+            <h4 className="mb-2 text-[11px] font-semibold uppercase tracking-wide text-foreground-muted">Recent runs — {meta.label}</h4>
+            {latest ? (
+              <table className="w-full text-left text-[11px]">
+                <thead>
+                  <tr className="border-b border-border text-[9px] uppercase text-foreground-muted">
+                    <th className="py-1 pr-2">State</th>
+                    <th className="py-1 pr-2">Rows</th>
+                    <th className="py-1 pr-2">Start</th>
+                    <th className="py-1 pr-2">End</th>
+                  </tr>
+                </thead>
+                <tbody>
+                  {(runs[active] || []).map((run) => (
+                    <tr key={run.run_id} className="border-b border-border/40">
+                      <td className="py-1 pr-2">{stateBadge(run.state)}</td>
+                      <td className="py-1 pr-2 font-mono text-foreground">{run.conf?.rows ?? '—'}</td>
+                      <td className="py-1 pr-2 text-foreground-muted">{run.start?.slice(11, 19) || '—'}</td>
+                      <td className="py-1 pr-2 text-foreground-muted">{run.end?.slice(11, 19) || '—'}</td>
+                    </tr>
+                  ))}
+                </tbody>
+              </table>
+            ) : (
+              <p className="text-[11px] text-foreground-muted">No runs yet.</p>
+            )}
+          </div>
+
+          {!embedded && (
+            <>
+              <div className="rounded-lg border border-border p-4">
+                <h4 className="mb-2 text-[11px] font-semibold uppercase tracking-wide text-foreground-muted">Live counts (Trino)</h4>
+                <div className="grid grid-cols-2 gap-2 sm:grid-cols-4">
+                  {COUNT_KEYS.map((k) => (
+                    <div key={k} className="rounded border border-border/60 px-2 py-1.5">
+                      <div className="text-[9px] uppercase text-foreground-muted">{k}</div>
+                      <div className="font-mono text-[12px] text-foreground">{counts[k]?.toLocaleString() ?? '…'}</div>
+                    </div>
+                  ))}
+                </div>
+              </div>
+
+              <div className="rounded-lg border border-border p-4">
+                <h4 className="mb-2 flex items-center gap-1.5 text-[11px] font-semibold uppercase tracking-wide text-foreground-faint">
+                  <ScrollText className="h-3.5 w-3.5" /> Agent activity
+                </h4>
+                {activity.length === 0 ? (
+                  <p className="text-[11px] text-foreground-muted">No agent actions logged yet.</p>
+                ) : (
+                  <ul className="space-y-1">
+                    {activity.map((a) => (
+                      <li key={a.id} className="flex items-start gap-2 text-[11px]">
+                        <span className="mt-0.5 text-foreground-muted">{a.ts?.slice(11, 19) || ''}</span>
+                        <span className="inline-flex items-center gap-1 rounded bg-violet-500/15 px-1 text-[9px] text-violet-300">
+                          <Bot className="h-2.5 w-2.5" />{a.agent_name}
+                        </span>
+                        <span className={cn('flex-1', a.level === 'err' ? 'text-danger' : 'text-foreground-muted')}>{a.message}</span>
+                      </li>
+                    ))}
+                  </ul>
+                )}
+              </div>
+            </>
+          )}
+
+          {embedded && (
+            <div className="grid gap-4 lg:grid-cols-2">
+              <div className="rounded-lg border border-border p-4">
+                <h4 className="mb-2 text-[11px] font-semibold uppercase tracking-wide text-foreground-muted">Live counts (Trino)</h4>
+                <div className="grid grid-cols-2 gap-2">
+                  {COUNT_KEYS.map((k) => (
+                    <div key={k} className="rounded border border-border/60 px-2 py-1.5">
+                      <div className="text-[9px] uppercase text-foreground-muted">{k}</div>
+                      <div className="font-mono text-[12px] text-foreground">{counts[k]?.toLocaleString() ?? '…'}</div>
+                    </div>
+                  ))}
+                </div>
+              </div>
+              <div className="rounded-lg border border-border p-4">
+                <h4 className="mb-2 flex items-center gap-1.5 text-[11px] font-semibold uppercase tracking-wide text-foreground-faint">
+                  <ScrollText className="h-3.5 w-3.5" /> Agent activity
+                </h4>
+                {activity.length === 0 ? (
+                  <p className="text-[11px] text-foreground-muted">No agent actions yet.</p>
+                ) : (
+                  <ul className="scrollbar-thin max-h-[140px] space-y-1 overflow-y-auto">
+                    {activity.slice(0, 8).map((a) => (
+                      <li key={a.id} className="flex items-start gap-2 text-[10px]">
+                        <span className="text-foreground-muted">{a.ts?.slice(11, 19) || ''}</span>
+                        <span className="flex-1 text-foreground">{a.message}</span>
+                      </li>
+                    ))}
+                  </ul>
+                )}
+              </div>
+            </div>
+          )}
+        </div>
+      </div>
+    </div>
+  )
+}

ui/src/components/features/DataGenView.tsx

@@ -1,303 +1,11 @@
-import { useCallback, useEffect, useRef, useState } from 'react'
-import { Database, Boxes, Activity, Cpu, Network, Layers, Play, Loader2, CheckCircle2, XCircle, RefreshCw, Bot, ScrollText } from 'lucide-react'
-import { cn } from '../../lib/utils'
-import { subTabActive, subTabIdle } from '../../lib/tabActive'
-
-type SourceKey = 'all' | 'postgres' | 'mysql' | 'mongodb' | 'cassandra' | 'neo4j'
-
-type SourceMeta = {
-  key: SourceKey
-  label: string
-  icon: typeof Database
-  accent: string
-  target: string
-  cdc: boolean
-  desc: string
-  defaultRows: number
-}
-
-const SOURCES: SourceMeta[] = [
-  { key: 'all', label: 'All sources', icon: Layers, accent: 'text-violet-400', target: 'alle 5 databases', cdc: true, desc: 'Genereer tegelijk in alle databases.', defaultRows: 5000 },
-  { key: 'postgres', label: 'PostgreSQL', icon: Database, accent: 'text-sky-400', target: 'sales_orders', cdc: true, desc: 'Sales orders. CDC via Debezium -> Kafka -> S3.', defaultRows: 5000 },
-  { key: 'mysql', label: 'MySQL', icon: Database, accent: 'text-amber-400', target: 'employee_events', cdc: true, desc: 'HR employee events. CDC via Debezium -> Kafka -> S3.', defaultRows: 5000 },
-  { key: 'mongodb', label: 'MongoDB', icon: Boxes, accent: 'text-emerald-400', target: 'supplychain.events', cdc: true, desc: 'Supply chain events. CDC via Debezium -> Kafka -> S3.', defaultRows: 5000 },
-  { key: 'cassandra', label: 'Cassandra', icon: Activity, accent: 'text-cyan-400', target: 'device_metrics', cdc: false, desc: 'Telemetry metrics. Zichtbaar via Trino.', defaultRows: 5000 },
-  { key: 'neo4j', label: 'Neo4j', icon: Network, accent: 'text-pink-400', target: 'Product/Supplier graph', cdc: false, desc: 'Graafdata (producten, leveranciers, relaties).', defaultRows: 2000 },
-]
-
-type RunInfo = { run_id?: string; state?: string; start?: string; end?: string; conf?: { rows?: number } }
-type AgentInfo = { agent_id: string; agent_name: string }
-type ActivityItem = { id: string; ts?: string; agent_id: string; agent_name: string; message: string; level: string }
+import { DataGenPanel } from './DataGenPanel'
 
 type Props = {
   onPulse: () => void
   onOpenPlatform: () => void
 }
 
-const COUNT_KEYS: SourceKey[] = ['postgres', 'mysql', 'mongodb', 'cassandra']
-
+/** Standalone page wrapper — prefer Data Sources UI → Generate tab */
 export function DataGenView({ onPulse, onOpenPlatform }: Props) {
-  const [active, setActive] = useState<SourceKey>('all')
-  const [rows, setRows] = useState<Record<SourceKey, number>>(
-    Object.fromEntries(SOURCES.map((s) => [s.key, s.defaultRows])) as Record<SourceKey, number>,
-  )
-  const [busy, setBusy] = useState<Record<string, boolean>>({})
-  const [runs, setRuns] = useState<Record<string, RunInfo[]>>({})
-  const [counts, setCounts] = useState<Record<string, number | null>>({})
-  const [msg, setMsg] = useState<string | null>(null)
-  const [agentMap, setAgentMap] = useState<Record<string, AgentInfo>>({})
-  const [activity, setActivity] = useState<ActivityItem[]>([])
-  const pollRef = useRef<Record<string, ReturnType<typeof setInterval>>>({})
-
-  const meta = SOURCES.find((s) => s.key === active)!
-
-  const loadCounts = useCallback(async () => {
-    try {
-      const r = await fetch('/api/pipeline/sync')
-      const j = await r.json()
-      if (j.ok) setCounts(j.counts || {})
-    } catch { /* */ }
-  }, [])
-
-  const loadActivity = useCallback(async () => {
-    try {
-      const r = await fetch('/api/pipeline/activity?limit=25')
-      const j = await r.json()
-      if (j.ok) setActivity(j.activity || [])
-    } catch { /* */ }
-  }, [])
-
-  const loadRuns = useCallback(async (source: SourceKey) => {
-    try {
-      const r = await fetch(`/api/pipeline/runs/${source}?limit=5`)
-      const j = await r.json()
-      if (j.ok) setRuns((prev) => ({ ...prev, [source]: j.runs || [] }))
-      return (j.runs || [])[0] as RunInfo | undefined
-    } catch {
-      return undefined
-    }
-  }, [])
-
-  useEffect(() => {
-    loadCounts()
-    loadActivity()
-    SOURCES.forEach((s) => loadRuns(s.key))
-    fetch('/api/pipeline/agents').then((r) => r.json()).then((j) => { if (j.ok) setAgentMap(j.agents || {}) }).catch(() => {})
-    const act = setInterval(loadActivity, 8000)
-    const poll = pollRef.current
-    return () => { Object.values(poll).forEach(clearInterval); clearInterval(act) }
-  }, [loadCounts, loadRuns, loadActivity])
-
-  const startPolling = useCallback((source: SourceKey) => {
-    if (pollRef.current[source]) clearInterval(pollRef.current[source])
-    pollRef.current[source] = setInterval(async () => {
-      const latest = await loadRuns(source)
-      if (latest && (latest.state === 'success' || latest.state === 'failed')) {
-        clearInterval(pollRef.current[source])
-        delete pollRef.current[source]
-        setBusy((b) => ({ ...b, [source]: false }))
-        loadCounts()
-        if (latest.state === 'success') setMsg(`${source}: klaar — nieuwe data gegenereerd`)
-        else setMsg(`${source}: run mislukt — check Airflow logs`)
-      }
-    }, 3000)
-  }, [loadRuns, loadCounts])
-
-  const generate = useCallback(async (source: SourceKey, autonomous = false) => {
-    setMsg(null)
-    setBusy((b) => ({ ...b, [source]: true }))
-    onPulse()
-    try {
-      const r = await fetch(`/api/pipeline/generate/${source}`, {
-        method: 'POST',
-        headers: { 'Content-Type': 'application/json' },
-        body: JSON.stringify({ rows: rows[source], autonomous }),
-      })
-      const j = await r.json()
-      if (!j.ok) {
-        setMsg(`Fout: ${j.error || 'kon niet triggeren'}`)
-        setBusy((b) => ({ ...b, [source]: false }))
-        return
-      }
-      const who = autonomous ? `${j.agent_name || 'Agent'} genereert zelf` : 'gestart'
-      setMsg(`${source}: ${who} (run ${String(j.run_id).slice(-8)})`)
-      startPolling(source)
-      setTimeout(loadActivity, 800)
-    } catch {
-      setMsg('API niet bereikbaar')
-      setBusy((b) => ({ ...b, [source]: false }))
-    }
-  }, [rows, onPulse, startPolling, loadActivity])
-
-  const stateBadge = (state?: string) => {
-    if (state === 'success') return <span className="inline-flex items-center gap-1 text-emerald-400"><CheckCircle2 className="h-3 w-3" /> success</span>
-    if (state === 'failed') return <span className="inline-flex items-center gap-1 text-danger"><XCircle className="h-3 w-3" /> failed</span>
-    if (state === 'running' || state === 'queued') return <span className="inline-flex items-center gap-1 text-amber-400"><Loader2 className="h-3 w-3 animate-spin" /> {state}</span>
-    return <span className="text-foreground-faint">{state || '—'}</span>
-  }
-
-  const latest = (runs[active] || [])[0]
-
-  return (
-    <div className="panel flex h-full min-h-0 flex-1 flex-col overflow-hidden">
-      <header className="flex shrink-0 flex-wrap items-center justify-between gap-2 border-b border-border px-4 py-3">
-        <div>
-          <h2 className="flex items-center gap-2 text-sm font-semibold text-foreground">
-            <Cpu className="h-4 w-4 text-violet-400" /> Data Generation
-          </h2>
-          <p className="text-[10px] text-foreground-muted">
-            Genereer per database nieuwe data en pulse de hele flow: bron -&gt; Debezium -&gt; Kafka -&gt; S3
-          </p>
-        </div>
-        <div className="flex gap-2">
-          <button type="button" onClick={() => { loadCounts(); SOURCES.forEach((s) => loadRuns(s.key)) }} className={cn('rounded-md px-3 py-1.5 text-[11px]', subTabIdle)}>
-            <RefreshCw className="inline h-3 w-3" /> Refresh
-          </button>
-          <button type="button" onClick={onOpenPlatform} className={cn('rounded-md px-3 py-1.5 text-[11px]', subTabActive)}>
-            Bekijk topology pulse
-          </button>
-        </div>
-      </header>
-
-      <div className="flex shrink-0 flex-wrap gap-1 border-b border-border px-3 py-2">
-        {SOURCES.map((s) => {
-          const Icon = s.icon
-          return (
-            <button
-              key={s.key}
-              type="button"
-              onClick={() => setActive(s.key)}
-              className={cn('inline-flex items-center gap-1.5 rounded-md px-3 py-1.5 text-[11px] font-medium', active === s.key ? subTabActive : subTabIdle)}
-            >
-              <Icon className={cn('h-3.5 w-3.5', s.accent)} />
-              {s.label}
-              {busy[s.key] && <Loader2 className="h-3 w-3 animate-spin text-amber-400" />}
-            </button>
-          )
-        })}
-      </div>
-
-      <div className="scrollbar-thin min-h-0 flex-1 overflow-y-auto p-4">
-        <div className="max-w-2xl space-y-4">
-          <div className="rounded-lg border border-border bg-surface-overlay/40 p-4">
-            <div className="mb-1 flex items-center gap-2">
-              <meta.icon className={cn('h-5 w-5', meta.accent)} />
-              <h3 className="text-sm font-semibold text-foreground">{meta.label}</h3>
-              {meta.cdc ? (
-                <span className="rounded bg-emerald-500/15 px-1.5 py-0.5 text-[9px] text-emerald-400">CDC actief</span>
-              ) : (
-                <span className="rounded bg-white/10 px-1.5 py-0.5 text-[9px] text-foreground-muted">geen CDC-stream</span>
-              )}
-              {agentMap[active] && (
-                <span className="inline-flex items-center gap-1 rounded bg-violet-500/15 px-1.5 py-0.5 text-[9px] text-violet-300">
-                  <Bot className="h-2.5 w-2.5" /> {agentMap[active].agent_name}
-                </span>
-              )}
-            </div>
-            <p className="mb-3 text-[11px] text-foreground-muted">{meta.desc} Doel: <span className="font-mono">{meta.target}</span></p>
-
-            <div className="flex flex-wrap items-end gap-3">
-              <label className="flex flex-col gap-1 text-[10px] uppercase tracking-wide text-foreground-faint">
-                Aantal rijen
-                <input
-                  type="number"
-                  min={1}
-                  max={2000000}
-                  value={rows[active]}
-                  onChange={(e) => setRows((r) => ({ ...r, [active]: Number(e.target.value) }))}
-                  className="w-40 rounded border border-border bg-background px-2 py-1.5 text-[12px] text-foreground"
-                />
-              </label>
-              <button
-                type="button"
-                disabled={!!busy[active]}
-                onClick={() => generate(active)}
-                className="inline-flex items-center gap-2 rounded-md bg-violet-500/90 px-4 py-2 text-[12px] font-semibold text-black hover:bg-violet-400 disabled:opacity-40"
-              >
-                {busy[active] ? <Loader2 className="h-4 w-4 animate-spin" /> : <Play className="h-4 w-4" />}
-                Genereer data
-              </button>
-              <button
-                type="button"
-                disabled={!!busy[active]}
-                onClick={() => generate(active, true)}
-                title="Laat de verantwoordelijke agent zelf data genereren (wordt gelogd)"
-                className="inline-flex items-center gap-2 rounded-md border border-violet-400/50 px-3 py-2 text-[12px] font-medium text-violet-300 hover:bg-violet-500/10 disabled:opacity-40"
-              >
-                <Bot className="h-4 w-4" />
-                Laat agent genereren
-              </button>
-              {active !== 'all' && COUNT_KEYS.includes(active) && (
-                <div className="text-[11px] text-foreground-muted">
-                  Huidige rijen: <span className="font-mono text-foreground">{counts[active]?.toLocaleString() ?? '…'}</span>
-                </div>
-              )}
-            </div>
-            {msg && <p className="mt-3 text-[11px] text-foreground-muted">{msg}</p>}
-          </div>
-
-          <div className="rounded-lg border border-border p-4">
-            <h4 className="mb-2 text-[11px] font-semibold uppercase tracking-wide text-foreground-faint">Recente runs — {meta.label}</h4>
-            {latest ? (
-              <table className="w-full text-left text-[11px]">
-                <thead>
-                  <tr className="border-b border-border text-[9px] uppercase text-foreground-faint">
-                    <th className="py-1 pr-2">State</th>
-                    <th className="py-1 pr-2">Rows</th>
-                    <th className="py-1 pr-2">Start</th>
-                    <th className="py-1 pr-2">Eind</th>
-                  </tr>
-                </thead>
-                <tbody>
-                  {(runs[active] || []).map((run) => (
-                    <tr key={run.run_id} className="border-b border-border/40">
-                      <td className="py-1 pr-2">{stateBadge(run.state)}</td>
-                      <td className="py-1 pr-2 font-mono text-foreground-muted">{run.conf?.rows ?? '—'}</td>
-                      <td className="py-1 pr-2 text-foreground-faint">{run.start?.slice(11, 19) || '—'}</td>
-                      <td className="py-1 pr-2 text-foreground-faint">{run.end?.slice(11, 19) || '—'}</td>
-                    </tr>
-                  ))}
-                </tbody>
-              </table>
-            ) : (
-              <p className="text-[11px] text-foreground-faint">Nog geen runs.</p>
-            )}
-          </div>
-
-          <div className="rounded-lg border border-border p-4">
-            <h4 className="mb-2 text-[11px] font-semibold uppercase tracking-wide text-foreground-faint">Live tellingen (Trino)</h4>
-            <div className="grid grid-cols-2 gap-2 sm:grid-cols-4">
-              {COUNT_KEYS.map((k) => (
-                <div key={k} className="rounded border border-border/60 px-2 py-1.5">
-                  <div className="text-[9px] uppercase text-foreground-faint">{k}</div>
-                  <div className="font-mono text-[12px] text-foreground">{counts[k]?.toLocaleString() ?? '…'}</div>
-                </div>
-              ))}
-            </div>
-          </div>
-
-          <div className="rounded-lg border border-border p-4">
-            <h4 className="mb-2 flex items-center gap-1.5 text-[11px] font-semibold uppercase tracking-wide text-foreground-faint">
-              <ScrollText className="h-3.5 w-3.5" /> Agent-activiteit (wat de agents deden)
-            </h4>
-            {activity.length === 0 ? (
-              <p className="text-[11px] text-foreground-faint">Nog geen agent-acties gelogd.</p>
-            ) : (
-              <ul className="space-y-1">
-                {activity.map((a) => (
-                  <li key={a.id} className="flex items-start gap-2 text-[11px]">
-                    <span className="mt-0.5 text-foreground-faint">{a.ts?.slice(11, 19) || ''}</span>
-                    <span className="inline-flex items-center gap-1 rounded bg-violet-500/15 px-1 text-[9px] text-violet-300">
-                      <Bot className="h-2.5 w-2.5" />{a.agent_name}
-                    </span>
-                    <span className={cn('flex-1', a.level === 'err' ? 'text-danger' : 'text-foreground-muted')}>{a.message}</span>
-                  </li>
-                ))}
-              </ul>
-            )}
-          </div>
-        </div>
-      </div>
-    </div>
-  )
+  return <DataGenPanel onPulse={onPulse} onOpenPlatform={onOpenPlatform} />
 }

ui/src/components/features/DataHubView.tsx

@@ -0,0 +1,60 @@
+import { useEffect, useState } from 'react'
+import { Database, Server } from 'lucide-react'
+import { DataSourcesView } from './DataSourcesView'
+import { HadoopSourcesView } from './HadoopSourcesView'
+import type { SourceEngine } from '../../lib/dataSourceCatalog'
+import { cn } from '../../lib/utils'
+import { subTabActive, subTabIdle } from '../../lib/tabActive'
+
+type HubTab = 'sources' | 'hadoop'
+
+type Props = {
+  focusEngine?: SourceEngine | null
+  initialTab?: HubTab
+  onPulse?: () => void
+}
+
+export function DataHubView({ focusEngine, initialTab = 'sources', onPulse }: Props) {
+  const [tab, setTab] = useState<HubTab>(initialTab)
+
+  useEffect(() => {
+    setTab(initialTab)
+  }, [initialTab])
+
+  return (
+    <div className="flex h-full min-h-0 flex-col overflow-hidden">
+      <div className="panel mx-3 mt-3 flex shrink-0 items-center gap-1 px-2 py-1.5">
+        <button
+          type="button"
+          onClick={() => setTab('sources')}
+          className={cn(
+            'inline-flex items-center gap-1.5 rounded-md px-3 py-1.5 text-[11px] font-medium transition-all',
+            tab === 'sources' ? subTabActive : subTabIdle,
+          )}
+        >
+          <Database className="h-3.5 w-3.5" /> Source Databases
+        </button>
+        <button
+          type="button"
+          onClick={() => setTab('hadoop')}
+          className={cn(
+            'inline-flex items-center gap-1.5 rounded-md px-3 py-1.5 text-[11px] font-medium transition-all',
+            tab === 'hadoop' ? subTabActive : subTabIdle,
+          )}
+        >
+          <Server className="h-3.5 w-3.5" /> Hadoop
+        </button>
+        <span className="ml-auto text-[9px] text-foreground-muted">
+          {tab === 'sources' ? 'PostgreSQL · MySQL · MongoDB · Cassandra · Neo4j' : 'HDFS · Hive · Iceberg · Spark · Kafka pipeline'}
+        </span>
+      </div>
+      <div className="min-h-0 flex-1 overflow-hidden">
+        {tab === 'sources' ? (
+          <DataSourcesView focusEngine={focusEngine} onPulse={onPulse} />
+        ) : (
+          <HadoopSourcesView onPulse={onPulse} />
+        )}
+      </div>
+    </div>
+  )
+}

ui/src/components/features/DataSourcesView.tsx

@@ -6,15 +6,20 @@ import {
   FolderTree,
   Loader2,
   Network,
+  Play,
   RefreshCw,
   Server,
   Table2,
   TerminalSquare,
 } from 'lucide-react'
 import { Badge } from '../ui/Badge'
+import { DbBrandIcon, dbBrandColor } from '../ui/DbBrandIcon'
+import { DataBrowserGrid } from './DataBrowserGrid'
+import { DataGenPanel } from './DataGenPanel'
 import { DbShell } from './DbShell'
 import { Neo4jGraphView } from './Neo4jGraphView'
 import { SqlWorkbench } from './SqlWorkbench'
+import { LakehouseWorkbench } from './SparkView'
 import {
   getSourceMeta,
   SOURCE_CATALOG,
@@ -39,16 +44,33 @@ type SampleResponse = {
   row_count?: number
   elapsed_ms?: number
   error?: string
+  primary_keys?: string[]
+  cdc?: boolean
+  editable?: boolean
+  offset?: number
+  limit?: number
+  total_count?: number | null
 }
 
 type Props = {
   focusEngine?: SourceEngine | null
+  onPulse?: () => void
 }
 
-const SUB_TABS: { id: SourceSubTab; label: string; icon: typeof FolderTree; neo4jOnly?: boolean }[] = [
+const ENGINE_CATALOG: Record<SourceEngine, string | undefined> = {
+  postgres: 'postgres_sales',
+  mysql: 'mysql_hr',
+  mongodb: 'mongodb_supplychain',
+  cassandra: 'cassandra_telemetry',
+  neo4j: undefined,
+}
+
+const SUB_TABS: { id: SourceSubTab; label: string; icon: typeof FolderTree; neo4jOnly?: boolean; hideNeo4j?: boolean }[] = [
   { id: 'browser', label: 'Browser', icon: FolderTree },
   { id: 'graph', label: 'Graph', icon: Network, neo4jOnly: true },
   { id: 'console', label: 'Query Console', icon: Database },
+  { id: 'workbench', label: 'Workbench', icon: Activity, hideNeo4j: true },
+  { id: 'generate', label: 'Generate', icon: Play },
   { id: 'shell', label: 'Shell', icon: TerminalSquare },
 ]
 
@@ -59,7 +81,7 @@ function fmtCount(n?: number | null) {
   return String(n)
 }
 
-export function DataSourcesView({ focusEngine }: Props) {
+export function DataSourcesView({ focusEngine, onPulse }: Props) {
   const [active, setActive] = useState<SourceEngine>(focusEngine || 'postgres')
   const [subTab, setSubTab] = useState<SourceSubTab>('browser')
   const [health, setHealth] = useState<HealthMap>({})
@@ -68,6 +90,8 @@ export function DataSourcesView({ focusEngine }: Props) {
   const [selectedObject, setSelectedObject] = useState<CatalogObject | null>(null)
   const [sample, setSample] = useState<SampleResponse | null>(null)
   const [sampleLoading, setSampleLoading] = useState(false)
+  const [page, setPage] = useState(1)
+  const [pageSize, setPageSize] = useState(100)
 
   const meta = getSourceMeta(active)
 
@@ -104,11 +128,14 @@ export function DataSourcesView({ focusEngine }: Props) {
     }
   }, [])
 
-  const loadSample = useCallback(async (engine: SourceEngine, obj: CatalogObject) => {
+  const loadSample = useCallback(async (engine: SourceEngine, obj: CatalogObject, pg = page, ps = pageSize) => {
     setSampleLoading(true)
     setSample(null)
+    const offset = (pg - 1) * ps
     try {
-      const r = await fetch(`/api/sql/sample/${engine}?object=${encodeURIComponent(obj.fqn)}&limit=50`)
+      const r = await fetch(
+        `/api/sql/sample/${engine}?object=${encodeURIComponent(obj.fqn)}&limit=${ps}&offset=${offset}`,
+      )
       const j = await r.json()
       setSample(j)
     } catch {
@@ -116,7 +143,7 @@ export function DataSourcesView({ focusEngine }: Props) {
     } finally {
       setSampleLoading(false)
     }
-  }, [])
+  }, [page, pageSize])
 
   useEffect(() => { loadHealth() }, [loadHealth])
   useEffect(() => {
@@ -124,21 +151,26 @@ export function DataSourcesView({ focusEngine }: Props) {
   }, [active, subTab, loadCatalog])
 
   useEffect(() => {
-    if (selectedObject && subTab === 'browser') loadSample(active, selectedObject)
-  }, [selectedObject, active, subTab, loadSample])
+    setPage(1)
+  }, [selectedObject?.fqn, active])
+
+  useEffect(() => {
+    if (selectedObject && subTab === 'browser') loadSample(active, selectedObject, page, pageSize)
+  }, [selectedObject, active, subTab, page, pageSize, loadSample])
 
   const refreshAll = () => {
     loadHealth()
-    if (subTab === 'browser') loadCatalog(active)
-    else if (subTab === 'graph' && active === 'neo4j') { /* Neo4jGraphView reloads itself */ }
-    else if (selectedObject) loadSample(active, selectedObject)
+    if (subTab === 'browser') {
+      loadCatalog(active)
+      if (selectedObject) loadSample(active, selectedObject, page, pageSize)
+    } else if (subTab === 'graph' && active === 'neo4j') { /* Neo4jGraphView reloads itself */ }
+    else if (selectedObject) loadSample(active, selectedObject, page, pageSize)
   }
 
-  const visibleSubTabs = SUB_TABS.filter((t) => !t.neo4jOnly || active === 'neo4j')
+  const visibleSubTabs = SUB_TABS.filter((t) => (!t.neo4jOnly || active === 'neo4j') && !(t.hideNeo4j && active === 'neo4j'))
 
   return (
     <div className="flex h-full min-h-0 flex-col gap-2 p-3">
-      {/* Header */}
       <header className="panel flex shrink-0 flex-wrap items-center justify-between gap-3 px-4 py-3">
         <div>
           <h1 className="flex items-center gap-2 text-base font-semibold text-foreground">
@@ -146,7 +178,7 @@ export function DataSourcesView({ focusEngine }: Props) {
             Data Sources UI
           </h1>
           <p className="text-[11px] text-foreground-muted">
-            Enterprise data browser — schema exploration, query console &amp; interactive shells for all source databases
+            Browse, filter, edit &amp; query all source databases — edits on CDC sources flow to Live Changes
           </p>
         </div>
         <button type="button" onClick={refreshAll} className={cn('inline-flex items-center gap-1.5 rounded-md px-3 py-1.5 text-[11px]', subTabIdle)}>
@@ -154,170 +186,159 @@ export function DataSourcesView({ focusEngine }: Props) {
         </button>
       </header>
 
-      <div className="flex min-h-0 flex-1 gap-2 overflow-hidden">
-        {/* Left rail — database cards */}
-        <aside className="panel flex w-[220px] shrink-0 flex-col overflow-hidden">
-          <div className="shrink-0 border-b border-border px-3 py-2">
-            <p className="text-[9px] font-semibold uppercase tracking-widest text-foreground-faint">Source Databases</p>
-          </div>
-          <div className="scrollbar-thin min-h-0 flex-1 space-y-1 overflow-y-auto p-2">
-            {SOURCE_CATALOG.map((src) => {
-              const Icon = src.icon
-              const up = health[src.engine]?.ok
-              const selected = active === src.engine
-              return (
-                <button
-                  key={src.engine}
-                  type="button"
-                  onClick={() => { setActive(src.engine); setSubTab('browser') }}
-                  className={cn(
-                    'flex w-full flex-col gap-1 rounded-lg border p-2.5 text-left transition-all',
-                    selected ? cn(src.border, src.accentBg, 'shadow-sm') : 'border-transparent hover:border-border hover:bg-surface-overlay',
-                  )}
-                >
-                  <div className="flex items-center justify-between">
-                    <span className={cn('flex items-center gap-1.5 text-[12px] font-semibold', selected ? src.accent : 'text-foreground')}>
-                      <Icon className="h-4 w-4" />
-                      {src.label}
-                    </span>
-                    <span className={cn('h-2 w-2 rounded-full', up === true ? 'bg-emerald-400 shadow-[0_0_6px_rgba(52,211,153,0.6)]' : up === false ? 'bg-red-400' : 'bg-foreground-faint')} title={up ? 'Online' : up === false ? 'Offline' : 'Unknown'} />
-                  </div>
-                  <p className="text-[9px] leading-snug text-foreground-muted">{src.description}</p>
-                  <div className="flex flex-wrap gap-1">
-                    <Badge variant="default">{src.host}:{src.port}</Badge>
-                    {src.cdc && <Badge variant="accent">CDC</Badge>}
-                  </div>
-                </button>
-              )
-            })}
-          </div>
-        </aside>
-
-        {/* Main panel */}
-        <div className="panel flex min-h-0 min-w-0 flex-1 flex-col overflow-hidden">
-          {/* Engine header */}
-          <div className={cn('flex shrink-0 flex-wrap items-center justify-between gap-2 border-b border-border px-4 py-2.5', meta.accentBg)}>
-            <div>
-              <h2 className={cn('flex items-center gap-2 text-sm font-semibold', meta.accent)}>
-                <meta.icon className="h-4 w-4" />
-                {meta.label}
-                {catalog?.version && (
-                  <span className="font-mono text-[10px] font-normal text-foreground-muted">v{catalog.version.split(' ')[0]?.slice(0, 20)}</span>
+      {/* Horizontal database selector */}
+      <div className="panel shrink-0 px-3 py-2">
+        <p className="mb-2 text-[9px] font-semibold uppercase tracking-widest text-foreground-muted">Source Databases</p>
+        <div className="scroll-x-stable scrollbar-thin flex gap-2 pb-1">
+          {SOURCE_CATALOG.map((src) => {
+            const up = health[src.engine]?.ok
+            const selected = active === src.engine
+            const brand = dbBrandColor(src.engine)
+            return (
+              <button
+                key={src.engine}
+                type="button"
+                onClick={() => setActive(src.engine)}
+                className={cn(
+                  'flex min-w-[148px] shrink-0 flex-col gap-1.5 rounded-lg border px-3 py-2.5 text-left transition-all',
+                  selected
+                    ? 'shadow-md'
+                    : 'border-border/60 hover:border-border hover:bg-surface-overlay',
                 )}
-              </h2>
-              <p className="font-mono text-[10px] text-foreground-muted">
-                {meta.host}:{meta.port} · {meta.database} · container {meta.container}
-              </p>
-            </div>
-            <div className="flex gap-1">
-              {visibleSubTabs.map(({ id, label, icon: Icon }) => (
-                <button
-                  key={id}
-                  type="button"
-                  onClick={() => setSubTab(id)}
-                  className={cn('inline-flex items-center gap-1 rounded-md px-2.5 py-1 text-[10px] font-medium', subTab === id ? subTabActive : subTabIdle)}
-                >
-                  <Icon className="h-3 w-3" /> {label}
-                </button>
-              ))}
-            </div>
-          </div>
-
-          {/* Sub-tab content */}
-          <div className="min-h-0 flex-1 overflow-hidden">
-            {subTab === 'browser' && (
-              <div className="flex h-full min-h-0">
-                {/* Object tree */}
-                <div className="flex w-[280px] shrink-0 flex-col border-r border-border/60">
-                  <div className="flex shrink-0 items-center justify-between border-b border-border/60 px-3 py-2">
-                    <span className="text-[10px] font-semibold uppercase tracking-wider text-foreground-faint">Objects</span>
-                    {catalogLoading && <Loader2 className="h-3 w-3 animate-spin text-foreground-muted" />}
-                  </div>
-                  <div className="scrollbar-thin min-h-0 flex-1 overflow-y-auto p-1.5">
-                    {catalog?.objects?.map((obj) => (
-                      <button
-                        key={obj.fqn}
-                        type="button"
-                        onClick={() => setSelectedObject(obj)}
-                        className={cn(
-                          'mb-0.5 flex w-full items-center gap-1.5 rounded px-2 py-1.5 text-left text-[10px] transition-colors',
-                          selectedObject?.fqn === obj.fqn ? subTabActive : 'hover:bg-surface-overlay',
-                        )}
-                      >
-                        {obj.type === 'node_label' ? <Activity className="h-3 w-3 shrink-0 text-pink-400" /> : <Table2 className="h-3 w-3 shrink-0 text-foreground-muted" />}
-                        <div className="min-w-0 flex-1">
-                          <p className="truncate font-medium text-foreground">{obj.name}</p>
-                          <p className="truncate font-mono text-[8px] text-foreground-faint">{obj.schema}{obj.type === 'relationship' ? ' · rel' : ''}</p>
-                        </div>
-                        <span className="shrink-0 font-mono text-[9px] text-foreground-muted">{fmtCount(obj.row_count)}</span>
-                        <ChevronRight className="h-3 w-3 shrink-0 text-foreground-faint" />
-                      </button>
-                    ))}
-                    {!catalogLoading && !catalog?.objects?.length && (
-                      <p className="p-4 text-center text-[10px] text-foreground-faint">No objects found</p>
+                style={selected ? { borderColor: brand, backgroundColor: `${brand}18`, boxShadow: `0 0 0 1px ${brand}40` } : undefined}
+              >
+                <div className="flex items-center justify-between gap-2">
+                  <span className="flex items-center gap-2 text-[12px] font-semibold text-foreground">
+                    <DbBrandIcon engine={src.engine} size={22} />
+                    {src.label}
+                  </span>
+                  <span
+                    className={cn(
+                      'h-2 w-2 shrink-0 rounded-full',
+                      up === true ? 'bg-emerald-400 shadow-[0_0_6px_rgba(52,211,153,0.6)]' : up === false ? 'bg-red-400' : 'bg-foreground-faint',
                     )}
-                  </div>
+                    title={up ? 'Online' : up === false ? 'Offline' : 'Unknown'}
+                  />
                 </div>
+                <p className="line-clamp-2 text-[9px] leading-snug text-foreground-muted">{src.description}</p>
+                <div className="flex flex-wrap gap-1">
+                  <Badge variant="default">{src.host}:{src.port}</Badge>
+                  {src.cdc && <Badge variant="accent">CDC</Badge>}
+                </div>
+              </button>
+            )
+          })}
+        </div>
+      </div>
 
-                {/* Sample data grid */}
-                <div className="flex min-h-0 min-w-0 flex-1 flex-col">
-                  <div className="flex shrink-0 items-center justify-between border-b border-border/60 px-3 py-2">
-                    <span className="text-[10px] font-semibold text-foreground">
-                      {selectedObject ? (
-                        <>Sample: <span className="font-mono text-docker">{selectedObject.fqn}</span></>
-                      ) : 'Select an object'}
-                    </span>
-                    {sampleLoading && <Loader2 className="h-3 w-3 animate-spin" />}
-                  </div>
-                  <div className="scrollbar-thin min-h-0 flex-1 overflow-auto p-2">
-                    {sample?.ok && sample.columns && (
-                      <>
-                        <p className="mb-1 font-mono text-[9px] text-foreground-faint">
-                          {sample.row_count} rows · {sample.elapsed_ms}ms
-                        </p>
-                        <table className="w-full text-left font-mono text-[10px]">
-                          <thead>
-                            <tr className="sticky top-0 border-b border-border bg-surface-raised text-docker">
-                              {sample.columns.map((c) => <th key={c} className="px-2 py-1">{c}</th>)}
-                            </tr>
-                          </thead>
-                          <tbody>
-                            {sample.rows?.map((row, i) => (
-                              <tr key={i} className="border-b border-border/30 hover:bg-white/5">
-                                {row.map((cell, j) => (
-                                  <td key={j} className="max-w-[200px] truncate px-2 py-1 text-foreground-muted">
-                                    {cell === null || cell === undefined ? 'NULL' : String(cell)}
-                                  </td>
-                                ))}
-                              </tr>
-                            ))}
-                          </tbody>
-                        </table>
-                      </>
-                    )}
-                    {sample && !sample.ok && (
-                      <p className="p-4 text-[11px] text-danger">{sample.error || 'Failed to load sample'}</p>
-                    )}
-                    {!selectedObject && !sampleLoading && (
-                      <p className="py-8 text-center text-[11px] text-foreground-faint">Select a table, collection or label to preview data</p>
-                    )}
-                  </div>
+      {/* Main panel — full width */}
+      <div className="panel flex min-h-0 min-w-0 flex-1 flex-col overflow-hidden">
+        <div
+          className="flex shrink-0 flex-wrap items-center justify-between gap-2 border-b border-border px-4 py-2.5"
+          style={{ backgroundColor: `${dbBrandColor(active)}12` }}
+        >
+          <div>
+            <h2 className="flex items-center gap-2 text-sm font-semibold text-foreground">
+              <DbBrandIcon engine={active} size={18} />
+              {meta.label}
+              {catalog?.version && (
+                <span className="font-mono text-[10px] font-normal text-foreground-muted">v{catalog.version.split(' ')[0]?.slice(0, 20)}</span>
+              )}
+            </h2>
+            <p className="font-mono text-[10px] text-foreground-muted">
+              {meta.host}:{meta.port} · {meta.database} · container {meta.container}
+            </p>
+          </div>
+          <div className="flex flex-wrap gap-1">
+            {visibleSubTabs.map(({ id, label, icon: Icon }) => (
+              <button
+                key={id}
+                type="button"
+                onClick={() => setSubTab(id)}
+                className={cn('inline-flex items-center gap-1 rounded-md px-2.5 py-1 text-[10px] font-medium', subTab === id ? subTabActive : subTabIdle)}
+              >
+                <Icon className="h-3 w-3" /> {label}
+              </button>
+            ))}
+          </div>
+        </div>
+
+        <div className="flex min-h-0 flex-1 flex-col overflow-hidden">
+          {subTab === 'browser' && (
+            <div className="flex h-full min-h-0 flex-1 overflow-hidden">
+              <div className="flex w-[260px] shrink-0 flex-col border-r border-border/60">
+                <div className="flex shrink-0 items-center justify-between border-b border-border/60 px-3 py-2">
+                  <span className="text-[10px] font-semibold uppercase tracking-wider text-foreground-muted">Objects</span>
+                  {catalogLoading && <Loader2 className="h-3 w-3 animate-spin text-foreground-muted" />}
+                </div>
+                <div className="scrollbar-thin min-h-0 flex-1 overflow-y-auto p-1.5">
+                  {catalog?.objects?.map((obj) => (
+                    <button
+                      key={obj.fqn}
+                      type="button"
+                      onClick={() => setSelectedObject(obj)}
+                      className={cn(
+                        'mb-0.5 flex w-full items-center gap-1.5 rounded px-2 py-1.5 text-left text-[10px] transition-colors',
+                        selectedObject?.fqn === obj.fqn ? subTabActive : 'hover:bg-surface-overlay',
+                      )}
+                    >
+                      {obj.type === 'node_label' ? (
+                        <Activity className="h-3 w-3 shrink-0" style={{ color: dbBrandColor('neo4j') }} />
+                      ) : (
+                        <Table2 className="h-3 w-3 shrink-0 text-foreground-muted" />
+                      )}
+                      <div className="min-w-0 flex-1">
+                        <p className="truncate font-medium text-foreground">{obj.name}</p>
+                        <p className="truncate font-mono text-[8px] text-foreground-muted">{obj.schema}{obj.type === 'relationship' ? ' · rel' : ''}</p>
+                      </div>
+                      <span className="shrink-0 font-mono text-[9px] text-foreground-muted">{fmtCount(obj.row_count)}</span>
+                      <ChevronRight className="h-3 w-3 shrink-0 text-foreground-muted" />
+                    </button>
+                  ))}
+                  {!catalogLoading && !catalog?.objects?.length && (
+                    <p className="p-4 text-center text-[10px] text-foreground-muted">No objects found</p>
+                  )}
                 </div>
               </div>
-            )}
 
-            {subTab === 'graph' && active === 'neo4j' && (
-              <Neo4jGraphView />
-            )}
+              <div className="flex h-full min-h-0 min-w-0 flex-1 flex-col overflow-hidden">
+                <div className="flex shrink-0 items-center border-b border-border/60 px-3 py-2">
+                  <span className="text-[10px] font-semibold text-foreground">
+                    {selectedObject ? (
+                      <>Data: <span className="font-mono text-docker">{selectedObject.fqn}</span></>
+                    ) : 'Select an object'}
+                  </span>
+                </div>
+                <DataBrowserGrid
+                  engine={active}
+                  objectFqn={selectedObject?.fqn || ''}
+                  sample={selectedObject ? sample : null}
+                  loading={sampleLoading}
+                  page={page}
+                  pageSize={pageSize}
+                  onPageChange={setPage}
+                  onPageSizeChange={(ps) => { setPageSize(ps); setPage(1) }}
+                  onReload={() => selectedObject && loadSample(active, selectedObject, page, pageSize)}
+                />
+              </div>
+            </div>
+          )}
 
-            {subTab === 'console' && (
-              <SqlWorkbench engine={active} />
-            )}
+          {subTab === 'graph' && active === 'neo4j' && <Neo4jGraphView />}
 
-            {subTab === 'shell' && (
-              <DbShell initialCommand={meta.shellCommand} />
-            )}
-          </div>
+          {subTab === 'console' && <SqlWorkbench engine={active} />}
+
+          {subTab === 'workbench' && (
+            <div className="flex min-h-0 flex-1 overflow-hidden">
+              <LakehouseWorkbench lockedCatalog={ENGINE_CATALOG[active]} />
+            </div>
+          )}
+
+          {subTab === 'generate' && onPulse && (
+            <DataGenPanel embedded focusSource={active} onPulse={onPulse} />
+          )}
+
+          {subTab === 'shell' && <DbShell initialCommand={meta.shellCommand} />}
         </div>
       </div>
     </div>

ui/src/components/features/HadoopSourcesView.tsx

@@ -0,0 +1,280 @@
+import { useCallback, useEffect, useState } from 'react'
+import {
+  Activity,
+  ChevronRight,
+  FolderTree,
+  GitBranch,
+  Loader2,
+  Play,
+  RefreshCw,
+  Server,
+  Table2,
+  TerminalSquare,
+  Zap,
+} from 'lucide-react'
+import { Badge } from '../ui/Badge'
+import { DataBrowserGrid } from './DataBrowserGrid'
+import { HdfsView } from './HdfsView'
+import { SparkView } from './SparkView'
+import { SqlWorkbench } from './SqlWorkbench'
+import type { CatalogObject } from '../../lib/dataSourceCatalog'
+import { cn } from '../../lib/utils'
+import { subTabActive, subTabIdle } from '../../lib/tabActive'
+import { runDataflowMovement, triggerStreamingPipeline } from '../../lib/api'
+
+type HadoopSubTab = 'browser' | 'files' | 'console' | 'spark' | 'pipeline'
+type CatalogResponse = { engine: string; version?: string; objects: CatalogObject[] }
+type SampleResponse = {
+  ok: boolean
+  columns?: string[]
+  rows?: unknown[][]
+  error?: string
+  total_count?: number | null
+  offset?: number
+  limit?: number
+  editable?: boolean
+}
+
+const SUB_TABS: { id: HadoopSubTab; label: string; icon: typeof FolderTree }[] = [
+  { id: 'browser', label: 'Tables', icon: Table2 },
+  { id: 'files', label: 'HDFS Files', icon: FolderTree },
+  { id: 'console', label: 'Query Console', icon: TerminalSquare },
+  { id: 'spark', label: 'Spark', icon: Activity },
+  { id: 'pipeline', label: 'Pipeline', icon: GitBranch },
+]
+
+export function HadoopSourcesView({ onPulse }: { onPulse?: () => void }) {
+  const [subTab, setSubTab] = useState<HadoopSubTab>('browser')
+  const [health, setHealth] = useState<{ ok?: boolean; error?: string } | null>(null)
+  const [catalog, setCatalog] = useState<CatalogResponse | null>(null)
+  const [catalogLoading, setCatalogLoading] = useState(false)
+  const [selectedObject, setSelectedObject] = useState<CatalogObject | null>(null)
+  const [sample, setSample] = useState<SampleResponse | null>(null)
+  const [sampleLoading, setSampleLoading] = useState(false)
+  const [page, setPage] = useState(1)
+  const [pageSize, setPageSize] = useState(100)
+  const [pipelineBusy, setPipelineBusy] = useState<string | null>(null)
+  const [pipelineMsg, setPipelineMsg] = useState<string | null>(null)
+
+  const loadHealth = useCallback(async () => {
+    try {
+      const r = await fetch('/api/sql/health/hadoop')
+      if (r.ok) setHealth(await r.json())
+    } catch { /* */ }
+  }, [])
+
+  const loadCatalog = useCallback(async () => {
+    setCatalogLoading(true)
+    try {
+      const r = await fetch('/api/sql/catalog/hadoop')
+      if (r.ok) {
+        const j: CatalogResponse = await r.json()
+        setCatalog(j)
+        const first = j.objects?.[0]
+        if (first) setSelectedObject(first)
+      }
+    } catch { /* */ } finally {
+      setCatalogLoading(false)
+    }
+  }, [])
+
+  const loadSample = useCallback(async (obj: CatalogObject, pg = page, ps = pageSize) => {
+    setSampleLoading(true)
+    const offset = (pg - 1) * ps
+    try {
+      const r = await fetch(
+        `/api/sql/sample/hadoop?object=${encodeURIComponent(obj.fqn)}&limit=${ps}&offset=${offset}`,
+      )
+      setSample(await r.json())
+    } catch {
+      setSample({ ok: false, error: 'Sample unavailable' })
+    } finally {
+      setSampleLoading(false)
+    }
+  }, [page, pageSize])
+
+  useEffect(() => { loadHealth() }, [loadHealth])
+  useEffect(() => {
+    if (subTab === 'browser') loadCatalog()
+  }, [subTab, loadCatalog])
+  useEffect(() => { setPage(1) }, [selectedObject?.fqn])
+  useEffect(() => {
+    if (selectedObject && subTab === 'browser') loadSample(selectedObject, page, pageSize)
+  }, [selectedObject, subTab, page, pageSize, loadSample])
+
+  const refreshAll = () => {
+    loadHealth()
+    if (subTab === 'browser') {
+      loadCatalog()
+      if (selectedObject) loadSample(selectedObject, page, pageSize)
+    }
+  }
+
+  const runPipeline = async (kind: 'full' | 'hdfs_kafka' | 'spark_s3') => {
+    setPipelineBusy(kind)
+    setPipelineMsg(null)
+    try {
+      if (kind === 'full') {
+        const r = await triggerStreamingPipeline('hadoop-lake')
+        const j = await r.json()
+        setPipelineMsg(j.ok ? `✓ Pipeline started: ${j.steps?.join(' → ') || 'ok'}` : j.error || 'Failed')
+      } else if (kind === 'hdfs_kafka') {
+        const r = await fetch('/api/pipeline/streaming/hdfs/to-kafka', {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/json' },
+          body: JSON.stringify({ source: 'trino', table: 'iceberg.hadoop.historical_sales_hdfs', topic: 'hdfs.historical.sales', limit: 2000 }),
+        })
+        const j = await r.json()
+        setPipelineMsg(j.ok ? `✓ ${j.rows_sent} rows → Kafka topic ${j.topic}` : j.error || 'Failed')
+      } else {
+        await runDataflowMovement('spark_to_s3')
+        setPipelineMsg('✓ Spark → S3 job triggered')
+      }
+      onPulse?.()
+    } catch {
+      setPipelineMsg('Pipeline failed')
+    } finally {
+      setPipelineBusy(null)
+    }
+  }
+
+  return (
+    <div className="flex h-full min-h-0 flex-col gap-2 p-3 pt-1">
+      <header className="panel flex shrink-0 flex-wrap items-center justify-between gap-3 px-4 py-3">
+        <div>
+          <h1 className="flex items-center gap-2 text-base font-semibold text-foreground">
+            <Server className="h-5 w-5 text-emerald-400" />
+            Hadoop Data Lake
+          </h1>
+          <p className="text-[11px] text-foreground-muted">
+            HDFS · Hive · Iceberg tables · Spark transforms · Kafka bridge → S3
+          </p>
+        </div>
+        <div className="flex items-center gap-2">
+          <Badge variant={health?.ok ? 'success' : 'warning'}>
+            {health?.ok ? 'Hadoop online' : health?.error?.slice(0, 40) || 'Checking…'}
+          </Badge>
+          <button type="button" onClick={refreshAll} className={cn('inline-flex items-center gap-1.5 rounded-md px-3 py-1.5 text-[11px]', subTabIdle)}>
+            <RefreshCw className="h-3.5 w-3.5" /> Refresh
+          </button>
+        </div>
+      </header>
+
+      <div className="panel flex shrink-0 gap-1 px-2 py-1.5">
+        {SUB_TABS.map(({ id, label, icon: Icon }) => (
+          <button
+            key={id}
+            type="button"
+            onClick={() => setSubTab(id)}
+            className={cn(
+              'inline-flex items-center gap-1.5 rounded-md px-2.5 py-1.5 text-[10px] font-medium',
+              subTab === id ? subTabActive : subTabIdle,
+            )}
+          >
+            <Icon className="h-3.5 w-3.5" /> {label}
+          </button>
+        ))}
+      </div>
+
+      {subTab === 'browser' && (
+        <div className="panel flex min-h-0 flex-1 gap-0 overflow-hidden">
+          <aside className="scrollbar-thin w-52 shrink-0 overflow-y-auto border-r border-border p-2">
+            <p className="mb-1 text-[9px] font-semibold uppercase tracking-wider text-foreground-muted">
+              {catalogLoading ? 'Loading…' : `${catalog?.objects?.length ?? 0} objects`}
+            </p>
+            {catalog?.objects?.map((obj) => (
+              <button
+                key={obj.fqn}
+                type="button"
+                onClick={() => setSelectedObject(obj)}
+                className={cn(
+                  'mb-0.5 flex w-full items-center gap-1 rounded px-2 py-1.5 text-left text-[10px] transition-colors',
+                  selectedObject?.fqn === obj.fqn ? 'bg-emerald-500/15 text-emerald-200' : 'hover:bg-surface-overlay text-foreground-muted',
+                )}
+              >
+                <ChevronRight className="h-3 w-3 shrink-0 opacity-50" />
+                <span className="min-w-0 truncate">
+                  <span className="block truncate font-medium text-foreground">{obj.name}</span>
+                  <span className="block truncate text-[8px] text-foreground-faint">{obj.schema}</span>
+                </span>
+              </button>
+            ))}
+          </aside>
+          <div className="min-h-0 min-w-0 flex-1 overflow-hidden p-2">
+            {selectedObject ? (
+              <DataBrowserGrid
+                engine="hadoop"
+                objectFqn={selectedObject.fqn}
+                sample={sample}
+                loading={sampleLoading}
+                page={page}
+                pageSize={pageSize}
+                onPageChange={setPage}
+                onPageSizeChange={(ps) => { setPageSize(ps); setPage(1) }}
+                onReload={() => selectedObject && loadSample(selectedObject, page, pageSize)}
+              />
+            ) : (
+              <p className="py-8 text-center text-sm text-foreground-muted">Select a table or HDFS path</p>
+            )}
+          </div>
+        </div>
+      )}
+
+      {subTab === 'files' && <HdfsView />}
+      {subTab === 'console' && <SqlWorkbench engine="hadoop" />}
+      {subTab === 'spark' && <SparkView embedded />}
+      {subTab === 'pipeline' && (
+        <div className="panel flex min-h-0 flex-1 flex-col gap-4 overflow-y-auto p-4">
+          <div>
+            <h3 className="mb-1 text-sm font-semibold text-foreground">Hadoop → Kafka → Spark → S3</h3>
+            <p className="text-[11px] text-foreground-muted">
+              Full lakehouse pipeline: export HDFS data to Kafka, Spark transforms into Iceberg/S3 curated layer.
+            </p>
+          </div>
+          <div className="grid gap-3 sm:grid-cols-3">
+            <PipelineCard
+              title="1. HDFS → Kafka"
+              desc="Export iceberg.hadoop.historical_sales_hdfs to Kafka topic hdfs.historical.sales"
+              busy={pipelineBusy === 'hdfs_kafka'}
+              onRun={() => runPipeline('hdfs_kafka')}
+            />
+            <PipelineCard
+              title="2. Spark transform"
+              desc="Run mask_to_curated / hadoop_to_trino via Airflow on Spark cluster"
+              busy={pipelineBusy === 'spark_s3'}
+              onRun={() => runPipeline('spark_s3')}
+            />
+            <PipelineCard
+              title="Full pipeline"
+              desc="HDFS → Kafka → Spark → Iceberg → S3 in one orchestrated run"
+              busy={pipelineBusy === 'full'}
+              onRun={() => runPipeline('full')}
+            />
+          </div>
+          {pipelineMsg && <p className="text-[11px] text-docker">{pipelineMsg}</p>}
+          <div className="rounded-lg border border-border bg-surface-overlay/30 p-3 font-mono text-[10px] text-foreground-muted">
+            hdfs:/data/historical/sales_orders → Kafka:hdfs.historical.sales → Spark → iceberg.hadoop → s3://data/hadoop/
+          </div>
+        </div>
+      )}
+    </div>
+  )
+}
+
+function PipelineCard({ title, desc, busy, onRun }: { title: string; desc: string; busy: boolean; onRun: () => void }) {
+  return (
+    <div className="rounded-lg border border-border bg-surface-overlay/20 p-3">
+      <h4 className="mb-1 text-[11px] font-semibold text-foreground">{title}</h4>
+      <p className="mb-2 min-h-[2.5rem] text-[10px] text-foreground-muted">{desc}</p>
+      <button
+        type="button"
+        disabled={busy}
+        onClick={onRun}
+        className="inline-flex items-center gap-1 rounded-md border border-emerald-400/50 bg-emerald-500/15 px-3 py-1.5 text-[10px] font-medium text-emerald-200 hover:bg-emerald-500/25 disabled:opacity-50"
+      >
+        {busy ? <Loader2 className="h-3.5 w-3.5 animate-spin" /> : <Play className="h-3.5 w-3.5" />}
+        Run
+      </button>
+    </div>
+  )
+}

ui/src/components/features/PlatformView.tsx

@@ -0,0 +1,66 @@
+import { useState } from 'react'
+import { LayoutDashboard, Presentation } from 'lucide-react'
+import { PlatformTopology } from './PlatformTopology'
+import { PresentationView } from './PresentationView'
+import type { AgentAnim, WorkloadData } from '../../types'
+import { cn } from '../../lib/utils'
+import { subTabActive, subTabIdle } from '../../lib/tabActive'
+
+type PlatformTab = 'topology' | 'presentation'
+
+type Props = {
+  workload: WorkloadData | null
+  animations: Record<string, AgentAnim>
+  selectedNodeId: string | null
+  onNodeClick: (nodeId: string) => void
+  pulse: boolean
+}
+
+const TABS: { id: PlatformTab; label: string; icon: typeof LayoutDashboard }[] = [
+  { id: 'topology', label: 'Topology', icon: LayoutDashboard },
+  { id: 'presentation', label: 'Presentation', icon: Presentation },
+]
+
+export function PlatformView({ workload, animations, selectedNodeId, onNodeClick, pulse }: Props) {
+  const [tab, setTab] = useState<PlatformTab>('topology')
+
+  return (
+    <div className="flex min-h-0 flex-1 flex-col overflow-hidden">
+      <div className="panel flex shrink-0 items-center gap-1 px-2 py-1.5">
+        {TABS.map(({ id, label, icon: Icon }) => (
+          <button
+            key={id}
+            type="button"
+            onClick={() => setTab(id)}
+            className={cn(
+              'inline-flex items-center gap-1.5 rounded-md px-3 py-1.5 text-[10px] font-medium transition-all',
+              tab === id ? subTabActive : subTabIdle,
+            )}
+          >
+            <Icon className="h-3.5 w-3.5" />
+            {label}
+          </button>
+        ))}
+        {tab === 'presentation' && (
+          <span className="ml-auto text-[9px] text-foreground-muted">
+            Live cluster deck · all running services
+          </span>
+        )}
+      </div>
+
+      <div className="min-h-0 flex-1 overflow-hidden pt-1">
+        {tab === 'topology' ? (
+          <PlatformTopology
+            workload={workload}
+            animations={animations}
+            selectedNodeId={selectedNodeId}
+            onNodeClick={onNodeClick}
+            pulse={pulse}
+          />
+        ) : (
+          <PresentationView embedded />
+        )}
+      </div>
+    </div>
+  )
+}

ui/src/components/features/PresentationView.tsx

@@ -1,8 +1,11 @@
 import { useCallback, useEffect, useRef, useState } from 'react'
+import { createPortal } from 'react-dom'
 import {
+  Expand,
   ExternalLink,
   FileUp,
   ImagePlus,
+  Maximize2,
   Monitor,
   Pencil,
   Plus,
@@ -17,6 +20,136 @@ import { cn } from '../../lib/utils'
 import { subTabActive, subTabIdle } from '../../lib/tabActive'
 
 type DeckSource = 'live' | 'data-maturity' | 'atc-platform' | string
+type SlideVariant = 'embedded' | 'normal' | 'present'
+type PresentMode = 'popup' | 'fullscreen' | null
+
+function SlidePanel({
+  slide,
+  slideIdx,
+  slideCount,
+  variant,
+}: {
+  slide: PresentationSlide
+  slideIdx: number
+  slideCount: number
+  variant: SlideVariant
+}) {
+  const isPresent = variant === 'present'
+  const isEmbedded = variant === 'embedded'
+
+  return (
+    <div className={cn('mx-auto w-full', isPresent ? 'max-w-6xl' : isEmbedded ? 'max-w-full' : 'max-w-5xl')}>
+      <div className={cn('flex', isPresent ? 'gap-8 lg:gap-12' : isEmbedded ? 'gap-4' : 'gap-4 md:gap-6')}>
+        <div className="min-w-0 flex-1">
+          <p className={cn(
+            'mb-1 font-medium uppercase tracking-widest text-docker/80',
+            isPresent ? 'text-xs md:text-sm' : isEmbedded ? 'text-[9px]' : 'text-[10px]',
+          )}>
+            {slide.kind || 'slide'} · {slideIdx + 1}/{slideCount}
+          </p>
+          <h1 className={cn(
+            'font-bold tracking-tight text-foreground',
+            isPresent ? 'mb-3 text-3xl leading-tight md:text-5xl lg:text-6xl'
+              : isEmbedded ? 'mb-1 text-lg leading-tight'
+              : 'mb-2 text-2xl md:text-4xl',
+          )}>
+            {slide.title}
+          </h1>
+          {slide.subtitle && (
+            <p className={cn(
+              'text-foreground-muted',
+              isPresent ? 'mb-6 text-lg md:text-2xl' : isEmbedded ? 'mb-2 text-xs' : 'mb-4 text-sm md:text-base',
+            )}>
+              {slide.subtitle}
+            </p>
+          )}
+          {'animation' in slide && slide.animation && (
+            <ArchitectureDiagram
+              animation={String(slide.animation)}
+              compact={isEmbedded}
+              present={isPresent}
+            />
+          )}
+          <ul className={cn(
+            'leading-relaxed text-foreground',
+            isPresent ? 'space-y-3 text-lg md:text-xl lg:text-2xl'
+              : isEmbedded ? 'space-y-1 text-xs'
+              : 'space-y-2 text-sm md:text-base',
+          )}>
+            {(slide.bullets || []).map((b: string, bi: number) => (
+              <li key={bi} className="flex gap-2"><span className="shrink-0 text-docker">▸</span><span>{b}</span></li>
+            ))}
+          </ul>
+        </div>
+        {slide.image && !isEmbedded && (
+          <div className={cn('hidden shrink-0 items-start', isPresent ? 'lg:flex' : 'md:flex')}>
+            <img
+              src={slide.image}
+              alt=""
+              className={cn(
+                'rounded-lg border border-border object-contain shadow-lg',
+                isPresent ? 'max-h-[55vh] max-w-[38vw]' : 'max-h-[46vh] max-w-[40vw]',
+              )}
+            />
+          </div>
+        )}
+      </div>
+      {slide.image && isPresent && (
+        <div className="mt-6 lg:hidden">
+          <img src={slide.image} alt="" className="max-h-[35vh] w-full rounded-lg border border-border object-contain shadow-lg" />
+        </div>
+      )}
+      {slide.image && !isPresent && (
+        <div className={cn('mt-3', isEmbedded ? '' : 'md:hidden')}>
+          <img
+            src={slide.image}
+            alt=""
+            className={cn('rounded-lg border border-border object-contain', isEmbedded ? 'max-h-[22vh] w-full' : 'max-h-[30vh]')}
+          />
+        </div>
+      )}
+    </div>
+  )
+}
+
+function SlideNav({
+  slideIdx,
+  slideCount,
+  embedded,
+  onPrev,
+  onNext,
+  onGo,
+  className,
+}: {
+  slideIdx: number
+  slideCount: number
+  embedded?: boolean
+  onPrev: () => void
+  onNext: () => void
+  onGo?: (idx: number) => void
+  className?: string
+}) {
+  return (
+    <div className={cn('flex shrink-0 items-center gap-2 border-t border-border bg-surface-raised/90 px-3 py-1.5', className)}>
+      <button type="button" disabled={slideIdx === 0} onClick={onPrev} className="rounded border border-border px-2 py-1 text-[10px] disabled:opacity-40 md:text-xs">← Prev</button>
+      {embedded ? (
+        <span className="flex-1 text-center font-mono text-[10px] text-foreground-muted md:text-xs">{slideIdx + 1} / {slideCount}</span>
+      ) : (
+        <div className="flex flex-1 flex-wrap justify-center gap-1">
+          {Array.from({ length: slideCount }, (_, i) => (
+            <button
+              key={i}
+              type="button"
+              onClick={() => onGo?.(i)}
+              className={cn('h-2 w-2 rounded-full', i === slideIdx ? 'scale-125 bg-docker' : 'bg-border')}
+            />
+          ))}
+        </div>
+      )}
+      <button type="button" disabled={slideIdx >= slideCount - 1} onClick={onNext} className="rounded border border-border px-2 py-1 text-[10px] disabled:opacity-40 md:text-xs">Next →</button>
+    </div>
+  )
+}
 
 const KIND_STYLES: Record<string, string> = {
   hero: 'from-blue-600/25 via-violet-600/20 to-emerald-600/15',
@@ -47,7 +180,7 @@ async function fetchDeck(id: DeckSource): Promise<PresentationData | null> {
   }
 }
 
-export function PresentationView() {
+export function PresentationView({ embedded = false }: { embedded?: boolean }) {
   const [source, setSource] = useState<DeckSource>('live')
   const [data, setData] = useState<PresentationData | null>(null)
   const [slideIdx, setSlideIdx] = useState(0)
@@ -63,8 +196,14 @@ export function PresentationView() {
   const [saving, setSaving] = useState(false)
   const [imgBusy, setImgBusy] = useState(false)
   const imgInputRef = useRef<HTMLInputElement>(null)
+  const presentRef = useRef<HTMLDivElement>(null)
+  const suppressFsPopup = useRef(false)
+
+  const [presentMode, setPresentMode] = useState<PresentMode>(null)
 
   const isCustom = customDecks.some((d) => d.id === source)
+  const canEditInPlace = source === 'live' || isCustom
+  const liveEdited = source === 'live' && Boolean(data?.edited)
 
   const refreshDeckList = useCallback(async () => {
     try {
@@ -105,20 +244,94 @@ export function PresentationView() {
     refreshDeckList()
   }, [source, load, refreshDeckList])
 
+  const slides = data?.slides || []
+  const slide: PresentationSlide | undefined = slides[slideIdx]
+
+  const closePresent = useCallback(() => {
+    suppressFsPopup.current = true
+    setPresentMode(null)
+    if (document.fullscreenElement) {
+      void document.exitFullscreen()
+    }
+    window.setTimeout(() => { suppressFsPopup.current = false }, 0)
+  }, [])
+
+  const openPresent = useCallback((mode: 'popup' | 'fullscreen') => {
+    if (!data?.slides?.length) return
+    setPresentMode(mode)
+  }, [data?.slides?.length])
+
+  const enterBrowserFullscreen = useCallback(async () => {
+    const el = presentRef.current
+    if (!el) return
+    try {
+      await el.requestFullscreen()
+      setPresentMode('fullscreen')
+    } catch {
+      setPresentMode('popup')
+    }
+  }, [])
+
   useEffect(() => {
-    if (editing) return
+    if (editing || presentMode) return
     const onKey = (e: KeyboardEvent) => {
       const n = data?.slides.length || 1
       if (e.key === 'ArrowRight' || e.key === ' ') { e.preventDefault(); setSlideIdx((i) => Math.min(n - 1, i + 1)) }
       if (e.key === 'ArrowLeft') setSlideIdx((i) => Math.max(0, i - 1))
-      if (e.key === 'f' || e.key === 'F') document.documentElement.requestFullscreen?.()
+      if ((e.key === 'f' || e.key === 'F') && data?.slides?.length) openPresent('fullscreen')
     }
     window.addEventListener('keydown', onKey)
     return () => window.removeEventListener('keydown', onKey)
-  }, [data?.slides.length, editing])
+  }, [data?.slides.length, editing, presentMode, openPresent])
 
-  const slides = data?.slides || []
-  const slide: PresentationSlide | undefined = slides[slideIdx]
+  useEffect(() => {
+    if (presentMode !== 'fullscreen') return
+    const id = window.requestAnimationFrame(() => { void enterBrowserFullscreen() })
+    return () => window.cancelAnimationFrame(id)
+  }, [presentMode, enterBrowserFullscreen])
+
+  useEffect(() => {
+    const onFsChange = () => {
+      if (!document.fullscreenElement && !suppressFsPopup.current) {
+        setPresentMode((mode) => (mode === 'fullscreen' ? 'popup' : mode))
+      }
+    }
+    document.addEventListener('fullscreenchange', onFsChange)
+    return () => document.removeEventListener('fullscreenchange', onFsChange)
+  }, [])
+
+  useEffect(() => {
+    if (!presentMode) return
+    const prev = document.body.style.overflow
+    document.body.style.overflow = 'hidden'
+    const onKey = (e: KeyboardEvent) => {
+      const n = slides.length || 1
+      if (e.key === 'Escape') {
+        e.preventDefault()
+        closePresent()
+        return
+      }
+      if (e.key === 'ArrowRight' || e.key === ' ') {
+        e.preventDefault()
+        setSlideIdx((i) => Math.min(n - 1, i + 1))
+      }
+      if (e.key === 'ArrowLeft') setSlideIdx((i) => Math.max(0, i - 1))
+      if (e.key === 'f' || e.key === 'F') {
+        e.preventDefault()
+        if (document.fullscreenElement) {
+          void document.exitFullscreen()
+          setPresentMode('popup')
+        } else {
+          void enterBrowserFullscreen()
+        }
+      }
+    }
+    window.addEventListener('keydown', onKey)
+    return () => {
+      document.body.style.overflow = prev
+      window.removeEventListener('keydown', onKey)
+    }
+  }, [presentMode, slides.length, closePresent, enterBrowserFullscreen])
 
   const editIdx = editing ? slideIdx : null
   const editSlides = draft?.slides || []
@@ -221,6 +434,26 @@ export function PresentationView() {
     setDraft(null)
   }
 
+  const resetLive = async () => {
+    if (!window.confirm('Reset Live Cluster to a fresh cluster snapshot? Your edits will be lost.')) return
+    setUploadMsg(null)
+    try {
+      const r = await fetch('/api/presentation/live/reset', { method: 'POST' })
+      const j = await r.json()
+      if (j.ok && j.deck) {
+        setData(j.deck)
+        setSlideIdx(0)
+        setEditing(false)
+        setDraft(null)
+        setUploadMsg('✓ Reset to cluster snapshot')
+      } else {
+        setUploadMsg(j.error || 'Reset failed')
+      }
+    } catch {
+      setUploadMsg('Reset failed — check connection')
+    }
+  }
+
   const patchSlide = (idx: number, patch: Partial<PresentationSlide>) => {
     setDraft((d) => {
       if (!d) return d
@@ -279,74 +512,174 @@ export function PresentationView() {
   ]
 
   return (
-    <div className="flex h-full min-h-[calc(100vh-140px)] flex-col overflow-hidden rounded-lg border border-border bg-surface-raised">
-      <div className="flex shrink-0 flex-wrap items-center justify-between gap-2 border-b border-border bg-surface-raised/90 px-3 py-2">
-        <div>
-          <h2 className="text-[11px] font-semibold uppercase tracking-wider text-foreground">Presentation</h2>
-          <p className="text-[9px] text-foreground-muted">
-            Live cluster · HTML templates · PPT upload · editable decks with text &amp; photos
-          </p>
-        </div>
-        <div className="flex flex-wrap gap-1">
+    <>
+    <div className={cn(
+      'flex min-h-0 flex-col overflow-hidden bg-surface-raised',
+      embedded ? 'h-full' : 'h-full min-h-[calc(100vh-140px)] rounded-lg border border-border',
+    )}>
+      {embedded ? (
+        <div className="flex shrink-0 flex-col gap-1 border-b border-border bg-surface-raised/90 px-2 py-1.5">
+          <div className="flex flex-wrap items-center justify-end gap-1">
+            {!editing && (
+              <>
+                <button type="button" onClick={() => newDeck(false)} className="inline-flex items-center gap-1 rounded border border-border px-2 py-0.5 text-[9px] hover:bg-surface-overlay">
+                  <Plus className="h-3 w-3" /> New
+                </button>
+                {canEditInPlace ? (
+                  <button type="button" onClick={startEdit} className="inline-flex items-center gap-1 rounded-md px-2 py-1 text-[9px] text-docker hover:bg-surface-overlay">
+                    <Pencil className="h-3 w-3" /> Edit
+                  </button>
+                ) : (
+                  <button type="button" onClick={() => newDeck(true)} className="inline-flex items-center gap-1 rounded border border-border px-2 py-0.5 text-[9px] hover:bg-surface-overlay">
+                    <Pencil className="h-3 w-3" /> Edit a copy
+                  </button>
+                )}
+                {liveEdited && (
+                  <button type="button" onClick={resetLive} className="rounded border border-border px-2 py-0.5 text-[9px] text-foreground-muted hover:bg-surface-overlay">
+                    Reset snapshot
+                  </button>
+                )}
+                <a href="/dq/" target="_blank" rel="noreferrer" className={cn('inline-flex items-center gap-1 rounded-md px-2 py-1 text-[9px]', subTabIdle)}>
+                  <Monitor className="h-3 w-3" /> DQ Portal
+                </a>
+                <a href="/docling/ui/" target="_blank" rel="noreferrer" className="inline-flex items-center gap-1 rounded border border-border px-2 py-0.5 text-[9px] hover:bg-surface-overlay">
+                  <ExternalLink className="h-3 w-3" /> Docling
+                </a>
+                <button type="button" onClick={() => load(source)} className="rounded border border-border px-2 py-0.5 text-[9px] hover:bg-surface-overlay">Refresh</button>
+                {!loading && slides.length > 0 && (
+                  <>
+                    <button type="button" onClick={() => openPresent('popup')} className="inline-flex items-center gap-1 rounded border border-border px-2 py-0.5 text-[9px] hover:bg-surface-overlay">
+                      <Maximize2 className="h-3 w-3" /> Popup
+                    </button>
+                    <button type="button" onClick={() => openPresent('fullscreen')} className={cn('inline-flex items-center gap-1 rounded-md px-2 py-1 text-[9px]', subTabIdle)}>
+                      <Expand className="h-3 w-3" /> Fullscreen
+                    </button>
+                  </>
+                )}
+                <button type="button" onClick={exportHtml} className={cn('rounded-md px-2 py-1 text-[9px]', subTabActive)}>Export HTML</button>
+              </>
+            )}
+            {editing && (
+              <>
+                <span className="inline-flex items-center rounded bg-docker/15 px-2 py-0.5 text-[9px] font-medium text-docker">Editing</span>
+                <button type="button" onClick={cancelEdit} className="inline-flex items-center gap-1 rounded border border-border px-2 py-0.5 text-[9px] hover:bg-surface-overlay">
+                  <X className="h-3 w-3" /> Cancel
+                </button>
+                <button type="button" onClick={saveDraft} disabled={saving} className={cn('inline-flex items-center gap-1 rounded-md px-2 py-1 text-[9px]', subTabActive, saving && 'opacity-50')}>
+                  <Save className="h-3 w-3" /> {saving ? 'Saving…' : 'Save'}
+                </button>
+              </>
+            )}
+          </div>
           {!editing && (
-            <>
-              <button type="button" onClick={() => newDeck(false)} className="inline-flex items-center gap-1 rounded border border-border px-2 py-0.5 text-[9px] hover:bg-surface-overlay">
-                <Plus className="h-3 w-3" /> New
-              </button>
-              {isCustom ? (
-                <button type="button" onClick={startEdit} className="inline-flex items-center gap-1 rounded-md px-2 py-1 text-[9px] text-docker hover:bg-surface-overlay">
-                  <Pencil className="h-3 w-3" /> Edit
+            <div className="scrollbar-thin flex shrink-0 gap-1 overflow-x-auto pb-0.5">
+              {tabs.map((t) => (
+                <button
+                  key={t.id}
+                  type="button"
+                  onClick={() => setSource(t.id)}
+                  className={cn(
+                    'shrink-0 rounded-md px-2 py-1 text-[10px] font-medium transition-all',
+                    source === t.id ? subTabActive : subTabIdle,
+                  )}
+                >
+                  {t.label}
                 </button>
-              ) : (
-                <button type="button" onClick={() => newDeck(true)} className="inline-flex items-center gap-1 rounded border border-border px-2 py-0.5 text-[9px] hover:bg-surface-overlay">
-                  <Pencil className="h-3 w-3" /> Edit a copy
-                </button>
-              )}
-              <a href="/dq/" target="_blank" rel="noreferrer" className={cn('inline-flex items-center gap-1 rounded-md px-2 py-1 text-[9px]', subTabIdle)}>
-                <Monitor className="h-3 w-3" /> DQ Portal
-              </a>
-              <a href="/docling/ui/" target="_blank" rel="noreferrer" className="inline-flex items-center gap-1 rounded border border-border px-2 py-0.5 text-[9px] hover:bg-surface-overlay">
-                <ExternalLink className="h-3 w-3" /> Docling
-              </a>
-              <button type="button" onClick={() => load(source)} className="rounded border border-border px-2 py-0.5 text-[9px] hover:bg-surface-overlay">Refresh</button>
-              <button type="button" onClick={exportHtml} className={cn('rounded-md px-2 py-1 text-[9px]', subTabActive)}>Export HTML</button>
-            </>
-          )}
-          {editing && (
-            <>
-              <span className="inline-flex items-center rounded bg-docker/15 px-2 py-0.5 text-[9px] font-medium text-docker">Editing</span>
-              <button type="button" onClick={cancelEdit} className="inline-flex items-center gap-1 rounded border border-border px-2 py-0.5 text-[9px] hover:bg-surface-overlay">
-                <X className="h-3 w-3" /> Cancel
-              </button>
-              <button type="button" onClick={saveDraft} disabled={saving} className={cn('inline-flex items-center gap-1 rounded-md px-2 py-1 text-[9px]', subTabActive, saving && 'opacity-50')}>
-                <Save className="h-3 w-3" /> {saving ? 'Saving…' : 'Save'}
-              </button>
-            </>
+              ))}
+              <label className={cn('ml-1 inline-flex shrink-0 cursor-pointer items-center gap-1 rounded-md border border-dashed border-border px-2 py-1 text-[10px]', uploading && 'opacity-50')}>
+                <Upload className="h-3 w-3" />
+                {uploading ? 'Uploading…' : 'PPT upload'}
+                <input type="file" accept=".ppt,.pptx,.pdf,.docx" className="hidden" disabled={uploading} onChange={(e) => e.target.files?.[0] && onUpload(e.target.files[0])} />
+              </label>
+            </div>
           )}
         </div>
-      </div>
+      ) : (
+        <>
+          <div className="flex shrink-0 flex-wrap items-center justify-between gap-2 border-b border-border bg-surface-raised/90 px-3 py-2">
+            <div>
+              <h2 className="text-[11px] font-semibold uppercase tracking-wider text-foreground">Presentation</h2>
+              <p className="text-[9px] text-foreground-muted">
+                Live cluster · HTML templates · PPT upload · editable decks with text &amp; photos
+              </p>
+            </div>
+            <div className="flex flex-wrap gap-1">
+              {!editing && (
+                <>
+                  <button type="button" onClick={() => newDeck(false)} className="inline-flex items-center gap-1 rounded border border-border px-2 py-0.5 text-[9px] hover:bg-surface-overlay">
+                    <Plus className="h-3 w-3" /> New
+                  </button>
+                  {canEditInPlace ? (
+                    <button type="button" onClick={startEdit} className="inline-flex items-center gap-1 rounded-md px-2 py-1 text-[9px] text-docker hover:bg-surface-overlay">
+                      <Pencil className="h-3 w-3" /> Edit
+                    </button>
+                  ) : (
+                    <button type="button" onClick={() => newDeck(true)} className="inline-flex items-center gap-1 rounded border border-border px-2 py-0.5 text-[9px] hover:bg-surface-overlay">
+                      <Pencil className="h-3 w-3" /> Edit a copy
+                    </button>
+                  )}
+                  {liveEdited && (
+                    <button type="button" onClick={resetLive} className="rounded border border-border px-2 py-0.5 text-[9px] text-foreground-muted hover:bg-surface-overlay">
+                      Reset snapshot
+                    </button>
+                  )}
+                  <a href="/dq/" target="_blank" rel="noreferrer" className={cn('inline-flex items-center gap-1 rounded-md px-2 py-1 text-[9px]', subTabIdle)}>
+                    <Monitor className="h-3 w-3" /> DQ Portal
+                  </a>
+                  <a href="/docling/ui/" target="_blank" rel="noreferrer" className="inline-flex items-center gap-1 rounded border border-border px-2 py-0.5 text-[9px] hover:bg-surface-overlay">
+                    <ExternalLink className="h-3 w-3" /> Docling
+                  </a>
+                  <button type="button" onClick={() => load(source)} className="rounded border border-border px-2 py-0.5 text-[9px] hover:bg-surface-overlay">Refresh</button>
+                  {!loading && slides.length > 0 && (
+                    <>
+                      <button type="button" onClick={() => openPresent('popup')} className="inline-flex items-center gap-1 rounded border border-border px-2 py-0.5 text-[9px] hover:bg-surface-overlay">
+                        <Maximize2 className="h-3 w-3" /> Popup
+                      </button>
+                      <button type="button" onClick={() => openPresent('fullscreen')} className={cn('inline-flex items-center gap-1 rounded-md px-2 py-1 text-[9px]', subTabIdle)}>
+                        <Expand className="h-3 w-3" /> Fullscreen
+                      </button>
+                    </>
+                  )}
+                  <button type="button" onClick={exportHtml} className={cn('rounded-md px-2 py-1 text-[9px]', subTabActive)}>Export HTML</button>
+                </>
+              )}
+              {editing && (
+                <>
+                  <span className="inline-flex items-center rounded bg-docker/15 px-2 py-0.5 text-[9px] font-medium text-docker">Editing</span>
+                  <button type="button" onClick={cancelEdit} className="inline-flex items-center gap-1 rounded border border-border px-2 py-0.5 text-[9px] hover:bg-surface-overlay">
+                    <X className="h-3 w-3" /> Cancel
+                  </button>
+                  <button type="button" onClick={saveDraft} disabled={saving} className={cn('inline-flex items-center gap-1 rounded-md px-2 py-1 text-[9px]', subTabActive, saving && 'opacity-50')}>
+                    <Save className="h-3 w-3" /> {saving ? 'Saving…' : 'Save'}
+                  </button>
+                </>
+              )}
+            </div>
+          </div>
 
-      {!editing && (
-        <div className="flex shrink-0 flex-wrap gap-1 border-b border-border bg-surface-overlay/40 px-2 py-1.5">
-          {tabs.map((t) => (
-            <button
-              key={t.id}
-              type="button"
-              onClick={() => setSource(t.id)}
-              className={cn(
-                'rounded-md px-2.5 py-1.5 text-[10px] font-medium transition-all',
-                source === t.id ? subTabActive : subTabIdle,
-              )}
-            >
-              {t.label}
-            </button>
-          ))}
-          <label className={cn('ml-auto inline-flex cursor-pointer items-center gap-1 rounded-md border border-dashed border-border px-2 py-1 text-[10px]', uploading && 'opacity-50')}>
-            <Upload className="h-3 w-3" />
-            {uploading ? 'Uploading…' : 'PPT upload'}
-            <input type="file" accept=".ppt,.pptx,.pdf,.docx" className="hidden" disabled={uploading} onChange={(e) => e.target.files?.[0] && onUpload(e.target.files[0])} />
-          </label>
-        </div>
+          {!editing && (
+            <div className="scrollbar-thin flex shrink-0 gap-1 overflow-x-auto border-b border-border bg-surface-overlay/40 px-2 py-1.5">
+              {tabs.map((t) => (
+                <button
+                  key={t.id}
+                  type="button"
+                  onClick={() => setSource(t.id)}
+                  className={cn(
+                    'shrink-0 rounded-md px-2.5 py-1.5 text-[10px] font-medium transition-all',
+                    source === t.id ? subTabActive : subTabIdle,
+                  )}
+                >
+                  {t.label}
+                </button>
+              ))}
+              <label className={cn('ml-auto inline-flex shrink-0 cursor-pointer items-center gap-1 rounded-md border border-dashed border-border px-2 py-1 text-[10px]', uploading && 'opacity-50')}>
+                <Upload className="h-3 w-3" />
+                {uploading ? 'Uploading…' : 'PPT upload'}
+                <input type="file" accept=".ppt,.pptx,.pdf,.docx" className="hidden" disabled={uploading} onChange={(e) => e.target.files?.[0] && onUpload(e.target.files[0])} />
+              </label>
+            </div>
+          )}
+        </>
       )}
 
       {uploadMsg && <p className="shrink-0 px-3 py-1 text-[10px] text-docker">{uploadMsg}</p>}
@@ -493,45 +826,103 @@ export function PresentationView() {
         </div>
       ) : (
         /* ─────────── VIEW MODE ─────────── */
-        <>
-          <div className={cn('relative flex min-h-0 flex-1 flex-col justify-center bg-gradient-to-br p-6 md:p-10', KIND_STYLES[slide.kind || 'narrative'] || KIND_STYLES.narrative)}>
-            <div className="flex max-w-5xl gap-6">
-              <div className="min-w-0 flex-1">
-                <p className="mb-1 text-[10px] font-medium uppercase tracking-widest text-docker/80">{slide.kind || 'slide'} · {slideIdx + 1}/{slides.length}</p>
-                <h1 className="mb-2 text-2xl font-bold tracking-tight text-foreground md:text-4xl">{slide.title}</h1>
-                {slide.subtitle && <p className="mb-4 text-sm text-foreground-muted md:text-base">{slide.subtitle}</p>}
-                {'animation' in slide && slide.animation && (
-                  <ArchitectureDiagram animation={String(slide.animation)} />
-                )}
-                <ul className="space-y-2 text-sm leading-relaxed text-foreground md:text-base">
-                  {(slide.bullets || []).map((b: string, bi: number) => (
-                    <li key={bi} className="flex gap-2"><span className="shrink-0 text-docker">▸</span><span>{b}</span></li>
-                  ))}
-                </ul>
-              </div>
-              {slide.image && (
-                <div className="hidden shrink-0 items-center md:flex">
-                  <img src={slide.image} alt="" className="max-h-[46vh] max-w-[40vw] rounded-lg border border-border object-contain shadow-lg" />
-                </div>
-              )}
-            </div>
-            {slide.image && (
-              <div className="mt-4 md:hidden">
-                <img src={slide.image} alt="" className="max-h-[30vh] rounded-lg border border-border object-contain" />
-              </div>
-            )}
+        <div className="flex min-h-0 flex-1 flex-col overflow-hidden">
+          <div className={cn(
+            'scrollbar-thin min-h-0 flex-1 overflow-y-auto bg-gradient-to-br',
+            embedded ? 'p-3' : 'p-6 md:p-10',
+            KIND_STYLES[slide.kind || 'narrative'] || KIND_STYLES.narrative,
+          )}>
+            <SlidePanel slide={slide} slideIdx={slideIdx} slideCount={slides.length} variant={embedded ? 'embedded' : 'normal'} />
           </div>
-          <div className="flex shrink-0 items-center gap-2 border-t border-border bg-surface-raised/90 px-3 py-2">
-            <button type="button" disabled={slideIdx === 0} onClick={() => setSlideIdx((i) => Math.max(0, i - 1))} className="rounded border border-border px-2 py-1 text-[10px] disabled:opacity-40">← Prev</button>
-            <div className="flex flex-1 flex-wrap justify-center gap-1">
-              {slides.map((_: PresentationSlide, i: number) => (
-                <button key={i} type="button" onClick={() => setSlideIdx(i)} className={cn('h-2 w-2 rounded-full', i === slideIdx ? 'scale-125 bg-docker' : 'bg-border')} />
-              ))}
-            </div>
-            <button type="button" disabled={slideIdx >= slides.length - 1} onClick={() => setSlideIdx((i) => Math.min(slides.length - 1, i + 1))} className="rounded border border-border px-2 py-1 text-[10px] disabled:opacity-40">Next →</button>
-          </div>
-        </>
+          <SlideNav
+            slideIdx={slideIdx}
+            slideCount={slides.length}
+            embedded={embedded}
+            onPrev={() => setSlideIdx((i) => Math.max(0, i - 1))}
+            onNext={() => setSlideIdx((i) => Math.min(slides.length - 1, i + 1))}
+            onGo={setSlideIdx}
+          />
+        </div>
       )}
     </div>
+    {presentMode && slide && createPortal(
+      <>
+        {presentMode === 'popup' && (
+          <button
+            type="button"
+            aria-label="Close presentation"
+            className="fixed inset-0 z-[199] bg-black/80 backdrop-blur-[2px]"
+            onClick={closePresent}
+          />
+        )}
+        <div
+          ref={presentRef}
+          className={cn(
+            'fixed z-[200] flex flex-col overflow-hidden bg-surface text-foreground shadow-2xl',
+            presentMode === 'popup'
+              ? 'inset-2 rounded-xl border border-border ring-1 ring-white/10 sm:inset-4 md:inset-8 lg:inset-10'
+              : 'inset-0',
+          )}
+        >
+          <div className="flex shrink-0 items-center justify-between gap-2 border-b border-border bg-surface-raised/95 px-4 py-2">
+            <div className="min-w-0 truncate text-xs text-foreground-muted md:text-sm">
+              {data?.title || 'Presentation'} · slide {slideIdx + 1}/{slides.length}
+            </div>
+            <div className="flex shrink-0 items-center gap-1">
+              {presentMode === 'popup' ? (
+                <button
+                  type="button"
+                  onClick={() => void enterBrowserFullscreen()}
+                  className="inline-flex items-center gap-1 rounded border border-border px-2 py-1 text-[10px] hover:bg-surface-overlay md:text-xs"
+                >
+                  <Expand className="h-3.5 w-3.5" /> Fullscreen
+                </button>
+              ) : (
+                <button
+                  type="button"
+                  onClick={() => {
+                    suppressFsPopup.current = true
+                    if (document.fullscreenElement) void document.exitFullscreen()
+                    setPresentMode('popup')
+                    window.setTimeout(() => { suppressFsPopup.current = false }, 0)
+                  }}
+                  className="inline-flex items-center gap-1 rounded border border-border px-2 py-1 text-[10px] hover:bg-surface-overlay md:text-xs"
+                >
+                  <Maximize2 className="h-3.5 w-3.5" /> Popup
+                </button>
+              )}
+              <button
+                type="button"
+                onClick={closePresent}
+                className="inline-flex items-center gap-1 rounded border border-border px-2 py-1 text-[10px] hover:bg-surface-overlay md:text-xs"
+              >
+                <X className="h-3.5 w-3.5" /> Close
+              </button>
+            </div>
+          </div>
+          <div className={cn(
+            'scrollbar-thin flex min-h-0 flex-1 flex-col overflow-hidden bg-gradient-to-br',
+            KIND_STYLES[slide.kind || 'narrative'] || KIND_STYLES.narrative,
+          )}>
+            <div className="min-h-0 flex-1 overflow-y-auto p-8 md:p-12 lg:p-16">
+              <SlidePanel slide={slide} slideIdx={slideIdx} slideCount={slides.length} variant="present" />
+            </div>
+            <SlideNav
+              slideIdx={slideIdx}
+              slideCount={slides.length}
+              onPrev={() => setSlideIdx((i) => Math.max(0, i - 1))}
+              onNext={() => setSlideIdx((i) => Math.min(slides.length - 1, i + 1))}
+              onGo={setSlideIdx}
+              className="border-t border-border/80 bg-surface-raised/95 px-4 py-2.5"
+            />
+          </div>
+          <p className="pointer-events-none absolute bottom-3 right-4 text-[10px] text-foreground-faint">
+            ← → navigate · F toggle fullscreen · Esc close
+          </p>
+        </div>
+      </>,
+      document.body,
+    )}
+    </>
   )
 }

ui/src/components/features/SparkKafkaPanel.tsx

@@ -0,0 +1,346 @@
+import { useCallback, useEffect, useState } from 'react'
+import {
+  Cpu,
+  ExternalLink,
+  Loader2,
+  Pause,
+  Play,
+  RefreshCw,
+  RotateCcw,
+  Zap,
+} from 'lucide-react'
+import type { StreamingStatus } from '../../types'
+import {
+  fetchStreamingStatus,
+  restartKafkaConnector,
+  pauseKafkaConnector,
+  resumeKafkaConnector,
+  triggerStreamingJob,
+} from '../../lib/api'
+import { cn } from '../../lib/utils'
+
+type Tab = 'spark' | 'kafka' | 'jobs'
+
+export function SparkKafkaPanel({
+  embedded,
+  selectedNodeId,
+  streaming: initialStreaming,
+  onRefreshGraph,
+}: {
+  embedded?: boolean
+  selectedNodeId?: string | null
+  streaming?: StreamingStatus | null
+  onRefreshGraph?: () => void
+}) {
+  const [tab, setTab] = useState<Tab>('spark')
+  const [streaming, setStreaming] = useState<StreamingStatus | null>(initialStreaming ?? null)
+  const [loading, setLoading] = useState(!initialStreaming)
+  const [busy, setBusy] = useState<string | null>(null)
+  const [jobConf, setJobConf] = useState<Record<string, string>>({})
+  const [selectedJob, setSelectedJob] = useState('spark_to_curated')
+  const [showSparkUi, setShowSparkUi] = useState(false)
+
+  const load = useCallback(async (refresh = false) => {
+    setLoading(true)
+    const s = await fetchStreamingStatus(refresh)
+    if (s) setStreaming(s)
+    setLoading(false)
+  }, [])
+
+  useEffect(() => {
+    if (initialStreaming) setStreaming(initialStreaming)
+  }, [initialStreaming])
+
+  useEffect(() => {
+    if (selectedNodeId === 'spark') setTab('spark')
+    if (selectedNodeId === 'kafka') setTab('kafka')
+  }, [selectedNodeId])
+
+  useEffect(() => {
+    const iv = setInterval(() => load(), 8000)
+    return () => clearInterval(iv)
+  }, [load])
+
+  const spark = streaming?.spark
+  const kafka = streaming?.kafka
+  const jobs = streaming?.jobs ?? []
+
+  const onJob = async (jobId: string) => {
+    setBusy(`job:${jobId}`)
+    try {
+      let conf: Record<string, unknown> = {}
+      const raw = jobConf[jobId]
+      if (raw?.trim()) {
+        conf = JSON.parse(raw)
+      }
+      await triggerStreamingJob(jobId, conf)
+      onRefreshGraph?.()
+      await load(true)
+    } catch {
+      /* ignore */
+    } finally {
+      setBusy(null)
+    }
+  }
+
+  const onConnector = async (name: string, action: 'restart' | 'pause' | 'resume') => {
+    setBusy(`conn:${name}:${action}`)
+    try {
+      if (action === 'restart') await restartKafkaConnector(name)
+      else if (action === 'pause') await pauseKafkaConnector(name)
+      else await resumeKafkaConnector(name)
+      await load(true)
+      onRefreshGraph?.()
+    } finally {
+      setBusy(null)
+    }
+  }
+
+  return (
+    <div className={cn(
+      'flex shrink-0 flex-col border-t border-border bg-surface/80',
+      embedded ? 'max-h-[42vh]' : 'max-h-[48vh]',
+    )}>
+      <div className="flex shrink-0 items-center justify-between gap-2 border-b border-border px-3 py-1.5">
+        <div className="flex items-center gap-1">
+          {(['spark', 'kafka', 'jobs'] as Tab[]).map((t) => (
+            <button
+              key={t}
+              type="button"
+              onClick={() => setTab(t)}
+              className={cn(
+                'rounded-md px-2.5 py-1 text-[10px] font-medium capitalize transition-colors',
+                tab === t
+                  ? 'bg-docker/20 text-docker ring-1 ring-docker/40'
+                  : 'text-foreground-muted hover:bg-surface-overlay hover:text-foreground',
+              )}
+            >
+              {t === 'jobs' ? 'Run jobs' : t === 'spark' ? 'Spark UI' : 'Kafka'}
+            </button>
+          ))}
+        </div>
+        <div className="flex items-center gap-1">
+          {spark?.ui_ok && (
+            <span className="hidden text-[9px] text-emerald-300 sm:inline">
+              Spark {spark.alive_workers}w · {spark.cores_used}/{spark.cores} cores
+            </span>
+          )}
+          {kafka?.connect_ok && (
+            <span className="hidden text-[9px] text-cyan-300 sm:inline">
+              {kafka.connectors?.length ?? 0} connectors
+            </span>
+          )}
+          <button
+            type="button"
+            onClick={() => load(true)}
+            className="inline-flex items-center gap-1 rounded border border-border px-1.5 py-0.5 text-[9px] text-foreground-muted hover:text-foreground"
+          >
+            <RefreshCw className={cn('h-3 w-3', loading && 'animate-spin')} /> Refresh
+          </button>
+        </div>
+      </div>
+
+      <div className="scrollbar-thin min-h-0 flex-1 overflow-y-auto p-3">
+        {tab === 'spark' && (
+          <div className="space-y-3">
+            <div className="flex flex-wrap items-center gap-2">
+              <Badge ok={spark?.ui_ok} label={spark?.ui_ok ? `Cluster ${spark.status}` : 'Spark UI offline'} />
+              <button
+                type="button"
+                onClick={() => setShowSparkUi((v) => !v)}
+                className="inline-flex items-center gap-1 rounded border border-border px-2 py-0.5 text-[9px] hover:bg-surface-overlay"
+              >
+                <Cpu className="h-3 w-3" /> {showSparkUi ? 'Hide' : 'Embed'} Spark UI
+              </button>
+              <a href="/spark-ui/" target="_blank" rel="noreferrer" className="inline-flex items-center gap-1 text-[9px] text-docker hover:underline">
+                Open full Spark UI <ExternalLink className="h-3 w-3" />
+              </a>
+            </div>
+
+            {showSparkUi && (
+              <iframe
+                title="Spark Master UI"
+                src="/spark-ui/"
+                className="h-[280px] w-full rounded-lg border border-border bg-black"
+              />
+            )}
+
+            <div className="grid gap-2 sm:grid-cols-3">
+              <Stat label="Workers" value={String(spark?.alive_workers ?? 0)} />
+              <Stat label="Cores used" value={`${spark?.cores_used ?? 0} / ${spark?.cores ?? 0}`} />
+              <Stat label="Memory MB" value={`${spark?.memory_used_mb ?? 0} / ${spark?.memory_mb ?? 0}`} />
+            </div>
+
+            {(spark?.workers?.length ?? 0) > 0 && (
+              <div>
+                <p className="mb-1 text-[9px] font-semibold uppercase tracking-wide text-foreground-muted">Workers</p>
+                <div className="overflow-x-auto rounded border border-border">
+                  <table className="w-full text-left text-[9px]">
+                    <thead className="bg-surface-overlay/60 text-foreground-muted">
+                      <tr>
+                        <th className="px-2 py-1">Host</th>
+                        <th className="px-2 py-1">State</th>
+                        <th className="px-2 py-1">Cores</th>
+                        <th className="px-2 py-1">Memory</th>
+                      </tr>
+                    </thead>
+                    <tbody>
+                      {spark!.workers!.map((w) => (
+                        <tr key={w.id} className="border-t border-border/60">
+                          <td className="px-2 py-1 font-mono">{w.host}</td>
+                          <td className="px-2 py-1 text-emerald-300">{w.state}</td>
+                          <td className="px-2 py-1">{w.cores_used}/{w.cores}</td>
+                          <td className="px-2 py-1">{w.memory_mb} MB</td>
+                        </tr>
+                      ))}
+                    </tbody>
+                  </table>
+                </div>
+              </div>
+            )}
+
+            <div>
+              <p className="mb-1 text-[9px] font-semibold uppercase tracking-wide text-foreground-muted">Active applications</p>
+              {(spark?.active_apps?.length ?? 0) === 0 ? (
+                <p className="text-[10px] text-foreground-muted">No running Spark apps — trigger a job below or start streaming on lake01.</p>
+              ) : (
+                <ul className="space-y-1">
+                  {spark!.active_apps!.map((a) => (
+                    <li key={a.id} className="rounded border border-border bg-surface-overlay/40 px-2 py-1 text-[10px]">
+                      <span className="font-semibold text-foreground">{a.name}</span>
+                      <span className="ml-2 font-mono text-foreground-muted">id={a.id} · {a.cores} cores</span>
+                    </li>
+                  ))}
+                </ul>
+              )}
+            </div>
+          </div>
+        )}
+
+        {tab === 'kafka' && (
+          <div className="space-y-3">
+            <div className="flex flex-wrap items-center gap-2">
+              <Badge ok={kafka?.ui_ok} label={kafka?.ui_ok ? `Cluster ${kafka.cluster?.name}` : 'Kafka UI offline'} />
+              <Badge ok={kafka?.connect_ok} label={`${kafka?.connectors?.length ?? 0} Connectors`} />
+              <a href="/kafka-ui/" target="_blank" rel="noreferrer" className="inline-flex items-center gap-1 text-[9px] text-docker hover:underline">
+                Kafka UI <ExternalLink className="h-3 w-3" />
+              </a>
+            </div>
+
+            <div>
+              <p className="mb-1 text-[9px] font-semibold uppercase tracking-wide text-foreground-muted">Debezium connectors</p>
+              <div className="space-y-1">
+                {(kafka?.connectors ?? []).map((c) => {
+                  const running = (c.state || '').toUpperCase() === 'RUNNING'
+                  const b = busy === `conn:${c.name}:restart` || busy === `conn:${c.name}:pause` || busy === `conn:${c.name}:resume`
+                  return (
+                    <div key={c.name} className="flex flex-wrap items-center gap-2 rounded border border-border bg-surface-overlay/30 px-2 py-1.5">
+                      <span className="min-w-0 flex-1 truncate font-mono text-[10px] text-foreground">{c.name}</span>
+                      <span className={cn('text-[9px] font-medium', running ? 'text-emerald-300' : 'text-amber-300')}>{c.state}</span>
+                      <button type="button" disabled={!!busy} onClick={() => onConnector(c.name, 'restart')} className="inline-flex items-center gap-0.5 rounded border border-border px-1.5 py-0.5 text-[8px] hover:bg-surface-overlay disabled:opacity-50">
+                        {b ? <Loader2 className="h-2.5 w-2.5 animate-spin" /> : <RotateCcw className="h-2.5 w-2.5" />} Restart
+                      </button>
+                      <button type="button" disabled={!!busy} onClick={() => onConnector(c.name, running ? 'pause' : 'resume')} className="inline-flex items-center gap-0.5 rounded border border-border px-1.5 py-0.5 text-[8px] hover:bg-surface-overlay disabled:opacity-50">
+                        {running ? <Pause className="h-2.5 w-2.5" /> : <Play className="h-2.5 w-2.5" />}
+                        {running ? 'Pause' : 'Resume'}
+                      </button>
+                    </div>
+                  )
+                })}
+              </div>
+            </div>
+
+            <div>
+              <p className="mb-1 text-[9px] font-semibold uppercase tracking-wide text-foreground-muted">Topics ({kafka?.topics?.length ?? 0})</p>
+              <div className="max-h-40 overflow-y-auto rounded border border-border">
+                <table className="w-full text-left text-[9px]">
+                  <thead className="sticky top-0 bg-surface-overlay/90 text-foreground-muted">
+                    <tr>
+                      <th className="px-2 py-1">Topic</th>
+                      <th className="px-2 py-1">Partitions</th>
+                    </tr>
+                  </thead>
+                  <tbody>
+                    {(kafka?.topics ?? []).slice(0, 30).map((t) => (
+                      <tr key={t.name} className="border-t border-border/50">
+                        <td className="max-w-[200px] truncate px-2 py-0.5 font-mono">{t.name}</td>
+                        <td className="px-2 py-0.5">{t.partitions ?? '—'}</td>
+                      </tr>
+                    ))}
+                  </tbody>
+                </table>
+              </div>
+            </div>
+          </div>
+        )}
+
+        {tab === 'jobs' && (
+          <div className="space-y-3">
+            <p className="text-[10px] text-foreground-muted">
+              Start lakehouse transforms manually via Airflow. Edit JSON config per job, then run.
+            </p>
+            <div className="flex flex-wrap gap-1">
+              {jobs.map((j) => (
+                <button
+                  key={j.id}
+                  type="button"
+                  onClick={() => setSelectedJob(j.id)}
+                  className={cn(
+                    'rounded border px-2 py-1 text-[9px] font-medium',
+                    selectedJob === j.id ? 'border-docker/50 bg-docker/15 text-docker' : 'border-border text-foreground-muted hover:text-foreground',
+                  )}
+                >
+                  {j.label}
+                </button>
+              ))}
+            </div>
+            {jobs.filter((j) => j.id === selectedJob).map((j) => (
+              <div key={j.id} className="space-y-2 rounded-lg border border-border bg-surface-overlay/30 p-2">
+                <p className="text-[10px] text-foreground">{j.description}</p>
+                <p className="font-mono text-[9px] text-foreground-muted">DAG: {j.dag_id}</p>
+                <label className="block">
+                  <span className="mb-0.5 block text-[8px] uppercase text-foreground-faint">Job config (JSON, editable)</span>
+                  <textarea
+                    value={jobConf[j.id] ?? JSON.stringify(j.default_conf ?? {}, null, 2)}
+                    onChange={(e) => setJobConf((prev) => ({ ...prev, [j.id]: e.target.value }))}
+                    rows={4}
+                    className="w-full rounded border border-border bg-surface px-2 py-1 font-mono text-[10px] text-foreground outline-none focus:border-docker"
+                  />
+                </label>
+                <button
+                  type="button"
+                  disabled={busy === `job:${j.id}`}
+                  onClick={() => onJob(j.id)}
+                  className="inline-flex items-center gap-1 rounded-md border border-emerald-400/50 bg-emerald-500/15 px-3 py-1.5 text-[10px] font-medium text-emerald-200 hover:bg-emerald-500/25 disabled:opacity-50"
+                >
+                  {busy === `job:${j.id}` ? <Loader2 className="h-3.5 w-3.5 animate-spin" /> : <Zap className="h-3.5 w-3.5" />}
+                  Start job
+                </button>
+              </div>
+            ))}
+          </div>
+        )}
+      </div>
+    </div>
+  )
+}
+
+function Badge({ ok, label }: { ok?: boolean; label: string }) {
+  return (
+    <span className={cn(
+      'inline-flex items-center rounded border px-1.5 py-0.5 text-[9px] font-medium',
+      ok ? 'border-emerald-400/40 bg-emerald-500/15 text-emerald-200' : 'border-amber-400/40 bg-amber-500/15 text-amber-200',
+    )}>
+      {label}
+    </span>
+  )
+}
+
+function Stat({ label, value }: { label: string; value: string }) {
+  return (
+    <div className="rounded border border-border bg-surface-overlay/40 px-2 py-1.5">
+      <p className="text-[8px] uppercase text-foreground-faint">{label}</p>
+      <p className="font-mono text-sm font-semibold text-foreground">{value}</p>
+    </div>
+  )
+}

ui/src/components/features/SparkView.tsx

@@ -0,0 +1,592 @@
+import { useCallback, useEffect, useRef, useState } from 'react'
+import {
+  Activity,
+  Cpu,
+  Database,
+  ExternalLink,
+  Gauge,
+  HardDrive,
+  Layers,
+  Loader2,
+  Play,
+  Plus,
+  RefreshCw,
+  Save,
+  Server,
+  Square,
+  Table2,
+  Trash2,
+  Zap,
+} from 'lucide-react'
+import type { SparkLive, SparkRun, SparkRunStats } from '../../types'
+import {
+  cancelSparkRun,
+  createSparkRun,
+  fetchSparkCatalogs,
+  fetchSparkColumns,
+  fetchSparkLive,
+  fetchSparkRun,
+  fetchSparkRuns,
+  fetchSparkSchemas,
+  fetchSparkTables,
+} from '../../lib/api'
+import { cn } from '../../lib/utils'
+import { subTabActive, subTabIdle } from '../../lib/tabActive'
+
+type Tab = 'workbench' | 'cluster' | 'runs' | 'ui'
+type Operation = 'preview' | 'filter' | 'aggregate' | 'profile' | 'join' | 'sql'
+type Column = { name: string; type: string }
+type Metric = { fn: string; col: string; alias?: string }
+
+const OPERATIONS: { id: Operation; label: string; desc: string }[] = [
+  { id: 'preview', label: 'Preview', desc: 'Sample rows from a table' },
+  { id: 'filter', label: 'Filter', desc: 'WHERE predicate on a table' },
+  { id: 'aggregate', label: 'Aggregate', desc: 'Group by + sum/avg/count/min/max' },
+  { id: 'profile', label: 'Profile', desc: 'Row count, distinct & non-null per column' },
+  { id: 'join', label: 'Join', desc: 'Join two tables on keys' },
+  { id: 'sql', label: 'SQL', desc: 'Run arbitrary distributed SQL' },
+]
+
+const AGG_FNS = ['count', 'sum', 'avg', 'min', 'max', 'approx_distinct', 'count_distinct']
+
+function fmtNum(n?: number | null) {
+  if (n == null) return '—'
+  if (n >= 1e9) return `${(n / 1e9).toFixed(2)}B`
+  if (n >= 1e6) return `${(n / 1e6).toFixed(2)}M`
+  if (n >= 1e3) return `${(n / 1e3).toFixed(1)}K`
+  return String(n)
+}
+function fmtBytes(n?: number | null) {
+  if (n == null) return '—'
+  let v = n
+  for (const u of ['B', 'KB', 'MB', 'GB', 'TB']) {
+    if (v < 1024) return `${v.toFixed(u === 'B' ? 0 : 1)} ${u}`
+    v /= 1024
+  }
+  return `${v.toFixed(1)} PB`
+}
+function fmtMs(n?: number | null) {
+  if (n == null) return '—'
+  if (n < 1000) return `${n} ms`
+  if (n < 60000) return `${(n / 1000).toFixed(1)} s`
+  return `${(n / 60000).toFixed(1)} m`
+}
+
+const STATE_COLOR: Record<string, string> = {
+  QUEUED: 'text-amber-300 bg-amber-500/15',
+  RUNNING: 'text-sky-300 bg-sky-500/15',
+  FINISHED: 'text-emerald-300 bg-emerald-500/15',
+  FAILED: 'text-rose-300 bg-rose-500/15',
+  CANCELED: 'text-foreground-muted bg-surface-overlay',
+}
+
+export function SparkView({ embedded }: { embedded?: boolean }) {
+  const [tab, setTab] = useState<Tab>('workbench')
+  const [live, setLive] = useState<SparkLive | null>(null)
+
+  const loadLive = useCallback(async () => {
+    const l = await fetchSparkLive()
+    if (l) setLive(l)
+  }, [])
+
+  useEffect(() => {
+    loadLive()
+    const iv = setInterval(loadLive, 4000)
+    return () => clearInterval(iv)
+  }, [loadLive])
+
+  const spark = live?.spark
+  const alive = spark?.ui_ok && (spark?.status || '').toUpperCase() === 'ALIVE'
+  const activeRuns = live?.active_runs ?? []
+
+  return (
+    <div className={cn('flex min-h-0 flex-1 flex-col gap-2', embedded ? 'p-2' : 'p-3')}>
+      <header className="panel flex shrink-0 flex-wrap items-center justify-between gap-3 px-4 py-3">
+        <div>
+          <h2 className="flex items-center gap-2 text-sm font-semibold text-foreground">
+            <Zap className="h-4 w-4 text-amber-400" />
+            Spark Lakehouse Workbench
+          </h2>
+          <p className="text-[10px] text-foreground-muted">
+            Select data · transform on the distributed engine · materialize to Iceberg / S3 · live cluster metrics
+          </p>
+        </div>
+        <div className="flex items-center gap-2">
+          <LiveChip label="Cluster" value={alive ? 'ALIVE' : 'down'} ok={!!alive} />
+          <LiveChip label="Cores" value={`${spark?.cores_used ?? 0}/${spark?.cores ?? 0}`} ok={(spark?.cores_used ?? 0) > 0} />
+          <LiveChip label="Active jobs" value={String(activeRuns.length)} ok={activeRuns.length > 0} />
+          <a href="/spark-ui/" target="_blank" rel="noreferrer" className="inline-flex items-center gap-1 rounded-md border border-border px-2 py-1 text-[10px] text-docker hover:bg-docker/10">
+            <ExternalLink className="h-3 w-3" /> Native UI
+          </a>
+        </div>
+      </header>
+
+      <div className="panel flex shrink-0 gap-1 px-2 py-1.5">
+        {(['workbench', 'cluster', 'runs', 'ui'] as Tab[]).map((t) => (
+          <button key={t} type="button" onClick={() => setTab(t)} className={cn('rounded-md px-2.5 py-1 text-[10px] font-medium capitalize', tab === t ? subTabActive : subTabIdle)}>
+            {t === 'ui' ? 'Spark UI' : t}
+            {t === 'runs' && activeRuns.length > 0 && <span className="ml-1 rounded-full bg-sky-500/30 px-1 text-[8px] text-sky-200">{activeRuns.length}</span>}
+          </button>
+        ))}
+      </div>
+
+      {tab === 'workbench' && <LakehouseWorkbench />}
+      {tab === 'cluster' && <ClusterPanel live={live} />}
+      {tab === 'runs' && <RunsPanel live={live} />}
+      {tab === 'ui' && (
+        <div className="panel min-h-0 flex-1 overflow-hidden p-1">
+          <iframe title="Spark Master UI" src="/spark-ui/" className="h-full min-h-[420px] w-full rounded-md border-0 bg-black" />
+        </div>
+      )}
+    </div>
+  )
+}
+
+/* ── Workbench: data selector + operation builder + live run ─────────────── */
+export function LakehouseWorkbench({ lockedCatalog }: { lockedCatalog?: string }) {
+  const [catalogs, setCatalogs] = useState<string[]>([])
+  const [catalog, setCatalog] = useState(lockedCatalog || 'iceberg')
+  const [schemas, setSchemas] = useState<string[]>([])
+  const [schema, setSchema] = useState('')
+  const [tables, setTables] = useState<{ name: string; fqn: string }[]>([])
+  const [table, setTable] = useState('')
+  const [columns, setColumns] = useState<Column[]>([])
+
+  const [op, setOp] = useState<Operation>('preview')
+  const [limit, setLimit] = useState(200)
+  const [where, setWhere] = useState('')
+  const [groupBy, setGroupBy] = useState<string[]>([])
+  const [metrics, setMetrics] = useState<Metric[]>([{ fn: 'count', col: '*' }])
+  const [profileCols, setProfileCols] = useState<string[]>([])
+  const [sql, setSql] = useState('SELECT region, count(*) AS orders, sum(amount) AS revenue\nFROM iceberg.hadoop.historical_sales_hdfs\nGROUP BY region\nORDER BY revenue DESC')
+  const [rightTable, setRightTable] = useState('')
+  const [leftKey, setLeftKey] = useState('')
+  const [rightKey, setRightKey] = useState('')
+  const [joinType, setJoinType] = useState('INNER')
+
+  const [matEnabled, setMatEnabled] = useState(false)
+  const [matSchema, setMatSchema] = useState('hadoop')
+  const [matTable, setMatTable] = useState('')
+  const [matMode, setMatMode] = useState('create')
+
+  const [run, setRun] = useState<SparkRun | null>(null)
+  const [submitting, setSubmitting] = useState(false)
+  const [err, setErr] = useState<string | null>(null)
+  const pollRef = useRef<ReturnType<typeof setInterval> | null>(null)
+
+  useEffect(() => {
+    if (lockedCatalog) { setCatalog(lockedCatalog); return }
+    fetchSparkCatalogs().then((c) => {
+      setCatalogs(c)
+      if (c.length && !c.includes(catalog)) setCatalog(c[0])
+    })
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [lockedCatalog])
+
+  useEffect(() => {
+    if (!catalog) return
+    setSchema(''); setTables([]); setTable(''); setColumns([])
+    fetchSparkSchemas(catalog).then((s) => { setSchemas(s); if (s.length) setSchema(s[0]) })
+  }, [catalog])
+
+  useEffect(() => {
+    if (!catalog || !schema) return
+    setTable(''); setColumns([])
+    fetchSparkTables(catalog, schema).then((t) => { setTables(t); if (t.length) setTable(t[0].fqn) })
+  }, [catalog, schema])
+
+  useEffect(() => {
+    if (!table) { setColumns([]); return }
+    fetchSparkColumns(table).then(setColumns)
+    setGroupBy([]); setProfileCols([])
+  }, [table])
+
+  useEffect(() => () => { if (pollRef.current) clearInterval(pollRef.current) }, [])
+
+  const startPolling = useCallback((runId: string) => {
+    if (pollRef.current) clearInterval(pollRef.current)
+    pollRef.current = setInterval(async () => {
+      const j = await fetchSparkRun(runId)
+      if (j?.run) {
+        setRun(j.run)
+        if (['FINISHED', 'FAILED', 'CANCELED'].includes(j.run.state)) {
+          if (pollRef.current) clearInterval(pollRef.current)
+        }
+      }
+    }, 700)
+  }, [])
+
+  const onRun = async () => {
+    setErr(null); setSubmitting(true); setRun(null)
+    const body: Record<string, unknown> = { operation: op, limit }
+    if (op !== 'sql' && op !== 'join') body.table = table
+    if (op === 'filter') body.where = where
+    if (op === 'aggregate') { body.table = table; body.group_by = groupBy; body.metrics = metrics }
+    if (op === 'profile') { body.table = table; body.columns = profileCols.length ? profileCols : columns.slice(0, 8).map((c) => c.name) }
+    if (op === 'sql') body.sql = sql
+    if (op === 'join') {
+      body.left = table; body.right = rightTable
+      body.left_key = leftKey; body.right_key = rightKey; body.join_type = joinType
+    }
+    if (matEnabled && matTable) body.materialize = { enabled: true, schema: matSchema, table: matTable, mode: matMode }
+    try {
+      const r = await createSparkRun(body)
+      if (!r.ok || !r.run_id) { setErr(r.error || 'Failed to submit'); setSubmitting(false); return }
+      startPolling(r.run_id)
+    } catch {
+      setErr('Submit failed')
+    } finally {
+      setSubmitting(false)
+    }
+  }
+
+  const onCancel = async () => { if (run) await cancelSparkRun(run.id) }
+
+  const colNames = columns.map((c) => c.name)
+
+  return (
+    <div className="flex min-h-0 flex-1 gap-2 overflow-hidden">
+      {/* data selector */}
+      <aside className="panel flex w-60 shrink-0 flex-col gap-2 overflow-y-auto p-3">
+        <h3 className="flex items-center gap-1.5 text-[11px] font-semibold text-foreground">
+          <Database className="h-3.5 w-3.5 text-emerald-400" /> Data
+        </h3>
+        {!lockedCatalog && (
+          <Field label="Catalog">
+            <select value={catalog} onChange={(e) => setCatalog(e.target.value)} className={selectCls}>
+              {catalogs.map((c) => <option key={c} value={c}>{c}</option>)}
+            </select>
+          </Field>
+        )}
+        <Field label="Schema">
+          <select value={schema} onChange={(e) => setSchema(e.target.value)} className={selectCls}>
+            {schemas.map((s) => <option key={s} value={s}>{s}</option>)}
+          </select>
+        </Field>
+        <Field label="Table">
+          <select value={table} onChange={(e) => setTable(e.target.value)} className={selectCls}>
+            {tables.map((t) => <option key={t.fqn} value={t.fqn}>{t.name}</option>)}
+          </select>
+        </Field>
+        <div className="min-h-0 flex-1">
+          <p className="mb-1 text-[9px] font-semibold uppercase tracking-wider text-foreground-muted">
+            {columns.length} columns
+          </p>
+          <div className="scrollbar-thin space-y-0.5 overflow-y-auto">
+            {columns.map((c) => (
+              <div key={c.name} className="flex items-center justify-between gap-1 rounded px-1.5 py-0.5 text-[9px] hover:bg-surface-overlay">
+                <span className="truncate font-mono text-foreground">{c.name}</span>
+                <span className="shrink-0 text-[8px] text-foreground-faint">{c.type}</span>
+              </div>
+            ))}
+          </div>
+        </div>
+      </aside>
+
+      {/* builder + run */}
+      <div className="flex min-h-0 flex-1 flex-col gap-2 overflow-hidden">
+        <div className="panel shrink-0 space-y-2 p-3">
+          <div className="flex flex-wrap gap-1">
+            {OPERATIONS.map((o) => (
+              <button key={o.id} type="button" onClick={() => setOp(o.id)} title={o.desc}
+                className={cn('rounded-md px-2.5 py-1 text-[10px] font-medium', op === o.id ? subTabActive : subTabIdle)}>
+                {o.label}
+              </button>
+            ))}
+          </div>
+
+          {op === 'sql' && (
+            <textarea value={sql} onChange={(e) => setSql(e.target.value)} rows={4} spellCheck={false}
+              className="w-full rounded-md border border-border bg-[#0d1117] p-2 font-mono text-[11px] text-emerald-100 outline-none" />
+          )}
+
+          {op === 'filter' && (
+            <Field label="WHERE predicate">
+              <input value={where} onChange={(e) => setWhere(e.target.value)} placeholder="amount > 1000 AND region = 'EU'" className={inputCls} />
+            </Field>
+          )}
+
+          {op === 'aggregate' && (
+            <div className="space-y-2">
+              <Field label="Group by">
+                <MultiChips options={colNames} selected={groupBy} onToggle={(c) =>
+                  setGroupBy((g) => g.includes(c) ? g.filter((x) => x !== c) : [...g, c])} />
+              </Field>
+              <div>
+                <p className="mb-1 text-[9px] font-semibold uppercase tracking-wider text-foreground-muted">Metrics</p>
+                {metrics.map((m, i) => (
+                  <div key={i} className="mb-1 flex items-center gap-1">
+                    <select value={m.fn} onChange={(e) => setMetrics((ms) => ms.map((x, j) => j === i ? { ...x, fn: e.target.value } : x))} className={cn(selectCls, 'w-32')}>
+                      {AGG_FNS.map((f) => <option key={f} value={f}>{f}</option>)}
+                    </select>
+                    <select value={m.col} onChange={(e) => setMetrics((ms) => ms.map((x, j) => j === i ? { ...x, col: e.target.value } : x))} className={cn(selectCls, 'flex-1')}>
+                      <option value="*">*</option>
+                      {colNames.map((c) => <option key={c} value={c}>{c}</option>)}
+                    </select>
+                    <button type="button" onClick={() => setMetrics((ms) => ms.filter((_, j) => j !== i))} className="rounded p-1 text-foreground-muted hover:text-rose-300">
+                      <Trash2 className="h-3 w-3" />
+                    </button>
+                  </div>
+                ))}
+                <button type="button" onClick={() => setMetrics((ms) => [...ms, { fn: 'sum', col: colNames[0] || '*' }])}
+                  className="inline-flex items-center gap-1 rounded border border-border px-2 py-0.5 text-[9px] text-foreground-muted hover:text-foreground">
+                  <Plus className="h-3 w-3" /> Add metric
+                </button>
+              </div>
+            </div>
+          )}
+
+          {op === 'profile' && (
+            <Field label="Columns (default: first 8)">
+              <MultiChips options={colNames} selected={profileCols} onToggle={(c) =>
+                setProfileCols((g) => g.includes(c) ? g.filter((x) => x !== c) : [...g, c])} />
+            </Field>
+          )}
+
+          {op === 'join' && (
+            <div className="grid grid-cols-2 gap-2">
+              <Field label="Right table (fqn)">
+                <input value={rightTable} onChange={(e) => setRightTable(e.target.value)} placeholder="postgres_sales.public.customers" className={inputCls} />
+              </Field>
+              <Field label="Join type">
+                <select value={joinType} onChange={(e) => setJoinType(e.target.value)} className={selectCls}>
+                  {['INNER', 'LEFT', 'RIGHT', 'FULL'].map((t) => <option key={t} value={t}>{t}</option>)}
+                </select>
+              </Field>
+              <Field label="Left key">
+                <select value={leftKey} onChange={(e) => setLeftKey(e.target.value)} className={selectCls}>
+                  <option value="">—</option>
+                  {colNames.map((c) => <option key={c} value={c}>{c}</option>)}
+                </select>
+              </Field>
+              <Field label="Right key">
+                <input value={rightKey} onChange={(e) => setRightKey(e.target.value)} placeholder="customer_id" className={inputCls} />
+              </Field>
+            </div>
+          )}
+
+          {/* materialize + run row */}
+          <div className="flex flex-wrap items-end gap-2 border-t border-border pt-2">
+            <label className="flex items-center gap-1.5 text-[10px] text-foreground">
+              <input type="checkbox" checked={matEnabled} onChange={(e) => setMatEnabled(e.target.checked)} />
+              <Save className="h-3 w-3 text-violet-300" /> Materialize → Iceberg/S3
+            </label>
+            {matEnabled && (
+              <>
+                <input value={matSchema} onChange={(e) => setMatSchema(e.target.value)} placeholder="schema" className={cn(inputCls, 'w-24')} />
+                <input value={matTable} onChange={(e) => setMatTable(e.target.value)} placeholder="new_table" className={cn(inputCls, 'w-32')} />
+                <select value={matMode} onChange={(e) => setMatMode(e.target.value)} className={cn(selectCls, 'w-28')}>
+                  <option value="create">create</option>
+                  <option value="replace">replace</option>
+                  <option value="insert">insert into</option>
+                </select>
+              </>
+            )}
+            {!matEnabled && (
+              <Field label="Limit" inline>
+                <input type="number" value={limit} onChange={(e) => setLimit(Math.max(1, Math.min(500, +e.target.value)))} className={cn(inputCls, 'w-20')} />
+              </Field>
+            )}
+            <div className="ml-auto flex gap-2">
+              {run && run.state === 'RUNNING' && (
+                <button type="button" onClick={onCancel} className="inline-flex items-center gap-1 rounded-md border border-rose-400/50 bg-rose-500/15 px-3 py-1.5 text-[11px] text-rose-200 hover:bg-rose-500/25">
+                  <Square className="h-3.5 w-3.5" /> Cancel
+                </button>
+              )}
+              <button type="button" onClick={onRun} disabled={submitting}
+                className="inline-flex items-center gap-1.5 rounded-md border border-amber-400/50 bg-amber-500/15 px-4 py-1.5 text-[11px] font-medium text-amber-200 hover:bg-amber-500/25 disabled:opacity-50">
+                {submitting ? <Loader2 className="h-4 w-4 animate-spin" /> : <Play className="h-4 w-4" />}
+                Run on Spark
+              </button>
+            </div>
+          </div>
+          {err && <p className="text-[11px] text-rose-300">{err}</p>}
+        </div>
+
+        {/* live run + results */}
+        <div className="panel flex min-h-0 flex-1 flex-col overflow-hidden">
+          {run ? <RunLive run={run} /> : (
+            <div className="flex flex-1 items-center justify-center text-[11px] text-foreground-muted">
+              Build an operation and Run to see the live execution matrix.
+            </div>
+          )}
+        </div>
+      </div>
+    </div>
+  )
+}
+
+function RunLive({ run }: { run: SparkRun }) {
+  const s = run.stats || {}
+  const pct = s.progress_pct ?? (run.state === 'FINISHED' ? 100 : 0)
+  return (
+    <div className="flex min-h-0 flex-1 flex-col">
+      <div className="shrink-0 border-b border-border p-3">
+        <div className="mb-2 flex items-center justify-between gap-2">
+          <div className="min-w-0">
+            <p className="truncate text-[11px] font-semibold text-foreground">{run.label}</p>
+            {run.target && <p className="truncate font-mono text-[9px] text-violet-300">→ {run.target}</p>}
+          </div>
+          <span className={cn('rounded-full px-2 py-0.5 text-[10px] font-medium', STATE_COLOR[run.state] || STATE_COLOR.QUEUED)}>
+            {run.state}
+          </span>
+        </div>
+        <div className="h-1.5 w-full overflow-hidden rounded-full bg-surface-overlay">
+          <div className="h-full rounded-full bg-gradient-to-r from-sky-400 to-emerald-400 transition-all" style={{ width: `${pct}%` }} />
+        </div>
+        <SparkMatrix stats={s} />
+        {run.error && <p className="mt-2 rounded bg-rose-500/10 p-2 font-mono text-[10px] text-rose-300">{run.error}</p>}
+      </div>
+      <div className="min-h-0 flex-1 overflow-auto">
+        {run.columns && run.columns.length > 0 ? (
+          <table className="w-full border-collapse text-[10px]">
+            <thead className="sticky top-0 bg-surface">
+              <tr>{run.columns.map((c) => <th key={c} className="border-b border-border px-2 py-1 text-left font-semibold text-foreground-muted">{c}</th>)}</tr>
+            </thead>
+            <tbody>
+              {(run.rows || []).map((row, i) => (
+                <tr key={i} className="hover:bg-surface-overlay/40">
+                  {row.map((cell, j) => <td key={j} className="border-b border-border/50 px-2 py-1 font-mono text-foreground">{cell == null ? '∅' : String(cell)}</td>)}
+                </tr>
+              ))}
+            </tbody>
+          </table>
+        ) : run.state === 'FINISHED' && run.target ? (
+          <p className="p-3 text-[11px] text-emerald-300">✓ Materialized to {run.target}{run.update_type ? ` (${run.update_type})` : ''}</p>
+        ) : null}
+      </div>
+    </div>
+  )
+}
+
+function SparkMatrix({ stats }: { stats: SparkRunStats }) {
+  const cells: { label: string; value: string; icon: typeof Cpu }[] = [
+    { label: 'Splits', value: `${fmtNum(stats.completed_splits)}/${fmtNum(stats.total_splits)}`, icon: Layers },
+    { label: 'Running', value: fmtNum(stats.running_splits), icon: Activity },
+    { label: 'Rows', value: fmtNum(stats.processed_rows), icon: Table2 },
+    { label: 'Input', value: fmtBytes(stats.processed_bytes), icon: HardDrive },
+    { label: 'CPU', value: fmtMs(stats.cpu_time_ms), icon: Cpu },
+    { label: 'Wall', value: fmtMs(stats.elapsed_ms ?? stats.wall_time_ms), icon: Gauge },
+    { label: 'Peak mem', value: fmtBytes(stats.peak_memory_bytes), icon: Server },
+    { label: 'Nodes', value: fmtNum(stats.nodes), icon: Cpu },
+  ]
+  return (
+    <div className="mt-2 grid grid-cols-4 gap-1.5 lg:grid-cols-8">
+      {cells.map((c) => (
+        <div key={c.label} className="rounded-md border border-border bg-surface-overlay/30 px-2 py-1.5">
+          <div className="flex items-center gap-1 text-[8px] uppercase tracking-wider text-foreground-muted">
+            <c.icon className="h-2.5 w-2.5" /> {c.label}
+          </div>
+          <p className="font-mono text-[11px] font-semibold tabular-nums text-foreground">{c.value}</p>
+        </div>
+      ))}
+    </div>
+  )
+}
+
+/* ── Cluster panel ───────────────────────────────────────────────────────── */
+function ClusterPanel({ live }: { live: SparkLive | null }) {
+  const spark = live?.spark
+  return (
+    <div className="panel grid min-h-0 flex-1 gap-3 overflow-y-auto p-4 sm:grid-cols-2 lg:grid-cols-4">
+      <Metric label="Workers" value={String(spark?.alive_workers ?? 0)} icon={Server} />
+      <Metric label="Cores" value={`${spark?.cores_used ?? 0} / ${spark?.cores ?? 0}`} icon={Cpu} />
+      <Metric label="Memory" value={`${fmtNum(spark?.memory_used_mb)} / ${fmtNum(spark?.memory_mb)} MB`} icon={Activity} />
+      <Metric label="Active jobs" value={String(live?.active_runs?.length ?? 0)} icon={Zap} />
+      <div className="col-span-full">
+        <h3 className="mb-2 text-[11px] font-semibold uppercase tracking-wider text-foreground-muted">Workers</h3>
+        <div className="grid gap-2 sm:grid-cols-2 lg:grid-cols-3">
+          {(spark?.workers || []).map((w) => (
+            <div key={w.id} className="rounded-lg border border-border bg-surface-overlay/30 p-3">
+              <p className="truncate font-mono text-[10px] text-foreground">{w.host || w.id}</p>
+              <p className="text-[9px] text-foreground-muted">{w.cores_used}/{w.cores} cores · {w.memory_mb} MB · {w.state}</p>
+            </div>
+          ))}
+          {!spark?.workers?.length && <p className="text-[10px] text-foreground-muted">No worker details</p>}
+        </div>
+      </div>
+    </div>
+  )
+}
+
+/* ── Runs history ────────────────────────────────────────────────────────── */
+function RunsPanel({ live }: { live: SparkLive | null }) {
+  const [runs, setRuns] = useState<SparkRun[]>([])
+  useEffect(() => {
+    fetchSparkRuns().then(setRuns)
+    const iv = setInterval(() => fetchSparkRuns().then(setRuns), 3000)
+    return () => clearInterval(iv)
+  }, [])
+  const list = runs.length ? runs : (live?.recent_runs ?? [])
+  return (
+    <div className="panel min-h-0 flex-1 overflow-y-auto p-3">
+      {list.length === 0 ? (
+        <p className="p-4 text-center text-[11px] text-foreground-muted">No runs yet. Submit an operation from the Workbench.</p>
+      ) : (
+        <div className="space-y-1.5">
+          {list.map((r) => (
+            <div key={r.id} className="flex items-center justify-between gap-3 rounded-lg border border-border bg-surface-overlay/20 px-3 py-2">
+              <div className="min-w-0">
+                <p className="truncate text-[11px] font-medium text-foreground">{r.label}</p>
+                <p className="truncate font-mono text-[9px] text-foreground-muted">{r.sql.slice(0, 90)}</p>
+              </div>
+              <div className="flex shrink-0 items-center gap-3 text-[9px] text-foreground-muted">
+                <span>{fmtNum(r.stats?.processed_rows)} rows</span>
+                <span>{fmtMs(r.stats?.elapsed_ms)}</span>
+                <span className={cn('rounded-full px-2 py-0.5 font-medium', STATE_COLOR[r.state] || STATE_COLOR.QUEUED)}>{r.state}</span>
+              </div>
+            </div>
+          ))}
+        </div>
+      )}
+    </div>
+  )
+}
+
+/* ── small UI helpers ────────────────────────────────────────────────────── */
+const selectCls = 'w-full rounded-md border border-border bg-surface px-2 py-1 text-[10px] text-foreground outline-none'
+const inputCls = 'rounded-md border border-border bg-surface px-2 py-1 text-[10px] text-foreground outline-none'
+
+function Field({ label, children, inline }: { label: string; children: React.ReactNode; inline?: boolean }) {
+  return (
+    <label className={cn('text-[9px] font-semibold uppercase tracking-wider text-foreground-muted', inline ? 'flex items-center gap-1.5' : 'block')}>
+      {label}
+      <div className={inline ? '' : 'mt-1'}>{children}</div>
+    </label>
+  )
+}
+
+function MultiChips({ options, selected, onToggle }: { options: string[]; selected: string[]; onToggle: (c: string) => void }) {
+  return (
+    <div className="flex max-h-24 flex-wrap gap-1 overflow-y-auto">
+      {options.map((o) => (
+        <button key={o} type="button" onClick={() => onToggle(o)}
+          className={cn('rounded border px-1.5 py-0.5 font-mono text-[9px]', selected.includes(o) ? 'border-emerald-400/50 bg-emerald-500/15 text-emerald-200' : 'border-border text-foreground-muted hover:bg-surface-overlay')}>
+          {o}
+        </button>
+      ))}
+      {!options.length && <span className="text-[9px] text-foreground-faint">select a table</span>}
+    </div>
+  )
+}
+
+function Metric({ label, value, icon: Icon }: { label: string; value: string; icon: typeof Cpu }) {
+  return (
+    <div className="rounded-lg border border-border bg-surface-overlay/25 p-3">
+      <div className="mb-1 flex items-center gap-1.5 text-[9px] uppercase tracking-wider text-foreground-muted">
+        <Icon className="h-3 w-3" /> {label}
+      </div>
+      <p className="text-lg font-semibold tabular-nums text-foreground">{value}</p>
+    </div>
+  )
+}
+
+function LiveChip({ label, value, ok }: { label: string; value: string; ok: boolean }) {
+  return (
+    <span className="inline-flex items-center gap-1 rounded-full border border-border px-2 py-0.5 text-[9px]">
+      <span className={cn('h-1.5 w-1.5 rounded-full', ok ? 'bg-emerald-400' : 'bg-amber-400')} />
+      <span className="text-foreground-muted">{label}</span>
+      <span className="font-mono font-semibold text-foreground">{value}</span>
+    </span>
+  )
+}

ui/src/components/features/SqlWorkbench.tsx

@@ -15,7 +15,7 @@ type SqlResult = {
   sql?: string
 }
 
-type Engine = SourceEngine | 'trino'
+type Engine = SourceEngine | 'trino' | 'hadoop'
 
 type Props = {
   engine: Engine
@@ -53,6 +53,12 @@ const ENGINE_META: Record<Engine, { title: string; sub: string; accent: string;
     accent: 'text-pink-400',
     queryLabel: 'Cypher',
   },
+  hadoop: {
+    title: 'Hadoop / Trino Console',
+    sub: 'Lakehouse · Trino 10.0.21.50:8089 · iceberg.hadoop · hive',
+    accent: 'text-emerald-400',
+    queryLabel: 'SQL',
+  },
   trino: {
     title: 'Trino SQL Console',
     sub: 'Lakehouse · 10.0.21.50:8089 · federated queries',

ui/src/components/layout/SideNav.tsx

@@ -1,4 +1,4 @@
-import { Database, DatabaseZap, HardDrive, Search, LayoutDashboard, MessageSquare, Presentation, Server, TerminalSquare, Cpu, Activity, GitBranch } from 'lucide-react'
+import { Database, DatabaseZap, HardDrive, Search, LayoutDashboard, MessageSquare, Server, TerminalSquare, Activity, GitBranch } from 'lucide-react'
 import type { GpuStatus, WorkloadData } from '../../types'
 import type { GpuLiveMetrics } from '../../hooks/useLiveMetrics'
 import { cn } from '../../lib/utils'
@@ -6,7 +6,7 @@ import { viewTabActive, viewTabIdle } from '../../lib/tabActive'
 import { GpuMatrixPanel } from '../features/GpuMatrixPanel'
 import { LabHealthPanel } from '../features/LabHealthPanel'
 
-type MainView = 'platform' | 'presentation' | 'dataquality' | 'knowledge' | 'storage' | 'hdfs' | 'search' | 'datagen' | 'approvals' | 'changes' | 'dataflow' | 'datasources'
+type MainView = 'platform' | 'dataquality' | 'knowledge' | 'storage' | 'hdfs' | 'search' | 'approvals' | 'changes' | 'dataflow' | 'datasources'
 
 type Props = {
   workload: WorkloadData | null
@@ -24,16 +24,13 @@ type Props = {
 
 const VIEWS: { id: MainView; label: string; icon: typeof LayoutDashboard }[] = [
   { id: 'platform', label: 'Data Platform', icon: LayoutDashboard },
-  { id: 'datasources', label: 'Data Sources UI', icon: Database },
-  { id: 'datagen', label: 'Data Generation', icon: Cpu },
+  { id: 'datasources', label: 'Data Hub', icon: Database },
   { id: 'changes', label: 'Live Changes', icon: Activity },
   { id: 'dataflow', label: 'Data Flow', icon: GitBranch },
-  { id: 'presentation', label: 'Presentation', icon: Presentation },
   { id: 'dataquality', label: 'Data Quality', icon: DatabaseZap },
   { id: 'knowledge', label: 'Knowledge Chat', icon: MessageSquare },
   { id: 'storage', label: 'Object Storage', icon: HardDrive },
-  { id: 'hdfs', label: 'Hadoop HDFS', icon: Server },
-  { id: 'search', label: 'Elasticsearch', icon: Search },
+    { id: 'search', label: 'Elasticsearch', icon: Search },
 ]
 
 export function SideNav({
@@ -52,10 +49,10 @@ export function SideNav({
   const matrixBoost = gpuBoost || mainView === 'knowledge'
 
   return (
-    <nav className="flex w-[240px] shrink-0 flex-col border-r border-border bg-surface-raised">
-      <section className="shrink-0 border-b border-border p-3">
-        <h2 className="mb-2 text-[9px] font-semibold uppercase tracking-widest text-foreground-faint">Views</h2>
-        <div className="space-y-1">
+    <nav className="flex h-full min-h-0 w-[240px] shrink-0 flex-col border-r border-border bg-surface-raised">
+      <section className="flex max-h-[42%] min-h-0 shrink-0 flex-col border-b border-border p-3">
+        <h2 className="mb-2 shrink-0 text-[9px] font-semibold uppercase tracking-widest text-foreground-muted">Views</h2>
+        <div className="scrollbar-thin min-h-0 flex-1 space-y-1 overflow-y-auto">
           {VIEWS.map(({ id, label, icon: Icon }) => (
             <button
               key={id}
@@ -81,21 +78,25 @@ export function SideNav({
         </div>
       </section>
 
-      <GpuMatrixPanel
-        gpu={gpu}
-        live={gpuLive}
-        boost={matrixBoost}
-        onSelectGpu={() => onSelectZone('gpu')}
-      />
+      <div className="flex min-h-0 flex-1 flex-col overflow-hidden">
+        <GpuMatrixPanel
+          gpu={gpu}
+          live={gpuLive}
+          boost={matrixBoost}
+          onSelectGpu={() => onSelectZone('gpu')}
+        />
 
-      <LabHealthPanel
-        workload={workload}
-        gpu={gpu}
-        selectedNodeId={selectedNodeId}
-        approvalCount={approvalCount}
-        onSelectZone={onSelectZone}
-        onOpenApprovals={onOpenApprovals}
-      />
+        <div className="scrollbar-thin min-h-0 flex-1 overflow-y-auto">
+          <LabHealthPanel
+            workload={workload}
+            gpu={gpu}
+            selectedNodeId={selectedNodeId}
+            approvalCount={approvalCount}
+            onSelectZone={onSelectZone}
+            onOpenApprovals={onOpenApprovals}
+          />
+        </div>
+      </div>
     </nav>
   )
 }

ui/src/components/ui/DbBrandIcon.tsx

@@ -0,0 +1,56 @@
+import { cn } from '../../lib/utils'
+import type { SourceEngine } from '../../lib/dataSourceCatalog'
+
+const BRAND: Record<SourceEngine, { color: string; label: string }> = {
+  postgres: { color: '#336791', label: 'PG' },
+  mysql: { color: '#00758F', label: 'MY' },
+  mongodb: { color: '#47A248', label: 'MG' },
+  cassandra: { color: '#1287B1', label: 'CS' },
+  neo4j: { color: '#018BFF', label: 'NJ' },
+}
+
+type Props = {
+  engine: SourceEngine
+  size?: number
+  className?: string
+}
+
+export function DbBrandIcon({ engine, size = 20, className }: Props) {
+  const b = BRAND[engine]
+  const s = size
+  return (
+    <svg
+      width={s}
+      height={s}
+      viewBox="0 0 24 24"
+      className={cn('shrink-0', className)}
+      aria-hidden
+    >
+      <circle cx="12" cy="12" r="11" fill={b.color} />
+      {engine === 'postgres' && (
+        <path fill="#fff" d="M7 8h10v1.5H7V8zm0 3.5h10V13H7v-1.5zm0 3.5h7V16H7v-1z" opacity="0.95" />
+      )}
+      {engine === 'mysql' && (
+        <path fill="#F29111" d="M12 5c-3 0-5 2-5 4.5 0 2 1.5 3.5 3.5 4.5-1 .5-1.5 1.5-1.5 2.5 0 2 2 3.5 4.5 3.5s4.5-1.5 4.5-3.5c0-1-.5-2-1.5-2.5 2-1 3.5-2.5 3.5-4.5C17 7 15 5 12 5z" />
+      )}
+      {engine === 'mongodb' && (
+        <path fill="#fff" d="M12 6c-2.5 2-4 5-4 8.5 0 2 .5 3.5 1.5 4.5.5-2 1.5-3.5 2.5-4.5 1 1 2 2.5 2.5 4.5 1-1 1.5-2.5 1.5-4.5C16 11 14.5 8 12 6z" />
+      )}
+      {engine === 'cassandra' && (
+        <path fill="#fff" d="M12 5l6 3.5v7L12 19l-6-3.5v-7L12 5zm0 2.2L8.5 9v4L12 14.8l3.5-1.8V9L12 7.2z" opacity="0.95" />
+      )}
+      {engine === 'neo4j' && (
+        <>
+          <circle cx="8" cy="10" r="2.2" fill="#fff" />
+          <circle cx="16" cy="10" r="2.2" fill="#fff" />
+          <circle cx="12" cy="16" r="2.2" fill="#fff" />
+          <path stroke="#fff" strokeWidth="1.2" d="M9.5 10.8 11 14M14.5 10.8 13 14M9.8 10 14.2 10" />
+        </>
+      )}
+    </svg>
+  )
+}
+
+export function dbBrandColor(engine: SourceEngine): string {
+  return BRAND[engine].color
+}

ui/src/context/ThemeContext.tsx

@@ -12,12 +12,12 @@ const STORAGE_KEY = 'atc-command-center-theme'
 
 function readStored(): Theme {
   const v = localStorage.getItem(STORAGE_KEY)
-  return v === 'dark' || v === 'light' ? v : 'light'
+  return v === 'dark' || v === 'light' ? v : 'dark'
 }
 
 export function ThemeProvider({ children }: { children: ReactNode }) {
   const [theme, setTheme] = useState<Theme>(() => {
-    if (typeof window === 'undefined') return 'light'
+    if (typeof window === 'undefined') return 'dark'
     return readStored()
   })
 

ui/src/hooks/useCommandCenter.ts

@@ -60,7 +60,7 @@ export function useCommandCenter() {
   const [selectedNode, setSelectedNode] = useState<TopologyNode | null>(null)
   const [nodeDetail, setNodeDetail] = useState<NodeDetail | null>(null)
   const [nodeBusy, setNodeBusy] = useState(false)
-  const [mainView, setMainView] = useState<'platform' | 'approvals' | 'presentation' | 'dataquality' | 'knowledge' | 'storage' | 'hdfs' | 'search' | 'datagen' | 'changes' | 'dataflow' | 'datasources'>('platform')
+  const [mainView, setMainView] = useState<'platform' | 'approvals' | 'dataquality' | 'knowledge' | 'storage' | 'hdfs' | 'search' | 'changes' | 'dataflow' | 'datasources'>('platform')
   const [changes, setChanges] = useState<CdcChange[]>([])
   const [genPulse, setGenPulse] = useState(false)
   const genPulseTimer = useRef<ReturnType<typeof setTimeout> | null>(null)

ui/src/lib/api.ts

@@ -9,6 +9,9 @@ import type {
   GpuStatus,
   Movement,
   PiiDataset,
+  SparkRun,
+  SparkLive,
+  StreamingStatus,
   StatusData,
   TerminalLine,
   WorkloadData,
@@ -178,3 +181,100 @@ export async function decideApproval(id: string, approved: boolean, decidedBy: s
     body: JSON.stringify({ approved, decided_by: decidedBy, note }),
   })
 }
+
+export async function fetchStreamingStatus(refresh = false): Promise<StreamingStatus | null> {
+  return fetchJson<StreamingStatus>(`/api/pipeline/streaming/status${refresh ? '?refresh=true' : ''}`, 20000)
+}
+
+export function triggerStreamingJob(jobId: string, conf?: Record<string, unknown>) {
+  return fetch(`/api/pipeline/streaming/jobs/${jobId}/trigger`, {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({ conf: conf ?? {} }),
+  })
+}
+
+export function restartKafkaConnector(name: string) {
+  return fetch(`/api/pipeline/streaming/kafka/connectors/${encodeURIComponent(name)}/restart`, { method: 'POST' })
+}
+
+export function pauseKafkaConnector(name: string) {
+  return fetch(`/api/pipeline/streaming/kafka/connectors/${encodeURIComponent(name)}/pause`, { method: 'POST' })
+}
+
+export function resumeKafkaConnector(name: string) {
+  return fetch(`/api/pipeline/streaming/kafka/connectors/${encodeURIComponent(name)}/resume`, { method: 'POST' })
+}
+
+export function triggerStreamingPipeline(pipelineId: string) {
+  return fetch(`/api/pipeline/streaming/pipeline/${encodeURIComponent(pipelineId)}`, { method: 'POST' })
+}
+
+export function exportHdfsToKafka(body?: { path?: string; topic?: string; limit?: number }) {
+  return fetch('/api/pipeline/streaming/hdfs/to-kafka', {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify(body ?? {}),
+  })
+}
+
+export function setStreamingFlow(action: 'pause' | 'resume' | 'stop') {
+  return fetch(`/api/pipeline/streaming/flow/${action}`, { method: 'POST' })
+}
+
+// ── Spark Workbench ──────────────────────────────────────────────
+export async function fetchSparkCatalogs(): Promise<string[]> {
+  const j = await fetchJson<{ catalogs?: string[] }>('/api/spark/catalogs', 15000)
+  return j?.catalogs ?? []
+}
+
+export async function fetchSparkSchemas(catalog: string): Promise<string[]> {
+  const j = await fetchJson<{ schemas?: string[] }>(`/api/spark/schemas?catalog=${encodeURIComponent(catalog)}`, 15000)
+  return j?.schemas ?? []
+}
+
+export async function fetchSparkTables(catalog: string, schema: string): Promise<{ name: string; fqn: string }[]> {
+  const j = await fetchJson<{ tables?: { name: string; fqn: string }[] }>(
+    `/api/spark/tables?catalog=${encodeURIComponent(catalog)}&schema=${encodeURIComponent(schema)}`, 15000)
+  return j?.tables ?? []
+}
+
+export async function fetchSparkColumns(table: string): Promise<{ name: string; type: string }[]> {
+  const j = await fetchJson<{ columns?: { name: string; type: string }[] }>(
+    `/api/spark/columns?table=${encodeURIComponent(table)}`, 15000)
+  return j?.columns ?? []
+}
+
+export async function createSparkRun(body: Record<string, unknown>): Promise<{ ok: boolean; run_id?: string; sql?: string; error?: string; target?: string | null }> {
+  const r = await fetch('/api/spark/run', {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify(body),
+  })
+  return r.json()
+}
+
+export async function fetchSparkRun(runId: string) {
+  return fetchJson<{ ok: boolean; run?: SparkRun }>(`/api/spark/run/${runId}`, 15000)
+}
+
+export function cancelSparkRun(runId: string) {
+  return fetch(`/api/spark/run/${runId}/cancel`, { method: 'POST' })
+}
+
+export async function fetchSparkLive() {
+  return fetchJson<SparkLive>('/api/spark/live', 15000)
+}
+
+export async function fetchSparkRuns(): Promise<SparkRun[]> {
+  const j = await fetchJson<{ runs?: SparkRun[] }>('/api/spark/runs', 15000)
+  return j?.runs ?? []
+}
+
+export function toggleCustodianOffload(enabled?: boolean) {
+  return fetch('/api/agent-ops/custodian/toggle', {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify(enabled === undefined ? {} : { enabled }),
+  })
+}

ui/src/lib/dataSourceCatalog.ts

@@ -3,7 +3,7 @@ import { Activity, Boxes, Database, Network } from 'lucide-react'
 
 export type SourceEngine = 'postgres' | 'mysql' | 'mongodb' | 'cassandra' | 'neo4j'
 
-export type SourceSubTab = 'browser' | 'console' | 'shell' | 'graph'
+export type SourceSubTab = 'browser' | 'console' | 'shell' | 'graph' | 'generate' | 'workbench'
 
 export type CatalogObject = {
   type: string

ui/src/styles/globals.css

@@ -11,9 +11,9 @@
     --surface-muted: 221 225 230;
     --border: 210 218 228;
     --border-strong: 180 192 208;
-    --foreground: 26 31 38;
-    --foreground-muted: 95 107 122;
-    --foreground-faint: 139 149 165;
+    --foreground: 12 18 28;
+    --foreground-muted: 38 48 62;
+    --foreground-faint: 62 72 88;
     --shadow-panel: 0 1px 3px rgba(15, 40, 80, 0.06), 0 4px 12px rgba(36, 150, 237, 0.06);
     --shadow-docker: 0 0 0 1px rgba(36, 150, 237, 0.12), 0 4px 14px rgba(36, 150, 237, 0.1);
     --topo-canvas: linear-gradient(145deg, #dbeafe 0%, #e0f2fe 35%, #ede9fe 70%, #ecfdf5 100%);
@@ -33,9 +33,9 @@
     --surface-muted: 36 58 92;
     --border: 48 74 112;
     --border-strong: 64 96 140;
-    --foreground: 232 241 255;
-    --foreground-muted: 148 175 212;
-    --foreground-faint: 100 130 168;
+    --foreground: 255 255 255;
+    --foreground-muted: 228 234 244;
+    --foreground-faint: 195 205 220;
     --shadow-panel: 0 1px 0 rgba(147, 197, 253, 0.06) inset, 0 8px 24px rgba(0, 0, 0, 0.35);
     --shadow-docker: 0 0 0 1px rgba(56, 189, 248, 0.2), 0 4px 16px rgba(14, 116, 214, 0.25);
     --topo-canvas: linear-gradient(145deg, #0c1929 0%, #132f4c 40%, #1a365d 75%, #0f2847 100%);

ui/src/types.ts

@@ -108,11 +108,61 @@ export type DataflowGraph = {
   ok: boolean
   nodes: DataflowNode[]
   edges: DataflowEdge[]
+  flow?: string
   pii_summary: { datasets?: number; pii_columns?: number; masked_columns?: number; unmasked_columns?: number }
   cdc: { connected?: boolean; consumed?: number; window_total?: number }
+  streaming?: StreamingStatus | null
   ts: number
 }
 
+export type StreamingSparkWorker = {
+  id?: string
+  host?: string
+  cores?: number
+  cores_used?: number
+  memory_mb?: number
+  state?: string
+  webui?: string
+}
+
+export type StreamingSparkApp = {
+  id?: string
+  name?: string
+  cores?: number
+  memory_mb?: number
+  submitdate?: string
+  duration_ms?: number
+  user?: string
+}
+
+export type StreamingStatus = {
+  ok?: boolean
+  spark?: {
+    ui_url?: string
+    ui_ok?: boolean
+    status?: string
+    workers?: StreamingSparkWorker[]
+    alive_workers?: number
+    cores?: number
+    cores_used?: number
+    memory_mb?: number
+    memory_used_mb?: number
+    active_apps?: StreamingSparkApp[]
+    completed_apps?: StreamingSparkApp[]
+  }
+  kafka?: {
+    ui_url?: string
+    ui_ok?: boolean
+    connect_ok?: boolean
+    cluster?: { name?: string; status?: string; broker_count?: number; topic_count?: number }
+    topics?: { name?: string; partitions?: number; replicas?: number; messages?: number }[]
+    connectors?: { name: string; state?: string; worker?: string; tasks?: { id?: number; state?: string }[]; type?: string }[]
+  }
+  edges?: Record<string, boolean>
+  jobs?: { id: string; label: string; dag_id: string; description?: string; default_conf?: Record<string, unknown> }[]
+  ts?: number
+}
+
 export type Movement = {
   id: string
   label: string
@@ -334,6 +384,53 @@ export type PresentationData = {
   pipeline_active?: boolean
   slides: PresentationSlide[]
   slide_count: number
+  edited?: boolean
+  override_ts?: string
+  id?: string
+  source?: string
   workload?: WorkloadData
   topologies?: Record<string, TopologyViewData>
 }
+
+export type SparkRunStats = {
+  state?: string
+  nodes?: number
+  total_splits?: number
+  queued_splits?: number
+  running_splits?: number
+  completed_splits?: number
+  processed_rows?: number
+  processed_bytes?: number
+  physical_input_bytes?: number
+  peak_memory_bytes?: number
+  cpu_time_ms?: number
+  wall_time_ms?: number
+  elapsed_ms?: number
+  progress_pct?: number
+}
+
+export type SparkRun = {
+  id: string
+  operation: string
+  label: string
+  sql: string
+  target?: string | null
+  state: string
+  engine_state?: string
+  stats?: SparkRunStats
+  columns?: string[]
+  rows?: unknown[][]
+  row_count?: number | null
+  error?: string | null
+  started_at?: number
+  ended_at?: number | null
+  update_type?: string
+}
+
+export type SparkLive = {
+  ok?: boolean
+  spark?: StreamingStatus['spark']
+  active_runs?: SparkRun[]
+  recent_runs?: SparkRun[]
+  ts?: number
+}