feat: Spark Workbench everywhere, autonomous Hadoop offload & LLM masking-aware

- Data Hub with Hadoop tab (HDFS/Iceberg browser, Spark, pipeline) - Databricks-style Lakehouse Workbench (Trino engine, live exec matrix, materialize to Iceberg/S3); reused & embedded in every source-DB UI - HDFS -> Kafka -> Spark -> Iceberg/S3 pipeline; WebHDFS hostname resolver - Data Flow master pulse switch (Run/Pause/Stop) gating animated edges - Data Custodian autonomous Hadoop offload loop (batch counterpart to CDC), pulsing source -> HDFS edges; toggle in Data Flow - LLM now autonomously aware of all latest platform changes (live platform context) and enforces masking policy: never reveals masked PII, still answers helpfully with aggregates/explanations
2026-06-27 19:37:50 +00:00
parent 5828113f53
commit 46b9c50e73
39 changed files with 5476 additions and 725 deletions
@@ -0,0 +1,144 @@
+"""Live 'platform capabilities + recent changes + masking guidance' block for the LLM.
+
+This is rebuilt on every question from live in-process state, so the assistant is
+always autonomously aware of the latest things running in the lab (Data Hub,
+Spark Workbench, HDFS→Kafka→Spark→S3 pipeline, autonomous agents) and of the
+exact masking policy currently in force.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+def _fmt_ts(ts: Any) -> str:
+    try:
+        return str(ts)[:19]
+    except Exception:
+        return "?"
+
+
+def build_platform_section() -> str:
+    lines: list[str] = ["=== PLATFORM CAPABILITIES & RECENT CHANGES (live) ==="]
+
+    lines += [
+        "Command Center features currently deployed:",
+        "  - Data Hub: tabbed UI with 'Source Databases' (PostgreSQL, MySQL, MongoDB, Cassandra, Neo4j) and 'Hadoop' (HDFS files, Hive/Iceberg tables, Spark, pipeline).",
+        "  - Spark Lakehouse Workbench (Databricks-style): pick any federated table, run preview/filter/aggregate/profile/join/SQL on the distributed engine, and materialize results to Iceberg (S3/HDFS-backed). Live execution matrix: splits, rows, bytes, CPU, wall-time, peak memory, nodes.",
+        "  - The same workbench is embedded in each source-database UI (scoped to that source's Trino catalog).",
+        "  - Data Flow: live lineage graph with a master pulse switch (Run / Pause / Stop) that starts/stops the animated flow.",
+        "  - Pipeline: HDFS (Iceberg historical_sales) → Kafka topic hdfs.historical.sales → Spark transform → Iceberg curated → S3.",
+        "Autonomous agents (run continuously, toggleable):",
+        "  - Data Custodian (DML): generates live INSERT/UPDATE/DELETE on the source DBs so Debezium CDC streams to Kafka.",
+        "  - Data Custodian (Hadoop offload): periodically offloads recent source rows into the Hadoop Iceberg lake (iceberg.hadoop.*_offload), the batch counterpart to CDC.",
+        "  - ETL agent: autonomously triggers data movements (HDFS→Iceberg, mask→curated, generators).",
+    ]
+
+    # live streaming + flow
+    try:
+        from streaming_ops import build_streaming_status, flow_snapshot  # type: ignore
+        flow = flow_snapshot()
+        lines.append(f"Data Flow pulse: {flow.get('mode')}")
+    except Exception:
+        pass
+
+    # autonomous agent state
+    try:
+        from agent_ops import _state, _etl_state, _custodian_state  # type: ignore
+        lines.append(
+            f"DML agent: {'on' if _state.get('enabled') else 'off'} "
+            f"(ops_total={_state.get('ops_total')}, last={_state.get('last_op')})"
+        )
+        lines.append(
+            f"ETL agent: {'on' if _etl_state.get('enabled') else 'off'} "
+            f"(runs={_etl_state.get('runs_total')}, last={_etl_state.get('last')})"
+        )
+        cust = _custodian_state
+        lines.append(
+            f"Custodian Hadoop offload: {'on' if cust.get('enabled') else 'off'} "
+            f"(offloads={cust.get('runs_total')}, last={cust.get('last')})"
+        )
+    except Exception:
+        pass
+
+    # recent movements
+    try:
+        from movements import last_runs  # type: ignore
+        runs = last_runs()
+        if runs:
+            lines.append("Recent data-movement runs:")
+            for mid, r in list(runs.items())[-6:]:
+                lines.append(f"  - {mid}: {r.get('state')} rows={r.get('rows')} {_fmt_ts(r.get('ended_at'))}")
+    except Exception:
+        pass
+
+    # recent spark workbench runs
+    try:
+        from spark_workbench import _runs as wb_runs, _run_order  # type: ignore
+        recent = [wb_runs[r] for r in _run_order[-6:] if r in wb_runs]
+        if recent:
+            lines.append("Recent Spark Workbench runs:")
+            for r in recent:
+                st = (r.get("stats") or {})
+                tgt = f" → {r.get('target')}" if r.get("target") else ""
+                lines.append(
+                    f"  - {r.get('label')}: {r.get('state')} rows={st.get('processed_rows')}{tgt}"
+                )
+    except Exception:
+        pass
+
+    return "\n".join(lines)
+
+
+def build_masking_section() -> str:
+    """Exact masking policy + strict guidance so the LLM can answer about masked
+    data without ever revealing masked raw values."""
+    lines: list[str] = ["=== DATA MASKING POLICY (enforced) ==="]
+    masked: list[str] = []
+    unmasked: list[str] = []
+    try:
+        from pii_catalog import get_pii  # type: ignore
+        data = get_pii()
+        for d in data.get("datasets", []):
+            for c in d.get("pii_columns", []):
+                tag = f"{d.get('label')}.{c.get('name')} [{c.get('category')}]"
+                (masked if c.get("masked") else unmasked).append(tag)
+        summ = data.get("summary", {})
+        lines.append(
+            f"PII columns: {summ.get('pii_columns', 0)} total — "
+            f"{summ.get('masked_columns', 0)} masked, {summ.get('unmasked_columns', 0)} visible."
+        )
+    except Exception as exc:
+        lines.append(f"(masking catalog unavailable: {exc})")
+
+    if masked:
+        lines.append("MASKED columns (raw values are withheld — token 🔒 MASKED):")
+        for m in masked[:40]:
+            lines.append(f"  - {m}")
+    if unmasked:
+        lines.append("Visible PII columns (operator opted out of masking):")
+        for u in unmasked[:40]:
+            lines.append(f"  - {u}")
+
+    lines += [
+        "",
+        "How to handle masked data when answering:",
+        "  1. NEVER reveal, guess, reconstruct or print the raw value of a MASKED column. If a value comes in as '🔒 MASKED', keep it masked.",
+        "  2. DO still answer helpfully: confirm the column exists and is masked for privacy/governance, and explain why (PII protection policy).",
+        "  3. You MAY use and report non-sensitive aggregates, counts, distributions and derived metrics over masked columns (e.g. 'there are N distinct customers') as long as no individual raw value is exposed.",
+        "  4. Tell the operator they can unmask a specific column from the Data Flow PII overlay if they have the authority, and that the curated/masked Iceberg layer is physically masked and cannot be unmasked.",
+        "  5. Unmasked PII columns may be shown, but flag that they are sensitive.",
+    ]
+    return "\n".join(lines)
+
+
+def build_llm_addendum() -> str:
+    try:
+        platform = build_platform_section()
+    except Exception as exc:
+        platform = f"(platform section error: {exc})"
+    try:
+        masking = build_masking_section()
+    except Exception as exc:
+        masking = f"(masking section error: {exc})"
+    return "\n\n".join([platform, masking])