feat: Spark Workbench everywhere, autonomous Hadoop offload & LLM masking-aware

- Data Hub with Hadoop tab (HDFS/Iceberg browser, Spark, pipeline) - Databricks-style Lakehouse Workbench (Trino engine, live exec matrix, materialize to Iceberg/S3); reused & embedded in every source-DB UI - HDFS -> Kafka -> Spark -> Iceberg/S3 pipeline; WebHDFS hostname resolver - Data Flow master pulse switch (Run/Pause/Stop) gating animated edges - Data Custodian autonomous Hadoop offload loop (batch counterpart to CDC), pulsing source -> HDFS edges; toggle in Data Flow - LLM now autonomously aware of all latest platform changes (live platform context) and enforces masking policy: never reveals masked PII, still answers helpfully with aggregates/explanations
2026-06-27 19:37:50 +00:00
parent 5828113f53
commit 46b9c50e73
39 changed files with 5476 additions and 725 deletions
@@ -0,0 +1,108 @@
+"""HDFS → Kafka export and full hadoop-lake pipeline orchestration."""
+
+from __future__ import annotations
+
+import csv
+import io
+import json
+import os
+import time
+from typing import Any
+
+from aiokafka import AIOKafkaProducer
+from webhdfs_util import open_bytes
+
+KAFKA_BOOTSTRAP = os.getenv("KAFKA_BOOTSTRAP", "10.0.21.36:9092")
+DEFAULT_TRINO_TABLE = os.getenv("HADOOP_EXPORT_TABLE", "iceberg.hadoop.historical_sales_hdfs")
+DEFAULT_HDFS_FILE = os.getenv("HADOOP_EXPORT_HDFS", "/data/historical/sales_orders/year=2020/part-0.csv")
+
+_last_hdfs_export: dict[str, Any] = {"ts": 0.0, "rows": 0, "topic": None}
+
+
+def hdfs_export_snapshot() -> dict[str, Any]:
+    age = time.time() - float(_last_hdfs_export.get("ts") or 0)
+    return {
+        **(_last_hdfs_export or {}),
+        "recent": age < 120,
+        "age_s": round(age, 1) if _last_hdfs_export.get("ts") else None,
+    }
+
+
+def _read_hdfs_csv(path: str, limit: int) -> tuple[list[str], list[list[str]]]:
+    raw = open_bytes(path).decode("utf-8", errors="replace")
+    rows = list(csv.reader(io.StringIO(raw)))
+    if not rows:
+        return [], []
+    header = [c.strip() for c in rows[0]]
+    data = [[cell.strip() for cell in row] for row in rows[1: 1 + limit]]
+    return header, data
+
+
+def _read_trino_table(table: str, limit: int) -> tuple[list[str], list[list[Any]]]:
+    from sql_console import _run_trino
+
+    result = _run_trino(f"SELECT * FROM {table} LIMIT {limit}", limit)
+    if not result.get("ok"):
+        raise RuntimeError(result.get("error") or "Trino query failed")
+    return list(result.get("columns") or []), list(result.get("rows") or [])
+
+
+async def export_hdfs_to_kafka(
+    path: str | None = None,
+    topic: str = "hdfs.historical.sales",
+    limit: int = 2000,
+    source: str = "trino",
+    table: str | None = None,
+    feed: Any = None,
+) -> dict[str, Any]:
+    global _last_hdfs_export
+    header: list[str] = []
+    data: list[list[Any]] = []
+    src_label = source
+
+    try:
+        if source == "trino":
+            tbl = table or DEFAULT_TRINO_TABLE
+            header, data = _read_trino_table(tbl, limit)
+            src_label = tbl
+        else:
+            p = path or DEFAULT_HDFS_FILE
+            header, data = _read_hdfs_csv(p, limit)
+            src_label = p
+    except Exception as exc:
+        if source == "trino" and path:
+            try:
+                header, data = _read_hdfs_csv(path, limit)
+                src_label = path
+            except Exception:
+                return {"ok": False, "error": str(exc)[:400]}
+        else:
+            return {"ok": False, "error": str(exc)[:400]}
+
+    if not data:
+        return {"ok": False, "error": "No data rows to export"}
+
+    producer = AIOKafkaProducer(
+        bootstrap_servers=KAFKA_BOOTSTRAP,
+        value_serializer=lambda v: json.dumps(v, default=str).encode("utf-8"),
+    )
+    await producer.start()
+    rows_sent = 0
+    try:
+        for i, row in enumerate(data):
+            payload = {
+                header[j] if j < len(header) else f"col_{j}": row[j] if j < len(row) else None
+                for j in range(max(len(header), len(row)))
+            }
+            payload["_source"] = "hadoop"
+            payload["_origin"] = src_label
+            payload["_row"] = i + 1
+            await producer.send_and_wait(topic, payload)
+            rows_sent += 1
+    finally:
+        await producer.stop()
+
+    _last_hdfs_export = {"ts": time.time(), "rows": rows_sent, "topic": topic, "source": src_label}
+    if feed:
+        feed("hadoop-ranger", f"[hadoop→kafka] {rows_sent} rows from {src_label} → {topic}", "info")
+    return {"ok": True, "rows_sent": rows_sent, "topic": topic, "source": src_label, "columns": header}