ATC/atc-agents
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Spark Workbench everywhere, autonomous Hadoop offload & LLM masking-aware

Browse Source 
- Data Hub with Hadoop tab (HDFS/Iceberg browser, Spark, pipeline)
- Databricks-style Lakehouse Workbench (Trino engine, live exec matrix,
  materialize to Iceberg/S3); reused & embedded in every source-DB UI
- HDFS -> Kafka -> Spark -> Iceberg/S3 pipeline; WebHDFS hostname resolver
- Data Flow master pulse switch (Run/Pause/Stop) gating animated edges
- Data Custodian autonomous Hadoop offload loop (batch counterpart to CDC),
  pulsing source -> HDFS edges; toggle in Data Flow
- LLM now autonomously aware of all latest platform changes (live platform
  context) and enforces masking policy: never reveals masked PII, still
  answers helpfully with aggregates/explanations
This commit is contained in:
mo
2026-06-27 19:37:50 +00:00
parent 5828113f53
commit 46b9c50e73
39 changed files with 5476 additions and 725 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/dataflow.py
+54 -6
View File
@@ -38,7 +38,9 @@ NODES: list[dict[str, Any]] = [
{"id": "mysql", "label": "MySQL", "sub": "employee_events", "kind": "source", "x": 28, "y": 38},
{"id": "mongodb", "label": "MongoDB", "sub": "events", "kind": "source", "x": 28, "y": 60},
# col 2 — change data capture
{"id": "kafka", "label": "Kafka · Debezium", "sub": "CDC topics", "kind": "stream", "x": 47, "y": 34},
{"id": "kafka", "label": "Kafka · Debezium", "sub": "CDC topics", "kind": "stream", "x": 47, "y": 24},
{"id": "spark", "label": "Apache Spark", "sub": "Streaming · batch", "kind": "compute", "x": 62, "y": 38,
"url": "/spark-ui/"},
# col 3 — storage / lakehouse
{"id": "s3_cdc", "label": "S3 CDC Archive", "sub": "object store", "kind": "sink", "x": 67, "y": 13},
{"id": "iceberg_curated", "label": "Iceberg · curated_masked", "sub": "masked PII", "kind": "lakehouse", "x": 67, "y": 45},
@@ -58,7 +60,13 @@ EDGES: list[dict[str, Any]] = [
{"from": "postgres", "to": "kafka", "kind": "cdc"},
{"from": "mysql", "to": "kafka", "kind": "cdc"},
{"from": "mongodb", "to": "kafka", "kind": "cdc"},
{"from": "kafka", "to": "spark", "kind": "stream"},
{"from": "spark", "to": "iceberg_curated", "kind": "movement", "movement_id": "spark_to_curated"},
{"from": "spark", "to": "s3_cdc", "kind": "movement", "movement_id": "spark_to_s3"},
{"from": "kafka", "to": "s3_cdc", "kind": "archive"},
{"from": "postgres", "to": "hdfs", "kind": "archive", "offload": True},
{"from": "mysql", "to": "hdfs", "kind": "archive", "offload": True},
{"from": "hdfs", "to": "kafka", "kind": "stream", "movement_id": "hdfs_to_kafka"},
{"from": "hdfs", "to": "iceberg_hadoop", "kind": "movement", "movement_id": "hadoop_to_trino"},
{"from": "postgres", "to": "iceberg_curated", "kind": "mask", "movement_id": "mask_to_curated"},
{"from": "mysql", "to": "iceberg_curated", "kind": "mask", "movement_id": "mask_to_curated"},
@@ -112,7 +120,7 @@ def _iceberg_hadoop_count() -> int | None:
return v
def _build() -> dict[str, Any]:
async def _build() -> dict[str, Any]:
# Live signals
try:
from movements import last_runs
@@ -129,6 +137,14 @@ def _build() -> dict[str, Any]:
pii = get_pii()
except Exception:
pii = {"datasets": []}
try:
from streaming_ops import build_streaming_status
streaming = await build_streaming_status()
except Exception:
streaming = {}
spark = streaming.get("spark") or {}
kafka = streaming.get("kafka") or {}
edge_live = streaming.get("edges") or {}
pii_by_node = {d["node_id"]: d for d in pii.get("datasets", [])}
nodes = []
@@ -138,9 +154,17 @@ def _build() -> dict[str, Any]:
metric = None
if n["id"] == "openmetadata":
metric = f"{pii.get('summary', {}).get('pii_columns', 0)} PII cols cataloged"
if n["id"] == "kafka":
metric = f"{cdc.get('window_total', 0)} chg/15m · {cdc.get('consumed', 0)} total"
node["level"] = "ok" if cdc.get("connected") else "warn"
elif n["id"] == "kafka":
topics = len(kafka.get("topics") or [])
conn_n = len(kafka.get("connectors") or [])
metric = f"{cdc.get('window_total', 0)} chg/15m · {topics} topics · {conn_n} connectors"
node["level"] = "ok" if cdc.get("connected") and kafka.get("ui_ok") else "warn"
elif n["id"] == "spark":
apps = len(spark.get("active_apps") or [])
cores = spark.get("cores") or 0
used = spark.get("cores_used") or 0
metric = f"{spark.get('alive_workers', 0)} workers · {used}/{cores} cores · {apps} apps"
node["level"] = "ok" if spark.get("ui_ok") and (spark.get("status") or "").upper() == "ALIVE" else "warn"
elif n["id"] in ("postgres", "mysql", "mongodb"):
metric = f"{cdc.get('by_source', {}).get(n['id'], 0)} CDC/15m"
elif n["id"] == "iceberg_hadoop":
@@ -161,6 +185,12 @@ def _build() -> dict[str, Any]:
node["metric"] = metric
nodes.append(node)
try:
from streaming_ops import flow_mode
_flow = flow_mode()
except Exception:
_flow = "running"
edges = []
for e in EDGES:
edge = dict(e)
@@ -173,6 +203,22 @@ def _build() -> dict[str, Any]:
edge["active"] = lr.get("state") == "running"
if e["kind"] == "cdc":
edge["active"] = cdc.get("by_source", {}).get(e["from"], 0) > 0
elif e.get("from") == "hdfs" and e.get("to") == "kafka":
edge["active"] = bool(edge_live.get("hdfs→kafka"))
elif e.get("from") == "kafka" and e.get("to") == "spark":
edge["active"] = bool(edge_live.get("kafka→spark"))
elif e.get("from") == "spark" and e.get("to") == "iceberg_curated":
edge["active"] = bool(edge_live.get("spark→iceberg")) or edge.get("active")
elif e.get("from") == "spark" and e.get("to") == "s3_cdc":
edge["active"] = bool(edge_live.get("spark→s3")) or edge.get("active")
if e.get("offload"):
try:
from agent_ops import custodian_recent
edge["active"] = custodian_recent()
except Exception:
pass
if _flow != "running":
edge["active"] = False
edges.append(edge)
return {
@@ -181,6 +227,8 @@ def _build() -> dict[str, Any]:
"edges": edges,
"pii_summary": pii.get("summary", {}),
"cdc": {"connected": cdc.get("connected"), "consumed": cdc.get("consumed"), "window_total": cdc.get("window_total")},
"streaming": streaming,
"flow": _flow,
"ts": time.time(),
}
@@ -190,7 +238,7 @@ async def get_dataflow(refresh: bool = False) -> JSONResponse:
now = time.time()
if not refresh and _cache["data"] and now - _cache["ts"] < _TTL:
return JSONResponse(_cache["data"])
data = _build()
data = await _build()
_cache["data"] = data
_cache["ts"] = now
return JSONResponse(data)