feat: Spark Workbench everywhere, autonomous Hadoop offload & LLM masking-aware
- Data Hub with Hadoop tab (HDFS/Iceberg browser, Spark, pipeline) - Databricks-style Lakehouse Workbench (Trino engine, live exec matrix, materialize to Iceberg/S3); reused & embedded in every source-DB UI - HDFS -> Kafka -> Spark -> Iceberg/S3 pipeline; WebHDFS hostname resolver - Data Flow master pulse switch (Run/Pause/Stop) gating animated edges - Data Custodian autonomous Hadoop offload loop (batch counterpart to CDC), pulsing source -> HDFS edges; toggle in Data Flow - LLM now autonomously aware of all latest platform changes (live platform context) and enforces masking policy: never reveals masked PII, still answers helpfully with aggregates/explanations
This commit is contained in:
mo
+1
-1
@@ -4,7 +4,7 @@ WORKDIR /app
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/*
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
COPY main.py lab_context.py agent_terminal.py workload.py node_registry.py node_ops.py topology_views.py supervisor.py approval_service.py db.py dockhand_envs.py presentation.py database_inventory.py presentation_upload.py presentation_static.py storage_s3.py elasticsearch_api.py sql_console.py hdfs_api.py ssh_terminal.py pipeline_ops.py hadoop_analytics.py agent_ops.py cdc_consumer.py movements.py dataflow.py pii_catalog.py hive_bench_seed.json .
|
||||
COPY main.py lab_context.py agent_terminal.py workload.py node_registry.py node_ops.py topology_views.py supervisor.py approval_service.py db.py dockhand_envs.py presentation.py database_inventory.py presentation_upload.py presentation_static.py storage_s3.py elasticsearch_api.py sql_console.py hdfs_api.py ssh_terminal.py pipeline_ops.py hadoop_analytics.py agent_ops.py cdc_consumer.py movements.py dataflow.py streaming_ops.py spark_workbench.py hadoop_sql.py hdfs_kafka.py webhdfs_util.py pii_catalog.py platform_context.py hive_bench_seed.json .
|
||||
RUN mkdir -p /data
|
||||
ENV DATABASE_URL=sqlite:////data/atc-agents.db
|
||||
EXPOSE 3201
|
||||
|
||||
+88
-1
@@ -328,11 +328,98 @@ async def etl_agent_loop() -> None:
|
||||
await asyncio.sleep(max(30.0, float(_etl_state["interval"])))
|
||||
|
||||
|
||||
|
||||
# ── Custodian Hadoop offload (batch counterpart to CDC) ─────────────────────
|
||||
_CUST_INTERVAL = float(os.getenv("CUSTODIAN_OFFLOAD_INTERVAL_SECONDS", "120"))
|
||||
_CUST_BATCH = int(os.getenv("CUSTODIAN_OFFLOAD_BATCH", "200"))
|
||||
_CUST_TARGETS = [
|
||||
{"label": "postgres sales_orders", "src": "postgres_sales.public.sales_orders",
|
||||
"target": "iceberg.hadoop.sales_orders_offload"},
|
||||
{"label": "mysql employee_events", "src": "mysql_hr.hr.employee_events",
|
||||
"target": "iceberg.hadoop.employee_events_offload"},
|
||||
]
|
||||
_custodian_state: dict[str, Any] = {
|
||||
"enabled": os.getenv("CUSTODIAN_OFFLOAD_ENABLED", "1") not in ("0", "false", "False", ""),
|
||||
"interval": _CUST_INTERVAL,
|
||||
"targets": [c["target"] for c in _CUST_TARGETS],
|
||||
"idx": 0,
|
||||
"runs_total": 0,
|
||||
"last": None,
|
||||
"started": False,
|
||||
}
|
||||
|
||||
|
||||
async def _custodian_offload_once(idx: int | None = None) -> dict[str, Any]:
|
||||
"""Offload a batch of source rows into the Hadoop Iceberg lake via Trino."""
|
||||
from spark_workbench import _trino_collect
|
||||
i = _custodian_state["idx"] if idx is None else idx
|
||||
tgt = _CUST_TARGETS[i % len(_CUST_TARGETS)]
|
||||
_custodian_state["idx"] = i + 1
|
||||
await _trino_collect(
|
||||
f"CREATE TABLE IF NOT EXISTS {tgt['target']} AS SELECT * FROM {tgt['src']} WHERE 1=0", 1)
|
||||
ins = await _trino_collect(
|
||||
f"INSERT INTO {tgt['target']} SELECT * FROM {tgt['src']} LIMIT {_CUST_BATCH}", 1)
|
||||
ok = bool(ins.get("ok"))
|
||||
_custodian_state["runs_total"] += 1
|
||||
_custodian_state["last"] = {
|
||||
"target": tgt["target"], "src": tgt["src"], "ok": ok,
|
||||
"rows": _CUST_BATCH if ok else 0,
|
||||
"ts": datetime.now(timezone.utc).isoformat(), "error": ins.get("error"),
|
||||
}
|
||||
if ok:
|
||||
await _emit(f"[custodian-offload] {tgt['label']} → {tgt['target']}: offloaded ~{_CUST_BATCH} rows to Hadoop", "info")
|
||||
else:
|
||||
await _emit(f"[custodian-offload] {tgt['label']} failed: {str(ins.get('error'))[:120]}", "err")
|
||||
return _custodian_state["last"]
|
||||
|
||||
|
||||
async def custodian_offload_loop() -> None:
|
||||
_custodian_state["started"] = True
|
||||
await asyncio.sleep(45)
|
||||
await _emit("[custodian-offload] Autonomous Hadoop offload online — batching source data into the Iceberg lake", "info")
|
||||
while True:
|
||||
try:
|
||||
if _custodian_state["enabled"]:
|
||||
await _custodian_offload_once()
|
||||
except Exception as exc:
|
||||
_custodian_state["last"] = {"error": str(exc), "ts": datetime.now(timezone.utc).isoformat()}
|
||||
await asyncio.sleep(max(30.0, float(_custodian_state["interval"])))
|
||||
|
||||
|
||||
def custodian_recent() -> bool:
|
||||
last = _custodian_state.get("last") or {}
|
||||
ts = last.get("ts")
|
||||
if not ts or not last.get("ok"):
|
||||
return False
|
||||
try:
|
||||
from datetime import datetime as _dt
|
||||
t = _dt.fromisoformat(str(ts).replace("Z", "+00:00"))
|
||||
window = max(60.0, float(_custodian_state["interval"]) * 1.5)
|
||||
return (datetime.now(timezone.utc) - t).total_seconds() < window
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
# ── Endpoints ────────────────────────────────────────────────────────────────
|
||||
@router.get("/status")
|
||||
async def status() -> JSONResponse:
|
||||
pools = {k: len(v) for k, v in _pools.items()}
|
||||
return JSONResponse({"ok": True, "pools": pools, "etl": _etl_state, **_state})
|
||||
return JSONResponse({"ok": True, "pools": pools, "etl": _etl_state, "custodian": _custodian_state, **_state})
|
||||
|
||||
|
||||
@router.post("/custodian/toggle")
|
||||
async def custodian_toggle(body: dict[str, Any] = Body(default={})) -> JSONResponse:
|
||||
if "enabled" in body:
|
||||
_custodian_state["enabled"] = bool(body["enabled"])
|
||||
else:
|
||||
_custodian_state["enabled"] = not _custodian_state["enabled"]
|
||||
if "interval" in body:
|
||||
try:
|
||||
_custodian_state["interval"] = max(30.0, float(body["interval"]))
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
await _emit(f"[custodian-offload] Hadoop offload {'ENABLED' if _custodian_state['enabled'] else 'PAUSED'} by operator", "warn")
|
||||
return JSONResponse({"ok": True, "enabled": _custodian_state["enabled"], "interval": _custodian_state["interval"]})
|
||||
|
||||
|
||||
@router.post("/etl/toggle")
|
||||
|
||||
+54
-6
@@ -38,7 +38,9 @@ NODES: list[dict[str, Any]] = [
|
||||
{"id": "mysql", "label": "MySQL", "sub": "employee_events", "kind": "source", "x": 28, "y": 38},
|
||||
{"id": "mongodb", "label": "MongoDB", "sub": "events", "kind": "source", "x": 28, "y": 60},
|
||||
# col 2 — change data capture
|
||||
{"id": "kafka", "label": "Kafka · Debezium", "sub": "CDC topics", "kind": "stream", "x": 47, "y": 34},
|
||||
{"id": "kafka", "label": "Kafka · Debezium", "sub": "CDC topics", "kind": "stream", "x": 47, "y": 24},
|
||||
{"id": "spark", "label": "Apache Spark", "sub": "Streaming · batch", "kind": "compute", "x": 62, "y": 38,
|
||||
"url": "/spark-ui/"},
|
||||
# col 3 — storage / lakehouse
|
||||
{"id": "s3_cdc", "label": "S3 CDC Archive", "sub": "object store", "kind": "sink", "x": 67, "y": 13},
|
||||
{"id": "iceberg_curated", "label": "Iceberg · curated_masked", "sub": "masked PII", "kind": "lakehouse", "x": 67, "y": 45},
|
||||
@@ -58,7 +60,13 @@ EDGES: list[dict[str, Any]] = [
|
||||
{"from": "postgres", "to": "kafka", "kind": "cdc"},
|
||||
{"from": "mysql", "to": "kafka", "kind": "cdc"},
|
||||
{"from": "mongodb", "to": "kafka", "kind": "cdc"},
|
||||
{"from": "kafka", "to": "spark", "kind": "stream"},
|
||||
{"from": "spark", "to": "iceberg_curated", "kind": "movement", "movement_id": "spark_to_curated"},
|
||||
{"from": "spark", "to": "s3_cdc", "kind": "movement", "movement_id": "spark_to_s3"},
|
||||
{"from": "kafka", "to": "s3_cdc", "kind": "archive"},
|
||||
{"from": "postgres", "to": "hdfs", "kind": "archive", "offload": True},
|
||||
{"from": "mysql", "to": "hdfs", "kind": "archive", "offload": True},
|
||||
{"from": "hdfs", "to": "kafka", "kind": "stream", "movement_id": "hdfs_to_kafka"},
|
||||
{"from": "hdfs", "to": "iceberg_hadoop", "kind": "movement", "movement_id": "hadoop_to_trino"},
|
||||
{"from": "postgres", "to": "iceberg_curated", "kind": "mask", "movement_id": "mask_to_curated"},
|
||||
{"from": "mysql", "to": "iceberg_curated", "kind": "mask", "movement_id": "mask_to_curated"},
|
||||
@@ -112,7 +120,7 @@ def _iceberg_hadoop_count() -> int | None:
|
||||
return v
|
||||
|
||||
|
||||
def _build() -> dict[str, Any]:
|
||||
async def _build() -> dict[str, Any]:
|
||||
# Live signals
|
||||
try:
|
||||
from movements import last_runs
|
||||
@@ -129,6 +137,14 @@ def _build() -> dict[str, Any]:
|
||||
pii = get_pii()
|
||||
except Exception:
|
||||
pii = {"datasets": []}
|
||||
try:
|
||||
from streaming_ops import build_streaming_status
|
||||
streaming = await build_streaming_status()
|
||||
except Exception:
|
||||
streaming = {}
|
||||
spark = streaming.get("spark") or {}
|
||||
kafka = streaming.get("kafka") or {}
|
||||
edge_live = streaming.get("edges") or {}
|
||||
pii_by_node = {d["node_id"]: d for d in pii.get("datasets", [])}
|
||||
|
||||
nodes = []
|
||||
@@ -138,9 +154,17 @@ def _build() -> dict[str, Any]:
|
||||
metric = None
|
||||
if n["id"] == "openmetadata":
|
||||
metric = f"{pii.get('summary', {}).get('pii_columns', 0)} PII cols cataloged"
|
||||
if n["id"] == "kafka":
|
||||
metric = f"{cdc.get('window_total', 0)} chg/15m · {cdc.get('consumed', 0)} total"
|
||||
node["level"] = "ok" if cdc.get("connected") else "warn"
|
||||
elif n["id"] == "kafka":
|
||||
topics = len(kafka.get("topics") or [])
|
||||
conn_n = len(kafka.get("connectors") or [])
|
||||
metric = f"{cdc.get('window_total', 0)} chg/15m · {topics} topics · {conn_n} connectors"
|
||||
node["level"] = "ok" if cdc.get("connected") and kafka.get("ui_ok") else "warn"
|
||||
elif n["id"] == "spark":
|
||||
apps = len(spark.get("active_apps") or [])
|
||||
cores = spark.get("cores") or 0
|
||||
used = spark.get("cores_used") or 0
|
||||
metric = f"{spark.get('alive_workers', 0)} workers · {used}/{cores} cores · {apps} apps"
|
||||
node["level"] = "ok" if spark.get("ui_ok") and (spark.get("status") or "").upper() == "ALIVE" else "warn"
|
||||
elif n["id"] in ("postgres", "mysql", "mongodb"):
|
||||
metric = f"{cdc.get('by_source', {}).get(n['id'], 0)} CDC/15m"
|
||||
elif n["id"] == "iceberg_hadoop":
|
||||
@@ -161,6 +185,12 @@ def _build() -> dict[str, Any]:
|
||||
node["metric"] = metric
|
||||
nodes.append(node)
|
||||
|
||||
try:
|
||||
from streaming_ops import flow_mode
|
||||
_flow = flow_mode()
|
||||
except Exception:
|
||||
_flow = "running"
|
||||
|
||||
edges = []
|
||||
for e in EDGES:
|
||||
edge = dict(e)
|
||||
@@ -173,6 +203,22 @@ def _build() -> dict[str, Any]:
|
||||
edge["active"] = lr.get("state") == "running"
|
||||
if e["kind"] == "cdc":
|
||||
edge["active"] = cdc.get("by_source", {}).get(e["from"], 0) > 0
|
||||
elif e.get("from") == "hdfs" and e.get("to") == "kafka":
|
||||
edge["active"] = bool(edge_live.get("hdfs→kafka"))
|
||||
elif e.get("from") == "kafka" and e.get("to") == "spark":
|
||||
edge["active"] = bool(edge_live.get("kafka→spark"))
|
||||
elif e.get("from") == "spark" and e.get("to") == "iceberg_curated":
|
||||
edge["active"] = bool(edge_live.get("spark→iceberg")) or edge.get("active")
|
||||
elif e.get("from") == "spark" and e.get("to") == "s3_cdc":
|
||||
edge["active"] = bool(edge_live.get("spark→s3")) or edge.get("active")
|
||||
if e.get("offload"):
|
||||
try:
|
||||
from agent_ops import custodian_recent
|
||||
edge["active"] = custodian_recent()
|
||||
except Exception:
|
||||
pass
|
||||
if _flow != "running":
|
||||
edge["active"] = False
|
||||
edges.append(edge)
|
||||
|
||||
return {
|
||||
@@ -181,6 +227,8 @@ def _build() -> dict[str, Any]:
|
||||
"edges": edges,
|
||||
"pii_summary": pii.get("summary", {}),
|
||||
"cdc": {"connected": cdc.get("connected"), "consumed": cdc.get("consumed"), "window_total": cdc.get("window_total")},
|
||||
"streaming": streaming,
|
||||
"flow": _flow,
|
||||
"ts": time.time(),
|
||||
}
|
||||
|
||||
@@ -190,7 +238,7 @@ async def get_dataflow(refresh: bool = False) -> JSONResponse:
|
||||
now = time.time()
|
||||
if not refresh and _cache["data"] and now - _cache["ts"] < _TTL:
|
||||
return JSONResponse(_cache["data"])
|
||||
data = _build()
|
||||
data = await _build()
|
||||
_cache["data"] = data
|
||||
_cache["ts"] = now
|
||||
return JSONResponse(data)
|
||||
|
||||
@@ -0,0 +1,152 @@
|
||||
"""Hadoop / HDFS / Iceberg catalog and sampling for the Data Hub."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import io
|
||||
import os
|
||||
from typing import Any, Callable
|
||||
|
||||
from webhdfs_util import open_bytes
|
||||
|
||||
HDFS_PATHS = [
|
||||
"/data/historical/sales_orders",
|
||||
"/data/historical/sales_orders/year=2020/part-0.csv",
|
||||
]
|
||||
|
||||
ICEBERG_SCHEMAS = ["hadoop", "curated_masked", "curated"]
|
||||
|
||||
HADOOP_SAMPLES: list[dict[str, str]] = [
|
||||
{"id": "hd1", "label": "Iceberg historical sales", "sql": "SELECT * FROM iceberg.hadoop.historical_sales_hdfs LIMIT 20"},
|
||||
{"id": "hd2", "label": "Row count historical", "sql": "SELECT count(*) FROM iceberg.hadoop.historical_sales_hdfs"},
|
||||
{"id": "hd3", "label": "Curated masked sample", "sql": "SELECT * FROM iceberg.curated_masked.sales_orders_masked LIMIT 15"},
|
||||
{"id": "hd4", "label": "Hive lake schemas", "sql": "SHOW SCHEMAS FROM hive"},
|
||||
{"id": "hd5", "label": "Iceberg hadoop tables", "sql": "SHOW TABLES FROM iceberg.hadoop"},
|
||||
]
|
||||
|
||||
|
||||
def catalog_hadoop(run_trino: Callable[[str, int], dict[str, Any]]) -> dict[str, Any]:
|
||||
objects: list[dict[str, Any]] = []
|
||||
|
||||
for schema in ICEBERG_SCHEMAS:
|
||||
show = run_trino(f"SHOW TABLES FROM iceberg.{schema}", 500)
|
||||
if not show.get("ok"):
|
||||
continue
|
||||
for row in show.get("rows") or []:
|
||||
name = str(row[0])
|
||||
fqn = f"iceberg.{schema}.{name}"
|
||||
cnt = run_trino(f"SELECT count(*) FROM iceberg.{schema}.{name}", 1)
|
||||
count = None
|
||||
if cnt.get("ok") and cnt.get("rows"):
|
||||
try:
|
||||
count = int(cnt["rows"][0][0])
|
||||
except (TypeError, ValueError, IndexError):
|
||||
pass
|
||||
objects.append({
|
||||
"type": "table",
|
||||
"schema": f"iceberg.{schema}",
|
||||
"name": name,
|
||||
"fqn": fqn,
|
||||
"row_count": count,
|
||||
})
|
||||
|
||||
for path in HDFS_PATHS:
|
||||
label = path.rstrip("/").rsplit("/", 1)[-1]
|
||||
objects.append({
|
||||
"type": "file",
|
||||
"schema": "hdfs",
|
||||
"name": label,
|
||||
"fqn": f"hdfs:{path}",
|
||||
"row_count": None,
|
||||
})
|
||||
|
||||
return {"engine": "hadoop", "version": "HDFS + Iceberg", "objects": objects}
|
||||
|
||||
|
||||
def sample_hdfs_csv(path: str, limit: int, offset: int, tabular: Callable[..., dict[str, Any]]) -> dict[str, Any]:
|
||||
try:
|
||||
text = open_bytes(path).decode("utf-8", errors="replace")
|
||||
reader = csv.reader(io.StringIO(text))
|
||||
all_rows = list(reader)
|
||||
if not all_rows:
|
||||
return tabular([], [], 0, row_count=0)
|
||||
columns = [c.strip() for c in all_rows[0]]
|
||||
data = all_rows[1 + offset: 1 + offset + limit]
|
||||
rows = [[cell.strip() for cell in row] for row in data]
|
||||
return tabular(columns, rows, 0, row_count=max(0, len(all_rows) - 1))
|
||||
except Exception as exc:
|
||||
return {"ok": False, "error": str(exc)[:500]}
|
||||
|
||||
|
||||
def sample_hadoop(
|
||||
object_name: str,
|
||||
limit: int,
|
||||
offset: int,
|
||||
run_trino: Callable[[str, int], dict[str, Any]],
|
||||
tabular: Callable[..., dict[str, Any]],
|
||||
) -> dict[str, Any]:
|
||||
if object_name.startswith("hdfs:"):
|
||||
return sample_hdfs_csv(object_name[5:], limit, offset, tabular)
|
||||
sql = f"SELECT * FROM {object_name} OFFSET {offset} LIMIT {limit}"
|
||||
return run_trino(sql, limit)
|
||||
|
||||
|
||||
def table_row_count_hadoop(object_name: str, run_trino: Callable[[str, int], dict[str, Any]]) -> int | None:
|
||||
if object_name.startswith("hdfs:"):
|
||||
try:
|
||||
text = open_bytes(object_name[5:]).decode("utf-8", errors="replace")
|
||||
return max(0, sum(1 for _ in csv.reader(io.StringIO(text))) - 1)
|
||||
except Exception:
|
||||
return None
|
||||
cnt = run_trino(f"SELECT count(*) FROM {object_name}", 1)
|
||||
if cnt.get("ok") and cnt.get("rows"):
|
||||
try:
|
||||
return int(cnt["rows"][0][0])
|
||||
except (TypeError, ValueError, IndexError):
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def health_hadoop(run_trino: Callable[[str, int], dict[str, Any]]) -> dict[str, Any]:
|
||||
from webhdfs_util import HDFS_NN_URL
|
||||
import httpx
|
||||
|
||||
nn_ok = False
|
||||
live_dn = None
|
||||
err_parts: list[str] = []
|
||||
try:
|
||||
with httpx.Client(timeout=6.0) as client:
|
||||
r = client.get(f"{HDFS_NN_URL}/jmx?qry=Hadoop:service=NameNode,name=FSNamesystemState")
|
||||
beans = (r.json().get("beans") or [{}])[0]
|
||||
nn_ok = r.status_code < 400
|
||||
live_dn = beans.get("NumLiveDataNodes")
|
||||
except Exception as exc:
|
||||
err_parts.append(f"NN: {exc}")
|
||||
|
||||
trino_ok = False
|
||||
try:
|
||||
tr = run_trino("SELECT 1", 1)
|
||||
trino_ok = bool(tr.get("ok"))
|
||||
if not trino_ok and tr.get("error"):
|
||||
err_parts.append(f"Trino: {tr['error'][:80]}")
|
||||
except Exception as exc:
|
||||
err_parts.append(f"Trino: {exc}")
|
||||
|
||||
ok = nn_ok and trino_ok
|
||||
return {
|
||||
"ok": ok,
|
||||
"namenode": HDFS_NN_URL,
|
||||
"live_datanodes": live_dn,
|
||||
"trino_ok": trino_ok,
|
||||
"error": "; ".join(err_parts) if err_parts else None,
|
||||
}
|
||||
|
||||
|
||||
def connection_info() -> dict[str, str]:
|
||||
from webhdfs_util import HDFS_NN_URL, HDFS_USER
|
||||
return {
|
||||
"namenode": HDFS_NN_URL,
|
||||
"user": HDFS_USER,
|
||||
"trino": os.getenv("TRINO_URL", "http://10.0.21.50:8089"),
|
||||
"spark": os.getenv("SPARK_UI_URL", "http://10.0.21.50:8080"),
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
"""HDFS → Kafka export and full hadoop-lake pipeline orchestration."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from aiokafka import AIOKafkaProducer
|
||||
from webhdfs_util import open_bytes
|
||||
|
||||
KAFKA_BOOTSTRAP = os.getenv("KAFKA_BOOTSTRAP", "10.0.21.36:9092")
|
||||
DEFAULT_TRINO_TABLE = os.getenv("HADOOP_EXPORT_TABLE", "iceberg.hadoop.historical_sales_hdfs")
|
||||
DEFAULT_HDFS_FILE = os.getenv("HADOOP_EXPORT_HDFS", "/data/historical/sales_orders/year=2020/part-0.csv")
|
||||
|
||||
_last_hdfs_export: dict[str, Any] = {"ts": 0.0, "rows": 0, "topic": None}
|
||||
|
||||
|
||||
def hdfs_export_snapshot() -> dict[str, Any]:
|
||||
age = time.time() - float(_last_hdfs_export.get("ts") or 0)
|
||||
return {
|
||||
**(_last_hdfs_export or {}),
|
||||
"recent": age < 120,
|
||||
"age_s": round(age, 1) if _last_hdfs_export.get("ts") else None,
|
||||
}
|
||||
|
||||
|
||||
def _read_hdfs_csv(path: str, limit: int) -> tuple[list[str], list[list[str]]]:
|
||||
raw = open_bytes(path).decode("utf-8", errors="replace")
|
||||
rows = list(csv.reader(io.StringIO(raw)))
|
||||
if not rows:
|
||||
return [], []
|
||||
header = [c.strip() for c in rows[0]]
|
||||
data = [[cell.strip() for cell in row] for row in rows[1: 1 + limit]]
|
||||
return header, data
|
||||
|
||||
|
||||
def _read_trino_table(table: str, limit: int) -> tuple[list[str], list[list[Any]]]:
|
||||
from sql_console import _run_trino
|
||||
|
||||
result = _run_trino(f"SELECT * FROM {table} LIMIT {limit}", limit)
|
||||
if not result.get("ok"):
|
||||
raise RuntimeError(result.get("error") or "Trino query failed")
|
||||
return list(result.get("columns") or []), list(result.get("rows") or [])
|
||||
|
||||
|
||||
async def export_hdfs_to_kafka(
|
||||
path: str | None = None,
|
||||
topic: str = "hdfs.historical.sales",
|
||||
limit: int = 2000,
|
||||
source: str = "trino",
|
||||
table: str | None = None,
|
||||
feed: Any = None,
|
||||
) -> dict[str, Any]:
|
||||
global _last_hdfs_export
|
||||
header: list[str] = []
|
||||
data: list[list[Any]] = []
|
||||
src_label = source
|
||||
|
||||
try:
|
||||
if source == "trino":
|
||||
tbl = table or DEFAULT_TRINO_TABLE
|
||||
header, data = _read_trino_table(tbl, limit)
|
||||
src_label = tbl
|
||||
else:
|
||||
p = path or DEFAULT_HDFS_FILE
|
||||
header, data = _read_hdfs_csv(p, limit)
|
||||
src_label = p
|
||||
except Exception as exc:
|
||||
if source == "trino" and path:
|
||||
try:
|
||||
header, data = _read_hdfs_csv(path, limit)
|
||||
src_label = path
|
||||
except Exception:
|
||||
return {"ok": False, "error": str(exc)[:400]}
|
||||
else:
|
||||
return {"ok": False, "error": str(exc)[:400]}
|
||||
|
||||
if not data:
|
||||
return {"ok": False, "error": "No data rows to export"}
|
||||
|
||||
producer = AIOKafkaProducer(
|
||||
bootstrap_servers=KAFKA_BOOTSTRAP,
|
||||
value_serializer=lambda v: json.dumps(v, default=str).encode("utf-8"),
|
||||
)
|
||||
await producer.start()
|
||||
rows_sent = 0
|
||||
try:
|
||||
for i, row in enumerate(data):
|
||||
payload = {
|
||||
header[j] if j < len(header) else f"col_{j}": row[j] if j < len(row) else None
|
||||
for j in range(max(len(header), len(row)))
|
||||
}
|
||||
payload["_source"] = "hadoop"
|
||||
payload["_origin"] = src_label
|
||||
payload["_row"] = i + 1
|
||||
await producer.send_and_wait(topic, payload)
|
||||
rows_sent += 1
|
||||
finally:
|
||||
await producer.stop()
|
||||
|
||||
_last_hdfs_export = {"ts": time.time(), "rows": rows_sent, "topic": topic, "source": src_label}
|
||||
if feed:
|
||||
feed("hadoop-ranger", f"[hadoop→kafka] {rows_sent} rows from {src_label} → {topic}", "info")
|
||||
return {"ok": True, "rows_sent": rows_sent, "topic": topic, "source": src_label, "columns": header}
|
||||
+43
-1
@@ -26,13 +26,16 @@ from agent_terminal import (
|
||||
from lab_context import collect_full_lab_context, format_context_for_agent
|
||||
from presentation import build_presentation_payload, render_presentation_html
|
||||
from presentation_upload import (
|
||||
clear_live_override,
|
||||
create_deck,
|
||||
delete_deck,
|
||||
get_asset_path,
|
||||
get_deck,
|
||||
get_live_override,
|
||||
list_decks,
|
||||
save_deck,
|
||||
save_image,
|
||||
save_live_override,
|
||||
save_upload,
|
||||
)
|
||||
from presentation_static import get_static_deck, list_static_decks
|
||||
@@ -42,11 +45,13 @@ from pipeline_ops import router as pipeline_router
|
||||
from hadoop_analytics import router as hadoop_router
|
||||
from elasticsearch_api import router as elasticsearch_router
|
||||
from sql_console import router as sql_router
|
||||
from agent_ops import router as agent_ops_router, agent_dml_loop, etl_agent_loop
|
||||
from agent_ops import router as agent_ops_router, agent_dml_loop, etl_agent_loop, custodian_offload_loop
|
||||
from cdc_consumer import router as cdc_router, cdc_consumer_loop
|
||||
from movements import router as movements_router
|
||||
from movements import MOVEMENT_BY_ID, trigger_and_watch
|
||||
from dataflow import router as dataflow_router
|
||||
from streaming_ops import router as streaming_router
|
||||
from spark_workbench import router as spark_workbench_router
|
||||
from pii_catalog import router as pii_router
|
||||
from ssh_terminal import ssh_session
|
||||
from node_registry import NODE_IDS, NODE_AGENT, NODE_REGISTRY, is_node_id
|
||||
@@ -339,6 +344,11 @@ async def gather_agent_context(
|
||||
sup = " [supervisor]" if a.get("supervisor") else ""
|
||||
agent_lines.append(f" - {a['name']} ({a['id']}){sup}: {a['role']}")
|
||||
ctx = ctx + "\n".join(agent_lines)
|
||||
try:
|
||||
from platform_context import build_llm_addendum
|
||||
ctx = ctx + "\n\n" + build_llm_addendum()
|
||||
except Exception:
|
||||
pass
|
||||
if log:
|
||||
await log("info", "fetch", f"▸ Context assembled: {len(ctx)} chars for LLM")
|
||||
return ctx
|
||||
@@ -363,6 +373,8 @@ Rules:
|
||||
- Use ONLY the live data below — do not invent hosts, ports, numbers or connector names.
|
||||
- Use exact container/connector names from the data (e.g. mysql-hr-connector, not "Debezium").
|
||||
- If something is DOWN or 0 GB, say so honestly.
|
||||
- Respect the data masking policy: NEVER reveal, guess or reconstruct raw values of MASKED columns (they arrive as the token 🔒 MASKED). You MUST still answer helpfully — confirm the column is masked for privacy/governance, explain why, and you may use non-sensitive aggregates/counts over it.
|
||||
- You are fully aware of all latest platform changes via the section PLATFORM CAPABILITIES & RECENT CHANGES below; use it to answer questions about recent changes, the Spark Workbench, the Hadoop pipeline, the Data Flow pulse switch and the autonomous agents (DML, ETL, Custodian Hadoop offload).
|
||||
- Be concise and helpful (max ~10 sentences); bullet lists are fine when they aid clarity.
|
||||
|
||||
--- LIVE LAB DATA (primary domain first, then full stack) ---
|
||||
@@ -720,6 +732,7 @@ async def lifespan(app: FastAPI):
|
||||
dml_task = asyncio.create_task(agent_dml_loop())
|
||||
cdc_task = asyncio.create_task(cdc_consumer_loop())
|
||||
etl_task = asyncio.create_task(etl_agent_loop())
|
||||
cust_task = asyncio.create_task(custodian_offload_loop())
|
||||
add_feed("infra-sentinel", "ATC Command Center API online", "info")
|
||||
yield
|
||||
task.cancel()
|
||||
@@ -741,6 +754,8 @@ app.include_router(agent_ops_router)
|
||||
app.include_router(cdc_router)
|
||||
app.include_router(movements_router)
|
||||
app.include_router(dataflow_router)
|
||||
app.include_router(streaming_router)
|
||||
app.include_router(spark_workbench_router)
|
||||
app.include_router(pii_router)
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
@@ -794,7 +809,18 @@ async def get_presentation_data(*, use_cache: bool = True) -> dict[str, Any]:
|
||||
gpu = await collect_gpu()
|
||||
snap = await collect_full_lab_context(gpu_data=gpu, include_inventory=False)
|
||||
data = build_presentation_payload(snap)
|
||||
override = get_live_override()
|
||||
if override and override.get("slides"):
|
||||
data["title"] = override.get("title") or data.get("title")
|
||||
data["subtitle"] = override.get("subtitle") or data.get("subtitle", "")
|
||||
data["slides"] = override["slides"]
|
||||
data["slide_count"] = len(override["slides"])
|
||||
data["edited"] = True
|
||||
data["override_ts"] = override.get("ts")
|
||||
else:
|
||||
data["edited"] = False
|
||||
data["source"] = "live"
|
||||
data["id"] = "live"
|
||||
_presentation_cache["ts"] = now
|
||||
_presentation_cache["data"] = data
|
||||
return data
|
||||
@@ -867,6 +893,7 @@ async def create_presentation_deck(body: dict[str, Any] | None = Body(default=No
|
||||
"bullets": list(s.get("bullets") or []),
|
||||
"image": s.get("image") or "",
|
||||
"kind": s.get("kind") or "narrative",
|
||||
**({"animation": s["animation"]} if s.get("animation") else {}),
|
||||
}
|
||||
for i, s in enumerate(src["slides"], start=1)
|
||||
]
|
||||
@@ -878,12 +905,27 @@ async def create_presentation_deck(body: dict[str, Any] | None = Body(default=No
|
||||
|
||||
@app.put("/api/presentation/decks/{deck_id}")
|
||||
async def update_presentation_deck(deck_id: str, body: dict[str, Any] = Body(...)):
|
||||
if deck_id == "live":
|
||||
save_live_override(body)
|
||||
_presentation_cache["ts"] = 0
|
||||
_presentation_cache["data"] = None
|
||||
payload = await get_presentation_data(use_cache=False)
|
||||
return {"ok": True, "deck": payload}
|
||||
deck = save_deck(deck_id, body)
|
||||
if not deck:
|
||||
return {"error": "deck not found or not editable"}
|
||||
return {"ok": True, "deck": deck}
|
||||
|
||||
|
||||
@app.post("/api/presentation/live/reset")
|
||||
async def reset_live_presentation():
|
||||
clear_live_override()
|
||||
_presentation_cache["ts"] = 0
|
||||
_presentation_cache["data"] = None
|
||||
payload = await get_presentation_data(use_cache=False)
|
||||
return {"ok": True, "deck": payload}
|
||||
|
||||
|
||||
@app.delete("/api/presentation/decks/{deck_id}")
|
||||
async def remove_presentation_deck(deck_id: str):
|
||||
return {"ok": delete_deck(deck_id)}
|
||||
|
||||
@@ -39,6 +39,10 @@ MOVEMENTS: list[dict[str, Any]] = [
|
||||
{"id": "gen_mongodb", "label": "Generate → MongoDB", "kind": "generate",
|
||||
"dag_id": "gen_mongodb", "agent": "data-custodian", "from": "generator", "to": "mongodb",
|
||||
"default_conf": {"rows": 3000}},
|
||||
{"id": "hdfs_to_kafka", "label": "HDFS → Kafka export", "kind": "stream",
|
||||
"dag_id": None, "agent": "hadoop-ranger", "from": "hdfs", "to": "kafka",
|
||||
"api": "/api/pipeline/streaming/hdfs/to-kafka",
|
||||
"default_conf": {"source": "trino", "table": "iceberg.hadoop.historical_sales_hdfs", "topic": "hdfs.historical.sales"}},
|
||||
{"id": "hadoop_to_trino", "label": "HDFS → Iceberg (Trino)", "kind": "movement",
|
||||
"dag_id": "hadoop_to_trino", "agent": "hadoop-ranger", "from": "hdfs", "to": "iceberg_hadoop",
|
||||
"default_conf": {"mode": "refresh"},
|
||||
@@ -47,6 +51,14 @@ MOVEMENTS: list[dict[str, Any]] = [
|
||||
"dag_id": "mask_to_curated", "agent": "lakehouse-ops", "from": "sources", "to": "iceberg_curated",
|
||||
"default_conf": {},
|
||||
"count_sql": "SELECT count(*) FROM iceberg.curated_masked.sales_orders_masked"},
|
||||
{"id": "spark_to_s3", "label": "Spark → S3 curated", "kind": "movement",
|
||||
"dag_id": "mask_to_curated", "agent": "lakehouse-ops", "from": "spark", "to": "s3_cdc",
|
||||
"default_conf": {"target": "s3"},
|
||||
"count_sql": "SELECT count(*) FROM iceberg.curated_masked.sales_orders_masked"},
|
||||
{"id": "spark_to_curated", "label": "Spark → Iceberg curated", "kind": "movement",
|
||||
"dag_id": "mask_to_curated", "agent": "lakehouse-ops", "from": "spark", "to": "iceberg_curated",
|
||||
"default_conf": {},
|
||||
"count_sql": "SELECT count(*) FROM iceberg.curated_masked.sales_orders_masked"},
|
||||
]
|
||||
MOVEMENT_BY_ID = {m["id"]: m for m in MOVEMENTS}
|
||||
|
||||
@@ -116,6 +128,30 @@ async def trigger_and_watch(mid: str, conf: dict[str, Any] | None = None, *, aut
|
||||
mv = MOVEMENT_BY_ID.get(mid)
|
||||
if not mv:
|
||||
return {"ok": False, "error": f"unknown movement {mid}"}
|
||||
if mv.get("api"):
|
||||
try:
|
||||
payload = {**(mv.get("default_conf") or {}), **(conf or {})}
|
||||
t0 = time.time()
|
||||
async with httpx.AsyncClient(timeout=120.0) as client:
|
||||
r = await client.post(f"http://127.0.0.1:8000{mv['api']}", json=payload)
|
||||
dur = round(time.time() - t0, 1)
|
||||
body = r.json() if r.headers.get("content-type", "").startswith("application/json") else {}
|
||||
state = "success" if r.status_code < 400 and body.get("ok", True) else "failed"
|
||||
rows = body.get("rows_sent") or body.get("rows")
|
||||
run = {
|
||||
"movement_id": mid, "state": state, "duration_s": dur, "rows": rows,
|
||||
"ended_at": datetime.now(timezone.utc).isoformat(), "conf": payload,
|
||||
}
|
||||
_last_runs[mid] = run
|
||||
await _publish({"type": "movement", **run})
|
||||
lvl = "info" if state == "success" else "err"
|
||||
_feed(agent, f"[etl] {mv['label']}: {state} in {dur}s", lvl)
|
||||
return {"ok": state == "success", **run}
|
||||
except Exception as exc:
|
||||
_last_runs[mid] = {**_last_runs.get(mid, {}), "state": "failed", "error": str(exc)}
|
||||
_feed(agent, f"[etl] {mv['label']}: error {str(exc)[:120]}", "err")
|
||||
return {"ok": False, "error": str(exc)}
|
||||
|
||||
conf = {**(mv.get("default_conf") or {}), **(conf or {})}
|
||||
agent = mv["agent"]
|
||||
count_sql = mv.get("count_sql")
|
||||
|
||||
+6
-6
@@ -114,7 +114,7 @@ def _feed(agent_id: str, message: str, level: str = "info") -> None:
|
||||
async def _watch_run(source: str, dag_id: str, run_id: str, agent_id: str, rows: int | None) -> None:
|
||||
"""Poll an Airflow run to completion and log the outcome to the feed."""
|
||||
name = AGENT_NAME.get(agent_id, agent_id)
|
||||
label = f"{rows} rijen" if rows else "data"
|
||||
label = f"{rows} rows" if rows else "data"
|
||||
try:
|
||||
async with httpx.AsyncClient() as client:
|
||||
tok = await _airflow_token(client)
|
||||
@@ -130,10 +130,10 @@ async def _watch_run(source: str, dag_id: str, run_id: str, agent_id: str, rows:
|
||||
except Exception:
|
||||
continue
|
||||
if state == "success":
|
||||
_feed(agent_id, f"[datagen] {name} genereerde {label} in {source} — klaar, data stroomt via CDC naar Kafka/S3", "info")
|
||||
_feed(agent_id, f"[datagen] {name} generated {label} in {source} — complete, data flowing via CDC to Kafka/S3", "info")
|
||||
return
|
||||
if state == "failed":
|
||||
_feed(agent_id, f"[datagen] {name}: generatie voor {source} is mislukt (zie Airflow logs)", "err")
|
||||
_feed(agent_id, f"[datagen] {name}: generation for {source} failed (see Airflow logs)", "err")
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
@@ -205,12 +205,12 @@ async def generate(source: str, body: dict[str, Any] = Body(default={})) -> JSON
|
||||
timeout=15,
|
||||
)
|
||||
if r.status_code >= 400:
|
||||
_feed(agent_id, f"[datagen] {name}: kon generatie voor {source} niet starten (Airflow {r.status_code})", "err")
|
||||
_feed(agent_id, f"[datagen] {name}: could not start generation for {source} (Airflow {r.status_code})", "err")
|
||||
return JSONResponse({"ok": False, "error": f"Airflow {r.status_code}: {r.text[:300]}"}, status_code=200)
|
||||
j = r.json()
|
||||
run_id = j.get("dag_run_id")
|
||||
verb = "genereert zelf" if autonomous else "startte generatie:"
|
||||
rows_txt = f"{conf['rows']} rijen" if conf.get("rows") else "data"
|
||||
verb = "generating autonomously" if autonomous else "started generation of"
|
||||
rows_txt = f"{conf['rows']} rows" if conf.get("rows") else "data"
|
||||
_feed(agent_id, f"[datagen] {name} {verb} {rows_txt} in {source}", "info")
|
||||
if run_id:
|
||||
asyncio.create_task(_watch_run(source, dag_id, run_id, agent_id, conf.get("rows")))
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
"""Live 'platform capabilities + recent changes + masking guidance' block for the LLM.
|
||||
|
||||
This is rebuilt on every question from live in-process state, so the assistant is
|
||||
always autonomously aware of the latest things running in the lab (Data Hub,
|
||||
Spark Workbench, HDFS→Kafka→Spark→S3 pipeline, autonomous agents) and of the
|
||||
exact masking policy currently in force.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
|
||||
def _fmt_ts(ts: Any) -> str:
|
||||
try:
|
||||
return str(ts)[:19]
|
||||
except Exception:
|
||||
return "?"
|
||||
|
||||
|
||||
def build_platform_section() -> str:
|
||||
lines: list[str] = ["=== PLATFORM CAPABILITIES & RECENT CHANGES (live) ==="]
|
||||
|
||||
lines += [
|
||||
"Command Center features currently deployed:",
|
||||
" - Data Hub: tabbed UI with 'Source Databases' (PostgreSQL, MySQL, MongoDB, Cassandra, Neo4j) and 'Hadoop' (HDFS files, Hive/Iceberg tables, Spark, pipeline).",
|
||||
" - Spark Lakehouse Workbench (Databricks-style): pick any federated table, run preview/filter/aggregate/profile/join/SQL on the distributed engine, and materialize results to Iceberg (S3/HDFS-backed). Live execution matrix: splits, rows, bytes, CPU, wall-time, peak memory, nodes.",
|
||||
" - The same workbench is embedded in each source-database UI (scoped to that source's Trino catalog).",
|
||||
" - Data Flow: live lineage graph with a master pulse switch (Run / Pause / Stop) that starts/stops the animated flow.",
|
||||
" - Pipeline: HDFS (Iceberg historical_sales) → Kafka topic hdfs.historical.sales → Spark transform → Iceberg curated → S3.",
|
||||
"Autonomous agents (run continuously, toggleable):",
|
||||
" - Data Custodian (DML): generates live INSERT/UPDATE/DELETE on the source DBs so Debezium CDC streams to Kafka.",
|
||||
" - Data Custodian (Hadoop offload): periodically offloads recent source rows into the Hadoop Iceberg lake (iceberg.hadoop.*_offload), the batch counterpart to CDC.",
|
||||
" - ETL agent: autonomously triggers data movements (HDFS→Iceberg, mask→curated, generators).",
|
||||
]
|
||||
|
||||
# live streaming + flow
|
||||
try:
|
||||
from streaming_ops import build_streaming_status, flow_snapshot # type: ignore
|
||||
flow = flow_snapshot()
|
||||
lines.append(f"Data Flow pulse: {flow.get('mode')}")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# autonomous agent state
|
||||
try:
|
||||
from agent_ops import _state, _etl_state, _custodian_state # type: ignore
|
||||
lines.append(
|
||||
f"DML agent: {'on' if _state.get('enabled') else 'off'} "
|
||||
f"(ops_total={_state.get('ops_total')}, last={_state.get('last_op')})"
|
||||
)
|
||||
lines.append(
|
||||
f"ETL agent: {'on' if _etl_state.get('enabled') else 'off'} "
|
||||
f"(runs={_etl_state.get('runs_total')}, last={_etl_state.get('last')})"
|
||||
)
|
||||
cust = _custodian_state
|
||||
lines.append(
|
||||
f"Custodian Hadoop offload: {'on' if cust.get('enabled') else 'off'} "
|
||||
f"(offloads={cust.get('runs_total')}, last={cust.get('last')})"
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# recent movements
|
||||
try:
|
||||
from movements import last_runs # type: ignore
|
||||
runs = last_runs()
|
||||
if runs:
|
||||
lines.append("Recent data-movement runs:")
|
||||
for mid, r in list(runs.items())[-6:]:
|
||||
lines.append(f" - {mid}: {r.get('state')} rows={r.get('rows')} {_fmt_ts(r.get('ended_at'))}")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# recent spark workbench runs
|
||||
try:
|
||||
from spark_workbench import _runs as wb_runs, _run_order # type: ignore
|
||||
recent = [wb_runs[r] for r in _run_order[-6:] if r in wb_runs]
|
||||
if recent:
|
||||
lines.append("Recent Spark Workbench runs:")
|
||||
for r in recent:
|
||||
st = (r.get("stats") or {})
|
||||
tgt = f" → {r.get('target')}" if r.get("target") else ""
|
||||
lines.append(
|
||||
f" - {r.get('label')}: {r.get('state')} rows={st.get('processed_rows')}{tgt}"
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def build_masking_section() -> str:
|
||||
"""Exact masking policy + strict guidance so the LLM can answer about masked
|
||||
data without ever revealing masked raw values."""
|
||||
lines: list[str] = ["=== DATA MASKING POLICY (enforced) ==="]
|
||||
masked: list[str] = []
|
||||
unmasked: list[str] = []
|
||||
try:
|
||||
from pii_catalog import get_pii # type: ignore
|
||||
data = get_pii()
|
||||
for d in data.get("datasets", []):
|
||||
for c in d.get("pii_columns", []):
|
||||
tag = f"{d.get('label')}.{c.get('name')} [{c.get('category')}]"
|
||||
(masked if c.get("masked") else unmasked).append(tag)
|
||||
summ = data.get("summary", {})
|
||||
lines.append(
|
||||
f"PII columns: {summ.get('pii_columns', 0)} total — "
|
||||
f"{summ.get('masked_columns', 0)} masked, {summ.get('unmasked_columns', 0)} visible."
|
||||
)
|
||||
except Exception as exc:
|
||||
lines.append(f"(masking catalog unavailable: {exc})")
|
||||
|
||||
if masked:
|
||||
lines.append("MASKED columns (raw values are withheld — token 🔒 MASKED):")
|
||||
for m in masked[:40]:
|
||||
lines.append(f" - {m}")
|
||||
if unmasked:
|
||||
lines.append("Visible PII columns (operator opted out of masking):")
|
||||
for u in unmasked[:40]:
|
||||
lines.append(f" - {u}")
|
||||
|
||||
lines += [
|
||||
"",
|
||||
"How to handle masked data when answering:",
|
||||
" 1. NEVER reveal, guess, reconstruct or print the raw value of a MASKED column. If a value comes in as '🔒 MASKED', keep it masked.",
|
||||
" 2. DO still answer helpfully: confirm the column exists and is masked for privacy/governance, and explain why (PII protection policy).",
|
||||
" 3. You MAY use and report non-sensitive aggregates, counts, distributions and derived metrics over masked columns (e.g. 'there are N distinct customers') as long as no individual raw value is exposed.",
|
||||
" 4. Tell the operator they can unmask a specific column from the Data Flow PII overlay if they have the authority, and that the curated/masked Iceberg layer is physically masked and cannot be unmasked.",
|
||||
" 5. Unmasked PII columns may be shown, but flag that they are sensitive.",
|
||||
]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def build_llm_addendum() -> str:
|
||||
try:
|
||||
platform = build_platform_section()
|
||||
except Exception as exc:
|
||||
platform = f"(platform section error: {exc})"
|
||||
try:
|
||||
masking = build_masking_section()
|
||||
except Exception as exc:
|
||||
masking = f"(masking section error: {exc})"
|
||||
return "\n\n".join([platform, masking])
|
||||
+164
-33
@@ -18,6 +18,30 @@ def _slide(slide_id: str, title: str, subtitle: str, bullets: list[str], **extra
|
||||
return {"id": slide_id, "title": title, "subtitle": subtitle, "bullets": bullets, **extra}
|
||||
|
||||
|
||||
def _node_slide(nid: str, extra_bullets: list[str] | None = None) -> dict[str, Any]:
|
||||
reg = NODE_REGISTRY.get(nid, {})
|
||||
bullets: list[str] = []
|
||||
if reg.get("description"):
|
||||
bullets.append(reg["description"])
|
||||
bullets.append(f"VM: {reg.get('vm', '?')} (VMID {reg.get('vmid', '?')}) @ {reg.get('ip', '?')}")
|
||||
for link in reg.get("links", [])[:5]:
|
||||
bullets.append(f"{link.get('label', 'Link')}: {link.get('url', '')}")
|
||||
for ep in reg.get("endpoints", [])[:6]:
|
||||
bullets.append(f"{ep.get('name', '?')}: {ep.get('host', '?')}:{ep.get('port', '?')}")
|
||||
if extra_bullets:
|
||||
bullets.extend(extra_bullets)
|
||||
kind = "gpu" if nid == "gpu" else "command" if nid == "command" else "zone"
|
||||
if nid in ("openmetadata", "elastic"):
|
||||
kind = "architecture"
|
||||
return _slide(
|
||||
f"node-{nid}",
|
||||
reg.get("label", nid),
|
||||
f"{reg.get('role', 'service').upper()} · {reg.get('vm', '')}",
|
||||
bullets[:14],
|
||||
kind=kind,
|
||||
)
|
||||
|
||||
|
||||
def build_presentation_payload(snap: dict[str, Any]) -> dict[str, Any]:
|
||||
workload = build_workload_payload(snap)
|
||||
topologies = workload.get("topologies") or build_all_topologies(snap)
|
||||
@@ -28,20 +52,24 @@ def build_presentation_payload(snap: dict[str, Any]) -> dict[str, Any]:
|
||||
hadoop = snap.get("hadoop", {})
|
||||
objectscale = snap.get("objectscale", {})
|
||||
command = snap.get("command_center", {})
|
||||
databases = snap.get("databases", {})
|
||||
docker = snap.get("docker", {})
|
||||
lakehouse = snap.get("lakehouse", {})
|
||||
governance = snap.get("governance") or {}
|
||||
|
||||
slides: list[dict[str, Any]] = []
|
||||
|
||||
slides.append(_slide(
|
||||
"title",
|
||||
"Dell ATC Data Lab",
|
||||
"Live demo & presentation — Command Center",
|
||||
"Live infrastructure presentation — Command Center",
|
||||
[
|
||||
f"Snapshot: {snap.get('ts', 'now')}",
|
||||
f"Pipeline: {'ACTIVE' if totals.get('pipeline_active') else 'INACTIVE'}",
|
||||
f"Apps running: {totals.get('apps_running', 0)}/{totals.get('apps_total', 0)}",
|
||||
f"CDC connectors: {totals.get('connectors', 0)}",
|
||||
f"CDC connectors: {totals.get('connectors', 0)} · Source DBs: 5 engines on db02",
|
||||
f"LLM: {gpu.get('model') or 'offline'} ({gpu.get('gpu_count', 0)}× V100)",
|
||||
"Command Center → http://10.0.21.33/",
|
||||
"Command Center → http://10.0.21.33/ · Data Platform tab → Presentation",
|
||||
],
|
||||
kind="hero",
|
||||
))
|
||||
@@ -51,53 +79,151 @@ def build_presentation_payload(snap: dict[str, Any]) -> dict[str, Any]:
|
||||
"Mission",
|
||||
"End-to-end modern data platform on Dell infrastructure",
|
||||
[
|
||||
"Ingest change data from operational databases (PostgreSQL, MySQL, MongoDB, Cassandra)",
|
||||
"Stream via Kafka & Debezium into the lakehouse (Spark, Trino, Iceberg)",
|
||||
"Land curated data on ObjectScale S3 — query with Trino & visualize in Superset",
|
||||
"Parallel HDFS cluster for batch / legacy workloads",
|
||||
"GPU lab powers autonomous ops agents with local vLLM inference",
|
||||
"This dashboard orchestrates agents, approvals, and live cluster visibility",
|
||||
"Ingest CDC from PostgreSQL, MySQL, MongoDB (+ Cassandra & Neo4j for analytics)",
|
||||
"Stream via Kafka & Debezium into Spark, Trino & Iceberg on the lakehouse",
|
||||
"Land curated data on ObjectScale S3 — federated SQL with Trino",
|
||||
"Govern with OpenMetadata — catalog, lineage, PII classification",
|
||||
"Search & observe via Elasticsearch/Kibana · BI via Superset",
|
||||
"Parallel 9-node Hadoop HDFS cluster for batch workloads",
|
||||
"GPU lab (4× V100) powers autonomous agents with local vLLM inference",
|
||||
"This Command Center orchestrates agents, approvals & live visibility",
|
||||
],
|
||||
kind="narrative",
|
||||
))
|
||||
|
||||
slides.append(_slide(
|
||||
"command-center-ui",
|
||||
"Command Center UI",
|
||||
"Everything you operate from this dashboard",
|
||||
[
|
||||
"Data Platform — interactive topology + live presentation deck",
|
||||
"Data Sources UI — browse, filter & edit all 5 source databases",
|
||||
"Live Changes — real-time Debezium CDC event stream",
|
||||
"Data Flow — OpenMetadata catalog, lineage & pipeline map",
|
||||
"Data Quality — Docling document QA + RAG ingest",
|
||||
"Knowledge Chat — GPU-backed RAG over lab documentation",
|
||||
"Object Storage · HDFS · Elasticsearch · SSH terminal",
|
||||
"Agent fleet · Approval inbox · Live GPU matrix",
|
||||
],
|
||||
kind="command",
|
||||
))
|
||||
|
||||
arch = topologies.get("architecture") or workload.get("topology") or {}
|
||||
arch_nodes = arch.get("nodes", [])
|
||||
slides.append(_slide(
|
||||
"architecture",
|
||||
"Data Platform Architecture",
|
||||
arch.get("subtitle", "Sources → Ingestion → Compute → Storage → Consumers"),
|
||||
[f"{n.get('label', n.get('id'))}: {n.get('subtitle', n.get('role', ''))}" for n in arch_nodes[:14]],
|
||||
[f"{n.get('label', n.get('id'))}: {n.get('subtitle', n.get('role', ''))}" for n in arch_nodes[:16]],
|
||||
kind="topology",
|
||||
topology=arch,
|
||||
))
|
||||
|
||||
db_lines = [
|
||||
f"Host atc-db02 (10.0.21.51) — {databases.get('running', 0)}/{databases.get('total', 0)} containers up",
|
||||
"postgres_sales — PostgreSQL sales_orders (CDC → Debezium)",
|
||||
"mysql_hr — MySQL employee_events (CDC → Debezium)",
|
||||
"mongodb_supplychain — MongoDB supplychain.events (CDC → Debezium)",
|
||||
"cassandra_telemetry — Cassandra device_metrics (Trino federated)",
|
||||
"neo4j_graph — Product/Supplier graph · 4.5M nodes",
|
||||
]
|
||||
for c in (databases.get("containers") or [])[:8]:
|
||||
db_lines.append(f" · {c.get('name', '?')}: {c.get('state', '?')}")
|
||||
slides.append(_slide(
|
||||
"source-databases",
|
||||
"Source Databases",
|
||||
"DB Vault · atc-db02 · 10.0.21.51",
|
||||
db_lines,
|
||||
kind="topology",
|
||||
))
|
||||
|
||||
pipeline = topologies.get("pipeline", {})
|
||||
connector_lines = [
|
||||
f" · {cs['name']}: {cs.get('state', '?')}"
|
||||
for cs in (etl.get("connector_status") or [])[:6]
|
||||
for cs in (etl.get("connector_status") or [])[:8]
|
||||
]
|
||||
slides.append(_slide(
|
||||
"pipeline",
|
||||
"CDC Pipeline",
|
||||
pipeline.get("subtitle", "Airflow → DB → Debezium → Kafka → Lakehouse → S3"),
|
||||
[
|
||||
f"Airflow: {'healthy' if etl.get('airflow_healthy') else 'degraded'} ({etl.get('airflow_url', '')})",
|
||||
f"Kafka UI: {'UP' if etl.get('kafka_ui_ok') else 'DOWN'}",
|
||||
f"Airflow: {'healthy' if etl.get('airflow_healthy') else 'degraded'} — {etl.get('airflow_url', 'http://10.0.21.55:8080')}",
|
||||
f"Kafka UI: {'UP' if etl.get('kafka_ui_ok') else 'DOWN'} — http://10.0.21.36:9000",
|
||||
f"Debezium Connect: http://10.0.21.50:8083",
|
||||
f"Connectors: {', '.join(etl.get('connectors') or []) or 'none'}",
|
||||
*connector_lines,
|
||||
f"Spark UI: {'UP' if etl.get('spark_ui_ok') else 'DOWN'}",
|
||||
f"ObjectScale: {'reachable' if objectscale.get('reachable') else 'down'} bucket={objectscale.get('bucket', 'data')}",
|
||||
f"Spark UI: {'UP' if etl.get('spark_ui_ok') else 'DOWN'} — http://10.0.21.50:8080",
|
||||
f"ObjectScale S3: {'reachable' if objectscale.get('reachable') else 'down'} — bucket={objectscale.get('bucket', 'data')}",
|
||||
],
|
||||
kind="topology",
|
||||
topology=pipeline,
|
||||
))
|
||||
|
||||
lake_lines = [
|
||||
f"atc-lake01 @ {lakehouse.get('host', '10.0.21.50')} — {lakehouse.get('running', 0)}/{lakehouse.get('total', 0)} containers",
|
||||
f"Trino: {'UP' if lakehouse.get('trino_ok') else 'DOWN'} — http://10.0.21.50:8089",
|
||||
"Spark — batch & streaming compute",
|
||||
"Kafka Connect + Debezium — CDC ingestion",
|
||||
"s3-kafka-consumer — events → ObjectScale S3",
|
||||
"Iceberg catalog — bronze → silver → gold tables",
|
||||
]
|
||||
for c in (lakehouse.get("containers") or [])[:8]:
|
||||
lake_lines.append(f" · {c.get('name', '?')}: {c.get('state', '?')}")
|
||||
slides.append(_slide(
|
||||
"lakehouse",
|
||||
"Lakehouse Hub",
|
||||
"Spark · Trino · Iceberg · Kafka Connect",
|
||||
lake_lines,
|
||||
kind="topology",
|
||||
))
|
||||
|
||||
docker_lines = [
|
||||
f"atc-docker01 @ 10.0.21.45 — {docker.get('running', 0)}/{docker.get('total', 0)} containers",
|
||||
"Homepage — http://10.0.21.45",
|
||||
"Dockhand — container management http://10.0.21.45:8082",
|
||||
"Apache Superset — BI dashboards http://10.0.21.45:8088",
|
||||
"Forgejo / Gitea · monitoring · nginx · redis",
|
||||
]
|
||||
for c in (docker.get("containers") or [])[:10]:
|
||||
docker_lines.append(f" · {c.get('name', '?')}: {c.get('state', '?')}")
|
||||
slides.append(_slide(
|
||||
"docker-rack",
|
||||
"Docker Rack & Analytics",
|
||||
"Platform services on atc-docker01",
|
||||
docker_lines,
|
||||
kind="zone",
|
||||
))
|
||||
|
||||
cdc = governance.get("cdc") or {}
|
||||
gov_lines = [
|
||||
f"OpenMetadata UI: {governance.get('openmetadata_url', 'http://10.0.21.47:8585')}",
|
||||
"Ingestion Airflow: http://10.0.21.47:8080",
|
||||
"Catalog · lineage · data quality · PII auto-classification (Presidio NER)",
|
||||
]
|
||||
if "error" not in cdc:
|
||||
by_src = ", ".join(f"{k}={v}" for k, v in (cdc.get("by_source") or {}).items()) or "none"
|
||||
gov_lines.append(f"CDC stream: connected={cdc.get('connected')} · {cdc.get('window_total', 0)} changes/15m ({by_src})")
|
||||
ps = governance.get("pii_summary") or {}
|
||||
if "error" not in ps:
|
||||
gov_lines.append(f"PII: {ps.get('pii_columns', 0)} columns · {ps.get('masked_columns', 0)} masked")
|
||||
for ln in (governance.get("lineage") or [])[:4]:
|
||||
gov_lines.append(f"Lineage: {ln}")
|
||||
slides.append(_slide(
|
||||
"governance",
|
||||
"Governance & Metadata",
|
||||
"OpenMetadata · CDC · PII · Lineage",
|
||||
gov_lines,
|
||||
kind="architecture",
|
||||
))
|
||||
|
||||
slides.append(_node_slide("openmetadata"))
|
||||
slides.append(_node_slide("elastic"))
|
||||
|
||||
for zone in zones:
|
||||
apps = zone.get("apps") or []
|
||||
app_lines = [
|
||||
f"{a['name']}: {a['state']}" + (f" ({a.get('host', '')})" if a.get("host") else "")
|
||||
for a in apps[:10]
|
||||
for a in apps[:12]
|
||||
]
|
||||
slides.append(_slide(
|
||||
f"zone-{zone['id']}",
|
||||
@@ -111,32 +237,36 @@ def build_presentation_payload(snap: dict[str, Any]) -> dict[str, Any]:
|
||||
zone=zone,
|
||||
))
|
||||
|
||||
infra_nodes = [
|
||||
nid for nid in NODE_REGISTRY
|
||||
if nid not in ("mo-commander", "bart-commander", "network-watcher", "mcp-coordinator")
|
||||
PRESENTATION_NODES = [
|
||||
"airflow", "db", "debezium", "kafka", "lakehouse", "s3",
|
||||
"docker", "hadoop", "gpu", "command",
|
||||
]
|
||||
slides.append(_slide(
|
||||
"infrastructure",
|
||||
"infrastructure-map",
|
||||
"Infrastructure Map",
|
||||
"Proxmox VMs & services across VLAN 20/21",
|
||||
[
|
||||
f"{NODE_REGISTRY[nid]['label']} — {NODE_REGISTRY[nid].get('vm')} "
|
||||
f"(VMID {NODE_REGISTRY[nid].get('vmid', '?')}) @ {NODE_REGISTRY[nid].get('ip')}"
|
||||
for nid in infra_nodes
|
||||
for nid in PRESENTATION_NODES if nid in NODE_REGISTRY
|
||||
],
|
||||
kind="registry",
|
||||
))
|
||||
|
||||
for nid in PRESENTATION_NODES:
|
||||
if nid in NODE_REGISTRY and nid not in ("docker", "db", "lakehouse"):
|
||||
slides.append(_node_slide(nid))
|
||||
|
||||
dn_lines = [
|
||||
f" · {dn['host']}: {dn.get('used_gb', 0)} GB — {dn.get('state', '')}"
|
||||
for dn in (hadoop.get("datanodes") or [])[:5]
|
||||
for dn in (hadoop.get("datanodes") or [])[:6]
|
||||
]
|
||||
slides.append(_slide(
|
||||
"hadoop",
|
||||
"Hadoop HDFS",
|
||||
"9-node parallel storage cluster",
|
||||
[
|
||||
f"NameNode: {'UP' if hadoop.get('reachable') else 'DOWN'} — {hadoop.get('namenode', '')}",
|
||||
f"NameNode: {'UP' if hadoop.get('reachable') else 'DOWN'} — {hadoop.get('namenode', 'http://10.0.21.61:9870')}",
|
||||
f"Capacity: {hadoop.get('capacity_used_gb', '?')} / {hadoop.get('capacity_total_gb', '?')} GB",
|
||||
f"DataNodes: {hadoop.get('live_datanodes', 0)} live, {hadoop.get('dead_datanodes', 0)} dead",
|
||||
f"Files: {hadoop.get('files_total', 0)}, Blocks: {hadoop.get('blocks_total', 0)}",
|
||||
@@ -158,7 +288,8 @@ def build_presentation_payload(snap: dict[str, Any]) -> dict[str, Any]:
|
||||
[
|
||||
f"Inference: {'ON' if gpu.get('inference_active') else 'OFF'}",
|
||||
f"API: {snap.get('gpu', {}).get('vllm_url') or 'http://10.0.20.106:8001/v1'}",
|
||||
"Manager: http://10.0.20.106:9000",
|
||||
"GPU Lab UI: http://10.0.20.106:9000",
|
||||
"Kibana/Elastic: http://10.0.21.46:5601",
|
||||
*gpu_lines,
|
||||
],
|
||||
kind="gpu",
|
||||
@@ -171,7 +302,7 @@ def build_presentation_payload(snap: dict[str, Any]) -> dict[str, Any]:
|
||||
[
|
||||
"ETL Guardian — Airflow, Kafka, Debezium, connectors",
|
||||
"Data Custodian — PostgreSQL, MySQL, MongoDB, Cassandra, Neo4j",
|
||||
"Lakehouse Ops — Spark, Trino, Iceberg, ObjectScale S3",
|
||||
"Lakehouse Ops — Spark, Trino, Iceberg, ObjectScale S3, OpenMetadata",
|
||||
"Hadoop Ranger — HDFS NameNode, DataNodes, block health",
|
||||
"Infra Sentinel — Docker rack, GPU lab, Command Center",
|
||||
"All agents receive LIVE cluster snapshot in every LLM prompt",
|
||||
@@ -182,13 +313,13 @@ def build_presentation_payload(snap: dict[str, Any]) -> dict[str, Any]:
|
||||
cc_apps = [f"{c['name']}: {c['state']}" for c in (command.get("containers") or [])]
|
||||
slides.append(_slide(
|
||||
"command",
|
||||
"Command Center",
|
||||
"VM 304 — this presentation runs here",
|
||||
"Command Center Stack",
|
||||
"VM 304 — this dashboard runs here",
|
||||
[
|
||||
f"Host: {command.get('host', '10.0.21.33')} (VMID {command.get('vmid', 304)})",
|
||||
f"Stack: {command.get('running', 0)}/{command.get('total', 0)} containers",
|
||||
f"Stack: {command.get('running', 0)}/{command.get('total', 0)} containers (api, ui, caddy, redis, postgres)",
|
||||
*cc_apps,
|
||||
"WebSocket ops feed · Approval inbox · Agent terminals",
|
||||
"WebSocket ops feed · Approval inbox · Agent terminals · Data Sources UI",
|
||||
],
|
||||
kind="command",
|
||||
))
|
||||
@@ -198,12 +329,12 @@ def build_presentation_payload(snap: dict[str, Any]) -> dict[str, Any]:
|
||||
"Live Demo Tips",
|
||||
"Use this deck during customer presentations",
|
||||
[
|
||||
"Press ← → or click dots to navigate slides",
|
||||
"F = fullscreen presentation mode",
|
||||
"Press ← → or click dots to navigate slides · F = fullscreen",
|
||||
"Data Platform tab → Presentation sub-tab (this deck)",
|
||||
"Data Platform → Topology for interactive pipeline map",
|
||||
"Export HTML opens a standalone deck for projectors / offline",
|
||||
"Ask agents in the Command Bar — they see full cluster context",
|
||||
"Switch to Data Platform tab for interactive topology",
|
||||
"GPU Lab chat: http://10.0.20.106:9000/chat",
|
||||
"Trigger data generation in Data Sources UI → Generate tab",
|
||||
],
|
||||
kind="cta",
|
||||
))
|
||||
|
||||
+69
-20
@@ -13,6 +13,7 @@ from typing import Any
|
||||
import httpx
|
||||
|
||||
PRESENTATIONS_DIR = Path(os.getenv("PRESENTATIONS_DIR", "/data/presentations"))
|
||||
LIVE_OVERRIDE_DIR = PRESENTATIONS_DIR / "live-override"
|
||||
DOCLING_URL = os.getenv("DOCLING_URL", "http://docling-serve:5001").rstrip("/")
|
||||
|
||||
|
||||
@@ -165,6 +166,65 @@ def get_deck(deck_id: str) -> dict[str, Any] | None:
|
||||
return json.loads(path.read_text())
|
||||
|
||||
|
||||
def get_live_override() -> dict[str, Any] | None:
|
||||
path = LIVE_OVERRIDE_DIR / "meta.json"
|
||||
if not path.exists():
|
||||
return None
|
||||
try:
|
||||
return json.loads(path.read_text())
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def save_live_override(body: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Persist user edits for the Live Cluster deck."""
|
||||
LIVE_OVERRIDE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
(LIVE_OVERRIDE_DIR / "assets").mkdir(parents=True, exist_ok=True)
|
||||
existing = get_live_override() or {"id": "live", "source": "live-override", "editable": True}
|
||||
slides = _clean_slides(body.get("slides"))
|
||||
payload = {
|
||||
**existing,
|
||||
"id": "live",
|
||||
"source": "live-override",
|
||||
"editable": True,
|
||||
"title": str(body.get("title") or existing.get("title") or "Live Cluster")[:120],
|
||||
"subtitle": str(body.get("subtitle") or existing.get("subtitle") or "")[:300],
|
||||
"ts": datetime.now(timezone.utc).isoformat(),
|
||||
"slides": slides,
|
||||
"slide_count": len(slides),
|
||||
}
|
||||
(LIVE_OVERRIDE_DIR / "meta.json").write_text(json.dumps(payload, indent=2, default=str))
|
||||
return payload
|
||||
|
||||
|
||||
def clear_live_override() -> bool:
|
||||
import shutil
|
||||
if not LIVE_OVERRIDE_DIR.exists():
|
||||
return True
|
||||
shutil.rmtree(LIVE_OVERRIDE_DIR, ignore_errors=True)
|
||||
return True
|
||||
|
||||
|
||||
def _clean_slides(incoming_slides: Any) -> list[dict[str, Any]]:
|
||||
clean_slides: list[dict[str, Any]] = []
|
||||
for i, s in enumerate(incoming_slides or [], start=1):
|
||||
if not isinstance(s, dict):
|
||||
continue
|
||||
bullets = [str(b).strip()[:400] for b in (s.get("bullets") or []) if str(b).strip()]
|
||||
slide: dict[str, Any] = {
|
||||
"id": str(s.get("id") or f"slide-{i}"),
|
||||
"title": str(s.get("title") or f"Slide {i}")[:200],
|
||||
"subtitle": str(s.get("subtitle") or "")[:300],
|
||||
"bullets": bullets,
|
||||
"image": str(s.get("image") or "")[:300],
|
||||
"kind": str(s.get("kind") or "narrative")[:40],
|
||||
}
|
||||
if s.get("animation"):
|
||||
slide["animation"] = str(s.get("animation"))[:40]
|
||||
clean_slides.append(slide)
|
||||
return clean_slides or [_blank_slide(1)]
|
||||
|
||||
|
||||
def _blank_slide(idx: int = 1) -> dict[str, Any]:
|
||||
return {
|
||||
"id": f"slide-{idx}",
|
||||
@@ -208,22 +268,7 @@ def save_deck(deck_id: str, body: dict[str, Any]) -> dict[str, Any] | None:
|
||||
return None
|
||||
existing = json.loads(meta_path.read_text())
|
||||
|
||||
incoming_slides = body.get("slides")
|
||||
clean_slides: list[dict[str, Any]] = []
|
||||
for i, s in enumerate(incoming_slides or [], start=1):
|
||||
if not isinstance(s, dict):
|
||||
continue
|
||||
bullets = [str(b).strip()[:400] for b in (s.get("bullets") or []) if str(b).strip()]
|
||||
clean_slides.append({
|
||||
"id": str(s.get("id") or f"slide-{i}"),
|
||||
"title": str(s.get("title") or f"Slide {i}")[:200],
|
||||
"subtitle": str(s.get("subtitle") or "")[:300],
|
||||
"bullets": bullets,
|
||||
"image": str(s.get("image") or "")[:300],
|
||||
"kind": str(s.get("kind") or "narrative")[:40],
|
||||
})
|
||||
if not clean_slides:
|
||||
clean_slides = [_blank_slide(1)]
|
||||
clean_slides = _clean_slides(body.get("slides"))
|
||||
|
||||
existing.update({
|
||||
"title": str(body.get("title") or existing.get("title") or "Untitled deck")[:120],
|
||||
@@ -250,9 +295,12 @@ def delete_deck(deck_id: str) -> bool:
|
||||
|
||||
def save_image(deck_id: str, filename: str, content: bytes) -> dict[str, Any] | None:
|
||||
"""Store an image in the deck's assets folder; return its served URL."""
|
||||
deck_dir = PRESENTATIONS_DIR / deck_id
|
||||
if not deck_dir.exists():
|
||||
return None
|
||||
if deck_id == "live":
|
||||
deck_dir = LIVE_OVERRIDE_DIR
|
||||
else:
|
||||
deck_dir = PRESENTATIONS_DIR / deck_id
|
||||
if not deck_dir.exists():
|
||||
return None
|
||||
assets = deck_dir / "assets"
|
||||
assets.mkdir(parents=True, exist_ok=True)
|
||||
ext = ""
|
||||
@@ -265,7 +313,8 @@ def save_image(deck_id: str, filename: str, content: bytes) -> dict[str, Any] |
|
||||
|
||||
def get_asset_path(deck_id: str, name: str) -> Path | None:
|
||||
safe = _safe_name(name)
|
||||
path = PRESENTATIONS_DIR / deck_id / "assets" / safe
|
||||
base = LIVE_OVERRIDE_DIR if deck_id == "live" else PRESENTATIONS_DIR / deck_id
|
||||
path = base / "assets" / safe
|
||||
if not path.exists() or not path.is_file():
|
||||
return None
|
||||
return path
|
||||
|
||||
@@ -0,0 +1,435 @@
|
||||
"""Databricks-style lakehouse workbench for the Command Center.
|
||||
|
||||
Lets the user interactively select data from any federated source (Iceberg,
|
||||
Hive/HDFS, Postgres, MySQL, Mongo, Cassandra, Kafka) and run distributed
|
||||
transformations on the lakehouse compute layer:
|
||||
|
||||
explore -> transform (aggregate / filter / join / profile) -> materialize to Iceberg (S3/HDFS)
|
||||
|
||||
Execution streams live engine metrics (state, splits, rows, bytes, CPU, wall,
|
||||
peak memory, nodes) so the UI can show a live "what the cluster is doing" matrix
|
||||
exactly like Databricks' Spark UI. Compute runs on the lakehouse engine (Trino
|
||||
coordinator on the Spark cluster host) which distributes work across the
|
||||
workers; Spark batch DAGs remain available via the jobs API.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from fastapi import APIRouter, Body, Query
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
TRINO_URL = os.getenv("TRINO_URL", "http://10.0.21.50:8089").rstrip("/")
|
||||
TRINO_USER = os.getenv("TRINO_USER", "mo")
|
||||
SPARK_UI_URL = os.getenv("SPARK_UI_URL", "http://10.0.21.50:8080").rstrip("/")
|
||||
|
||||
router = APIRouter(prefix="/api/spark", tags=["spark-workbench"])
|
||||
|
||||
_runs: dict[str, dict[str, Any]] = {}
|
||||
_run_order: list[str] = []
|
||||
_MAX_RUNS = 40
|
||||
_MAX_RESULT_ROWS = 500
|
||||
|
||||
_IDENT_RE = re.compile(r"^[A-Za-z0-9_.\" ]+$")
|
||||
_TABLE_RE = re.compile(r"^[A-Za-z0-9_.\"]+$")
|
||||
|
||||
AGG_FUNCS = {"count", "sum", "avg", "min", "max", "approx_distinct", "count_distinct"}
|
||||
JOIN_TYPES = {"INNER", "LEFT", "RIGHT", "FULL"}
|
||||
|
||||
|
||||
def _feed(agent_id: str, message: str, level: str = "info") -> None:
|
||||
try:
|
||||
from main import add_feed
|
||||
add_feed(agent_id, message, level)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _qident(name: str) -> str:
|
||||
name = name.strip().strip('"')
|
||||
if not re.match(r"^[A-Za-z0-9_]+$", name):
|
||||
raise ValueError(f"invalid identifier: {name}")
|
||||
return f'"{name}"'
|
||||
|
||||
|
||||
def _safe_table(name: str) -> str:
|
||||
name = (name or "").strip()
|
||||
if not name or not _TABLE_RE.match(name):
|
||||
raise ValueError(f"invalid table reference: {name}")
|
||||
return name
|
||||
|
||||
|
||||
# ── Trino helpers ────────────────────────────────────────────────────────────
|
||||
def _trino_headers() -> dict[str, str]:
|
||||
return {"X-Trino-User": TRINO_USER, "Content-Type": "text/plain"}
|
||||
|
||||
|
||||
async def _trino_collect(sql: str, limit: int = 200, timeout: float = 60.0) -> dict[str, Any]:
|
||||
"""Synchronous-style helper: run a query and return all rows (for catalog ops)."""
|
||||
columns: list[str] = []
|
||||
rows: list[list[Any]] = []
|
||||
async with httpx.AsyncClient(timeout=timeout) as client:
|
||||
r = await client.post(f"{TRINO_URL}/v1/statement", content=sql.encode(), headers=_trino_headers())
|
||||
if r.status_code >= 400:
|
||||
return {"ok": False, "error": r.text[:400]}
|
||||
data = r.json()
|
||||
while True:
|
||||
if data.get("error"):
|
||||
return {"ok": False, "error": str(data["error"])[:400]}
|
||||
if data.get("columns") and not columns:
|
||||
columns = [c["name"] for c in data["columns"]]
|
||||
for row in data.get("data") or []:
|
||||
rows.append(row)
|
||||
if len(rows) >= limit:
|
||||
break
|
||||
nxt = data.get("nextUri")
|
||||
if not nxt or len(rows) >= limit:
|
||||
if nxt:
|
||||
try:
|
||||
await client.delete(nxt, headers=_trino_headers())
|
||||
except Exception:
|
||||
pass
|
||||
break
|
||||
data = (await client.get(nxt, headers=_trino_headers())).json()
|
||||
return {"ok": True, "columns": columns, "rows": rows}
|
||||
|
||||
|
||||
def _norm_stats(stats: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"state": stats.get("state"),
|
||||
"nodes": stats.get("nodes"),
|
||||
"total_splits": stats.get("totalSplits"),
|
||||
"queued_splits": stats.get("queuedSplits"),
|
||||
"running_splits": stats.get("runningSplits"),
|
||||
"completed_splits": stats.get("completedSplits"),
|
||||
"processed_rows": stats.get("processedRows"),
|
||||
"processed_bytes": stats.get("processedBytes"),
|
||||
"physical_input_bytes": stats.get("physicalInputBytes"),
|
||||
"peak_memory_bytes": stats.get("peakMemoryBytes"),
|
||||
"cpu_time_ms": stats.get("cpuTimeMillis"),
|
||||
"wall_time_ms": stats.get("wallTimeMillis"),
|
||||
"elapsed_ms": stats.get("elapsedTimeMillis"),
|
||||
"progress_pct": round(
|
||||
(stats.get("completedSplits") or 0) / stats["totalSplits"] * 100, 1
|
||||
) if stats.get("totalSplits") else (100.0 if stats.get("state") == "FINISHED" else 0.0),
|
||||
}
|
||||
|
||||
|
||||
async def _execute_run(run_id: str, sql: str, returns_rows: bool, pre_sql: str | None = None) -> None:
|
||||
run = _runs[run_id]
|
||||
run["state"] = "RUNNING"
|
||||
columns: list[str] = []
|
||||
rows: list[list[Any]] = []
|
||||
try:
|
||||
if pre_sql:
|
||||
pre = await _trino_collect(pre_sql, 1)
|
||||
if not pre.get("ok"):
|
||||
run.update(state="FAILED", error=pre.get("error"), ended_at=time.time())
|
||||
_feed("lakehouse-ops", f"[workbench] {run['label']}: pre-step failed", "err")
|
||||
return
|
||||
async with httpx.AsyncClient(timeout=None) as client:
|
||||
r = await client.post(f"{TRINO_URL}/v1/statement", content=sql.encode(), headers=_trino_headers())
|
||||
if r.status_code >= 400:
|
||||
run.update(state="FAILED", error=r.text[:500], ended_at=time.time())
|
||||
_feed("lakehouse-ops", f"[workbench] {run['label']}: failed ({r.status_code})", "err")
|
||||
return
|
||||
data = r.json()
|
||||
run["query_id"] = data.get("id")
|
||||
while True:
|
||||
if run.get("cancel_requested"):
|
||||
nxt = data.get("nextUri")
|
||||
if nxt:
|
||||
try:
|
||||
await client.delete(nxt, headers=_trino_headers())
|
||||
except Exception:
|
||||
pass
|
||||
run.update(state="CANCELED", ended_at=time.time())
|
||||
_feed("lakehouse-ops", f"[workbench] {run['label']}: canceled", "warn")
|
||||
return
|
||||
if data.get("stats"):
|
||||
run["stats"] = _norm_stats(data["stats"])
|
||||
st = data["stats"].get("state")
|
||||
if st:
|
||||
run["engine_state"] = st
|
||||
if data.get("error"):
|
||||
run.update(state="FAILED", error=str(data["error"])[:500], ended_at=time.time())
|
||||
_feed("lakehouse-ops", f"[workbench] {run['label']}: {str(data['error'])[:120]}", "err")
|
||||
return
|
||||
if data.get("columns") and not columns:
|
||||
columns = [c["name"] for c in data["columns"]]
|
||||
run["columns"] = columns
|
||||
for row in data.get("data") or []:
|
||||
if returns_rows and len(rows) < _MAX_RESULT_ROWS:
|
||||
rows.append(row)
|
||||
if data.get("updateType"):
|
||||
run["update_type"] = data.get("updateType")
|
||||
nxt = data.get("nextUri")
|
||||
if not nxt:
|
||||
break
|
||||
data = (await client.get(nxt, headers=_trino_headers())).json()
|
||||
run["next_uri"] = nxt
|
||||
run["rows"] = rows
|
||||
run["row_count"] = len(rows)
|
||||
run.update(state="FINISHED", ended_at=time.time())
|
||||
if run.get("stats"):
|
||||
run["stats"]["state"] = "FINISHED"
|
||||
run["stats"]["progress_pct"] = 100.0
|
||||
msg = f"[workbench] {run['label']}: finished"
|
||||
if run.get("target"):
|
||||
msg = f"[workbench] {run['label']}: materialized → {run['target']}"
|
||||
_feed("lakehouse-ops", msg, "info")
|
||||
except Exception as exc:
|
||||
run.update(state="FAILED", error=str(exc)[:500], ended_at=time.time())
|
||||
_feed("lakehouse-ops", f"[workbench] {run['label']}: error {str(exc)[:120]}", "err")
|
||||
|
||||
|
||||
# ── SQL builders ─────────────────────────────────────────────────────────────
|
||||
def _build_sql(body: dict[str, Any]) -> tuple[str, bool, str, str | None, str | None]:
|
||||
"""Return (sql, returns_rows, label, materialize_target, pre_sql)."""
|
||||
op = body.get("operation", "preview")
|
||||
limit = max(1, min(int(body.get("limit", 200)), _MAX_RESULT_ROWS))
|
||||
materialize = body.get("materialize") or {}
|
||||
target = None
|
||||
|
||||
if op == "sql":
|
||||
sql = (body.get("sql") or "").strip().rstrip(";")
|
||||
if not sql:
|
||||
raise ValueError("empty SQL")
|
||||
select_sql = sql
|
||||
label = "Custom SQL"
|
||||
returns_rows = sql.lower().lstrip().startswith(("select", "show", "describe", "with", "explain"))
|
||||
|
||||
elif op == "preview":
|
||||
table = _safe_table(body.get("table"))
|
||||
select_sql = f"SELECT * FROM {table} LIMIT {limit}"
|
||||
label = f"Preview {table}"
|
||||
returns_rows = True
|
||||
|
||||
elif op == "filter":
|
||||
table = _safe_table(body.get("table"))
|
||||
where = (body.get("where") or "").strip()
|
||||
clause = f" WHERE {where}" if where else ""
|
||||
select_sql = f"SELECT * FROM {table}{clause} LIMIT {limit}"
|
||||
label = f"Filter {table}"
|
||||
returns_rows = True
|
||||
|
||||
elif op == "aggregate":
|
||||
table = _safe_table(body.get("table"))
|
||||
group_by = [c for c in (body.get("group_by") or []) if c]
|
||||
metrics = body.get("metrics") or []
|
||||
select_parts: list[str] = [_qident(c) for c in group_by]
|
||||
for m in metrics:
|
||||
fn = (m.get("fn") or "count").lower()
|
||||
if fn not in AGG_FUNCS:
|
||||
raise ValueError(f"unsupported function {fn}")
|
||||
col = m.get("col")
|
||||
alias = m.get("alias") or (f"{fn}_{col}" if col else fn)
|
||||
if fn == "count" and (not col or col == "*"):
|
||||
expr = "count(*)"
|
||||
elif fn == "count_distinct":
|
||||
expr = f"count(DISTINCT {_qident(col)})"
|
||||
else:
|
||||
expr = f"{fn}({_qident(col)})"
|
||||
select_parts.append(f"{expr} AS {_qident(alias)}")
|
||||
if not select_parts:
|
||||
select_parts = ["count(*) AS cnt"]
|
||||
gb = f" GROUP BY {', '.join(_qident(c) for c in group_by)}" if group_by else ""
|
||||
order = ""
|
||||
if group_by:
|
||||
order = f" ORDER BY {', '.join(_qident(c) for c in group_by)}"
|
||||
select_sql = f"SELECT {', '.join(select_parts)} FROM {table}{gb}{order} LIMIT {limit}"
|
||||
label = f"Aggregate {table}"
|
||||
returns_rows = True
|
||||
|
||||
elif op == "join":
|
||||
left = _safe_table(body.get("left"))
|
||||
right = _safe_table(body.get("right"))
|
||||
jt = (body.get("join_type") or "INNER").upper()
|
||||
if jt not in JOIN_TYPES:
|
||||
raise ValueError(f"invalid join type {jt}")
|
||||
lk = _qident(body.get("left_key"))
|
||||
rk = _qident(body.get("right_key"))
|
||||
select_sql = (
|
||||
f"SELECT l.*, r.* FROM {left} l {jt} JOIN {right} r "
|
||||
f"ON l.{lk} = r.{rk} LIMIT {limit}"
|
||||
)
|
||||
label = f"Join {left} ⋈ {right}"
|
||||
returns_rows = True
|
||||
|
||||
elif op == "profile":
|
||||
table = _safe_table(body.get("table"))
|
||||
cols = [c for c in (body.get("columns") or []) if c][:12]
|
||||
parts = ["count(*) AS row_count"]
|
||||
for c in cols:
|
||||
qc = _qident(c)
|
||||
parts.append(f"approx_distinct({qc}) AS {_qident(c + '_distinct')}")
|
||||
parts.append(f"count({qc}) AS {_qident(c + '_nonnull')}")
|
||||
select_sql = f"SELECT {', '.join(parts)} FROM {table}"
|
||||
label = f"Profile {table}"
|
||||
returns_rows = True
|
||||
|
||||
else:
|
||||
raise ValueError(f"unknown operation {op}")
|
||||
|
||||
if materialize.get("enabled"):
|
||||
schema = materialize.get("schema", "hadoop")
|
||||
name = materialize.get("table")
|
||||
if not name:
|
||||
raise ValueError("materialize target table required")
|
||||
target = f"iceberg.{_qident(schema).strip(chr(34))}.{_qident(name).strip(chr(34))}"
|
||||
mode = (materialize.get("mode") or "create").lower()
|
||||
pre = None
|
||||
if mode == "replace":
|
||||
pre = f"DROP TABLE IF EXISTS {target}"
|
||||
ddl = f"CREATE TABLE {target} AS {select_sql}"
|
||||
elif mode == "insert":
|
||||
ddl = f"INSERT INTO {target} {select_sql}"
|
||||
else:
|
||||
ddl = f"CREATE TABLE {target} AS {select_sql}"
|
||||
return ddl, False, f"Materialize → {target}", target, pre
|
||||
|
||||
return select_sql, returns_rows, label, None, None
|
||||
|
||||
|
||||
# ── catalog endpoints ────────────────────────────────────────────────────────
|
||||
@router.get("/catalogs")
|
||||
async def list_catalogs() -> JSONResponse:
|
||||
res = await _trino_collect("SHOW CATALOGS", 100)
|
||||
if not res.get("ok"):
|
||||
return JSONResponse(res, status_code=502)
|
||||
cats = [r[0] for r in res["rows"] if r[0] not in ("system",)]
|
||||
return JSONResponse({"ok": True, "catalogs": cats})
|
||||
|
||||
|
||||
@router.get("/schemas")
|
||||
async def list_schemas(catalog: str = Query(...)) -> JSONResponse:
|
||||
cat = _safe_table(catalog)
|
||||
res = await _trino_collect(f"SHOW SCHEMAS FROM {cat}", 200)
|
||||
if not res.get("ok"):
|
||||
return JSONResponse(res, status_code=502)
|
||||
skip = {"information_schema"}
|
||||
schemas = [r[0] for r in res["rows"] if r[0] not in skip]
|
||||
return JSONResponse({"ok": True, "catalog": catalog, "schemas": schemas})
|
||||
|
||||
|
||||
@router.get("/tables")
|
||||
async def list_tables(catalog: str = Query(...), schema: str = Query(...)) -> JSONResponse:
|
||||
cat = _safe_table(catalog)
|
||||
sch = _safe_table(schema)
|
||||
res = await _trino_collect(f"SHOW TABLES FROM {cat}.{sch}", 500)
|
||||
if not res.get("ok"):
|
||||
return JSONResponse(res, status_code=502)
|
||||
tables = [{"name": r[0], "fqn": f"{catalog}.{schema}.{r[0]}"} for r in res["rows"]]
|
||||
return JSONResponse({"ok": True, "catalog": catalog, "schema": schema, "tables": tables})
|
||||
|
||||
|
||||
@router.get("/columns")
|
||||
async def list_columns(table: str = Query(...)) -> JSONResponse:
|
||||
tbl = _safe_table(table)
|
||||
res = await _trino_collect(f"DESCRIBE {tbl}", 500)
|
||||
if not res.get("ok"):
|
||||
return JSONResponse(res, status_code=502)
|
||||
cols = [{"name": r[0], "type": r[1] if len(r) > 1 else ""} for r in res["rows"]]
|
||||
return JSONResponse({"ok": True, "table": table, "columns": cols})
|
||||
|
||||
|
||||
# ── run endpoints ────────────────────────────────────────────────────────────
|
||||
@router.post("/run")
|
||||
async def create_run(body: dict[str, Any] = Body(...)) -> JSONResponse:
|
||||
try:
|
||||
sql, returns_rows, label, target, pre_sql = _build_sql(body)
|
||||
except ValueError as exc:
|
||||
return JSONResponse({"ok": False, "error": str(exc)}, status_code=400)
|
||||
|
||||
run_id = uuid.uuid4().hex[:12]
|
||||
_runs[run_id] = {
|
||||
"id": run_id,
|
||||
"operation": body.get("operation", "preview"),
|
||||
"label": label,
|
||||
"sql": sql,
|
||||
"target": target,
|
||||
"state": "QUEUED",
|
||||
"engine_state": "QUEUED",
|
||||
"stats": {},
|
||||
"columns": [],
|
||||
"rows": [],
|
||||
"row_count": None,
|
||||
"error": None,
|
||||
"started_at": time.time(),
|
||||
"ended_at": None,
|
||||
"cancel_requested": False,
|
||||
}
|
||||
_run_order.append(run_id)
|
||||
while len(_run_order) > _MAX_RUNS:
|
||||
old = _run_order.pop(0)
|
||||
_runs.pop(old, None)
|
||||
|
||||
_feed("lakehouse-ops", f"[workbench] {label}: submitted", "info")
|
||||
asyncio.create_task(_execute_run(run_id, sql, returns_rows, pre_sql))
|
||||
return JSONResponse({"ok": True, "run_id": run_id, "sql": sql, "label": label, "target": target})
|
||||
|
||||
|
||||
def _run_public(run: dict[str, Any], include_rows: bool = True) -> dict[str, Any]:
|
||||
out = {k: v for k, v in run.items() if k not in ("next_uri", "cancel_requested")}
|
||||
if not include_rows:
|
||||
out.pop("rows", None)
|
||||
return out
|
||||
|
||||
|
||||
@router.get("/run/{run_id}")
|
||||
async def get_run(run_id: str) -> JSONResponse:
|
||||
run = _runs.get(run_id)
|
||||
if not run:
|
||||
return JSONResponse({"ok": False, "error": "unknown run"}, status_code=404)
|
||||
return JSONResponse({"ok": True, "run": _run_public(run)})
|
||||
|
||||
|
||||
@router.post("/run/{run_id}/cancel")
|
||||
async def cancel_run(run_id: str) -> JSONResponse:
|
||||
run = _runs.get(run_id)
|
||||
if not run:
|
||||
return JSONResponse({"ok": False, "error": "unknown run"}, status_code=404)
|
||||
run["cancel_requested"] = True
|
||||
return JSONResponse({"ok": True, "run_id": run_id, "state": "canceling"})
|
||||
|
||||
|
||||
@router.get("/runs")
|
||||
async def list_runs() -> JSONResponse:
|
||||
out = [_run_public(_runs[r], include_rows=False) for r in reversed(_run_order) if r in _runs]
|
||||
return JSONResponse({"ok": True, "runs": out})
|
||||
|
||||
|
||||
@router.get("/live")
|
||||
async def spark_live() -> JSONResponse:
|
||||
"""Live cluster + active-run matrix for the workbench dashboard."""
|
||||
spark: dict[str, Any] = {}
|
||||
try:
|
||||
from streaming_ops import collect_spark
|
||||
spark = await collect_spark()
|
||||
except Exception as exc:
|
||||
spark = {"error": str(exc)[:200]}
|
||||
|
||||
active = [
|
||||
_run_public(_runs[r], include_rows=False)
|
||||
for r in reversed(_run_order)
|
||||
if r in _runs and _runs[r]["state"] in ("QUEUED", "RUNNING")
|
||||
]
|
||||
recent = [
|
||||
_run_public(_runs[r], include_rows=False)
|
||||
for r in reversed(_run_order[-8:])
|
||||
if r in _runs
|
||||
]
|
||||
return JSONResponse({
|
||||
"ok": True,
|
||||
"spark": spark,
|
||||
"active_runs": active,
|
||||
"recent_runs": recent,
|
||||
"ts": time.time(),
|
||||
})
|
||||
+430
-24
@@ -48,6 +48,17 @@ TRINO_USER = os.getenv("TRINO_USER", "atc")
|
||||
SOURCE_ENGINES = ("postgres", "mysql", "mongodb", "cassandra", "neo4j")
|
||||
ENGINES = SOURCE_ENGINES + ("trino",)
|
||||
|
||||
from hadoop_sql import (
|
||||
HADOOP_SAMPLES,
|
||||
catalog_hadoop,
|
||||
connection_info as hadoop_connection_info,
|
||||
health_hadoop,
|
||||
sample_hadoop,
|
||||
table_row_count_hadoop,
|
||||
)
|
||||
|
||||
LAKE_ENGINES = ("hadoop",)
|
||||
|
||||
router = APIRouter(prefix="/api/sql", tags=["sql"])
|
||||
|
||||
SAMPLES: dict[str, list[dict[str, str]]] = {
|
||||
@@ -111,6 +122,7 @@ SAMPLES: dict[str, list[dict[str, str]]] = {
|
||||
{"id": "nj9", "label": "Schema visualization", "sql": "CALL db.schema.visualization()"},
|
||||
{"id": "nj10", "label": "Constraint info", "sql": "SHOW CONSTRAINTS"},
|
||||
],
|
||||
"hadoop": HADOOP_SAMPLES,
|
||||
"trino": [
|
||||
{"id": "tq1", "label": "Trino version", "sql": "SELECT version()"},
|
||||
{"id": "tq2", "label": "Catalogs", "sql": "SHOW CATALOGS"},
|
||||
@@ -332,10 +344,17 @@ def _run_trino(sql: str, limit: int = 200) -> dict[str, Any]:
|
||||
|
||||
|
||||
class SqlRequest(BaseModel):
|
||||
engine: str = Field(..., pattern="^(postgres|mysql|mongodb|trino|cassandra|neo4j)$")
|
||||
engine: str = Field(..., pattern="^(postgres|mysql|mongodb|trino|cassandra|neo4j|hadoop)$")
|
||||
sql: str = Field(..., min_length=1, max_length=8000)
|
||||
|
||||
|
||||
class RowUpdateRequest(BaseModel):
|
||||
engine: str = Field(..., pattern="^(postgres|mysql|mongodb|cassandra|neo4j)$")
|
||||
object: str = Field(..., min_length=1, max_length=256)
|
||||
pk: dict[str, Any] = Field(..., min_length=1)
|
||||
changes: dict[str, Any] = Field(..., min_length=1)
|
||||
|
||||
|
||||
def _connection_info(engine: str) -> dict[str, str]:
|
||||
if engine == "postgres":
|
||||
return {"host": DB_HOST, "port": str(PG_PORT), "database": PG_DB, "user": PG_USER}
|
||||
@@ -361,6 +380,8 @@ def _dispatch(engine: str, sql: str, limit: int = 200) -> dict[str, Any]:
|
||||
return _run_cassandra(sql, limit)
|
||||
if engine == "neo4j":
|
||||
return _run_neo4j(sql, limit)
|
||||
if engine == "hadoop":
|
||||
return _run_trino(sql, limit)
|
||||
return _run_trino(sql, limit)
|
||||
|
||||
|
||||
@@ -458,6 +479,15 @@ def _catalog_neo4j() -> dict[str, Any]:
|
||||
driver.close()
|
||||
|
||||
|
||||
|
||||
def _catalog_hadoop() -> dict[str, Any]:
|
||||
return catalog_hadoop(_run_trino)
|
||||
|
||||
|
||||
def _sample_hadoop(object_name: str, limit: int, offset: int = 0) -> dict[str, Any]:
|
||||
return sample_hadoop(object_name, limit, offset, _run_trino, _tabular)
|
||||
|
||||
|
||||
def _catalog(engine: str) -> dict[str, Any]:
|
||||
if engine == "postgres":
|
||||
return _catalog_postgres()
|
||||
@@ -469,37 +499,74 @@ def _catalog(engine: str) -> dict[str, Any]:
|
||||
return _catalog_cassandra()
|
||||
if engine == "neo4j":
|
||||
return _catalog_neo4j()
|
||||
if engine == "hadoop":
|
||||
return _catalog_hadoop()
|
||||
raise ValueError(f"Catalog not supported for {engine}")
|
||||
|
||||
|
||||
def _sample_postgres(object_name: str, limit: int) -> dict[str, Any]:
|
||||
def _sample_postgres(object_name: str, limit: int, offset: int = 0) -> dict[str, Any]:
|
||||
if "." in object_name:
|
||||
schema, table = object_name.split(".", 1)
|
||||
sql = f'SELECT * FROM "{schema}"."{table}" LIMIT {limit}'
|
||||
sql = f'SELECT * FROM "{schema}"."{table}" OFFSET {int(offset)} LIMIT {int(limit)}'
|
||||
else:
|
||||
sql = f'SELECT * FROM public."{object_name}" LIMIT {limit}'
|
||||
sql = f'SELECT * FROM public."{object_name}" OFFSET {int(offset)} LIMIT {int(limit)}'
|
||||
return _run_postgres(sql, limit)
|
||||
|
||||
|
||||
def _sample_mysql(object_name: str, limit: int) -> dict[str, Any]:
|
||||
def _sample_mysql(object_name: str, limit: int, offset: int = 0) -> dict[str, Any]:
|
||||
table = object_name.split(".")[-1]
|
||||
return _run_mysql(f"SELECT * FROM `{table}` LIMIT {limit}", limit)
|
||||
return _run_mysql(f"SELECT * FROM `{table}` LIMIT {int(limit)} OFFSET {int(offset)}", limit)
|
||||
|
||||
|
||||
def _sample_mongodb(object_name: str, limit: int) -> dict[str, Any]:
|
||||
def _sample_mongodb(object_name: str, limit: int, offset: int = 0) -> dict[str, Any]:
|
||||
if "." in object_name:
|
||||
db_name, coll = object_name.split(".", 1)
|
||||
else:
|
||||
db_name, coll = MONGO_DB, object_name
|
||||
return _run_mongo(f"FIND {db_name}.{coll} LIMIT {limit}", limit)
|
||||
t0 = time.perf_counter()
|
||||
client = _mongo_client()
|
||||
try:
|
||||
docs = list(client[db_name][coll].find({}).skip(int(offset)).limit(int(limit)))
|
||||
elapsed_ms = int((time.perf_counter() - t0) * 1000)
|
||||
if not docs:
|
||||
return _tabular(["result"], [["(empty)"]], elapsed_ms)
|
||||
columns = sorted({k for d in docs for k in d})
|
||||
rows = [[_fmt(d.get(c)) for c in columns] for d in docs]
|
||||
return _tabular(columns, rows, elapsed_ms, object=object_name)
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
def _sample_cassandra(object_name: str, limit: int) -> dict[str, Any]:
|
||||
def _sample_cassandra(object_name: str, limit: int, offset: int = 0) -> dict[str, Any]:
|
||||
if "." in object_name:
|
||||
ks, table = object_name.split(".", 1)
|
||||
else:
|
||||
ks, table = CASS_KS, object_name
|
||||
return _run_cassandra(f"SELECT * FROM {ks}.{table} LIMIT {limit}", limit)
|
||||
t0 = time.perf_counter()
|
||||
cluster = _cass_cluster()
|
||||
session = cluster.connect()
|
||||
try:
|
||||
stmt = f"SELECT * FROM {ks}.{table}"
|
||||
result = session.execute(stmt, timeout=60)
|
||||
skipped = 0
|
||||
picked: list[Any] = []
|
||||
columns: list[str] = []
|
||||
for row in result:
|
||||
if skipped < offset:
|
||||
skipped += 1
|
||||
continue
|
||||
if not columns:
|
||||
columns = list(row._fields)
|
||||
picked.append(row)
|
||||
if len(picked) >= limit:
|
||||
break
|
||||
elapsed_ms = int((time.perf_counter() - t0) * 1000)
|
||||
if not picked:
|
||||
return _tabular(["result"], [["(empty)"]], elapsed_ms)
|
||||
rows = [[_fmt(getattr(r, c)) for c in columns] for r in picked]
|
||||
return _tabular(columns, rows, elapsed_ms, truncated=len(picked) >= limit)
|
||||
finally:
|
||||
cluster.shutdown()
|
||||
|
||||
|
||||
def _graph_neo4j(edge_limit: int = 60, rel_type: str | None = None) -> dict[str, Any]:
|
||||
@@ -582,9 +649,9 @@ def _graph_neo4j(edge_limit: int = 60, rel_type: str | None = None) -> dict[str,
|
||||
driver.close()
|
||||
|
||||
|
||||
def _sample_neo4j(object_name: str, limit: int) -> dict[str, Any]:
|
||||
def _sample_neo4j(object_name: str, limit: int, offset: int = 0) -> dict[str, Any]:
|
||||
label = object_name.split(".")[-1]
|
||||
cypher = f"MATCH (n:`{label}`) RETURN n LIMIT {limit}"
|
||||
cypher = f"MATCH (n:`{label}`) RETURN n SKIP {int(offset)} LIMIT {int(limit)}"
|
||||
t0 = time.perf_counter()
|
||||
driver = _neo4j_driver()
|
||||
try:
|
||||
@@ -601,24 +668,319 @@ def _sample_neo4j(object_name: str, limit: int) -> dict[str, Any]:
|
||||
driver.close()
|
||||
|
||||
|
||||
def _sample(engine: str, object_name: str, limit: int) -> dict[str, Any]:
|
||||
def _sample(engine: str, object_name: str, limit: int, offset: int = 0) -> dict[str, Any]:
|
||||
if engine == "postgres":
|
||||
return _sample_postgres(object_name, limit)
|
||||
return _sample_postgres(object_name, limit, offset)
|
||||
if engine == "mysql":
|
||||
return _sample_mysql(object_name, limit)
|
||||
return _sample_mysql(object_name, limit, offset)
|
||||
if engine == "mongodb":
|
||||
return _sample_mongodb(object_name, limit)
|
||||
return _sample_mongodb(object_name, limit, offset)
|
||||
if engine == "cassandra":
|
||||
return _sample_cassandra(object_name, limit)
|
||||
return _sample_cassandra(object_name, limit, offset)
|
||||
if engine == "neo4j":
|
||||
return _sample_neo4j(object_name, limit)
|
||||
return _sample_neo4j(object_name, limit, offset)
|
||||
if engine == "hadoop":
|
||||
return _sample_hadoop(object_name, limit, offset)
|
||||
raise ValueError(f"Sample not supported for {engine}")
|
||||
|
||||
|
||||
def _table_row_count(engine: str, object_name: str) -> int | None:
|
||||
try:
|
||||
if engine == "postgres":
|
||||
schema, table = _parse_fqn(engine, object_name)
|
||||
conn = psycopg2.connect(
|
||||
host=DB_HOST, port=PG_PORT, user=PG_USER, password=PG_PASS, dbname=PG_DB, connect_timeout=8,
|
||||
)
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute(f'SELECT count(*) FROM "{schema}"."{table}"')
|
||||
return int(cur.fetchone()[0])
|
||||
finally:
|
||||
conn.close()
|
||||
if engine == "mysql":
|
||||
schema, table = _parse_fqn(engine, object_name)
|
||||
conn = pymysql.connect(
|
||||
host=DB_HOST, port=MYSQL_PORT, user=MYSQL_USER, password=MYSQL_PASS,
|
||||
database=MYSQL_DB, connect_timeout=8,
|
||||
)
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute(f"SELECT count(*) FROM `{table}`")
|
||||
return int(cur.fetchone()[0])
|
||||
finally:
|
||||
conn.close()
|
||||
if engine == "mongodb":
|
||||
db_name, coll = _parse_fqn(engine, object_name)
|
||||
client = _mongo_client()
|
||||
try:
|
||||
return int(client[db_name][coll].estimated_document_count())
|
||||
finally:
|
||||
client.close()
|
||||
if engine == "neo4j":
|
||||
label = object_name.split(".")[-1]
|
||||
driver = _neo4j_driver()
|
||||
try:
|
||||
with driver.session() as session:
|
||||
return int(session.run(f"MATCH (n:`{label}`) RETURN count(n) AS c").single()["c"])
|
||||
finally:
|
||||
driver.close()
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def _parse_fqn(engine: str, object_name: str) -> tuple[str, str]:
|
||||
if engine == "postgres":
|
||||
if "." in object_name:
|
||||
return object_name.split(".", 1)
|
||||
return "public", object_name
|
||||
if engine == "mysql":
|
||||
if "." in object_name:
|
||||
return object_name.split(".", 1)
|
||||
return MYSQL_DB, object_name
|
||||
if engine == "mongodb":
|
||||
if "." in object_name:
|
||||
return object_name.split(".", 1)
|
||||
return MONGO_DB, object_name
|
||||
if engine == "cassandra":
|
||||
if "." in object_name:
|
||||
return object_name.split(".", 1)
|
||||
return CASS_KS, object_name
|
||||
return "graph", object_name.split(".")[-1]
|
||||
|
||||
|
||||
def _object_primary_keys(engine: str, object_name: str) -> list[str]:
|
||||
try:
|
||||
if engine == "postgres":
|
||||
schema, table = _parse_fqn(engine, object_name)
|
||||
conn = psycopg2.connect(
|
||||
host=DB_HOST, port=PG_PORT, user=PG_USER, password=PG_PASS, dbname=PG_DB, connect_timeout=8,
|
||||
)
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT a.attname
|
||||
FROM pg_constraint c
|
||||
JOIN pg_class t ON t.oid = c.conrelid
|
||||
JOIN pg_namespace n ON n.oid = t.relnamespace
|
||||
JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = ANY(c.conkey)
|
||||
WHERE c.contype = 'p' AND n.nspname = %s AND t.relname = %s
|
||||
ORDER BY array_position(c.conkey, a.attnum)
|
||||
""",
|
||||
(schema, table),
|
||||
)
|
||||
return [r[0] for r in cur.fetchall()]
|
||||
finally:
|
||||
conn.close()
|
||||
if engine == "mysql":
|
||||
schema, table = _parse_fqn(engine, object_name)
|
||||
conn = pymysql.connect(
|
||||
host=DB_HOST, port=MYSQL_PORT, user=MYSQL_USER, password=MYSQL_PASS,
|
||||
database=MYSQL_DB, connect_timeout=8,
|
||||
)
|
||||
try:
|
||||
cur = conn.cursor()
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT COLUMN_NAME FROM information_schema.KEY_COLUMN_USAGE
|
||||
WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s AND CONSTRAINT_NAME = 'PRIMARY'
|
||||
ORDER BY ORDINAL_POSITION
|
||||
""",
|
||||
(schema, table),
|
||||
)
|
||||
return [r[0] for r in cur.fetchall()]
|
||||
finally:
|
||||
conn.close()
|
||||
if engine == "mongodb":
|
||||
return ["_id"]
|
||||
if engine == "cassandra":
|
||||
ks, table = _parse_fqn(engine, object_name)
|
||||
cluster = _cass_cluster()
|
||||
session = cluster.connect()
|
||||
try:
|
||||
rows = session.execute(
|
||||
"SELECT column_name FROM system_schema.columns "
|
||||
"WHERE keyspace_name=%s AND table_name=%s AND kind IN ('partition_key','clustering') "
|
||||
"ORDER BY position",
|
||||
(ks, table),
|
||||
)
|
||||
return [r.column_name for r in rows]
|
||||
finally:
|
||||
cluster.shutdown()
|
||||
if engine == "neo4j":
|
||||
label = object_name.split(".")[-1]
|
||||
if label == "Product":
|
||||
return ["product_id"]
|
||||
if label == "Supplier":
|
||||
return ["supplier_id"]
|
||||
except Exception:
|
||||
pass
|
||||
return []
|
||||
|
||||
|
||||
def _coerce_value(val: Any) -> Any:
|
||||
if val is None or val == "":
|
||||
return None
|
||||
if isinstance(val, (int, float, bool)):
|
||||
return val
|
||||
s = str(val)
|
||||
if s.lower() == "null":
|
||||
return None
|
||||
if re.fullmatch(r"-?\d+", s):
|
||||
try:
|
||||
return int(s)
|
||||
except ValueError:
|
||||
pass
|
||||
if re.fullmatch(r"-?\d+\.\d+", s):
|
||||
try:
|
||||
return float(s)
|
||||
except ValueError:
|
||||
pass
|
||||
return s
|
||||
|
||||
|
||||
def _safe_ident(name: str) -> bool:
|
||||
return bool(re.fullmatch(r"[a-zA-Z_][a-zA-Z0-9_]*", name))
|
||||
|
||||
|
||||
def _update_postgres(object_name: str, pk: dict[str, Any], changes: dict[str, Any]) -> str:
|
||||
schema, table = _parse_fqn("postgres", object_name)
|
||||
if not changes or not pk:
|
||||
raise ValueError("Primary key and changes required")
|
||||
for c in list(changes) + list(pk):
|
||||
if not _safe_ident(c):
|
||||
raise ValueError(f"Invalid column name: {c}")
|
||||
set_sql = ", ".join(f'"{c}"=%s' for c in changes)
|
||||
where_sql = " AND ".join(f'"{c}"=%s' for c in pk)
|
||||
sql = f'UPDATE "{schema}"."{table}" SET {set_sql} WHERE {where_sql}'
|
||||
params = list(changes.values()) + list(pk.values())
|
||||
conn = psycopg2.connect(
|
||||
host=DB_HOST, port=PG_PORT, user=PG_USER, password=PG_PASS, dbname=PG_DB, connect_timeout=8,
|
||||
)
|
||||
try:
|
||||
conn.autocommit = True
|
||||
cur = conn.cursor()
|
||||
cur.execute(sql, params)
|
||||
if cur.rowcount == 0:
|
||||
raise ValueError("No row matched primary key")
|
||||
return sql
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _update_mysql(object_name: str, pk: dict[str, Any], changes: dict[str, Any]) -> str:
|
||||
_schema, table = _parse_fqn("mysql", object_name)
|
||||
if not changes or not pk:
|
||||
raise ValueError("Primary key and changes required")
|
||||
for c in list(changes) + list(pk):
|
||||
if not _safe_ident(c):
|
||||
raise ValueError(f"Invalid column name: {c}")
|
||||
set_sql = ", ".join(f"`{c}`=%s" for c in changes)
|
||||
where_sql = " AND ".join(f"`{c}`=%s" for c in pk)
|
||||
sql = f"UPDATE `{table}` SET {set_sql} WHERE {where_sql}"
|
||||
params = list(changes.values()) + list(pk.values())
|
||||
conn = pymysql.connect(
|
||||
host=DB_HOST, port=MYSQL_PORT, user=MYSQL_USER, password=MYSQL_PASS,
|
||||
database=MYSQL_DB, connect_timeout=8,
|
||||
)
|
||||
try:
|
||||
conn.autocommit = True
|
||||
cur = conn.cursor()
|
||||
cur.execute(sql, params)
|
||||
if cur.rowcount == 0:
|
||||
raise ValueError("No row matched primary key")
|
||||
return sql
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _update_mongodb(object_name: str, pk: dict[str, Any], changes: dict[str, Any]) -> str:
|
||||
db_name, coll = _parse_fqn("mongodb", object_name)
|
||||
if not pk or not changes:
|
||||
raise ValueError("Primary key and changes required")
|
||||
client = _mongo_client()
|
||||
try:
|
||||
from bson import ObjectId
|
||||
filt = dict(pk)
|
||||
if "_id" in filt and isinstance(filt["_id"], str) and len(filt["_id"]) == 24:
|
||||
try:
|
||||
filt["_id"] = ObjectId(filt["_id"])
|
||||
except Exception:
|
||||
pass
|
||||
res = client[db_name][coll].update_one(filt, {"$set": changes})
|
||||
if res.matched_count == 0:
|
||||
raise ValueError("No document matched primary key")
|
||||
return f"UPDATE {db_name}.{coll} {filt}"
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
|
||||
def _update_cassandra(object_name: str, pk: dict[str, Any], changes: dict[str, Any]) -> str:
|
||||
ks, table = _parse_fqn("cassandra", object_name)
|
||||
if not pk or not changes:
|
||||
raise ValueError("Primary key and changes required")
|
||||
set_cql = ", ".join(f"{c}=%s" for c in changes)
|
||||
where_cql = " AND ".join(f"{c}=%s" for c in pk)
|
||||
cql = f"UPDATE {ks}.{table} SET {set_cql} WHERE {where_cql}"
|
||||
cluster = _cass_cluster()
|
||||
session = cluster.connect()
|
||||
try:
|
||||
session.execute(cql, list(changes.values()) + list(pk.values()))
|
||||
return cql
|
||||
finally:
|
||||
cluster.shutdown()
|
||||
|
||||
|
||||
def _update_neo4j(object_name: str, pk: dict[str, Any], changes: dict[str, Any]) -> str:
|
||||
label = object_name.split(".")[-1]
|
||||
if not pk or not changes:
|
||||
raise ValueError("Primary key and changes required")
|
||||
pk_col, pk_val = next(iter(pk.items()))
|
||||
if not _safe_ident(pk_col):
|
||||
raise ValueError(f"Invalid property name: {pk_col}")
|
||||
for c in changes:
|
||||
if not _safe_ident(c):
|
||||
raise ValueError(f"Invalid property name: {c}")
|
||||
set_frag = ", ".join(f"n.{c} = ${c}" for c in changes)
|
||||
cypher = f"MATCH (n:`{label}` {{{pk_col}: $pk}}) SET {set_frag} RETURN n"
|
||||
params: dict[str, Any] = {"pk": _coerce_value(pk_val)}
|
||||
params.update({c: _coerce_value(v) for c, v in changes.items()})
|
||||
driver = _neo4j_driver()
|
||||
try:
|
||||
with driver.session() as session:
|
||||
rec = session.run(cypher, **params).single()
|
||||
if not rec:
|
||||
raise ValueError("No node matched primary key")
|
||||
return cypher
|
||||
finally:
|
||||
driver.close()
|
||||
|
||||
|
||||
def _update_row(engine: str, object_name: str, pk: dict[str, Any], changes: dict[str, Any]) -> dict[str, Any]:
|
||||
pk = {k: _coerce_value(v) for k, v in pk.items()}
|
||||
changes = {k: _coerce_value(v) for k, v in changes.items()}
|
||||
if engine == "postgres":
|
||||
sql = _update_postgres(object_name, pk, changes)
|
||||
elif engine == "mysql":
|
||||
sql = _update_mysql(object_name, pk, changes)
|
||||
elif engine == "mongodb":
|
||||
sql = _update_mongodb(object_name, pk, changes)
|
||||
elif engine == "cassandra":
|
||||
sql = _update_cassandra(object_name, pk, changes)
|
||||
elif engine == "neo4j":
|
||||
sql = _update_neo4j(object_name, pk, changes)
|
||||
else:
|
||||
raise ValueError(f"Update not supported for {engine}")
|
||||
return {"ok": True, "engine": engine, "object": object_name, "statement": sql, "cdc": engine in ("postgres", "mysql", "mongodb")}
|
||||
|
||||
|
||||
@router.get("/samples/{engine}")
|
||||
async def get_samples(engine: str):
|
||||
if engine not in SAMPLES:
|
||||
if engine not in SAMPLES and engine not in LAKE_ENGINES:
|
||||
return JSONResponse({"error": "unknown engine"}, status_code=404)
|
||||
if engine == "hadoop":
|
||||
return {"engine": engine, "samples": SAMPLES[engine], "connection": hadoop_connection_info()}
|
||||
return {"engine": engine, "samples": SAMPLES[engine], "connection": _connection_info(engine)}
|
||||
|
||||
|
||||
@@ -647,9 +1009,20 @@ async def sql_health():
|
||||
return out
|
||||
|
||||
|
||||
|
||||
|
||||
@router.get("/health/{engine}")
|
||||
async def sql_health_engine(engine: str):
|
||||
if engine == "hadoop":
|
||||
return health_hadoop(_run_trino)
|
||||
if engine not in SOURCE_ENGINES:
|
||||
return JSONResponse({"error": "unknown engine"}, status_code=404)
|
||||
all_h = await sql_health()
|
||||
return all_h.get(engine) or {"ok": False, "error": "not found"}
|
||||
|
||||
@router.get("/catalog/{engine}")
|
||||
async def get_catalog(engine: str):
|
||||
if engine not in SOURCE_ENGINES:
|
||||
if engine not in SOURCE_ENGINES and engine not in LAKE_ENGINES:
|
||||
return JSONResponse({"error": "unknown engine"}, status_code=404)
|
||||
try:
|
||||
return _catalog(engine)
|
||||
@@ -658,14 +1031,31 @@ async def get_catalog(engine: str):
|
||||
|
||||
|
||||
@router.get("/sample/{engine}")
|
||||
async def get_sample(engine: str, object: str = Query(..., min_length=1), limit: int = Query(50, ge=1, le=200)):
|
||||
if engine not in SOURCE_ENGINES:
|
||||
async def get_sample(
|
||||
engine: str,
|
||||
object: str = Query(..., min_length=1),
|
||||
limit: int = Query(100, ge=1, le=500),
|
||||
offset: int = Query(0, ge=0),
|
||||
):
|
||||
if engine not in SOURCE_ENGINES and engine not in LAKE_ENGINES:
|
||||
return JSONResponse({"error": "unknown engine"}, status_code=404)
|
||||
try:
|
||||
result = _sample(engine, object, limit)
|
||||
result = _sample(engine, object, limit, offset)
|
||||
if not result.get("ok"):
|
||||
return JSONResponse(result, status_code=422)
|
||||
return {**result, "engine": engine, "object": object}
|
||||
pks = _object_primary_keys(engine, object)
|
||||
total = _table_row_count(engine, object)
|
||||
return {
|
||||
**result,
|
||||
"engine": engine,
|
||||
"object": object,
|
||||
"offset": offset,
|
||||
"limit": limit,
|
||||
"total_count": total,
|
||||
"primary_keys": pks,
|
||||
"cdc": engine in ("postgres", "mysql", "mongodb"),
|
||||
"editable": bool(pks),
|
||||
}
|
||||
except Exception as exc:
|
||||
return JSONResponse({"ok": False, "error": str(exc)[:500]}, status_code=502)
|
||||
|
||||
@@ -698,6 +1088,22 @@ async def execute_sql(body: SqlRequest):
|
||||
return JSONResponse({"ok": False, "error": str(exc)[:500]}, status_code=502)
|
||||
|
||||
|
||||
@router.post("/row/update")
|
||||
async def update_row(body: RowUpdateRequest):
|
||||
allowed_pks = _object_primary_keys(body.engine, body.object)
|
||||
if allowed_pks:
|
||||
for k in body.pk:
|
||||
if k not in allowed_pks:
|
||||
return JSONResponse({"ok": False, "error": f"Invalid primary key column '{k}'"}, status_code=400)
|
||||
for k in body.changes:
|
||||
if k in body.pk:
|
||||
return JSONResponse({"ok": False, "error": "Cannot change primary key columns"}, status_code=400)
|
||||
try:
|
||||
return _update_row(body.engine, body.object, body.pk, body.changes)
|
||||
except Exception as exc:
|
||||
return JSONResponse({"ok": False, "error": str(exc)[:500]}, status_code=422)
|
||||
|
||||
|
||||
@router.post("/benchmark")
|
||||
async def benchmark():
|
||||
"""Parallel analytics: PostgreSQL OLTP vs Trino distributed engine (5M sin() rows)."""
|
||||
|
||||
@@ -0,0 +1,435 @@
|
||||
"""Live Spark + Kafka visibility and manual job control for the Command Center."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from fastapi import APIRouter, Body
|
||||
from fastapi.responses import JSONResponse, Response
|
||||
|
||||
SPARK_UI_URL = os.getenv("SPARK_UI_URL", "http://10.0.21.50:8080").rstrip("/")
|
||||
KAFKA_UI_URL = os.getenv("KAFKA_UI_URL", "http://10.0.21.36:9000").rstrip("/")
|
||||
KAFKA_CONNECT_URL = os.getenv("KAFKA_CONNECT_URL", "http://10.0.21.50:8083").rstrip("/")
|
||||
AIRFLOW_URL = os.getenv("AIRFLOW_URL", "http://10.0.21.55:8080").rstrip("/")
|
||||
AIRFLOW_USER = os.getenv("AIRFLOW_USER", "admin")
|
||||
AIRFLOW_PASSWORD = os.getenv("AIRFLOW_PASSWORD", "")
|
||||
|
||||
from hdfs_kafka import export_hdfs_to_kafka, hdfs_export_snapshot
|
||||
|
||||
router = APIRouter(prefix="/api/pipeline/streaming", tags=["streaming"])
|
||||
|
||||
# Manual Spark / lakehouse jobs (Airflow DAGs on the lab)
|
||||
SPARK_JOBS: dict[str, dict[str, Any]] = {
|
||||
"spark_to_curated": {
|
||||
"label": "Spark → Iceberg curated (mask PII)",
|
||||
"dag_id": "mask_to_curated",
|
||||
"agent": "lakehouse-ops",
|
||||
"description": "Runs the mask_to_curated DAG — Spark/SQL transform into iceberg.curated_masked",
|
||||
"default_conf": {},
|
||||
"editable_fields": ["batch_size", "sources"],
|
||||
},
|
||||
"hadoop_to_trino": {
|
||||
"label": "HDFS → Iceberg (historical)",
|
||||
"dag_id": "hadoop_to_trino",
|
||||
"agent": "hadoop-ranger",
|
||||
"description": "Load historical_sales from HDFS into Iceberg via Trino/Spark pipeline",
|
||||
"default_conf": {"mode": "refresh"},
|
||||
"editable_fields": ["mode"],
|
||||
},
|
||||
"spark_to_s3": {
|
||||
"label": "Spark → S3 curated layer",
|
||||
"dag_id": "mask_to_curated",
|
||||
"agent": "lakehouse-ops",
|
||||
"description": "Spark transform into Iceberg curated tables mirrored to S3",
|
||||
"default_conf": {"target": "s3"},
|
||||
"editable_fields": ["target", "batch_size"],
|
||||
},
|
||||
"generate_all": {
|
||||
"label": "Seed all source DBs",
|
||||
"dag_id": "generate_data_all_databases",
|
||||
"agent": "data-custodian",
|
||||
"description": "Airflow DAG — generates rows into PostgreSQL, MySQL, MongoDB, Cassandra, Neo4j",
|
||||
"default_conf": {"rows": 3000},
|
||||
"editable_fields": ["rows"],
|
||||
},
|
||||
}
|
||||
|
||||
# Master pipeline pulse switch: running | paused | stopped
|
||||
_flow_state: dict[str, Any] = {"mode": "running", "since": time.time()}
|
||||
|
||||
def flow_mode() -> str:
|
||||
return _flow_state.get("mode", "running")
|
||||
|
||||
def flow_snapshot() -> dict[str, Any]:
|
||||
return {"mode": _flow_state.get("mode", "running"), "since": _flow_state.get("since")}
|
||||
|
||||
_cache: dict[str, Any] = {"ts": 0.0, "data": None}
|
||||
_TTL = 8.0
|
||||
_token_cache: dict[str, Any] = {"token": None, "exp": 0.0}
|
||||
|
||||
|
||||
def _feed(agent_id: str, message: str, level: str = "info") -> None:
|
||||
try:
|
||||
from main import add_feed
|
||||
add_feed(agent_id, message, level)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
async def _airflow_token(client: httpx.AsyncClient) -> str:
|
||||
now = time.time()
|
||||
if _token_cache["token"] and _token_cache["exp"] > now + 30:
|
||||
return _token_cache["token"]
|
||||
r = await client.post(
|
||||
f"{AIRFLOW_URL}/auth/token",
|
||||
json={"username": AIRFLOW_USER, "password": AIRFLOW_PASSWORD},
|
||||
timeout=10,
|
||||
)
|
||||
r.raise_for_status()
|
||||
tok = r.json()["access_token"]
|
||||
_token_cache["token"] = tok
|
||||
_token_cache["exp"] = now + 20 * 60
|
||||
return tok
|
||||
|
||||
|
||||
async def collect_spark() -> dict[str, Any]:
|
||||
out: dict[str, Any] = {
|
||||
"ui_url": SPARK_UI_URL,
|
||||
"ui_ok": False,
|
||||
"status": "UNKNOWN",
|
||||
"workers": [],
|
||||
"alive_workers": 0,
|
||||
"cores": 0,
|
||||
"cores_used": 0,
|
||||
"memory_mb": 0,
|
||||
"memory_used_mb": 0,
|
||||
"active_apps": [],
|
||||
"completed_apps": [],
|
||||
}
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=6.0) as client:
|
||||
r = await client.get(f"{SPARK_UI_URL}/json/")
|
||||
if r.status_code >= 400:
|
||||
return out
|
||||
d = r.json()
|
||||
out["ui_ok"] = True
|
||||
out["status"] = d.get("status") or "ALIVE"
|
||||
out["alive_workers"] = int(d.get("aliveworkers") or 0)
|
||||
out["cores"] = int(d.get("cores") or 0)
|
||||
out["cores_used"] = int(d.get("coresused") or 0)
|
||||
out["memory_mb"] = int(d.get("memory") or 0)
|
||||
out["memory_used_mb"] = int(d.get("memoryused") or 0)
|
||||
workers = d.get("workers") or []
|
||||
out["workers"] = [
|
||||
{
|
||||
"id": w.get("id"),
|
||||
"host": w.get("host"),
|
||||
"cores": w.get("cores"),
|
||||
"cores_used": w.get("coresused"),
|
||||
"memory_mb": w.get("memory"),
|
||||
"state": w.get("state"),
|
||||
"webui": w.get("webuiaddress"),
|
||||
}
|
||||
for w in workers
|
||||
]
|
||||
for app in (d.get("activeapps") or [])[:20]:
|
||||
out["active_apps"].append({
|
||||
"id": app.get("id"),
|
||||
"name": app.get("name"),
|
||||
"cores": app.get("cores"),
|
||||
"memory_mb": app.get("memory"),
|
||||
"submitdate": app.get("submitdate"),
|
||||
"duration_ms": app.get("duration"),
|
||||
"user": app.get("user"),
|
||||
})
|
||||
for app in (d.get("completedapps") or [])[:10]:
|
||||
out["completed_apps"].append({
|
||||
"id": app.get("id"),
|
||||
"name": app.get("name"),
|
||||
"duration_ms": app.get("duration"),
|
||||
})
|
||||
except Exception as exc:
|
||||
out["error"] = str(exc)[:200]
|
||||
return out
|
||||
|
||||
|
||||
async def collect_kafka() -> dict[str, Any]:
|
||||
out: dict[str, Any] = {
|
||||
"ui_url": KAFKA_UI_URL,
|
||||
"ui_ok": False,
|
||||
"connect_url": KAFKA_CONNECT_URL,
|
||||
"connect_ok": False,
|
||||
"cluster": {},
|
||||
"topics": [],
|
||||
"connectors": [],
|
||||
}
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=8.0) as client:
|
||||
cr = await client.get(f"{KAFKA_UI_URL}/api/clusters")
|
||||
if cr.status_code < 400:
|
||||
clusters = cr.json()
|
||||
out["ui_ok"] = True
|
||||
if clusters:
|
||||
name = clusters[0].get("name", "local")
|
||||
out["cluster"] = {
|
||||
"name": name,
|
||||
"status": clusters[0].get("status"),
|
||||
"broker_count": clusters[0].get("brokerCount"),
|
||||
"topic_count": clusters[0].get("topicCount"),
|
||||
"online_partitions": clusters[0].get("onlinePartitionCount"),
|
||||
}
|
||||
tr = await client.get(
|
||||
f"{KAFKA_UI_URL}/api/clusters/{name}/topics",
|
||||
params={"page": 1, "perPage": 50, "showInternal": False},
|
||||
)
|
||||
if tr.status_code < 400:
|
||||
topics = tr.json().get("topics") or []
|
||||
out["topics"] = [
|
||||
{
|
||||
"name": t.get("name"),
|
||||
"partitions": t.get("partitionCount"),
|
||||
"replicas": t.get("replicationFactor"),
|
||||
"messages": t.get("messagesCount"),
|
||||
}
|
||||
for t in topics[:50]
|
||||
]
|
||||
lr = await client.get(f"{KAFKA_CONNECT_URL}/connectors")
|
||||
if lr.status_code < 400:
|
||||
out["connect_ok"] = True
|
||||
names_raw = lr.json()
|
||||
names = names_raw if isinstance(names_raw, list) else []
|
||||
for cn in names[:20]:
|
||||
try:
|
||||
sr = await client.get(f"{KAFKA_CONNECT_URL}/connectors/{cn}/status")
|
||||
st = sr.json() if sr.status_code < 400 else {}
|
||||
conn = st.get("connector") or {}
|
||||
tasks = st.get("tasks") or []
|
||||
out["connectors"].append({
|
||||
"name": cn,
|
||||
"state": conn.get("state"),
|
||||
"worker": conn.get("worker_id"),
|
||||
"tasks": [{"id": t.get("id"), "state": t.get("state")} for t in tasks],
|
||||
"type": st.get("type"),
|
||||
})
|
||||
except Exception:
|
||||
out["connectors"].append({"name": cn, "state": "UNKNOWN"})
|
||||
except Exception as exc:
|
||||
out["error"] = str(exc)[:200]
|
||||
return out
|
||||
|
||||
|
||||
async def build_streaming_status() -> dict[str, Any]:
|
||||
spark, kafka = await asyncio.gather(collect_spark(), collect_kafka())
|
||||
cdc_active = False
|
||||
try:
|
||||
from cdc_consumer import snapshot as cdc_snapshot
|
||||
cdc = cdc_snapshot(15)
|
||||
cdc_active = bool(cdc.get("connected")) and int(cdc.get("window_total") or 0) > 0
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
connectors_running = sum(
|
||||
1 for c in kafka.get("connectors", [])
|
||||
if (c.get("state") or "").upper() == "RUNNING"
|
||||
)
|
||||
spark_alive = spark.get("ui_ok") and (spark.get("status") or "").upper() == "ALIVE"
|
||||
apps_running = len(spark.get("active_apps") or [])
|
||||
|
||||
hdfs_recent = hdfs_export_snapshot().get("recent")
|
||||
|
||||
edges = {
|
||||
"hdfs→kafka": bool(hdfs_recent),
|
||||
"kafka→spark": cdc_active and spark_alive or hdfs_recent,
|
||||
"spark→iceberg": apps_running > 0 or spark_alive,
|
||||
"spark→s3": apps_running > 0 or spark_alive,
|
||||
"sources→kafka": cdc_active,
|
||||
"connectors": connectors_running > 0,
|
||||
}
|
||||
|
||||
mode = flow_mode()
|
||||
if mode != "running":
|
||||
edges = {k: False for k in edges}
|
||||
|
||||
return {
|
||||
"ok": True,
|
||||
"flow": mode,
|
||||
"spark": spark,
|
||||
"kafka": kafka,
|
||||
"edges": edges,
|
||||
"jobs": [
|
||||
{**{"id": k}, **{kk: vv for kk, vv in v.items() if kk != "editable_fields"}}
|
||||
for k, v in SPARK_JOBS.items()
|
||||
],
|
||||
"ts": time.time(),
|
||||
}
|
||||
|
||||
|
||||
@router.get("/status")
|
||||
async def streaming_status(refresh: bool = False) -> JSONResponse:
|
||||
now = time.time()
|
||||
if not refresh and _cache["data"] and now - _cache["ts"] < _TTL:
|
||||
return JSONResponse(_cache["data"])
|
||||
data = await build_streaming_status()
|
||||
_cache["data"] = data
|
||||
_cache["ts"] = now
|
||||
return JSONResponse(data)
|
||||
|
||||
|
||||
@router.get("/jobs")
|
||||
async def list_spark_jobs() -> JSONResponse:
|
||||
return JSONResponse({"ok": True, "jobs": SPARK_JOBS})
|
||||
|
||||
|
||||
|
||||
|
||||
@router.post("/flow/{action}")
|
||||
async def set_flow(action: str) -> JSONResponse:
|
||||
mapping = {
|
||||
"pause": "paused",
|
||||
"stop": "stopped",
|
||||
"resume": "running",
|
||||
"start": "running",
|
||||
"run": "running",
|
||||
}
|
||||
if action not in mapping:
|
||||
return JSONResponse({"ok": False, "error": f"unknown action {action}"}, status_code=400)
|
||||
_flow_state["mode"] = mapping[action]
|
||||
_flow_state["since"] = time.time()
|
||||
_cache["ts"] = 0
|
||||
label = {"running": "resumed", "paused": "paused", "stopped": "stopped"}[mapping[action]]
|
||||
_feed("lakehouse-ops", f"[pipeline] data flow {label}", "info" if mapping[action] == "running" else "warn")
|
||||
return JSONResponse({"ok": True, "flow": flow_snapshot()})
|
||||
|
||||
|
||||
@router.get("/flow")
|
||||
async def get_flow() -> JSONResponse:
|
||||
return JSONResponse({"ok": True, "flow": flow_snapshot()})
|
||||
|
||||
@router.post("/jobs/{job_id}/trigger")
|
||||
async def trigger_spark_job(job_id: str, body: dict[str, Any] = Body(default={})) -> JSONResponse:
|
||||
job = SPARK_JOBS.get(job_id)
|
||||
if not job:
|
||||
return JSONResponse({"ok": False, "error": f"unknown job {job_id}"}, status_code=400)
|
||||
conf = {**(job.get("default_conf") or {}), **(body.get("conf") or {})}
|
||||
agent = job.get("agent", "lakehouse-ops")
|
||||
dag_id = job["dag_id"]
|
||||
_feed(agent, f"[spark] Triggering {job['label']} conf={conf}", "info")
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient() as client:
|
||||
tok = await _airflow_token(client)
|
||||
h = {"Authorization": f"Bearer {tok}"}
|
||||
r = await client.post(
|
||||
f"{AIRFLOW_URL}/api/v2/dags/{dag_id}/dagRuns",
|
||||
headers=h,
|
||||
json={"logical_date": None, "conf": conf},
|
||||
timeout=20,
|
||||
)
|
||||
if r.status_code >= 400:
|
||||
_feed(agent, f"[spark] {job['label']}: Airflow {r.status_code}", "err")
|
||||
return JSONResponse({"ok": False, "error": f"airflow {r.status_code}: {r.text[:300]}"}, status_code=502)
|
||||
run = r.json()
|
||||
run_id = run.get("dag_run_id")
|
||||
_feed(agent, f"[spark] {job['label']}: started run {run_id}", "info")
|
||||
return JSONResponse({"ok": True, "job_id": job_id, "dag_id": dag_id, "run_id": run_id, "conf": conf})
|
||||
except Exception as exc:
|
||||
_feed(agent, f"[spark] {job['label']}: {str(exc)[:120]}", "err")
|
||||
return JSONResponse({"ok": False, "error": str(exc)}, status_code=500)
|
||||
|
||||
|
||||
|
||||
|
||||
@router.post("/hdfs/to-kafka")
|
||||
async def hdfs_to_kafka(body: dict[str, Any] = Body(default={})) -> JSONResponse:
|
||||
path = body.get("path", "/data/historical/sales_orders")
|
||||
topic = body.get("topic", "hdfs.historical.sales")
|
||||
limit = int(body.get("limit", 2000))
|
||||
try:
|
||||
result = await export_hdfs_to_kafka(path=path, topic=topic, limit=limit, feed=_feed)
|
||||
if not result.get("ok"):
|
||||
return JSONResponse(result, status_code=502)
|
||||
_cache["ts"] = 0
|
||||
return JSONResponse(result)
|
||||
except Exception as exc:
|
||||
_feed("hadoop-ranger", f"[hdfs→kafka] failed: {str(exc)[:120]}", "err")
|
||||
return JSONResponse({"ok": False, "error": str(exc)}, status_code=500)
|
||||
|
||||
|
||||
@router.post("/pipeline/{pipeline_id}")
|
||||
async def run_streaming_pipeline(pipeline_id: str) -> JSONResponse:
|
||||
if pipeline_id != "hadoop-lake":
|
||||
return JSONResponse({"ok": False, "error": f"unknown pipeline {pipeline_id}"}, status_code=400)
|
||||
steps: list[str] = []
|
||||
try:
|
||||
exp = await export_hdfs_to_kafka(feed=_feed)
|
||||
if not exp.get("ok"):
|
||||
return JSONResponse({"ok": False, "error": exp.get("error"), "steps": steps}, status_code=502)
|
||||
steps.append(f"hdfs→kafka ({exp.get('rows_sent')} rows)")
|
||||
|
||||
async with httpx.AsyncClient() as client:
|
||||
tok = await _airflow_token(client)
|
||||
h = {"Authorization": f"Bearer {tok}"}
|
||||
for job_id, dag_id, agent in [
|
||||
("hadoop_to_trino", "hadoop_to_trino", "hadoop-ranger"),
|
||||
("spark_to_curated", "mask_to_curated", "lakehouse-ops"),
|
||||
]:
|
||||
r = await client.post(
|
||||
f"{AIRFLOW_URL}/api/v2/dags/{dag_id}/dagRuns",
|
||||
headers=h,
|
||||
json={"logical_date": None, "conf": {}},
|
||||
timeout=20,
|
||||
)
|
||||
if r.status_code >= 400:
|
||||
_feed(agent, f"[pipeline] {dag_id} failed {r.status_code}", "err")
|
||||
return JSONResponse({"ok": False, "error": f"{dag_id}: {r.text[:200]}", "steps": steps}, status_code=502)
|
||||
steps.append(dag_id)
|
||||
_feed(agent, f"[pipeline] started {dag_id}", "info")
|
||||
|
||||
_cache["ts"] = 0
|
||||
return JSONResponse({"ok": True, "pipeline": pipeline_id, "steps": steps, "export": exp})
|
||||
except Exception as exc:
|
||||
return JSONResponse({"ok": False, "error": str(exc), "steps": steps}, status_code=500)
|
||||
|
||||
@router.post("/kafka/connectors/{name}/restart")
|
||||
async def restart_kafka_connector(name: str) -> JSONResponse:
|
||||
_feed("etl-guardian", f"[kafka] Restarting connector {name}", "info")
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=15.0) as client:
|
||||
r = await client.post(f"{KAFKA_CONNECT_URL}/connectors/{name}/restart")
|
||||
if r.status_code >= 400:
|
||||
return JSONResponse({"ok": False, "error": r.text[:300]}, status_code=502)
|
||||
_feed("etl-guardian", f"[kafka] Connector {name} restart requested", "info")
|
||||
_cache["ts"] = 0
|
||||
return JSONResponse({"ok": True, "connector": name, "action": "restart"})
|
||||
except Exception as exc:
|
||||
return JSONResponse({"ok": False, "error": str(exc)}, status_code=500)
|
||||
|
||||
|
||||
@router.post("/kafka/connectors/{name}/pause")
|
||||
async def pause_kafka_connector(name: str) -> JSONResponse:
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=10.0) as client:
|
||||
r = await client.put(f"{KAFKA_CONNECT_URL}/connectors/{name}/pause")
|
||||
if r.status_code >= 400:
|
||||
return JSONResponse({"ok": False, "error": r.text[:300]}, status_code=502)
|
||||
_cache["ts"] = 0
|
||||
return JSONResponse({"ok": True, "connector": name, "action": "pause"})
|
||||
except Exception as exc:
|
||||
return JSONResponse({"ok": False, "error": str(exc)}, status_code=500)
|
||||
|
||||
|
||||
@router.post("/kafka/connectors/{name}/resume")
|
||||
async def resume_kafka_connector(name: str) -> JSONResponse:
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=10.0) as client:
|
||||
r = await client.put(f"{KAFKA_CONNECT_URL}/connectors/{name}/resume")
|
||||
if r.status_code >= 400:
|
||||
return JSONResponse({"ok": False, "error": r.text[:300]}, status_code=502)
|
||||
_cache["ts"] = 0
|
||||
return JSONResponse({"ok": True, "connector": name, "action": "resume"})
|
||||
except Exception as exc:
|
||||
return JSONResponse({"ok": False, "error": str(exc)}, status_code=500)
|
||||
@@ -0,0 +1,77 @@
|
||||
"""Shared WebHDFS helpers — resolve datanode redirects via NameNode JMX."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import time
|
||||
from typing import Any
|
||||
from urllib.parse import quote, urlparse, urlunparse
|
||||
|
||||
import httpx
|
||||
|
||||
HDFS_NN_URL = os.getenv("HDFS_NN_URL", "http://10.0.21.61:9870").rstrip("/")
|
||||
HDFS_USER = os.getenv("HDFS_USER", "hdfs")
|
||||
WEBHDFS = f"{HDFS_NN_URL}/webhdfs/v1"
|
||||
|
||||
_dn_cache: dict[str, Any] = {"ts": 0.0, "map": {}}
|
||||
|
||||
|
||||
def encode_path(path: str) -> str:
|
||||
p = path if path.startswith("/") else f"/{path}"
|
||||
return "/".join(quote(seg, safe="") for seg in p.split("/"))
|
||||
|
||||
|
||||
def datanode_host_map() -> dict[str, str]:
|
||||
now = time.time()
|
||||
if _dn_cache["map"] and now - _dn_cache["ts"] < 120:
|
||||
return _dn_cache["map"]
|
||||
mapping: dict[str, str] = {}
|
||||
try:
|
||||
with httpx.Client(timeout=8.0) as client:
|
||||
r = client.get(f"{HDFS_NN_URL}/jmx?qry=Hadoop:service=NameNode,name=NameNodeInfo")
|
||||
raw = (r.json().get("beans") or [{}])[0].get("LiveNodes") or "{}"
|
||||
if isinstance(raw, str):
|
||||
import json
|
||||
nodes = json.loads(raw)
|
||||
else:
|
||||
nodes = raw
|
||||
for key, info in nodes.items():
|
||||
host = key.split(":")[0]
|
||||
info_addr = (info or {}).get("infoAddr") or ""
|
||||
if info_addr:
|
||||
mapping[host] = info_addr # ip:9864
|
||||
except Exception:
|
||||
pass
|
||||
_dn_cache["map"] = mapping
|
||||
_dn_cache["ts"] = now
|
||||
return mapping
|
||||
|
||||
|
||||
def resolve_redirect(url: str) -> str:
|
||||
parsed = urlparse(url)
|
||||
host = parsed.hostname or ""
|
||||
if not host or host.replace(".", "").isdigit():
|
||||
return url
|
||||
dn = datanode_host_map().get(host)
|
||||
if not dn:
|
||||
return url
|
||||
ip, _, port = dn.partition(":")
|
||||
port = parsed.port or port or "9864"
|
||||
return urlunparse(parsed._replace(netloc=f"{ip}:{port}"))
|
||||
|
||||
|
||||
def open_bytes(path: str, max_bytes: int = 8_000_000) -> bytes:
|
||||
url = f"{WEBHDFS}{encode_path(path)}"
|
||||
with httpx.Client(timeout=90.0, follow_redirects=False) as client:
|
||||
r = client.get(url, params={"op": "OPEN", "user.name": HDFS_USER})
|
||||
if r.status_code in (301, 302, 307, 308):
|
||||
loc = r.headers.get("location") or ""
|
||||
if loc:
|
||||
r = client.get(resolve_redirect(loc), follow_redirects=True)
|
||||
elif r.status_code == 200 and r.headers.get("content-type", "").startswith("application/json"):
|
||||
loc = r.json().get("Location") or r.json().get("location") or ""
|
||||
if loc:
|
||||
r = client.get(resolve_redirect(loc), follow_redirects=True)
|
||||
if r.status_code >= 400:
|
||||
raise RuntimeError(r.text[:300])
|
||||
return r.content[:max_bytes]
|
||||
Reference in New Issue
Block a user