From 46b9c50e7306cd7e893543288d51859b660c0c34 Mon Sep 17 00:00:00 2001 From: mo Date: Sat, 27 Jun 2026 19:37:50 +0000 Subject: [PATCH] feat: Spark Workbench everywhere, autonomous Hadoop offload & LLM masking-aware - Data Hub with Hadoop tab (HDFS/Iceberg browser, Spark, pipeline) - Databricks-style Lakehouse Workbench (Trino engine, live exec matrix, materialize to Iceberg/S3); reused & embedded in every source-DB UI - HDFS -> Kafka -> Spark -> Iceberg/S3 pipeline; WebHDFS hostname resolver - Data Flow master pulse switch (Run/Pause/Stop) gating animated edges - Data Custodian autonomous Hadoop offload loop (batch counterpart to CDC), pulsing source -> HDFS edges; toggle in Data Flow - LLM now autonomously aware of all latest platform changes (live platform context) and enforces masking policy: never reveals masked PII, still answers helpfully with aggregates/explanations --- api/Dockerfile | 2 +- api/agent_ops.py | 89 ++- api/dataflow.py | 60 +- api/hadoop_sql.py | 152 +++++ api/hdfs_kafka.py | 108 ++++ api/main.py | 44 +- api/movements.py | 36 ++ api/pipeline_ops.py | 12 +- api/platform_context.py | 144 +++++ api/presentation.py | 197 +++++- api/presentation_upload.py | 89 ++- api/spark_workbench.py | 435 +++++++++++++ api/sql_console.py | 454 ++++++++++++- api/streaming_ops.py | 435 +++++++++++++ api/webhdfs_util.py | 77 +++ ui/index.html | 2 +- ui/nginx.conf | 25 + ui/src/App.tsx | 21 +- .../features/ArchitectureDiagram.tsx | 70 +- .../components/features/DataBrowserGrid.tsx | 376 +++++++++++ ui/src/components/features/DataFlowView.tsx | 69 +- ui/src/components/features/DataGenPanel.tsx | 370 +++++++++++ ui/src/components/features/DataGenView.tsx | 298 +-------- ui/src/components/features/DataHubView.tsx | 60 ++ .../components/features/DataSourcesView.tsx | 357 ++++++----- .../components/features/HadoopSourcesView.tsx | 280 ++++++++ ui/src/components/features/PlatformView.tsx | 66 ++ .../components/features/PresentationView.tsx | 603 +++++++++++++++--- .../components/features/SparkKafkaPanel.tsx | 346 ++++++++++ ui/src/components/features/SparkView.tsx | 592 +++++++++++++++++ ui/src/components/features/SqlWorkbench.tsx | 8 +- ui/src/components/layout/SideNav.tsx | 51 +- ui/src/components/ui/DbBrandIcon.tsx | 56 ++ ui/src/context/ThemeContext.tsx | 4 +- ui/src/hooks/useCommandCenter.ts | 2 +- ui/src/lib/api.ts | 100 +++ ui/src/lib/dataSourceCatalog.ts | 2 +- ui/src/styles/globals.css | 12 +- ui/src/types.ts | 97 +++ 39 files changed, 5476 insertions(+), 725 deletions(-) create mode 100644 api/hadoop_sql.py create mode 100644 api/hdfs_kafka.py create mode 100644 api/platform_context.py create mode 100644 api/spark_workbench.py create mode 100644 api/streaming_ops.py create mode 100644 api/webhdfs_util.py create mode 100644 ui/src/components/features/DataBrowserGrid.tsx create mode 100644 ui/src/components/features/DataGenPanel.tsx create mode 100644 ui/src/components/features/DataHubView.tsx create mode 100644 ui/src/components/features/HadoopSourcesView.tsx create mode 100644 ui/src/components/features/PlatformView.tsx create mode 100644 ui/src/components/features/SparkKafkaPanel.tsx create mode 100644 ui/src/components/features/SparkView.tsx create mode 100644 ui/src/components/ui/DbBrandIcon.tsx diff --git a/api/Dockerfile b/api/Dockerfile index 9e632cf..3ac5772 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -4,7 +4,7 @@ WORKDIR /app RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/* COPY requirements.txt . RUN pip install --no-cache-dir -r requirements.txt -COPY main.py lab_context.py agent_terminal.py workload.py node_registry.py node_ops.py topology_views.py supervisor.py approval_service.py db.py dockhand_envs.py presentation.py database_inventory.py presentation_upload.py presentation_static.py storage_s3.py elasticsearch_api.py sql_console.py hdfs_api.py ssh_terminal.py pipeline_ops.py hadoop_analytics.py agent_ops.py cdc_consumer.py movements.py dataflow.py pii_catalog.py hive_bench_seed.json . +COPY main.py lab_context.py agent_terminal.py workload.py node_registry.py node_ops.py topology_views.py supervisor.py approval_service.py db.py dockhand_envs.py presentation.py database_inventory.py presentation_upload.py presentation_static.py storage_s3.py elasticsearch_api.py sql_console.py hdfs_api.py ssh_terminal.py pipeline_ops.py hadoop_analytics.py agent_ops.py cdc_consumer.py movements.py dataflow.py streaming_ops.py spark_workbench.py hadoop_sql.py hdfs_kafka.py webhdfs_util.py pii_catalog.py platform_context.py hive_bench_seed.json . RUN mkdir -p /data ENV DATABASE_URL=sqlite:////data/atc-agents.db EXPOSE 3201 diff --git a/api/agent_ops.py b/api/agent_ops.py index 3c5e52e..c1c8033 100644 --- a/api/agent_ops.py +++ b/api/agent_ops.py @@ -328,11 +328,98 @@ async def etl_agent_loop() -> None: await asyncio.sleep(max(30.0, float(_etl_state["interval"]))) + +# ── Custodian Hadoop offload (batch counterpart to CDC) ───────────────────── +_CUST_INTERVAL = float(os.getenv("CUSTODIAN_OFFLOAD_INTERVAL_SECONDS", "120")) +_CUST_BATCH = int(os.getenv("CUSTODIAN_OFFLOAD_BATCH", "200")) +_CUST_TARGETS = [ + {"label": "postgres sales_orders", "src": "postgres_sales.public.sales_orders", + "target": "iceberg.hadoop.sales_orders_offload"}, + {"label": "mysql employee_events", "src": "mysql_hr.hr.employee_events", + "target": "iceberg.hadoop.employee_events_offload"}, +] +_custodian_state: dict[str, Any] = { + "enabled": os.getenv("CUSTODIAN_OFFLOAD_ENABLED", "1") not in ("0", "false", "False", ""), + "interval": _CUST_INTERVAL, + "targets": [c["target"] for c in _CUST_TARGETS], + "idx": 0, + "runs_total": 0, + "last": None, + "started": False, +} + + +async def _custodian_offload_once(idx: int | None = None) -> dict[str, Any]: + """Offload a batch of source rows into the Hadoop Iceberg lake via Trino.""" + from spark_workbench import _trino_collect + i = _custodian_state["idx"] if idx is None else idx + tgt = _CUST_TARGETS[i % len(_CUST_TARGETS)] + _custodian_state["idx"] = i + 1 + await _trino_collect( + f"CREATE TABLE IF NOT EXISTS {tgt['target']} AS SELECT * FROM {tgt['src']} WHERE 1=0", 1) + ins = await _trino_collect( + f"INSERT INTO {tgt['target']} SELECT * FROM {tgt['src']} LIMIT {_CUST_BATCH}", 1) + ok = bool(ins.get("ok")) + _custodian_state["runs_total"] += 1 + _custodian_state["last"] = { + "target": tgt["target"], "src": tgt["src"], "ok": ok, + "rows": _CUST_BATCH if ok else 0, + "ts": datetime.now(timezone.utc).isoformat(), "error": ins.get("error"), + } + if ok: + await _emit(f"[custodian-offload] {tgt['label']} → {tgt['target']}: offloaded ~{_CUST_BATCH} rows to Hadoop", "info") + else: + await _emit(f"[custodian-offload] {tgt['label']} failed: {str(ins.get('error'))[:120]}", "err") + return _custodian_state["last"] + + +async def custodian_offload_loop() -> None: + _custodian_state["started"] = True + await asyncio.sleep(45) + await _emit("[custodian-offload] Autonomous Hadoop offload online — batching source data into the Iceberg lake", "info") + while True: + try: + if _custodian_state["enabled"]: + await _custodian_offload_once() + except Exception as exc: + _custodian_state["last"] = {"error": str(exc), "ts": datetime.now(timezone.utc).isoformat()} + await asyncio.sleep(max(30.0, float(_custodian_state["interval"]))) + + +def custodian_recent() -> bool: + last = _custodian_state.get("last") or {} + ts = last.get("ts") + if not ts or not last.get("ok"): + return False + try: + from datetime import datetime as _dt + t = _dt.fromisoformat(str(ts).replace("Z", "+00:00")) + window = max(60.0, float(_custodian_state["interval"]) * 1.5) + return (datetime.now(timezone.utc) - t).total_seconds() < window + except Exception: + return False + + # ── Endpoints ──────────────────────────────────────────────────────────────── @router.get("/status") async def status() -> JSONResponse: pools = {k: len(v) for k, v in _pools.items()} - return JSONResponse({"ok": True, "pools": pools, "etl": _etl_state, **_state}) + return JSONResponse({"ok": True, "pools": pools, "etl": _etl_state, "custodian": _custodian_state, **_state}) + + +@router.post("/custodian/toggle") +async def custodian_toggle(body: dict[str, Any] = Body(default={})) -> JSONResponse: + if "enabled" in body: + _custodian_state["enabled"] = bool(body["enabled"]) + else: + _custodian_state["enabled"] = not _custodian_state["enabled"] + if "interval" in body: + try: + _custodian_state["interval"] = max(30.0, float(body["interval"])) + except (TypeError, ValueError): + pass + await _emit(f"[custodian-offload] Hadoop offload {'ENABLED' if _custodian_state['enabled'] else 'PAUSED'} by operator", "warn") + return JSONResponse({"ok": True, "enabled": _custodian_state["enabled"], "interval": _custodian_state["interval"]}) @router.post("/etl/toggle") diff --git a/api/dataflow.py b/api/dataflow.py index 666b1c6..15778aa 100644 --- a/api/dataflow.py +++ b/api/dataflow.py @@ -38,7 +38,9 @@ NODES: list[dict[str, Any]] = [ {"id": "mysql", "label": "MySQL", "sub": "employee_events", "kind": "source", "x": 28, "y": 38}, {"id": "mongodb", "label": "MongoDB", "sub": "events", "kind": "source", "x": 28, "y": 60}, # col 2 — change data capture - {"id": "kafka", "label": "Kafka · Debezium", "sub": "CDC topics", "kind": "stream", "x": 47, "y": 34}, + {"id": "kafka", "label": "Kafka · Debezium", "sub": "CDC topics", "kind": "stream", "x": 47, "y": 24}, + {"id": "spark", "label": "Apache Spark", "sub": "Streaming · batch", "kind": "compute", "x": 62, "y": 38, + "url": "/spark-ui/"}, # col 3 — storage / lakehouse {"id": "s3_cdc", "label": "S3 CDC Archive", "sub": "object store", "kind": "sink", "x": 67, "y": 13}, {"id": "iceberg_curated", "label": "Iceberg · curated_masked", "sub": "masked PII", "kind": "lakehouse", "x": 67, "y": 45}, @@ -58,7 +60,13 @@ EDGES: list[dict[str, Any]] = [ {"from": "postgres", "to": "kafka", "kind": "cdc"}, {"from": "mysql", "to": "kafka", "kind": "cdc"}, {"from": "mongodb", "to": "kafka", "kind": "cdc"}, + {"from": "kafka", "to": "spark", "kind": "stream"}, + {"from": "spark", "to": "iceberg_curated", "kind": "movement", "movement_id": "spark_to_curated"}, + {"from": "spark", "to": "s3_cdc", "kind": "movement", "movement_id": "spark_to_s3"}, {"from": "kafka", "to": "s3_cdc", "kind": "archive"}, + {"from": "postgres", "to": "hdfs", "kind": "archive", "offload": True}, + {"from": "mysql", "to": "hdfs", "kind": "archive", "offload": True}, + {"from": "hdfs", "to": "kafka", "kind": "stream", "movement_id": "hdfs_to_kafka"}, {"from": "hdfs", "to": "iceberg_hadoop", "kind": "movement", "movement_id": "hadoop_to_trino"}, {"from": "postgres", "to": "iceberg_curated", "kind": "mask", "movement_id": "mask_to_curated"}, {"from": "mysql", "to": "iceberg_curated", "kind": "mask", "movement_id": "mask_to_curated"}, @@ -112,7 +120,7 @@ def _iceberg_hadoop_count() -> int | None: return v -def _build() -> dict[str, Any]: +async def _build() -> dict[str, Any]: # Live signals try: from movements import last_runs @@ -129,6 +137,14 @@ def _build() -> dict[str, Any]: pii = get_pii() except Exception: pii = {"datasets": []} + try: + from streaming_ops import build_streaming_status + streaming = await build_streaming_status() + except Exception: + streaming = {} + spark = streaming.get("spark") or {} + kafka = streaming.get("kafka") or {} + edge_live = streaming.get("edges") or {} pii_by_node = {d["node_id"]: d for d in pii.get("datasets", [])} nodes = [] @@ -138,9 +154,17 @@ def _build() -> dict[str, Any]: metric = None if n["id"] == "openmetadata": metric = f"{pii.get('summary', {}).get('pii_columns', 0)} PII cols cataloged" - if n["id"] == "kafka": - metric = f"{cdc.get('window_total', 0)} chg/15m · {cdc.get('consumed', 0)} total" - node["level"] = "ok" if cdc.get("connected") else "warn" + elif n["id"] == "kafka": + topics = len(kafka.get("topics") or []) + conn_n = len(kafka.get("connectors") or []) + metric = f"{cdc.get('window_total', 0)} chg/15m · {topics} topics · {conn_n} connectors" + node["level"] = "ok" if cdc.get("connected") and kafka.get("ui_ok") else "warn" + elif n["id"] == "spark": + apps = len(spark.get("active_apps") or []) + cores = spark.get("cores") or 0 + used = spark.get("cores_used") or 0 + metric = f"{spark.get('alive_workers', 0)} workers · {used}/{cores} cores · {apps} apps" + node["level"] = "ok" if spark.get("ui_ok") and (spark.get("status") or "").upper() == "ALIVE" else "warn" elif n["id"] in ("postgres", "mysql", "mongodb"): metric = f"{cdc.get('by_source', {}).get(n['id'], 0)} CDC/15m" elif n["id"] == "iceberg_hadoop": @@ -161,6 +185,12 @@ def _build() -> dict[str, Any]: node["metric"] = metric nodes.append(node) + try: + from streaming_ops import flow_mode + _flow = flow_mode() + except Exception: + _flow = "running" + edges = [] for e in EDGES: edge = dict(e) @@ -173,6 +203,22 @@ def _build() -> dict[str, Any]: edge["active"] = lr.get("state") == "running" if e["kind"] == "cdc": edge["active"] = cdc.get("by_source", {}).get(e["from"], 0) > 0 + elif e.get("from") == "hdfs" and e.get("to") == "kafka": + edge["active"] = bool(edge_live.get("hdfs→kafka")) + elif e.get("from") == "kafka" and e.get("to") == "spark": + edge["active"] = bool(edge_live.get("kafka→spark")) + elif e.get("from") == "spark" and e.get("to") == "iceberg_curated": + edge["active"] = bool(edge_live.get("spark→iceberg")) or edge.get("active") + elif e.get("from") == "spark" and e.get("to") == "s3_cdc": + edge["active"] = bool(edge_live.get("spark→s3")) or edge.get("active") + if e.get("offload"): + try: + from agent_ops import custodian_recent + edge["active"] = custodian_recent() + except Exception: + pass + if _flow != "running": + edge["active"] = False edges.append(edge) return { @@ -181,6 +227,8 @@ def _build() -> dict[str, Any]: "edges": edges, "pii_summary": pii.get("summary", {}), "cdc": {"connected": cdc.get("connected"), "consumed": cdc.get("consumed"), "window_total": cdc.get("window_total")}, + "streaming": streaming, + "flow": _flow, "ts": time.time(), } @@ -190,7 +238,7 @@ async def get_dataflow(refresh: bool = False) -> JSONResponse: now = time.time() if not refresh and _cache["data"] and now - _cache["ts"] < _TTL: return JSONResponse(_cache["data"]) - data = _build() + data = await _build() _cache["data"] = data _cache["ts"] = now return JSONResponse(data) diff --git a/api/hadoop_sql.py b/api/hadoop_sql.py new file mode 100644 index 0000000..a2e7687 --- /dev/null +++ b/api/hadoop_sql.py @@ -0,0 +1,152 @@ +"""Hadoop / HDFS / Iceberg catalog and sampling for the Data Hub.""" + +from __future__ import annotations + +import csv +import io +import os +from typing import Any, Callable + +from webhdfs_util import open_bytes + +HDFS_PATHS = [ + "/data/historical/sales_orders", + "/data/historical/sales_orders/year=2020/part-0.csv", +] + +ICEBERG_SCHEMAS = ["hadoop", "curated_masked", "curated"] + +HADOOP_SAMPLES: list[dict[str, str]] = [ + {"id": "hd1", "label": "Iceberg historical sales", "sql": "SELECT * FROM iceberg.hadoop.historical_sales_hdfs LIMIT 20"}, + {"id": "hd2", "label": "Row count historical", "sql": "SELECT count(*) FROM iceberg.hadoop.historical_sales_hdfs"}, + {"id": "hd3", "label": "Curated masked sample", "sql": "SELECT * FROM iceberg.curated_masked.sales_orders_masked LIMIT 15"}, + {"id": "hd4", "label": "Hive lake schemas", "sql": "SHOW SCHEMAS FROM hive"}, + {"id": "hd5", "label": "Iceberg hadoop tables", "sql": "SHOW TABLES FROM iceberg.hadoop"}, +] + + +def catalog_hadoop(run_trino: Callable[[str, int], dict[str, Any]]) -> dict[str, Any]: + objects: list[dict[str, Any]] = [] + + for schema in ICEBERG_SCHEMAS: + show = run_trino(f"SHOW TABLES FROM iceberg.{schema}", 500) + if not show.get("ok"): + continue + for row in show.get("rows") or []: + name = str(row[0]) + fqn = f"iceberg.{schema}.{name}" + cnt = run_trino(f"SELECT count(*) FROM iceberg.{schema}.{name}", 1) + count = None + if cnt.get("ok") and cnt.get("rows"): + try: + count = int(cnt["rows"][0][0]) + except (TypeError, ValueError, IndexError): + pass + objects.append({ + "type": "table", + "schema": f"iceberg.{schema}", + "name": name, + "fqn": fqn, + "row_count": count, + }) + + for path in HDFS_PATHS: + label = path.rstrip("/").rsplit("/", 1)[-1] + objects.append({ + "type": "file", + "schema": "hdfs", + "name": label, + "fqn": f"hdfs:{path}", + "row_count": None, + }) + + return {"engine": "hadoop", "version": "HDFS + Iceberg", "objects": objects} + + +def sample_hdfs_csv(path: str, limit: int, offset: int, tabular: Callable[..., dict[str, Any]]) -> dict[str, Any]: + try: + text = open_bytes(path).decode("utf-8", errors="replace") + reader = csv.reader(io.StringIO(text)) + all_rows = list(reader) + if not all_rows: + return tabular([], [], 0, row_count=0) + columns = [c.strip() for c in all_rows[0]] + data = all_rows[1 + offset: 1 + offset + limit] + rows = [[cell.strip() for cell in row] for row in data] + return tabular(columns, rows, 0, row_count=max(0, len(all_rows) - 1)) + except Exception as exc: + return {"ok": False, "error": str(exc)[:500]} + + +def sample_hadoop( + object_name: str, + limit: int, + offset: int, + run_trino: Callable[[str, int], dict[str, Any]], + tabular: Callable[..., dict[str, Any]], +) -> dict[str, Any]: + if object_name.startswith("hdfs:"): + return sample_hdfs_csv(object_name[5:], limit, offset, tabular) + sql = f"SELECT * FROM {object_name} OFFSET {offset} LIMIT {limit}" + return run_trino(sql, limit) + + +def table_row_count_hadoop(object_name: str, run_trino: Callable[[str, int], dict[str, Any]]) -> int | None: + if object_name.startswith("hdfs:"): + try: + text = open_bytes(object_name[5:]).decode("utf-8", errors="replace") + return max(0, sum(1 for _ in csv.reader(io.StringIO(text))) - 1) + except Exception: + return None + cnt = run_trino(f"SELECT count(*) FROM {object_name}", 1) + if cnt.get("ok") and cnt.get("rows"): + try: + return int(cnt["rows"][0][0]) + except (TypeError, ValueError, IndexError): + pass + return None + + +def health_hadoop(run_trino: Callable[[str, int], dict[str, Any]]) -> dict[str, Any]: + from webhdfs_util import HDFS_NN_URL + import httpx + + nn_ok = False + live_dn = None + err_parts: list[str] = [] + try: + with httpx.Client(timeout=6.0) as client: + r = client.get(f"{HDFS_NN_URL}/jmx?qry=Hadoop:service=NameNode,name=FSNamesystemState") + beans = (r.json().get("beans") or [{}])[0] + nn_ok = r.status_code < 400 + live_dn = beans.get("NumLiveDataNodes") + except Exception as exc: + err_parts.append(f"NN: {exc}") + + trino_ok = False + try: + tr = run_trino("SELECT 1", 1) + trino_ok = bool(tr.get("ok")) + if not trino_ok and tr.get("error"): + err_parts.append(f"Trino: {tr['error'][:80]}") + except Exception as exc: + err_parts.append(f"Trino: {exc}") + + ok = nn_ok and trino_ok + return { + "ok": ok, + "namenode": HDFS_NN_URL, + "live_datanodes": live_dn, + "trino_ok": trino_ok, + "error": "; ".join(err_parts) if err_parts else None, + } + + +def connection_info() -> dict[str, str]: + from webhdfs_util import HDFS_NN_URL, HDFS_USER + return { + "namenode": HDFS_NN_URL, + "user": HDFS_USER, + "trino": os.getenv("TRINO_URL", "http://10.0.21.50:8089"), + "spark": os.getenv("SPARK_UI_URL", "http://10.0.21.50:8080"), + } diff --git a/api/hdfs_kafka.py b/api/hdfs_kafka.py new file mode 100644 index 0000000..c9d7849 --- /dev/null +++ b/api/hdfs_kafka.py @@ -0,0 +1,108 @@ +"""HDFS → Kafka export and full hadoop-lake pipeline orchestration.""" + +from __future__ import annotations + +import csv +import io +import json +import os +import time +from typing import Any + +from aiokafka import AIOKafkaProducer +from webhdfs_util import open_bytes + +KAFKA_BOOTSTRAP = os.getenv("KAFKA_BOOTSTRAP", "10.0.21.36:9092") +DEFAULT_TRINO_TABLE = os.getenv("HADOOP_EXPORT_TABLE", "iceberg.hadoop.historical_sales_hdfs") +DEFAULT_HDFS_FILE = os.getenv("HADOOP_EXPORT_HDFS", "/data/historical/sales_orders/year=2020/part-0.csv") + +_last_hdfs_export: dict[str, Any] = {"ts": 0.0, "rows": 0, "topic": None} + + +def hdfs_export_snapshot() -> dict[str, Any]: + age = time.time() - float(_last_hdfs_export.get("ts") or 0) + return { + **(_last_hdfs_export or {}), + "recent": age < 120, + "age_s": round(age, 1) if _last_hdfs_export.get("ts") else None, + } + + +def _read_hdfs_csv(path: str, limit: int) -> tuple[list[str], list[list[str]]]: + raw = open_bytes(path).decode("utf-8", errors="replace") + rows = list(csv.reader(io.StringIO(raw))) + if not rows: + return [], [] + header = [c.strip() for c in rows[0]] + data = [[cell.strip() for cell in row] for row in rows[1: 1 + limit]] + return header, data + + +def _read_trino_table(table: str, limit: int) -> tuple[list[str], list[list[Any]]]: + from sql_console import _run_trino + + result = _run_trino(f"SELECT * FROM {table} LIMIT {limit}", limit) + if not result.get("ok"): + raise RuntimeError(result.get("error") or "Trino query failed") + return list(result.get("columns") or []), list(result.get("rows") or []) + + +async def export_hdfs_to_kafka( + path: str | None = None, + topic: str = "hdfs.historical.sales", + limit: int = 2000, + source: str = "trino", + table: str | None = None, + feed: Any = None, +) -> dict[str, Any]: + global _last_hdfs_export + header: list[str] = [] + data: list[list[Any]] = [] + src_label = source + + try: + if source == "trino": + tbl = table or DEFAULT_TRINO_TABLE + header, data = _read_trino_table(tbl, limit) + src_label = tbl + else: + p = path or DEFAULT_HDFS_FILE + header, data = _read_hdfs_csv(p, limit) + src_label = p + except Exception as exc: + if source == "trino" and path: + try: + header, data = _read_hdfs_csv(path, limit) + src_label = path + except Exception: + return {"ok": False, "error": str(exc)[:400]} + else: + return {"ok": False, "error": str(exc)[:400]} + + if not data: + return {"ok": False, "error": "No data rows to export"} + + producer = AIOKafkaProducer( + bootstrap_servers=KAFKA_BOOTSTRAP, + value_serializer=lambda v: json.dumps(v, default=str).encode("utf-8"), + ) + await producer.start() + rows_sent = 0 + try: + for i, row in enumerate(data): + payload = { + header[j] if j < len(header) else f"col_{j}": row[j] if j < len(row) else None + for j in range(max(len(header), len(row))) + } + payload["_source"] = "hadoop" + payload["_origin"] = src_label + payload["_row"] = i + 1 + await producer.send_and_wait(topic, payload) + rows_sent += 1 + finally: + await producer.stop() + + _last_hdfs_export = {"ts": time.time(), "rows": rows_sent, "topic": topic, "source": src_label} + if feed: + feed("hadoop-ranger", f"[hadoop→kafka] {rows_sent} rows from {src_label} → {topic}", "info") + return {"ok": True, "rows_sent": rows_sent, "topic": topic, "source": src_label, "columns": header} diff --git a/api/main.py b/api/main.py index e91ecaa..e5e7ebd 100644 --- a/api/main.py +++ b/api/main.py @@ -26,13 +26,16 @@ from agent_terminal import ( from lab_context import collect_full_lab_context, format_context_for_agent from presentation import build_presentation_payload, render_presentation_html from presentation_upload import ( + clear_live_override, create_deck, delete_deck, get_asset_path, get_deck, + get_live_override, list_decks, save_deck, save_image, + save_live_override, save_upload, ) from presentation_static import get_static_deck, list_static_decks @@ -42,11 +45,13 @@ from pipeline_ops import router as pipeline_router from hadoop_analytics import router as hadoop_router from elasticsearch_api import router as elasticsearch_router from sql_console import router as sql_router -from agent_ops import router as agent_ops_router, agent_dml_loop, etl_agent_loop +from agent_ops import router as agent_ops_router, agent_dml_loop, etl_agent_loop, custodian_offload_loop from cdc_consumer import router as cdc_router, cdc_consumer_loop from movements import router as movements_router from movements import MOVEMENT_BY_ID, trigger_and_watch from dataflow import router as dataflow_router +from streaming_ops import router as streaming_router +from spark_workbench import router as spark_workbench_router from pii_catalog import router as pii_router from ssh_terminal import ssh_session from node_registry import NODE_IDS, NODE_AGENT, NODE_REGISTRY, is_node_id @@ -339,6 +344,11 @@ async def gather_agent_context( sup = " [supervisor]" if a.get("supervisor") else "" agent_lines.append(f" - {a['name']} ({a['id']}){sup}: {a['role']}") ctx = ctx + "\n".join(agent_lines) + try: + from platform_context import build_llm_addendum + ctx = ctx + "\n\n" + build_llm_addendum() + except Exception: + pass if log: await log("info", "fetch", f"▸ Context assembled: {len(ctx)} chars for LLM") return ctx @@ -363,6 +373,8 @@ Rules: - Use ONLY the live data below — do not invent hosts, ports, numbers or connector names. - Use exact container/connector names from the data (e.g. mysql-hr-connector, not "Debezium"). - If something is DOWN or 0 GB, say so honestly. +- Respect the data masking policy: NEVER reveal, guess or reconstruct raw values of MASKED columns (they arrive as the token 🔒 MASKED). You MUST still answer helpfully — confirm the column is masked for privacy/governance, explain why, and you may use non-sensitive aggregates/counts over it. +- You are fully aware of all latest platform changes via the section PLATFORM CAPABILITIES & RECENT CHANGES below; use it to answer questions about recent changes, the Spark Workbench, the Hadoop pipeline, the Data Flow pulse switch and the autonomous agents (DML, ETL, Custodian Hadoop offload). - Be concise and helpful (max ~10 sentences); bullet lists are fine when they aid clarity. --- LIVE LAB DATA (primary domain first, then full stack) --- @@ -720,6 +732,7 @@ async def lifespan(app: FastAPI): dml_task = asyncio.create_task(agent_dml_loop()) cdc_task = asyncio.create_task(cdc_consumer_loop()) etl_task = asyncio.create_task(etl_agent_loop()) + cust_task = asyncio.create_task(custodian_offload_loop()) add_feed("infra-sentinel", "ATC Command Center API online", "info") yield task.cancel() @@ -741,6 +754,8 @@ app.include_router(agent_ops_router) app.include_router(cdc_router) app.include_router(movements_router) app.include_router(dataflow_router) +app.include_router(streaming_router) +app.include_router(spark_workbench_router) app.include_router(pii_router) app.add_middleware( CORSMiddleware, @@ -794,7 +809,18 @@ async def get_presentation_data(*, use_cache: bool = True) -> dict[str, Any]: gpu = await collect_gpu() snap = await collect_full_lab_context(gpu_data=gpu, include_inventory=False) data = build_presentation_payload(snap) + override = get_live_override() + if override and override.get("slides"): + data["title"] = override.get("title") or data.get("title") + data["subtitle"] = override.get("subtitle") or data.get("subtitle", "") + data["slides"] = override["slides"] + data["slide_count"] = len(override["slides"]) + data["edited"] = True + data["override_ts"] = override.get("ts") + else: + data["edited"] = False data["source"] = "live" + data["id"] = "live" _presentation_cache["ts"] = now _presentation_cache["data"] = data return data @@ -867,6 +893,7 @@ async def create_presentation_deck(body: dict[str, Any] | None = Body(default=No "bullets": list(s.get("bullets") or []), "image": s.get("image") or "", "kind": s.get("kind") or "narrative", + **({"animation": s["animation"]} if s.get("animation") else {}), } for i, s in enumerate(src["slides"], start=1) ] @@ -878,12 +905,27 @@ async def create_presentation_deck(body: dict[str, Any] | None = Body(default=No @app.put("/api/presentation/decks/{deck_id}") async def update_presentation_deck(deck_id: str, body: dict[str, Any] = Body(...)): + if deck_id == "live": + save_live_override(body) + _presentation_cache["ts"] = 0 + _presentation_cache["data"] = None + payload = await get_presentation_data(use_cache=False) + return {"ok": True, "deck": payload} deck = save_deck(deck_id, body) if not deck: return {"error": "deck not found or not editable"} return {"ok": True, "deck": deck} +@app.post("/api/presentation/live/reset") +async def reset_live_presentation(): + clear_live_override() + _presentation_cache["ts"] = 0 + _presentation_cache["data"] = None + payload = await get_presentation_data(use_cache=False) + return {"ok": True, "deck": payload} + + @app.delete("/api/presentation/decks/{deck_id}") async def remove_presentation_deck(deck_id: str): return {"ok": delete_deck(deck_id)} diff --git a/api/movements.py b/api/movements.py index 93f25b4..15f7034 100644 --- a/api/movements.py +++ b/api/movements.py @@ -39,6 +39,10 @@ MOVEMENTS: list[dict[str, Any]] = [ {"id": "gen_mongodb", "label": "Generate → MongoDB", "kind": "generate", "dag_id": "gen_mongodb", "agent": "data-custodian", "from": "generator", "to": "mongodb", "default_conf": {"rows": 3000}}, + {"id": "hdfs_to_kafka", "label": "HDFS → Kafka export", "kind": "stream", + "dag_id": None, "agent": "hadoop-ranger", "from": "hdfs", "to": "kafka", + "api": "/api/pipeline/streaming/hdfs/to-kafka", + "default_conf": {"source": "trino", "table": "iceberg.hadoop.historical_sales_hdfs", "topic": "hdfs.historical.sales"}}, {"id": "hadoop_to_trino", "label": "HDFS → Iceberg (Trino)", "kind": "movement", "dag_id": "hadoop_to_trino", "agent": "hadoop-ranger", "from": "hdfs", "to": "iceberg_hadoop", "default_conf": {"mode": "refresh"}, @@ -47,6 +51,14 @@ MOVEMENTS: list[dict[str, Any]] = [ "dag_id": "mask_to_curated", "agent": "lakehouse-ops", "from": "sources", "to": "iceberg_curated", "default_conf": {}, "count_sql": "SELECT count(*) FROM iceberg.curated_masked.sales_orders_masked"}, + {"id": "spark_to_s3", "label": "Spark → S3 curated", "kind": "movement", + "dag_id": "mask_to_curated", "agent": "lakehouse-ops", "from": "spark", "to": "s3_cdc", + "default_conf": {"target": "s3"}, + "count_sql": "SELECT count(*) FROM iceberg.curated_masked.sales_orders_masked"}, + {"id": "spark_to_curated", "label": "Spark → Iceberg curated", "kind": "movement", + "dag_id": "mask_to_curated", "agent": "lakehouse-ops", "from": "spark", "to": "iceberg_curated", + "default_conf": {}, + "count_sql": "SELECT count(*) FROM iceberg.curated_masked.sales_orders_masked"}, ] MOVEMENT_BY_ID = {m["id"]: m for m in MOVEMENTS} @@ -116,6 +128,30 @@ async def trigger_and_watch(mid: str, conf: dict[str, Any] | None = None, *, aut mv = MOVEMENT_BY_ID.get(mid) if not mv: return {"ok": False, "error": f"unknown movement {mid}"} + if mv.get("api"): + try: + payload = {**(mv.get("default_conf") or {}), **(conf or {})} + t0 = time.time() + async with httpx.AsyncClient(timeout=120.0) as client: + r = await client.post(f"http://127.0.0.1:8000{mv['api']}", json=payload) + dur = round(time.time() - t0, 1) + body = r.json() if r.headers.get("content-type", "").startswith("application/json") else {} + state = "success" if r.status_code < 400 and body.get("ok", True) else "failed" + rows = body.get("rows_sent") or body.get("rows") + run = { + "movement_id": mid, "state": state, "duration_s": dur, "rows": rows, + "ended_at": datetime.now(timezone.utc).isoformat(), "conf": payload, + } + _last_runs[mid] = run + await _publish({"type": "movement", **run}) + lvl = "info" if state == "success" else "err" + _feed(agent, f"[etl] {mv['label']}: {state} in {dur}s", lvl) + return {"ok": state == "success", **run} + except Exception as exc: + _last_runs[mid] = {**_last_runs.get(mid, {}), "state": "failed", "error": str(exc)} + _feed(agent, f"[etl] {mv['label']}: error {str(exc)[:120]}", "err") + return {"ok": False, "error": str(exc)} + conf = {**(mv.get("default_conf") or {}), **(conf or {})} agent = mv["agent"] count_sql = mv.get("count_sql") diff --git a/api/pipeline_ops.py b/api/pipeline_ops.py index 555d92f..f6ed7f0 100644 --- a/api/pipeline_ops.py +++ b/api/pipeline_ops.py @@ -114,7 +114,7 @@ def _feed(agent_id: str, message: str, level: str = "info") -> None: async def _watch_run(source: str, dag_id: str, run_id: str, agent_id: str, rows: int | None) -> None: """Poll an Airflow run to completion and log the outcome to the feed.""" name = AGENT_NAME.get(agent_id, agent_id) - label = f"{rows} rijen" if rows else "data" + label = f"{rows} rows" if rows else "data" try: async with httpx.AsyncClient() as client: tok = await _airflow_token(client) @@ -130,10 +130,10 @@ async def _watch_run(source: str, dag_id: str, run_id: str, agent_id: str, rows: except Exception: continue if state == "success": - _feed(agent_id, f"[datagen] {name} genereerde {label} in {source} — klaar, data stroomt via CDC naar Kafka/S3", "info") + _feed(agent_id, f"[datagen] {name} generated {label} in {source} — complete, data flowing via CDC to Kafka/S3", "info") return if state == "failed": - _feed(agent_id, f"[datagen] {name}: generatie voor {source} is mislukt (zie Airflow logs)", "err") + _feed(agent_id, f"[datagen] {name}: generation for {source} failed (see Airflow logs)", "err") return except Exception: pass @@ -205,12 +205,12 @@ async def generate(source: str, body: dict[str, Any] = Body(default={})) -> JSON timeout=15, ) if r.status_code >= 400: - _feed(agent_id, f"[datagen] {name}: kon generatie voor {source} niet starten (Airflow {r.status_code})", "err") + _feed(agent_id, f"[datagen] {name}: could not start generation for {source} (Airflow {r.status_code})", "err") return JSONResponse({"ok": False, "error": f"Airflow {r.status_code}: {r.text[:300]}"}, status_code=200) j = r.json() run_id = j.get("dag_run_id") - verb = "genereert zelf" if autonomous else "startte generatie:" - rows_txt = f"{conf['rows']} rijen" if conf.get("rows") else "data" + verb = "generating autonomously" if autonomous else "started generation of" + rows_txt = f"{conf['rows']} rows" if conf.get("rows") else "data" _feed(agent_id, f"[datagen] {name} {verb} {rows_txt} in {source}", "info") if run_id: asyncio.create_task(_watch_run(source, dag_id, run_id, agent_id, conf.get("rows"))) diff --git a/api/platform_context.py b/api/platform_context.py new file mode 100644 index 0000000..a32ad19 --- /dev/null +++ b/api/platform_context.py @@ -0,0 +1,144 @@ +"""Live 'platform capabilities + recent changes + masking guidance' block for the LLM. + +This is rebuilt on every question from live in-process state, so the assistant is +always autonomously aware of the latest things running in the lab (Data Hub, +Spark Workbench, HDFS→Kafka→Spark→S3 pipeline, autonomous agents) and of the +exact masking policy currently in force. +""" + +from __future__ import annotations + +from typing import Any + + +def _fmt_ts(ts: Any) -> str: + try: + return str(ts)[:19] + except Exception: + return "?" + + +def build_platform_section() -> str: + lines: list[str] = ["=== PLATFORM CAPABILITIES & RECENT CHANGES (live) ==="] + + lines += [ + "Command Center features currently deployed:", + " - Data Hub: tabbed UI with 'Source Databases' (PostgreSQL, MySQL, MongoDB, Cassandra, Neo4j) and 'Hadoop' (HDFS files, Hive/Iceberg tables, Spark, pipeline).", + " - Spark Lakehouse Workbench (Databricks-style): pick any federated table, run preview/filter/aggregate/profile/join/SQL on the distributed engine, and materialize results to Iceberg (S3/HDFS-backed). Live execution matrix: splits, rows, bytes, CPU, wall-time, peak memory, nodes.", + " - The same workbench is embedded in each source-database UI (scoped to that source's Trino catalog).", + " - Data Flow: live lineage graph with a master pulse switch (Run / Pause / Stop) that starts/stops the animated flow.", + " - Pipeline: HDFS (Iceberg historical_sales) → Kafka topic hdfs.historical.sales → Spark transform → Iceberg curated → S3.", + "Autonomous agents (run continuously, toggleable):", + " - Data Custodian (DML): generates live INSERT/UPDATE/DELETE on the source DBs so Debezium CDC streams to Kafka.", + " - Data Custodian (Hadoop offload): periodically offloads recent source rows into the Hadoop Iceberg lake (iceberg.hadoop.*_offload), the batch counterpart to CDC.", + " - ETL agent: autonomously triggers data movements (HDFS→Iceberg, mask→curated, generators).", + ] + + # live streaming + flow + try: + from streaming_ops import build_streaming_status, flow_snapshot # type: ignore + flow = flow_snapshot() + lines.append(f"Data Flow pulse: {flow.get('mode')}") + except Exception: + pass + + # autonomous agent state + try: + from agent_ops import _state, _etl_state, _custodian_state # type: ignore + lines.append( + f"DML agent: {'on' if _state.get('enabled') else 'off'} " + f"(ops_total={_state.get('ops_total')}, last={_state.get('last_op')})" + ) + lines.append( + f"ETL agent: {'on' if _etl_state.get('enabled') else 'off'} " + f"(runs={_etl_state.get('runs_total')}, last={_etl_state.get('last')})" + ) + cust = _custodian_state + lines.append( + f"Custodian Hadoop offload: {'on' if cust.get('enabled') else 'off'} " + f"(offloads={cust.get('runs_total')}, last={cust.get('last')})" + ) + except Exception: + pass + + # recent movements + try: + from movements import last_runs # type: ignore + runs = last_runs() + if runs: + lines.append("Recent data-movement runs:") + for mid, r in list(runs.items())[-6:]: + lines.append(f" - {mid}: {r.get('state')} rows={r.get('rows')} {_fmt_ts(r.get('ended_at'))}") + except Exception: + pass + + # recent spark workbench runs + try: + from spark_workbench import _runs as wb_runs, _run_order # type: ignore + recent = [wb_runs[r] for r in _run_order[-6:] if r in wb_runs] + if recent: + lines.append("Recent Spark Workbench runs:") + for r in recent: + st = (r.get("stats") or {}) + tgt = f" → {r.get('target')}" if r.get("target") else "" + lines.append( + f" - {r.get('label')}: {r.get('state')} rows={st.get('processed_rows')}{tgt}" + ) + except Exception: + pass + + return "\n".join(lines) + + +def build_masking_section() -> str: + """Exact masking policy + strict guidance so the LLM can answer about masked + data without ever revealing masked raw values.""" + lines: list[str] = ["=== DATA MASKING POLICY (enforced) ==="] + masked: list[str] = [] + unmasked: list[str] = [] + try: + from pii_catalog import get_pii # type: ignore + data = get_pii() + for d in data.get("datasets", []): + for c in d.get("pii_columns", []): + tag = f"{d.get('label')}.{c.get('name')} [{c.get('category')}]" + (masked if c.get("masked") else unmasked).append(tag) + summ = data.get("summary", {}) + lines.append( + f"PII columns: {summ.get('pii_columns', 0)} total — " + f"{summ.get('masked_columns', 0)} masked, {summ.get('unmasked_columns', 0)} visible." + ) + except Exception as exc: + lines.append(f"(masking catalog unavailable: {exc})") + + if masked: + lines.append("MASKED columns (raw values are withheld — token 🔒 MASKED):") + for m in masked[:40]: + lines.append(f" - {m}") + if unmasked: + lines.append("Visible PII columns (operator opted out of masking):") + for u in unmasked[:40]: + lines.append(f" - {u}") + + lines += [ + "", + "How to handle masked data when answering:", + " 1. NEVER reveal, guess, reconstruct or print the raw value of a MASKED column. If a value comes in as '🔒 MASKED', keep it masked.", + " 2. DO still answer helpfully: confirm the column exists and is masked for privacy/governance, and explain why (PII protection policy).", + " 3. You MAY use and report non-sensitive aggregates, counts, distributions and derived metrics over masked columns (e.g. 'there are N distinct customers') as long as no individual raw value is exposed.", + " 4. Tell the operator they can unmask a specific column from the Data Flow PII overlay if they have the authority, and that the curated/masked Iceberg layer is physically masked and cannot be unmasked.", + " 5. Unmasked PII columns may be shown, but flag that they are sensitive.", + ] + return "\n".join(lines) + + +def build_llm_addendum() -> str: + try: + platform = build_platform_section() + except Exception as exc: + platform = f"(platform section error: {exc})" + try: + masking = build_masking_section() + except Exception as exc: + masking = f"(masking section error: {exc})" + return "\n\n".join([platform, masking]) diff --git a/api/presentation.py b/api/presentation.py index 79a2a44..6b73627 100644 --- a/api/presentation.py +++ b/api/presentation.py @@ -18,6 +18,30 @@ def _slide(slide_id: str, title: str, subtitle: str, bullets: list[str], **extra return {"id": slide_id, "title": title, "subtitle": subtitle, "bullets": bullets, **extra} +def _node_slide(nid: str, extra_bullets: list[str] | None = None) -> dict[str, Any]: + reg = NODE_REGISTRY.get(nid, {}) + bullets: list[str] = [] + if reg.get("description"): + bullets.append(reg["description"]) + bullets.append(f"VM: {reg.get('vm', '?')} (VMID {reg.get('vmid', '?')}) @ {reg.get('ip', '?')}") + for link in reg.get("links", [])[:5]: + bullets.append(f"{link.get('label', 'Link')}: {link.get('url', '')}") + for ep in reg.get("endpoints", [])[:6]: + bullets.append(f"{ep.get('name', '?')}: {ep.get('host', '?')}:{ep.get('port', '?')}") + if extra_bullets: + bullets.extend(extra_bullets) + kind = "gpu" if nid == "gpu" else "command" if nid == "command" else "zone" + if nid in ("openmetadata", "elastic"): + kind = "architecture" + return _slide( + f"node-{nid}", + reg.get("label", nid), + f"{reg.get('role', 'service').upper()} · {reg.get('vm', '')}", + bullets[:14], + kind=kind, + ) + + def build_presentation_payload(snap: dict[str, Any]) -> dict[str, Any]: workload = build_workload_payload(snap) topologies = workload.get("topologies") or build_all_topologies(snap) @@ -28,20 +52,24 @@ def build_presentation_payload(snap: dict[str, Any]) -> dict[str, Any]: hadoop = snap.get("hadoop", {}) objectscale = snap.get("objectscale", {}) command = snap.get("command_center", {}) + databases = snap.get("databases", {}) + docker = snap.get("docker", {}) + lakehouse = snap.get("lakehouse", {}) + governance = snap.get("governance") or {} slides: list[dict[str, Any]] = [] slides.append(_slide( "title", "Dell ATC Data Lab", - "Live demo & presentation — Command Center", + "Live infrastructure presentation — Command Center", [ f"Snapshot: {snap.get('ts', 'now')}", f"Pipeline: {'ACTIVE' if totals.get('pipeline_active') else 'INACTIVE'}", f"Apps running: {totals.get('apps_running', 0)}/{totals.get('apps_total', 0)}", - f"CDC connectors: {totals.get('connectors', 0)}", + f"CDC connectors: {totals.get('connectors', 0)} · Source DBs: 5 engines on db02", f"LLM: {gpu.get('model') or 'offline'} ({gpu.get('gpu_count', 0)}× V100)", - "Command Center → http://10.0.21.33/", + "Command Center → http://10.0.21.33/ · Data Platform tab → Presentation", ], kind="hero", )) @@ -51,53 +79,151 @@ def build_presentation_payload(snap: dict[str, Any]) -> dict[str, Any]: "Mission", "End-to-end modern data platform on Dell infrastructure", [ - "Ingest change data from operational databases (PostgreSQL, MySQL, MongoDB, Cassandra)", - "Stream via Kafka & Debezium into the lakehouse (Spark, Trino, Iceberg)", - "Land curated data on ObjectScale S3 — query with Trino & visualize in Superset", - "Parallel HDFS cluster for batch / legacy workloads", - "GPU lab powers autonomous ops agents with local vLLM inference", - "This dashboard orchestrates agents, approvals, and live cluster visibility", + "Ingest CDC from PostgreSQL, MySQL, MongoDB (+ Cassandra & Neo4j for analytics)", + "Stream via Kafka & Debezium into Spark, Trino & Iceberg on the lakehouse", + "Land curated data on ObjectScale S3 — federated SQL with Trino", + "Govern with OpenMetadata — catalog, lineage, PII classification", + "Search & observe via Elasticsearch/Kibana · BI via Superset", + "Parallel 9-node Hadoop HDFS cluster for batch workloads", + "GPU lab (4× V100) powers autonomous agents with local vLLM inference", + "This Command Center orchestrates agents, approvals & live visibility", ], kind="narrative", )) + slides.append(_slide( + "command-center-ui", + "Command Center UI", + "Everything you operate from this dashboard", + [ + "Data Platform — interactive topology + live presentation deck", + "Data Sources UI — browse, filter & edit all 5 source databases", + "Live Changes — real-time Debezium CDC event stream", + "Data Flow — OpenMetadata catalog, lineage & pipeline map", + "Data Quality — Docling document QA + RAG ingest", + "Knowledge Chat — GPU-backed RAG over lab documentation", + "Object Storage · HDFS · Elasticsearch · SSH terminal", + "Agent fleet · Approval inbox · Live GPU matrix", + ], + kind="command", + )) + arch = topologies.get("architecture") or workload.get("topology") or {} arch_nodes = arch.get("nodes", []) slides.append(_slide( "architecture", "Data Platform Architecture", arch.get("subtitle", "Sources → Ingestion → Compute → Storage → Consumers"), - [f"{n.get('label', n.get('id'))}: {n.get('subtitle', n.get('role', ''))}" for n in arch_nodes[:14]], + [f"{n.get('label', n.get('id'))}: {n.get('subtitle', n.get('role', ''))}" for n in arch_nodes[:16]], kind="topology", topology=arch, )) + db_lines = [ + f"Host atc-db02 (10.0.21.51) — {databases.get('running', 0)}/{databases.get('total', 0)} containers up", + "postgres_sales — PostgreSQL sales_orders (CDC → Debezium)", + "mysql_hr — MySQL employee_events (CDC → Debezium)", + "mongodb_supplychain — MongoDB supplychain.events (CDC → Debezium)", + "cassandra_telemetry — Cassandra device_metrics (Trino federated)", + "neo4j_graph — Product/Supplier graph · 4.5M nodes", + ] + for c in (databases.get("containers") or [])[:8]: + db_lines.append(f" · {c.get('name', '?')}: {c.get('state', '?')}") + slides.append(_slide( + "source-databases", + "Source Databases", + "DB Vault · atc-db02 · 10.0.21.51", + db_lines, + kind="topology", + )) + pipeline = topologies.get("pipeline", {}) connector_lines = [ f" · {cs['name']}: {cs.get('state', '?')}" - for cs in (etl.get("connector_status") or [])[:6] + for cs in (etl.get("connector_status") or [])[:8] ] slides.append(_slide( "pipeline", "CDC Pipeline", pipeline.get("subtitle", "Airflow → DB → Debezium → Kafka → Lakehouse → S3"), [ - f"Airflow: {'healthy' if etl.get('airflow_healthy') else 'degraded'} ({etl.get('airflow_url', '')})", - f"Kafka UI: {'UP' if etl.get('kafka_ui_ok') else 'DOWN'}", + f"Airflow: {'healthy' if etl.get('airflow_healthy') else 'degraded'} — {etl.get('airflow_url', 'http://10.0.21.55:8080')}", + f"Kafka UI: {'UP' if etl.get('kafka_ui_ok') else 'DOWN'} — http://10.0.21.36:9000", + f"Debezium Connect: http://10.0.21.50:8083", f"Connectors: {', '.join(etl.get('connectors') or []) or 'none'}", *connector_lines, - f"Spark UI: {'UP' if etl.get('spark_ui_ok') else 'DOWN'}", - f"ObjectScale: {'reachable' if objectscale.get('reachable') else 'down'} bucket={objectscale.get('bucket', 'data')}", + f"Spark UI: {'UP' if etl.get('spark_ui_ok') else 'DOWN'} — http://10.0.21.50:8080", + f"ObjectScale S3: {'reachable' if objectscale.get('reachable') else 'down'} — bucket={objectscale.get('bucket', 'data')}", ], kind="topology", topology=pipeline, )) + lake_lines = [ + f"atc-lake01 @ {lakehouse.get('host', '10.0.21.50')} — {lakehouse.get('running', 0)}/{lakehouse.get('total', 0)} containers", + f"Trino: {'UP' if lakehouse.get('trino_ok') else 'DOWN'} — http://10.0.21.50:8089", + "Spark — batch & streaming compute", + "Kafka Connect + Debezium — CDC ingestion", + "s3-kafka-consumer — events → ObjectScale S3", + "Iceberg catalog — bronze → silver → gold tables", + ] + for c in (lakehouse.get("containers") or [])[:8]: + lake_lines.append(f" · {c.get('name', '?')}: {c.get('state', '?')}") + slides.append(_slide( + "lakehouse", + "Lakehouse Hub", + "Spark · Trino · Iceberg · Kafka Connect", + lake_lines, + kind="topology", + )) + + docker_lines = [ + f"atc-docker01 @ 10.0.21.45 — {docker.get('running', 0)}/{docker.get('total', 0)} containers", + "Homepage — http://10.0.21.45", + "Dockhand — container management http://10.0.21.45:8082", + "Apache Superset — BI dashboards http://10.0.21.45:8088", + "Forgejo / Gitea · monitoring · nginx · redis", + ] + for c in (docker.get("containers") or [])[:10]: + docker_lines.append(f" · {c.get('name', '?')}: {c.get('state', '?')}") + slides.append(_slide( + "docker-rack", + "Docker Rack & Analytics", + "Platform services on atc-docker01", + docker_lines, + kind="zone", + )) + + cdc = governance.get("cdc") or {} + gov_lines = [ + f"OpenMetadata UI: {governance.get('openmetadata_url', 'http://10.0.21.47:8585')}", + "Ingestion Airflow: http://10.0.21.47:8080", + "Catalog · lineage · data quality · PII auto-classification (Presidio NER)", + ] + if "error" not in cdc: + by_src = ", ".join(f"{k}={v}" for k, v in (cdc.get("by_source") or {}).items()) or "none" + gov_lines.append(f"CDC stream: connected={cdc.get('connected')} · {cdc.get('window_total', 0)} changes/15m ({by_src})") + ps = governance.get("pii_summary") or {} + if "error" not in ps: + gov_lines.append(f"PII: {ps.get('pii_columns', 0)} columns · {ps.get('masked_columns', 0)} masked") + for ln in (governance.get("lineage") or [])[:4]: + gov_lines.append(f"Lineage: {ln}") + slides.append(_slide( + "governance", + "Governance & Metadata", + "OpenMetadata · CDC · PII · Lineage", + gov_lines, + kind="architecture", + )) + + slides.append(_node_slide("openmetadata")) + slides.append(_node_slide("elastic")) + for zone in zones: apps = zone.get("apps") or [] app_lines = [ f"{a['name']}: {a['state']}" + (f" ({a.get('host', '')})" if a.get("host") else "") - for a in apps[:10] + for a in apps[:12] ] slides.append(_slide( f"zone-{zone['id']}", @@ -111,32 +237,36 @@ def build_presentation_payload(snap: dict[str, Any]) -> dict[str, Any]: zone=zone, )) - infra_nodes = [ - nid for nid in NODE_REGISTRY - if nid not in ("mo-commander", "bart-commander", "network-watcher", "mcp-coordinator") + PRESENTATION_NODES = [ + "airflow", "db", "debezium", "kafka", "lakehouse", "s3", + "docker", "hadoop", "gpu", "command", ] slides.append(_slide( - "infrastructure", + "infrastructure-map", "Infrastructure Map", "Proxmox VMs & services across VLAN 20/21", [ f"{NODE_REGISTRY[nid]['label']} — {NODE_REGISTRY[nid].get('vm')} " f"(VMID {NODE_REGISTRY[nid].get('vmid', '?')}) @ {NODE_REGISTRY[nid].get('ip')}" - for nid in infra_nodes + for nid in PRESENTATION_NODES if nid in NODE_REGISTRY ], kind="registry", )) + for nid in PRESENTATION_NODES: + if nid in NODE_REGISTRY and nid not in ("docker", "db", "lakehouse"): + slides.append(_node_slide(nid)) + dn_lines = [ f" · {dn['host']}: {dn.get('used_gb', 0)} GB — {dn.get('state', '')}" - for dn in (hadoop.get("datanodes") or [])[:5] + for dn in (hadoop.get("datanodes") or [])[:6] ] slides.append(_slide( "hadoop", "Hadoop HDFS", "9-node parallel storage cluster", [ - f"NameNode: {'UP' if hadoop.get('reachable') else 'DOWN'} — {hadoop.get('namenode', '')}", + f"NameNode: {'UP' if hadoop.get('reachable') else 'DOWN'} — {hadoop.get('namenode', 'http://10.0.21.61:9870')}", f"Capacity: {hadoop.get('capacity_used_gb', '?')} / {hadoop.get('capacity_total_gb', '?')} GB", f"DataNodes: {hadoop.get('live_datanodes', 0)} live, {hadoop.get('dead_datanodes', 0)} dead", f"Files: {hadoop.get('files_total', 0)}, Blocks: {hadoop.get('blocks_total', 0)}", @@ -158,7 +288,8 @@ def build_presentation_payload(snap: dict[str, Any]) -> dict[str, Any]: [ f"Inference: {'ON' if gpu.get('inference_active') else 'OFF'}", f"API: {snap.get('gpu', {}).get('vllm_url') or 'http://10.0.20.106:8001/v1'}", - "Manager: http://10.0.20.106:9000", + "GPU Lab UI: http://10.0.20.106:9000", + "Kibana/Elastic: http://10.0.21.46:5601", *gpu_lines, ], kind="gpu", @@ -171,7 +302,7 @@ def build_presentation_payload(snap: dict[str, Any]) -> dict[str, Any]: [ "ETL Guardian — Airflow, Kafka, Debezium, connectors", "Data Custodian — PostgreSQL, MySQL, MongoDB, Cassandra, Neo4j", - "Lakehouse Ops — Spark, Trino, Iceberg, ObjectScale S3", + "Lakehouse Ops — Spark, Trino, Iceberg, ObjectScale S3, OpenMetadata", "Hadoop Ranger — HDFS NameNode, DataNodes, block health", "Infra Sentinel — Docker rack, GPU lab, Command Center", "All agents receive LIVE cluster snapshot in every LLM prompt", @@ -182,13 +313,13 @@ def build_presentation_payload(snap: dict[str, Any]) -> dict[str, Any]: cc_apps = [f"{c['name']}: {c['state']}" for c in (command.get("containers") or [])] slides.append(_slide( "command", - "Command Center", - "VM 304 — this presentation runs here", + "Command Center Stack", + "VM 304 — this dashboard runs here", [ f"Host: {command.get('host', '10.0.21.33')} (VMID {command.get('vmid', 304)})", - f"Stack: {command.get('running', 0)}/{command.get('total', 0)} containers", + f"Stack: {command.get('running', 0)}/{command.get('total', 0)} containers (api, ui, caddy, redis, postgres)", *cc_apps, - "WebSocket ops feed · Approval inbox · Agent terminals", + "WebSocket ops feed · Approval inbox · Agent terminals · Data Sources UI", ], kind="command", )) @@ -198,12 +329,12 @@ def build_presentation_payload(snap: dict[str, Any]) -> dict[str, Any]: "Live Demo Tips", "Use this deck during customer presentations", [ - "Press ← → or click dots to navigate slides", - "F = fullscreen presentation mode", + "Press ← → or click dots to navigate slides · F = fullscreen", + "Data Platform tab → Presentation sub-tab (this deck)", + "Data Platform → Topology for interactive pipeline map", "Export HTML opens a standalone deck for projectors / offline", "Ask agents in the Command Bar — they see full cluster context", - "Switch to Data Platform tab for interactive topology", - "GPU Lab chat: http://10.0.20.106:9000/chat", + "Trigger data generation in Data Sources UI → Generate tab", ], kind="cta", )) diff --git a/api/presentation_upload.py b/api/presentation_upload.py index 21fd5a8..cfc8521 100644 --- a/api/presentation_upload.py +++ b/api/presentation_upload.py @@ -13,6 +13,7 @@ from typing import Any import httpx PRESENTATIONS_DIR = Path(os.getenv("PRESENTATIONS_DIR", "/data/presentations")) +LIVE_OVERRIDE_DIR = PRESENTATIONS_DIR / "live-override" DOCLING_URL = os.getenv("DOCLING_URL", "http://docling-serve:5001").rstrip("/") @@ -165,6 +166,65 @@ def get_deck(deck_id: str) -> dict[str, Any] | None: return json.loads(path.read_text()) +def get_live_override() -> dict[str, Any] | None: + path = LIVE_OVERRIDE_DIR / "meta.json" + if not path.exists(): + return None + try: + return json.loads(path.read_text()) + except Exception: + return None + + +def save_live_override(body: dict[str, Any]) -> dict[str, Any]: + """Persist user edits for the Live Cluster deck.""" + LIVE_OVERRIDE_DIR.mkdir(parents=True, exist_ok=True) + (LIVE_OVERRIDE_DIR / "assets").mkdir(parents=True, exist_ok=True) + existing = get_live_override() or {"id": "live", "source": "live-override", "editable": True} + slides = _clean_slides(body.get("slides")) + payload = { + **existing, + "id": "live", + "source": "live-override", + "editable": True, + "title": str(body.get("title") or existing.get("title") or "Live Cluster")[:120], + "subtitle": str(body.get("subtitle") or existing.get("subtitle") or "")[:300], + "ts": datetime.now(timezone.utc).isoformat(), + "slides": slides, + "slide_count": len(slides), + } + (LIVE_OVERRIDE_DIR / "meta.json").write_text(json.dumps(payload, indent=2, default=str)) + return payload + + +def clear_live_override() -> bool: + import shutil + if not LIVE_OVERRIDE_DIR.exists(): + return True + shutil.rmtree(LIVE_OVERRIDE_DIR, ignore_errors=True) + return True + + +def _clean_slides(incoming_slides: Any) -> list[dict[str, Any]]: + clean_slides: list[dict[str, Any]] = [] + for i, s in enumerate(incoming_slides or [], start=1): + if not isinstance(s, dict): + continue + bullets = [str(b).strip()[:400] for b in (s.get("bullets") or []) if str(b).strip()] + slide: dict[str, Any] = { + "id": str(s.get("id") or f"slide-{i}"), + "title": str(s.get("title") or f"Slide {i}")[:200], + "subtitle": str(s.get("subtitle") or "")[:300], + "bullets": bullets, + "image": str(s.get("image") or "")[:300], + "kind": str(s.get("kind") or "narrative")[:40], + } + if s.get("animation"): + slide["animation"] = str(s.get("animation"))[:40] + clean_slides.append(slide) + return clean_slides or [_blank_slide(1)] + + def _blank_slide(idx: int = 1) -> dict[str, Any]: return { "id": f"slide-{idx}", @@ -208,22 +268,7 @@ def save_deck(deck_id: str, body: dict[str, Any]) -> dict[str, Any] | None: return None existing = json.loads(meta_path.read_text()) - incoming_slides = body.get("slides") - clean_slides: list[dict[str, Any]] = [] - for i, s in enumerate(incoming_slides or [], start=1): - if not isinstance(s, dict): - continue - bullets = [str(b).strip()[:400] for b in (s.get("bullets") or []) if str(b).strip()] - clean_slides.append({ - "id": str(s.get("id") or f"slide-{i}"), - "title": str(s.get("title") or f"Slide {i}")[:200], - "subtitle": str(s.get("subtitle") or "")[:300], - "bullets": bullets, - "image": str(s.get("image") or "")[:300], - "kind": str(s.get("kind") or "narrative")[:40], - }) - if not clean_slides: - clean_slides = [_blank_slide(1)] + clean_slides = _clean_slides(body.get("slides")) existing.update({ "title": str(body.get("title") or existing.get("title") or "Untitled deck")[:120], @@ -250,9 +295,12 @@ def delete_deck(deck_id: str) -> bool: def save_image(deck_id: str, filename: str, content: bytes) -> dict[str, Any] | None: """Store an image in the deck's assets folder; return its served URL.""" - deck_dir = PRESENTATIONS_DIR / deck_id - if not deck_dir.exists(): - return None + if deck_id == "live": + deck_dir = LIVE_OVERRIDE_DIR + else: + deck_dir = PRESENTATIONS_DIR / deck_id + if not deck_dir.exists(): + return None assets = deck_dir / "assets" assets.mkdir(parents=True, exist_ok=True) ext = "" @@ -265,7 +313,8 @@ def save_image(deck_id: str, filename: str, content: bytes) -> dict[str, Any] | def get_asset_path(deck_id: str, name: str) -> Path | None: safe = _safe_name(name) - path = PRESENTATIONS_DIR / deck_id / "assets" / safe + base = LIVE_OVERRIDE_DIR if deck_id == "live" else PRESENTATIONS_DIR / deck_id + path = base / "assets" / safe if not path.exists() or not path.is_file(): return None return path diff --git a/api/spark_workbench.py b/api/spark_workbench.py new file mode 100644 index 0000000..821e5a4 --- /dev/null +++ b/api/spark_workbench.py @@ -0,0 +1,435 @@ +"""Databricks-style lakehouse workbench for the Command Center. + +Lets the user interactively select data from any federated source (Iceberg, +Hive/HDFS, Postgres, MySQL, Mongo, Cassandra, Kafka) and run distributed +transformations on the lakehouse compute layer: + + explore -> transform (aggregate / filter / join / profile) -> materialize to Iceberg (S3/HDFS) + +Execution streams live engine metrics (state, splits, rows, bytes, CPU, wall, +peak memory, nodes) so the UI can show a live "what the cluster is doing" matrix +exactly like Databricks' Spark UI. Compute runs on the lakehouse engine (Trino +coordinator on the Spark cluster host) which distributes work across the +workers; Spark batch DAGs remain available via the jobs API. +""" + +from __future__ import annotations + +import asyncio +import os +import re +import time +import uuid +from typing import Any + +import httpx +from fastapi import APIRouter, Body, Query +from fastapi.responses import JSONResponse + +TRINO_URL = os.getenv("TRINO_URL", "http://10.0.21.50:8089").rstrip("/") +TRINO_USER = os.getenv("TRINO_USER", "mo") +SPARK_UI_URL = os.getenv("SPARK_UI_URL", "http://10.0.21.50:8080").rstrip("/") + +router = APIRouter(prefix="/api/spark", tags=["spark-workbench"]) + +_runs: dict[str, dict[str, Any]] = {} +_run_order: list[str] = [] +_MAX_RUNS = 40 +_MAX_RESULT_ROWS = 500 + +_IDENT_RE = re.compile(r"^[A-Za-z0-9_.\" ]+$") +_TABLE_RE = re.compile(r"^[A-Za-z0-9_.\"]+$") + +AGG_FUNCS = {"count", "sum", "avg", "min", "max", "approx_distinct", "count_distinct"} +JOIN_TYPES = {"INNER", "LEFT", "RIGHT", "FULL"} + + +def _feed(agent_id: str, message: str, level: str = "info") -> None: + try: + from main import add_feed + add_feed(agent_id, message, level) + except Exception: + pass + + +def _qident(name: str) -> str: + name = name.strip().strip('"') + if not re.match(r"^[A-Za-z0-9_]+$", name): + raise ValueError(f"invalid identifier: {name}") + return f'"{name}"' + + +def _safe_table(name: str) -> str: + name = (name or "").strip() + if not name or not _TABLE_RE.match(name): + raise ValueError(f"invalid table reference: {name}") + return name + + +# ── Trino helpers ──────────────────────────────────────────────────────────── +def _trino_headers() -> dict[str, str]: + return {"X-Trino-User": TRINO_USER, "Content-Type": "text/plain"} + + +async def _trino_collect(sql: str, limit: int = 200, timeout: float = 60.0) -> dict[str, Any]: + """Synchronous-style helper: run a query and return all rows (for catalog ops).""" + columns: list[str] = [] + rows: list[list[Any]] = [] + async with httpx.AsyncClient(timeout=timeout) as client: + r = await client.post(f"{TRINO_URL}/v1/statement", content=sql.encode(), headers=_trino_headers()) + if r.status_code >= 400: + return {"ok": False, "error": r.text[:400]} + data = r.json() + while True: + if data.get("error"): + return {"ok": False, "error": str(data["error"])[:400]} + if data.get("columns") and not columns: + columns = [c["name"] for c in data["columns"]] + for row in data.get("data") or []: + rows.append(row) + if len(rows) >= limit: + break + nxt = data.get("nextUri") + if not nxt or len(rows) >= limit: + if nxt: + try: + await client.delete(nxt, headers=_trino_headers()) + except Exception: + pass + break + data = (await client.get(nxt, headers=_trino_headers())).json() + return {"ok": True, "columns": columns, "rows": rows} + + +def _norm_stats(stats: dict[str, Any]) -> dict[str, Any]: + return { + "state": stats.get("state"), + "nodes": stats.get("nodes"), + "total_splits": stats.get("totalSplits"), + "queued_splits": stats.get("queuedSplits"), + "running_splits": stats.get("runningSplits"), + "completed_splits": stats.get("completedSplits"), + "processed_rows": stats.get("processedRows"), + "processed_bytes": stats.get("processedBytes"), + "physical_input_bytes": stats.get("physicalInputBytes"), + "peak_memory_bytes": stats.get("peakMemoryBytes"), + "cpu_time_ms": stats.get("cpuTimeMillis"), + "wall_time_ms": stats.get("wallTimeMillis"), + "elapsed_ms": stats.get("elapsedTimeMillis"), + "progress_pct": round( + (stats.get("completedSplits") or 0) / stats["totalSplits"] * 100, 1 + ) if stats.get("totalSplits") else (100.0 if stats.get("state") == "FINISHED" else 0.0), + } + + +async def _execute_run(run_id: str, sql: str, returns_rows: bool, pre_sql: str | None = None) -> None: + run = _runs[run_id] + run["state"] = "RUNNING" + columns: list[str] = [] + rows: list[list[Any]] = [] + try: + if pre_sql: + pre = await _trino_collect(pre_sql, 1) + if not pre.get("ok"): + run.update(state="FAILED", error=pre.get("error"), ended_at=time.time()) + _feed("lakehouse-ops", f"[workbench] {run['label']}: pre-step failed", "err") + return + async with httpx.AsyncClient(timeout=None) as client: + r = await client.post(f"{TRINO_URL}/v1/statement", content=sql.encode(), headers=_trino_headers()) + if r.status_code >= 400: + run.update(state="FAILED", error=r.text[:500], ended_at=time.time()) + _feed("lakehouse-ops", f"[workbench] {run['label']}: failed ({r.status_code})", "err") + return + data = r.json() + run["query_id"] = data.get("id") + while True: + if run.get("cancel_requested"): + nxt = data.get("nextUri") + if nxt: + try: + await client.delete(nxt, headers=_trino_headers()) + except Exception: + pass + run.update(state="CANCELED", ended_at=time.time()) + _feed("lakehouse-ops", f"[workbench] {run['label']}: canceled", "warn") + return + if data.get("stats"): + run["stats"] = _norm_stats(data["stats"]) + st = data["stats"].get("state") + if st: + run["engine_state"] = st + if data.get("error"): + run.update(state="FAILED", error=str(data["error"])[:500], ended_at=time.time()) + _feed("lakehouse-ops", f"[workbench] {run['label']}: {str(data['error'])[:120]}", "err") + return + if data.get("columns") and not columns: + columns = [c["name"] for c in data["columns"]] + run["columns"] = columns + for row in data.get("data") or []: + if returns_rows and len(rows) < _MAX_RESULT_ROWS: + rows.append(row) + if data.get("updateType"): + run["update_type"] = data.get("updateType") + nxt = data.get("nextUri") + if not nxt: + break + data = (await client.get(nxt, headers=_trino_headers())).json() + run["next_uri"] = nxt + run["rows"] = rows + run["row_count"] = len(rows) + run.update(state="FINISHED", ended_at=time.time()) + if run.get("stats"): + run["stats"]["state"] = "FINISHED" + run["stats"]["progress_pct"] = 100.0 + msg = f"[workbench] {run['label']}: finished" + if run.get("target"): + msg = f"[workbench] {run['label']}: materialized → {run['target']}" + _feed("lakehouse-ops", msg, "info") + except Exception as exc: + run.update(state="FAILED", error=str(exc)[:500], ended_at=time.time()) + _feed("lakehouse-ops", f"[workbench] {run['label']}: error {str(exc)[:120]}", "err") + + +# ── SQL builders ───────────────────────────────────────────────────────────── +def _build_sql(body: dict[str, Any]) -> tuple[str, bool, str, str | None, str | None]: + """Return (sql, returns_rows, label, materialize_target, pre_sql).""" + op = body.get("operation", "preview") + limit = max(1, min(int(body.get("limit", 200)), _MAX_RESULT_ROWS)) + materialize = body.get("materialize") or {} + target = None + + if op == "sql": + sql = (body.get("sql") or "").strip().rstrip(";") + if not sql: + raise ValueError("empty SQL") + select_sql = sql + label = "Custom SQL" + returns_rows = sql.lower().lstrip().startswith(("select", "show", "describe", "with", "explain")) + + elif op == "preview": + table = _safe_table(body.get("table")) + select_sql = f"SELECT * FROM {table} LIMIT {limit}" + label = f"Preview {table}" + returns_rows = True + + elif op == "filter": + table = _safe_table(body.get("table")) + where = (body.get("where") or "").strip() + clause = f" WHERE {where}" if where else "" + select_sql = f"SELECT * FROM {table}{clause} LIMIT {limit}" + label = f"Filter {table}" + returns_rows = True + + elif op == "aggregate": + table = _safe_table(body.get("table")) + group_by = [c for c in (body.get("group_by") or []) if c] + metrics = body.get("metrics") or [] + select_parts: list[str] = [_qident(c) for c in group_by] + for m in metrics: + fn = (m.get("fn") or "count").lower() + if fn not in AGG_FUNCS: + raise ValueError(f"unsupported function {fn}") + col = m.get("col") + alias = m.get("alias") or (f"{fn}_{col}" if col else fn) + if fn == "count" and (not col or col == "*"): + expr = "count(*)" + elif fn == "count_distinct": + expr = f"count(DISTINCT {_qident(col)})" + else: + expr = f"{fn}({_qident(col)})" + select_parts.append(f"{expr} AS {_qident(alias)}") + if not select_parts: + select_parts = ["count(*) AS cnt"] + gb = f" GROUP BY {', '.join(_qident(c) for c in group_by)}" if group_by else "" + order = "" + if group_by: + order = f" ORDER BY {', '.join(_qident(c) for c in group_by)}" + select_sql = f"SELECT {', '.join(select_parts)} FROM {table}{gb}{order} LIMIT {limit}" + label = f"Aggregate {table}" + returns_rows = True + + elif op == "join": + left = _safe_table(body.get("left")) + right = _safe_table(body.get("right")) + jt = (body.get("join_type") or "INNER").upper() + if jt not in JOIN_TYPES: + raise ValueError(f"invalid join type {jt}") + lk = _qident(body.get("left_key")) + rk = _qident(body.get("right_key")) + select_sql = ( + f"SELECT l.*, r.* FROM {left} l {jt} JOIN {right} r " + f"ON l.{lk} = r.{rk} LIMIT {limit}" + ) + label = f"Join {left} ⋈ {right}" + returns_rows = True + + elif op == "profile": + table = _safe_table(body.get("table")) + cols = [c for c in (body.get("columns") or []) if c][:12] + parts = ["count(*) AS row_count"] + for c in cols: + qc = _qident(c) + parts.append(f"approx_distinct({qc}) AS {_qident(c + '_distinct')}") + parts.append(f"count({qc}) AS {_qident(c + '_nonnull')}") + select_sql = f"SELECT {', '.join(parts)} FROM {table}" + label = f"Profile {table}" + returns_rows = True + + else: + raise ValueError(f"unknown operation {op}") + + if materialize.get("enabled"): + schema = materialize.get("schema", "hadoop") + name = materialize.get("table") + if not name: + raise ValueError("materialize target table required") + target = f"iceberg.{_qident(schema).strip(chr(34))}.{_qident(name).strip(chr(34))}" + mode = (materialize.get("mode") or "create").lower() + pre = None + if mode == "replace": + pre = f"DROP TABLE IF EXISTS {target}" + ddl = f"CREATE TABLE {target} AS {select_sql}" + elif mode == "insert": + ddl = f"INSERT INTO {target} {select_sql}" + else: + ddl = f"CREATE TABLE {target} AS {select_sql}" + return ddl, False, f"Materialize → {target}", target, pre + + return select_sql, returns_rows, label, None, None + + +# ── catalog endpoints ──────────────────────────────────────────────────────── +@router.get("/catalogs") +async def list_catalogs() -> JSONResponse: + res = await _trino_collect("SHOW CATALOGS", 100) + if not res.get("ok"): + return JSONResponse(res, status_code=502) + cats = [r[0] for r in res["rows"] if r[0] not in ("system",)] + return JSONResponse({"ok": True, "catalogs": cats}) + + +@router.get("/schemas") +async def list_schemas(catalog: str = Query(...)) -> JSONResponse: + cat = _safe_table(catalog) + res = await _trino_collect(f"SHOW SCHEMAS FROM {cat}", 200) + if not res.get("ok"): + return JSONResponse(res, status_code=502) + skip = {"information_schema"} + schemas = [r[0] for r in res["rows"] if r[0] not in skip] + return JSONResponse({"ok": True, "catalog": catalog, "schemas": schemas}) + + +@router.get("/tables") +async def list_tables(catalog: str = Query(...), schema: str = Query(...)) -> JSONResponse: + cat = _safe_table(catalog) + sch = _safe_table(schema) + res = await _trino_collect(f"SHOW TABLES FROM {cat}.{sch}", 500) + if not res.get("ok"): + return JSONResponse(res, status_code=502) + tables = [{"name": r[0], "fqn": f"{catalog}.{schema}.{r[0]}"} for r in res["rows"]] + return JSONResponse({"ok": True, "catalog": catalog, "schema": schema, "tables": tables}) + + +@router.get("/columns") +async def list_columns(table: str = Query(...)) -> JSONResponse: + tbl = _safe_table(table) + res = await _trino_collect(f"DESCRIBE {tbl}", 500) + if not res.get("ok"): + return JSONResponse(res, status_code=502) + cols = [{"name": r[0], "type": r[1] if len(r) > 1 else ""} for r in res["rows"]] + return JSONResponse({"ok": True, "table": table, "columns": cols}) + + +# ── run endpoints ──────────────────────────────────────────────────────────── +@router.post("/run") +async def create_run(body: dict[str, Any] = Body(...)) -> JSONResponse: + try: + sql, returns_rows, label, target, pre_sql = _build_sql(body) + except ValueError as exc: + return JSONResponse({"ok": False, "error": str(exc)}, status_code=400) + + run_id = uuid.uuid4().hex[:12] + _runs[run_id] = { + "id": run_id, + "operation": body.get("operation", "preview"), + "label": label, + "sql": sql, + "target": target, + "state": "QUEUED", + "engine_state": "QUEUED", + "stats": {}, + "columns": [], + "rows": [], + "row_count": None, + "error": None, + "started_at": time.time(), + "ended_at": None, + "cancel_requested": False, + } + _run_order.append(run_id) + while len(_run_order) > _MAX_RUNS: + old = _run_order.pop(0) + _runs.pop(old, None) + + _feed("lakehouse-ops", f"[workbench] {label}: submitted", "info") + asyncio.create_task(_execute_run(run_id, sql, returns_rows, pre_sql)) + return JSONResponse({"ok": True, "run_id": run_id, "sql": sql, "label": label, "target": target}) + + +def _run_public(run: dict[str, Any], include_rows: bool = True) -> dict[str, Any]: + out = {k: v for k, v in run.items() if k not in ("next_uri", "cancel_requested")} + if not include_rows: + out.pop("rows", None) + return out + + +@router.get("/run/{run_id}") +async def get_run(run_id: str) -> JSONResponse: + run = _runs.get(run_id) + if not run: + return JSONResponse({"ok": False, "error": "unknown run"}, status_code=404) + return JSONResponse({"ok": True, "run": _run_public(run)}) + + +@router.post("/run/{run_id}/cancel") +async def cancel_run(run_id: str) -> JSONResponse: + run = _runs.get(run_id) + if not run: + return JSONResponse({"ok": False, "error": "unknown run"}, status_code=404) + run["cancel_requested"] = True + return JSONResponse({"ok": True, "run_id": run_id, "state": "canceling"}) + + +@router.get("/runs") +async def list_runs() -> JSONResponse: + out = [_run_public(_runs[r], include_rows=False) for r in reversed(_run_order) if r in _runs] + return JSONResponse({"ok": True, "runs": out}) + + +@router.get("/live") +async def spark_live() -> JSONResponse: + """Live cluster + active-run matrix for the workbench dashboard.""" + spark: dict[str, Any] = {} + try: + from streaming_ops import collect_spark + spark = await collect_spark() + except Exception as exc: + spark = {"error": str(exc)[:200]} + + active = [ + _run_public(_runs[r], include_rows=False) + for r in reversed(_run_order) + if r in _runs and _runs[r]["state"] in ("QUEUED", "RUNNING") + ] + recent = [ + _run_public(_runs[r], include_rows=False) + for r in reversed(_run_order[-8:]) + if r in _runs + ] + return JSONResponse({ + "ok": True, + "spark": spark, + "active_runs": active, + "recent_runs": recent, + "ts": time.time(), + }) diff --git a/api/sql_console.py b/api/sql_console.py index aecacf3..19d0899 100644 --- a/api/sql_console.py +++ b/api/sql_console.py @@ -48,6 +48,17 @@ TRINO_USER = os.getenv("TRINO_USER", "atc") SOURCE_ENGINES = ("postgres", "mysql", "mongodb", "cassandra", "neo4j") ENGINES = SOURCE_ENGINES + ("trino",) +from hadoop_sql import ( + HADOOP_SAMPLES, + catalog_hadoop, + connection_info as hadoop_connection_info, + health_hadoop, + sample_hadoop, + table_row_count_hadoop, +) + +LAKE_ENGINES = ("hadoop",) + router = APIRouter(prefix="/api/sql", tags=["sql"]) SAMPLES: dict[str, list[dict[str, str]]] = { @@ -111,6 +122,7 @@ SAMPLES: dict[str, list[dict[str, str]]] = { {"id": "nj9", "label": "Schema visualization", "sql": "CALL db.schema.visualization()"}, {"id": "nj10", "label": "Constraint info", "sql": "SHOW CONSTRAINTS"}, ], + "hadoop": HADOOP_SAMPLES, "trino": [ {"id": "tq1", "label": "Trino version", "sql": "SELECT version()"}, {"id": "tq2", "label": "Catalogs", "sql": "SHOW CATALOGS"}, @@ -332,10 +344,17 @@ def _run_trino(sql: str, limit: int = 200) -> dict[str, Any]: class SqlRequest(BaseModel): - engine: str = Field(..., pattern="^(postgres|mysql|mongodb|trino|cassandra|neo4j)$") + engine: str = Field(..., pattern="^(postgres|mysql|mongodb|trino|cassandra|neo4j|hadoop)$") sql: str = Field(..., min_length=1, max_length=8000) +class RowUpdateRequest(BaseModel): + engine: str = Field(..., pattern="^(postgres|mysql|mongodb|cassandra|neo4j)$") + object: str = Field(..., min_length=1, max_length=256) + pk: dict[str, Any] = Field(..., min_length=1) + changes: dict[str, Any] = Field(..., min_length=1) + + def _connection_info(engine: str) -> dict[str, str]: if engine == "postgres": return {"host": DB_HOST, "port": str(PG_PORT), "database": PG_DB, "user": PG_USER} @@ -361,6 +380,8 @@ def _dispatch(engine: str, sql: str, limit: int = 200) -> dict[str, Any]: return _run_cassandra(sql, limit) if engine == "neo4j": return _run_neo4j(sql, limit) + if engine == "hadoop": + return _run_trino(sql, limit) return _run_trino(sql, limit) @@ -458,6 +479,15 @@ def _catalog_neo4j() -> dict[str, Any]: driver.close() + +def _catalog_hadoop() -> dict[str, Any]: + return catalog_hadoop(_run_trino) + + +def _sample_hadoop(object_name: str, limit: int, offset: int = 0) -> dict[str, Any]: + return sample_hadoop(object_name, limit, offset, _run_trino, _tabular) + + def _catalog(engine: str) -> dict[str, Any]: if engine == "postgres": return _catalog_postgres() @@ -469,37 +499,74 @@ def _catalog(engine: str) -> dict[str, Any]: return _catalog_cassandra() if engine == "neo4j": return _catalog_neo4j() + if engine == "hadoop": + return _catalog_hadoop() raise ValueError(f"Catalog not supported for {engine}") -def _sample_postgres(object_name: str, limit: int) -> dict[str, Any]: +def _sample_postgres(object_name: str, limit: int, offset: int = 0) -> dict[str, Any]: if "." in object_name: schema, table = object_name.split(".", 1) - sql = f'SELECT * FROM "{schema}"."{table}" LIMIT {limit}' + sql = f'SELECT * FROM "{schema}"."{table}" OFFSET {int(offset)} LIMIT {int(limit)}' else: - sql = f'SELECT * FROM public."{object_name}" LIMIT {limit}' + sql = f'SELECT * FROM public."{object_name}" OFFSET {int(offset)} LIMIT {int(limit)}' return _run_postgres(sql, limit) -def _sample_mysql(object_name: str, limit: int) -> dict[str, Any]: +def _sample_mysql(object_name: str, limit: int, offset: int = 0) -> dict[str, Any]: table = object_name.split(".")[-1] - return _run_mysql(f"SELECT * FROM `{table}` LIMIT {limit}", limit) + return _run_mysql(f"SELECT * FROM `{table}` LIMIT {int(limit)} OFFSET {int(offset)}", limit) -def _sample_mongodb(object_name: str, limit: int) -> dict[str, Any]: +def _sample_mongodb(object_name: str, limit: int, offset: int = 0) -> dict[str, Any]: if "." in object_name: db_name, coll = object_name.split(".", 1) else: db_name, coll = MONGO_DB, object_name - return _run_mongo(f"FIND {db_name}.{coll} LIMIT {limit}", limit) + t0 = time.perf_counter() + client = _mongo_client() + try: + docs = list(client[db_name][coll].find({}).skip(int(offset)).limit(int(limit))) + elapsed_ms = int((time.perf_counter() - t0) * 1000) + if not docs: + return _tabular(["result"], [["(empty)"]], elapsed_ms) + columns = sorted({k for d in docs for k in d}) + rows = [[_fmt(d.get(c)) for c in columns] for d in docs] + return _tabular(columns, rows, elapsed_ms, object=object_name) + finally: + client.close() -def _sample_cassandra(object_name: str, limit: int) -> dict[str, Any]: +def _sample_cassandra(object_name: str, limit: int, offset: int = 0) -> dict[str, Any]: if "." in object_name: ks, table = object_name.split(".", 1) else: ks, table = CASS_KS, object_name - return _run_cassandra(f"SELECT * FROM {ks}.{table} LIMIT {limit}", limit) + t0 = time.perf_counter() + cluster = _cass_cluster() + session = cluster.connect() + try: + stmt = f"SELECT * FROM {ks}.{table}" + result = session.execute(stmt, timeout=60) + skipped = 0 + picked: list[Any] = [] + columns: list[str] = [] + for row in result: + if skipped < offset: + skipped += 1 + continue + if not columns: + columns = list(row._fields) + picked.append(row) + if len(picked) >= limit: + break + elapsed_ms = int((time.perf_counter() - t0) * 1000) + if not picked: + return _tabular(["result"], [["(empty)"]], elapsed_ms) + rows = [[_fmt(getattr(r, c)) for c in columns] for r in picked] + return _tabular(columns, rows, elapsed_ms, truncated=len(picked) >= limit) + finally: + cluster.shutdown() def _graph_neo4j(edge_limit: int = 60, rel_type: str | None = None) -> dict[str, Any]: @@ -582,9 +649,9 @@ def _graph_neo4j(edge_limit: int = 60, rel_type: str | None = None) -> dict[str, driver.close() -def _sample_neo4j(object_name: str, limit: int) -> dict[str, Any]: +def _sample_neo4j(object_name: str, limit: int, offset: int = 0) -> dict[str, Any]: label = object_name.split(".")[-1] - cypher = f"MATCH (n:`{label}`) RETURN n LIMIT {limit}" + cypher = f"MATCH (n:`{label}`) RETURN n SKIP {int(offset)} LIMIT {int(limit)}" t0 = time.perf_counter() driver = _neo4j_driver() try: @@ -601,24 +668,319 @@ def _sample_neo4j(object_name: str, limit: int) -> dict[str, Any]: driver.close() -def _sample(engine: str, object_name: str, limit: int) -> dict[str, Any]: +def _sample(engine: str, object_name: str, limit: int, offset: int = 0) -> dict[str, Any]: if engine == "postgres": - return _sample_postgres(object_name, limit) + return _sample_postgres(object_name, limit, offset) if engine == "mysql": - return _sample_mysql(object_name, limit) + return _sample_mysql(object_name, limit, offset) if engine == "mongodb": - return _sample_mongodb(object_name, limit) + return _sample_mongodb(object_name, limit, offset) if engine == "cassandra": - return _sample_cassandra(object_name, limit) + return _sample_cassandra(object_name, limit, offset) if engine == "neo4j": - return _sample_neo4j(object_name, limit) + return _sample_neo4j(object_name, limit, offset) + if engine == "hadoop": + return _sample_hadoop(object_name, limit, offset) raise ValueError(f"Sample not supported for {engine}") +def _table_row_count(engine: str, object_name: str) -> int | None: + try: + if engine == "postgres": + schema, table = _parse_fqn(engine, object_name) + conn = psycopg2.connect( + host=DB_HOST, port=PG_PORT, user=PG_USER, password=PG_PASS, dbname=PG_DB, connect_timeout=8, + ) + try: + cur = conn.cursor() + cur.execute(f'SELECT count(*) FROM "{schema}"."{table}"') + return int(cur.fetchone()[0]) + finally: + conn.close() + if engine == "mysql": + schema, table = _parse_fqn(engine, object_name) + conn = pymysql.connect( + host=DB_HOST, port=MYSQL_PORT, user=MYSQL_USER, password=MYSQL_PASS, + database=MYSQL_DB, connect_timeout=8, + ) + try: + cur = conn.cursor() + cur.execute(f"SELECT count(*) FROM `{table}`") + return int(cur.fetchone()[0]) + finally: + conn.close() + if engine == "mongodb": + db_name, coll = _parse_fqn(engine, object_name) + client = _mongo_client() + try: + return int(client[db_name][coll].estimated_document_count()) + finally: + client.close() + if engine == "neo4j": + label = object_name.split(".")[-1] + driver = _neo4j_driver() + try: + with driver.session() as session: + return int(session.run(f"MATCH (n:`{label}`) RETURN count(n) AS c").single()["c"]) + finally: + driver.close() + except Exception: + pass + return None + + +def _parse_fqn(engine: str, object_name: str) -> tuple[str, str]: + if engine == "postgres": + if "." in object_name: + return object_name.split(".", 1) + return "public", object_name + if engine == "mysql": + if "." in object_name: + return object_name.split(".", 1) + return MYSQL_DB, object_name + if engine == "mongodb": + if "." in object_name: + return object_name.split(".", 1) + return MONGO_DB, object_name + if engine == "cassandra": + if "." in object_name: + return object_name.split(".", 1) + return CASS_KS, object_name + return "graph", object_name.split(".")[-1] + + +def _object_primary_keys(engine: str, object_name: str) -> list[str]: + try: + if engine == "postgres": + schema, table = _parse_fqn(engine, object_name) + conn = psycopg2.connect( + host=DB_HOST, port=PG_PORT, user=PG_USER, password=PG_PASS, dbname=PG_DB, connect_timeout=8, + ) + try: + cur = conn.cursor() + cur.execute( + """ + SELECT a.attname + FROM pg_constraint c + JOIN pg_class t ON t.oid = c.conrelid + JOIN pg_namespace n ON n.oid = t.relnamespace + JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = ANY(c.conkey) + WHERE c.contype = 'p' AND n.nspname = %s AND t.relname = %s + ORDER BY array_position(c.conkey, a.attnum) + """, + (schema, table), + ) + return [r[0] for r in cur.fetchall()] + finally: + conn.close() + if engine == "mysql": + schema, table = _parse_fqn(engine, object_name) + conn = pymysql.connect( + host=DB_HOST, port=MYSQL_PORT, user=MYSQL_USER, password=MYSQL_PASS, + database=MYSQL_DB, connect_timeout=8, + ) + try: + cur = conn.cursor() + cur.execute( + """ + SELECT COLUMN_NAME FROM information_schema.KEY_COLUMN_USAGE + WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s AND CONSTRAINT_NAME = 'PRIMARY' + ORDER BY ORDINAL_POSITION + """, + (schema, table), + ) + return [r[0] for r in cur.fetchall()] + finally: + conn.close() + if engine == "mongodb": + return ["_id"] + if engine == "cassandra": + ks, table = _parse_fqn(engine, object_name) + cluster = _cass_cluster() + session = cluster.connect() + try: + rows = session.execute( + "SELECT column_name FROM system_schema.columns " + "WHERE keyspace_name=%s AND table_name=%s AND kind IN ('partition_key','clustering') " + "ORDER BY position", + (ks, table), + ) + return [r.column_name for r in rows] + finally: + cluster.shutdown() + if engine == "neo4j": + label = object_name.split(".")[-1] + if label == "Product": + return ["product_id"] + if label == "Supplier": + return ["supplier_id"] + except Exception: + pass + return [] + + +def _coerce_value(val: Any) -> Any: + if val is None or val == "": + return None + if isinstance(val, (int, float, bool)): + return val + s = str(val) + if s.lower() == "null": + return None + if re.fullmatch(r"-?\d+", s): + try: + return int(s) + except ValueError: + pass + if re.fullmatch(r"-?\d+\.\d+", s): + try: + return float(s) + except ValueError: + pass + return s + + +def _safe_ident(name: str) -> bool: + return bool(re.fullmatch(r"[a-zA-Z_][a-zA-Z0-9_]*", name)) + + +def _update_postgres(object_name: str, pk: dict[str, Any], changes: dict[str, Any]) -> str: + schema, table = _parse_fqn("postgres", object_name) + if not changes or not pk: + raise ValueError("Primary key and changes required") + for c in list(changes) + list(pk): + if not _safe_ident(c): + raise ValueError(f"Invalid column name: {c}") + set_sql = ", ".join(f'"{c}"=%s' for c in changes) + where_sql = " AND ".join(f'"{c}"=%s' for c in pk) + sql = f'UPDATE "{schema}"."{table}" SET {set_sql} WHERE {where_sql}' + params = list(changes.values()) + list(pk.values()) + conn = psycopg2.connect( + host=DB_HOST, port=PG_PORT, user=PG_USER, password=PG_PASS, dbname=PG_DB, connect_timeout=8, + ) + try: + conn.autocommit = True + cur = conn.cursor() + cur.execute(sql, params) + if cur.rowcount == 0: + raise ValueError("No row matched primary key") + return sql + finally: + conn.close() + + +def _update_mysql(object_name: str, pk: dict[str, Any], changes: dict[str, Any]) -> str: + _schema, table = _parse_fqn("mysql", object_name) + if not changes or not pk: + raise ValueError("Primary key and changes required") + for c in list(changes) + list(pk): + if not _safe_ident(c): + raise ValueError(f"Invalid column name: {c}") + set_sql = ", ".join(f"`{c}`=%s" for c in changes) + where_sql = " AND ".join(f"`{c}`=%s" for c in pk) + sql = f"UPDATE `{table}` SET {set_sql} WHERE {where_sql}" + params = list(changes.values()) + list(pk.values()) + conn = pymysql.connect( + host=DB_HOST, port=MYSQL_PORT, user=MYSQL_USER, password=MYSQL_PASS, + database=MYSQL_DB, connect_timeout=8, + ) + try: + conn.autocommit = True + cur = conn.cursor() + cur.execute(sql, params) + if cur.rowcount == 0: + raise ValueError("No row matched primary key") + return sql + finally: + conn.close() + + +def _update_mongodb(object_name: str, pk: dict[str, Any], changes: dict[str, Any]) -> str: + db_name, coll = _parse_fqn("mongodb", object_name) + if not pk or not changes: + raise ValueError("Primary key and changes required") + client = _mongo_client() + try: + from bson import ObjectId + filt = dict(pk) + if "_id" in filt and isinstance(filt["_id"], str) and len(filt["_id"]) == 24: + try: + filt["_id"] = ObjectId(filt["_id"]) + except Exception: + pass + res = client[db_name][coll].update_one(filt, {"$set": changes}) + if res.matched_count == 0: + raise ValueError("No document matched primary key") + return f"UPDATE {db_name}.{coll} {filt}" + finally: + client.close() + + +def _update_cassandra(object_name: str, pk: dict[str, Any], changes: dict[str, Any]) -> str: + ks, table = _parse_fqn("cassandra", object_name) + if not pk or not changes: + raise ValueError("Primary key and changes required") + set_cql = ", ".join(f"{c}=%s" for c in changes) + where_cql = " AND ".join(f"{c}=%s" for c in pk) + cql = f"UPDATE {ks}.{table} SET {set_cql} WHERE {where_cql}" + cluster = _cass_cluster() + session = cluster.connect() + try: + session.execute(cql, list(changes.values()) + list(pk.values())) + return cql + finally: + cluster.shutdown() + + +def _update_neo4j(object_name: str, pk: dict[str, Any], changes: dict[str, Any]) -> str: + label = object_name.split(".")[-1] + if not pk or not changes: + raise ValueError("Primary key and changes required") + pk_col, pk_val = next(iter(pk.items())) + if not _safe_ident(pk_col): + raise ValueError(f"Invalid property name: {pk_col}") + for c in changes: + if not _safe_ident(c): + raise ValueError(f"Invalid property name: {c}") + set_frag = ", ".join(f"n.{c} = ${c}" for c in changes) + cypher = f"MATCH (n:`{label}` {{{pk_col}: $pk}}) SET {set_frag} RETURN n" + params: dict[str, Any] = {"pk": _coerce_value(pk_val)} + params.update({c: _coerce_value(v) for c, v in changes.items()}) + driver = _neo4j_driver() + try: + with driver.session() as session: + rec = session.run(cypher, **params).single() + if not rec: + raise ValueError("No node matched primary key") + return cypher + finally: + driver.close() + + +def _update_row(engine: str, object_name: str, pk: dict[str, Any], changes: dict[str, Any]) -> dict[str, Any]: + pk = {k: _coerce_value(v) for k, v in pk.items()} + changes = {k: _coerce_value(v) for k, v in changes.items()} + if engine == "postgres": + sql = _update_postgres(object_name, pk, changes) + elif engine == "mysql": + sql = _update_mysql(object_name, pk, changes) + elif engine == "mongodb": + sql = _update_mongodb(object_name, pk, changes) + elif engine == "cassandra": + sql = _update_cassandra(object_name, pk, changes) + elif engine == "neo4j": + sql = _update_neo4j(object_name, pk, changes) + else: + raise ValueError(f"Update not supported for {engine}") + return {"ok": True, "engine": engine, "object": object_name, "statement": sql, "cdc": engine in ("postgres", "mysql", "mongodb")} + + @router.get("/samples/{engine}") async def get_samples(engine: str): - if engine not in SAMPLES: + if engine not in SAMPLES and engine not in LAKE_ENGINES: return JSONResponse({"error": "unknown engine"}, status_code=404) + if engine == "hadoop": + return {"engine": engine, "samples": SAMPLES[engine], "connection": hadoop_connection_info()} return {"engine": engine, "samples": SAMPLES[engine], "connection": _connection_info(engine)} @@ -647,9 +1009,20 @@ async def sql_health(): return out + + +@router.get("/health/{engine}") +async def sql_health_engine(engine: str): + if engine == "hadoop": + return health_hadoop(_run_trino) + if engine not in SOURCE_ENGINES: + return JSONResponse({"error": "unknown engine"}, status_code=404) + all_h = await sql_health() + return all_h.get(engine) or {"ok": False, "error": "not found"} + @router.get("/catalog/{engine}") async def get_catalog(engine: str): - if engine not in SOURCE_ENGINES: + if engine not in SOURCE_ENGINES and engine not in LAKE_ENGINES: return JSONResponse({"error": "unknown engine"}, status_code=404) try: return _catalog(engine) @@ -658,14 +1031,31 @@ async def get_catalog(engine: str): @router.get("/sample/{engine}") -async def get_sample(engine: str, object: str = Query(..., min_length=1), limit: int = Query(50, ge=1, le=200)): - if engine not in SOURCE_ENGINES: +async def get_sample( + engine: str, + object: str = Query(..., min_length=1), + limit: int = Query(100, ge=1, le=500), + offset: int = Query(0, ge=0), +): + if engine not in SOURCE_ENGINES and engine not in LAKE_ENGINES: return JSONResponse({"error": "unknown engine"}, status_code=404) try: - result = _sample(engine, object, limit) + result = _sample(engine, object, limit, offset) if not result.get("ok"): return JSONResponse(result, status_code=422) - return {**result, "engine": engine, "object": object} + pks = _object_primary_keys(engine, object) + total = _table_row_count(engine, object) + return { + **result, + "engine": engine, + "object": object, + "offset": offset, + "limit": limit, + "total_count": total, + "primary_keys": pks, + "cdc": engine in ("postgres", "mysql", "mongodb"), + "editable": bool(pks), + } except Exception as exc: return JSONResponse({"ok": False, "error": str(exc)[:500]}, status_code=502) @@ -698,6 +1088,22 @@ async def execute_sql(body: SqlRequest): return JSONResponse({"ok": False, "error": str(exc)[:500]}, status_code=502) +@router.post("/row/update") +async def update_row(body: RowUpdateRequest): + allowed_pks = _object_primary_keys(body.engine, body.object) + if allowed_pks: + for k in body.pk: + if k not in allowed_pks: + return JSONResponse({"ok": False, "error": f"Invalid primary key column '{k}'"}, status_code=400) + for k in body.changes: + if k in body.pk: + return JSONResponse({"ok": False, "error": "Cannot change primary key columns"}, status_code=400) + try: + return _update_row(body.engine, body.object, body.pk, body.changes) + except Exception as exc: + return JSONResponse({"ok": False, "error": str(exc)[:500]}, status_code=422) + + @router.post("/benchmark") async def benchmark(): """Parallel analytics: PostgreSQL OLTP vs Trino distributed engine (5M sin() rows).""" diff --git a/api/streaming_ops.py b/api/streaming_ops.py new file mode 100644 index 0000000..4215c03 --- /dev/null +++ b/api/streaming_ops.py @@ -0,0 +1,435 @@ +"""Live Spark + Kafka visibility and manual job control for the Command Center.""" + +from __future__ import annotations + +import asyncio +import os +import time +from typing import Any + +import httpx +from fastapi import APIRouter, Body +from fastapi.responses import JSONResponse, Response + +SPARK_UI_URL = os.getenv("SPARK_UI_URL", "http://10.0.21.50:8080").rstrip("/") +KAFKA_UI_URL = os.getenv("KAFKA_UI_URL", "http://10.0.21.36:9000").rstrip("/") +KAFKA_CONNECT_URL = os.getenv("KAFKA_CONNECT_URL", "http://10.0.21.50:8083").rstrip("/") +AIRFLOW_URL = os.getenv("AIRFLOW_URL", "http://10.0.21.55:8080").rstrip("/") +AIRFLOW_USER = os.getenv("AIRFLOW_USER", "admin") +AIRFLOW_PASSWORD = os.getenv("AIRFLOW_PASSWORD", "") + +from hdfs_kafka import export_hdfs_to_kafka, hdfs_export_snapshot + +router = APIRouter(prefix="/api/pipeline/streaming", tags=["streaming"]) + +# Manual Spark / lakehouse jobs (Airflow DAGs on the lab) +SPARK_JOBS: dict[str, dict[str, Any]] = { + "spark_to_curated": { + "label": "Spark → Iceberg curated (mask PII)", + "dag_id": "mask_to_curated", + "agent": "lakehouse-ops", + "description": "Runs the mask_to_curated DAG — Spark/SQL transform into iceberg.curated_masked", + "default_conf": {}, + "editable_fields": ["batch_size", "sources"], + }, + "hadoop_to_trino": { + "label": "HDFS → Iceberg (historical)", + "dag_id": "hadoop_to_trino", + "agent": "hadoop-ranger", + "description": "Load historical_sales from HDFS into Iceberg via Trino/Spark pipeline", + "default_conf": {"mode": "refresh"}, + "editable_fields": ["mode"], + }, + "spark_to_s3": { + "label": "Spark → S3 curated layer", + "dag_id": "mask_to_curated", + "agent": "lakehouse-ops", + "description": "Spark transform into Iceberg curated tables mirrored to S3", + "default_conf": {"target": "s3"}, + "editable_fields": ["target", "batch_size"], + }, + "generate_all": { + "label": "Seed all source DBs", + "dag_id": "generate_data_all_databases", + "agent": "data-custodian", + "description": "Airflow DAG — generates rows into PostgreSQL, MySQL, MongoDB, Cassandra, Neo4j", + "default_conf": {"rows": 3000}, + "editable_fields": ["rows"], + }, +} + +# Master pipeline pulse switch: running | paused | stopped +_flow_state: dict[str, Any] = {"mode": "running", "since": time.time()} + +def flow_mode() -> str: + return _flow_state.get("mode", "running") + +def flow_snapshot() -> dict[str, Any]: + return {"mode": _flow_state.get("mode", "running"), "since": _flow_state.get("since")} + +_cache: dict[str, Any] = {"ts": 0.0, "data": None} +_TTL = 8.0 +_token_cache: dict[str, Any] = {"token": None, "exp": 0.0} + + +def _feed(agent_id: str, message: str, level: str = "info") -> None: + try: + from main import add_feed + add_feed(agent_id, message, level) + except Exception: + pass + + +async def _airflow_token(client: httpx.AsyncClient) -> str: + now = time.time() + if _token_cache["token"] and _token_cache["exp"] > now + 30: + return _token_cache["token"] + r = await client.post( + f"{AIRFLOW_URL}/auth/token", + json={"username": AIRFLOW_USER, "password": AIRFLOW_PASSWORD}, + timeout=10, + ) + r.raise_for_status() + tok = r.json()["access_token"] + _token_cache["token"] = tok + _token_cache["exp"] = now + 20 * 60 + return tok + + +async def collect_spark() -> dict[str, Any]: + out: dict[str, Any] = { + "ui_url": SPARK_UI_URL, + "ui_ok": False, + "status": "UNKNOWN", + "workers": [], + "alive_workers": 0, + "cores": 0, + "cores_used": 0, + "memory_mb": 0, + "memory_used_mb": 0, + "active_apps": [], + "completed_apps": [], + } + try: + async with httpx.AsyncClient(timeout=6.0) as client: + r = await client.get(f"{SPARK_UI_URL}/json/") + if r.status_code >= 400: + return out + d = r.json() + out["ui_ok"] = True + out["status"] = d.get("status") or "ALIVE" + out["alive_workers"] = int(d.get("aliveworkers") or 0) + out["cores"] = int(d.get("cores") or 0) + out["cores_used"] = int(d.get("coresused") or 0) + out["memory_mb"] = int(d.get("memory") or 0) + out["memory_used_mb"] = int(d.get("memoryused") or 0) + workers = d.get("workers") or [] + out["workers"] = [ + { + "id": w.get("id"), + "host": w.get("host"), + "cores": w.get("cores"), + "cores_used": w.get("coresused"), + "memory_mb": w.get("memory"), + "state": w.get("state"), + "webui": w.get("webuiaddress"), + } + for w in workers + ] + for app in (d.get("activeapps") or [])[:20]: + out["active_apps"].append({ + "id": app.get("id"), + "name": app.get("name"), + "cores": app.get("cores"), + "memory_mb": app.get("memory"), + "submitdate": app.get("submitdate"), + "duration_ms": app.get("duration"), + "user": app.get("user"), + }) + for app in (d.get("completedapps") or [])[:10]: + out["completed_apps"].append({ + "id": app.get("id"), + "name": app.get("name"), + "duration_ms": app.get("duration"), + }) + except Exception as exc: + out["error"] = str(exc)[:200] + return out + + +async def collect_kafka() -> dict[str, Any]: + out: dict[str, Any] = { + "ui_url": KAFKA_UI_URL, + "ui_ok": False, + "connect_url": KAFKA_CONNECT_URL, + "connect_ok": False, + "cluster": {}, + "topics": [], + "connectors": [], + } + try: + async with httpx.AsyncClient(timeout=8.0) as client: + cr = await client.get(f"{KAFKA_UI_URL}/api/clusters") + if cr.status_code < 400: + clusters = cr.json() + out["ui_ok"] = True + if clusters: + name = clusters[0].get("name", "local") + out["cluster"] = { + "name": name, + "status": clusters[0].get("status"), + "broker_count": clusters[0].get("brokerCount"), + "topic_count": clusters[0].get("topicCount"), + "online_partitions": clusters[0].get("onlinePartitionCount"), + } + tr = await client.get( + f"{KAFKA_UI_URL}/api/clusters/{name}/topics", + params={"page": 1, "perPage": 50, "showInternal": False}, + ) + if tr.status_code < 400: + topics = tr.json().get("topics") or [] + out["topics"] = [ + { + "name": t.get("name"), + "partitions": t.get("partitionCount"), + "replicas": t.get("replicationFactor"), + "messages": t.get("messagesCount"), + } + for t in topics[:50] + ] + lr = await client.get(f"{KAFKA_CONNECT_URL}/connectors") + if lr.status_code < 400: + out["connect_ok"] = True + names_raw = lr.json() + names = names_raw if isinstance(names_raw, list) else [] + for cn in names[:20]: + try: + sr = await client.get(f"{KAFKA_CONNECT_URL}/connectors/{cn}/status") + st = sr.json() if sr.status_code < 400 else {} + conn = st.get("connector") or {} + tasks = st.get("tasks") or [] + out["connectors"].append({ + "name": cn, + "state": conn.get("state"), + "worker": conn.get("worker_id"), + "tasks": [{"id": t.get("id"), "state": t.get("state")} for t in tasks], + "type": st.get("type"), + }) + except Exception: + out["connectors"].append({"name": cn, "state": "UNKNOWN"}) + except Exception as exc: + out["error"] = str(exc)[:200] + return out + + +async def build_streaming_status() -> dict[str, Any]: + spark, kafka = await asyncio.gather(collect_spark(), collect_kafka()) + cdc_active = False + try: + from cdc_consumer import snapshot as cdc_snapshot + cdc = cdc_snapshot(15) + cdc_active = bool(cdc.get("connected")) and int(cdc.get("window_total") or 0) > 0 + except Exception: + pass + + connectors_running = sum( + 1 for c in kafka.get("connectors", []) + if (c.get("state") or "").upper() == "RUNNING" + ) + spark_alive = spark.get("ui_ok") and (spark.get("status") or "").upper() == "ALIVE" + apps_running = len(spark.get("active_apps") or []) + + hdfs_recent = hdfs_export_snapshot().get("recent") + + edges = { + "hdfs→kafka": bool(hdfs_recent), + "kafka→spark": cdc_active and spark_alive or hdfs_recent, + "spark→iceberg": apps_running > 0 or spark_alive, + "spark→s3": apps_running > 0 or spark_alive, + "sources→kafka": cdc_active, + "connectors": connectors_running > 0, + } + + mode = flow_mode() + if mode != "running": + edges = {k: False for k in edges} + + return { + "ok": True, + "flow": mode, + "spark": spark, + "kafka": kafka, + "edges": edges, + "jobs": [ + {**{"id": k}, **{kk: vv for kk, vv in v.items() if kk != "editable_fields"}} + for k, v in SPARK_JOBS.items() + ], + "ts": time.time(), + } + + +@router.get("/status") +async def streaming_status(refresh: bool = False) -> JSONResponse: + now = time.time() + if not refresh and _cache["data"] and now - _cache["ts"] < _TTL: + return JSONResponse(_cache["data"]) + data = await build_streaming_status() + _cache["data"] = data + _cache["ts"] = now + return JSONResponse(data) + + +@router.get("/jobs") +async def list_spark_jobs() -> JSONResponse: + return JSONResponse({"ok": True, "jobs": SPARK_JOBS}) + + + + +@router.post("/flow/{action}") +async def set_flow(action: str) -> JSONResponse: + mapping = { + "pause": "paused", + "stop": "stopped", + "resume": "running", + "start": "running", + "run": "running", + } + if action not in mapping: + return JSONResponse({"ok": False, "error": f"unknown action {action}"}, status_code=400) + _flow_state["mode"] = mapping[action] + _flow_state["since"] = time.time() + _cache["ts"] = 0 + label = {"running": "resumed", "paused": "paused", "stopped": "stopped"}[mapping[action]] + _feed("lakehouse-ops", f"[pipeline] data flow {label}", "info" if mapping[action] == "running" else "warn") + return JSONResponse({"ok": True, "flow": flow_snapshot()}) + + +@router.get("/flow") +async def get_flow() -> JSONResponse: + return JSONResponse({"ok": True, "flow": flow_snapshot()}) + +@router.post("/jobs/{job_id}/trigger") +async def trigger_spark_job(job_id: str, body: dict[str, Any] = Body(default={})) -> JSONResponse: + job = SPARK_JOBS.get(job_id) + if not job: + return JSONResponse({"ok": False, "error": f"unknown job {job_id}"}, status_code=400) + conf = {**(job.get("default_conf") or {}), **(body.get("conf") or {})} + agent = job.get("agent", "lakehouse-ops") + dag_id = job["dag_id"] + _feed(agent, f"[spark] Triggering {job['label']} conf={conf}", "info") + + try: + async with httpx.AsyncClient() as client: + tok = await _airflow_token(client) + h = {"Authorization": f"Bearer {tok}"} + r = await client.post( + f"{AIRFLOW_URL}/api/v2/dags/{dag_id}/dagRuns", + headers=h, + json={"logical_date": None, "conf": conf}, + timeout=20, + ) + if r.status_code >= 400: + _feed(agent, f"[spark] {job['label']}: Airflow {r.status_code}", "err") + return JSONResponse({"ok": False, "error": f"airflow {r.status_code}: {r.text[:300]}"}, status_code=502) + run = r.json() + run_id = run.get("dag_run_id") + _feed(agent, f"[spark] {job['label']}: started run {run_id}", "info") + return JSONResponse({"ok": True, "job_id": job_id, "dag_id": dag_id, "run_id": run_id, "conf": conf}) + except Exception as exc: + _feed(agent, f"[spark] {job['label']}: {str(exc)[:120]}", "err") + return JSONResponse({"ok": False, "error": str(exc)}, status_code=500) + + + + +@router.post("/hdfs/to-kafka") +async def hdfs_to_kafka(body: dict[str, Any] = Body(default={})) -> JSONResponse: + path = body.get("path", "/data/historical/sales_orders") + topic = body.get("topic", "hdfs.historical.sales") + limit = int(body.get("limit", 2000)) + try: + result = await export_hdfs_to_kafka(path=path, topic=topic, limit=limit, feed=_feed) + if not result.get("ok"): + return JSONResponse(result, status_code=502) + _cache["ts"] = 0 + return JSONResponse(result) + except Exception as exc: + _feed("hadoop-ranger", f"[hdfs→kafka] failed: {str(exc)[:120]}", "err") + return JSONResponse({"ok": False, "error": str(exc)}, status_code=500) + + +@router.post("/pipeline/{pipeline_id}") +async def run_streaming_pipeline(pipeline_id: str) -> JSONResponse: + if pipeline_id != "hadoop-lake": + return JSONResponse({"ok": False, "error": f"unknown pipeline {pipeline_id}"}, status_code=400) + steps: list[str] = [] + try: + exp = await export_hdfs_to_kafka(feed=_feed) + if not exp.get("ok"): + return JSONResponse({"ok": False, "error": exp.get("error"), "steps": steps}, status_code=502) + steps.append(f"hdfs→kafka ({exp.get('rows_sent')} rows)") + + async with httpx.AsyncClient() as client: + tok = await _airflow_token(client) + h = {"Authorization": f"Bearer {tok}"} + for job_id, dag_id, agent in [ + ("hadoop_to_trino", "hadoop_to_trino", "hadoop-ranger"), + ("spark_to_curated", "mask_to_curated", "lakehouse-ops"), + ]: + r = await client.post( + f"{AIRFLOW_URL}/api/v2/dags/{dag_id}/dagRuns", + headers=h, + json={"logical_date": None, "conf": {}}, + timeout=20, + ) + if r.status_code >= 400: + _feed(agent, f"[pipeline] {dag_id} failed {r.status_code}", "err") + return JSONResponse({"ok": False, "error": f"{dag_id}: {r.text[:200]}", "steps": steps}, status_code=502) + steps.append(dag_id) + _feed(agent, f"[pipeline] started {dag_id}", "info") + + _cache["ts"] = 0 + return JSONResponse({"ok": True, "pipeline": pipeline_id, "steps": steps, "export": exp}) + except Exception as exc: + return JSONResponse({"ok": False, "error": str(exc), "steps": steps}, status_code=500) + +@router.post("/kafka/connectors/{name}/restart") +async def restart_kafka_connector(name: str) -> JSONResponse: + _feed("etl-guardian", f"[kafka] Restarting connector {name}", "info") + try: + async with httpx.AsyncClient(timeout=15.0) as client: + r = await client.post(f"{KAFKA_CONNECT_URL}/connectors/{name}/restart") + if r.status_code >= 400: + return JSONResponse({"ok": False, "error": r.text[:300]}, status_code=502) + _feed("etl-guardian", f"[kafka] Connector {name} restart requested", "info") + _cache["ts"] = 0 + return JSONResponse({"ok": True, "connector": name, "action": "restart"}) + except Exception as exc: + return JSONResponse({"ok": False, "error": str(exc)}, status_code=500) + + +@router.post("/kafka/connectors/{name}/pause") +async def pause_kafka_connector(name: str) -> JSONResponse: + try: + async with httpx.AsyncClient(timeout=10.0) as client: + r = await client.put(f"{KAFKA_CONNECT_URL}/connectors/{name}/pause") + if r.status_code >= 400: + return JSONResponse({"ok": False, "error": r.text[:300]}, status_code=502) + _cache["ts"] = 0 + return JSONResponse({"ok": True, "connector": name, "action": "pause"}) + except Exception as exc: + return JSONResponse({"ok": False, "error": str(exc)}, status_code=500) + + +@router.post("/kafka/connectors/{name}/resume") +async def resume_kafka_connector(name: str) -> JSONResponse: + try: + async with httpx.AsyncClient(timeout=10.0) as client: + r = await client.put(f"{KAFKA_CONNECT_URL}/connectors/{name}/resume") + if r.status_code >= 400: + return JSONResponse({"ok": False, "error": r.text[:300]}, status_code=502) + _cache["ts"] = 0 + return JSONResponse({"ok": True, "connector": name, "action": "resume"}) + except Exception as exc: + return JSONResponse({"ok": False, "error": str(exc)}, status_code=500) diff --git a/api/webhdfs_util.py b/api/webhdfs_util.py new file mode 100644 index 0000000..a224142 --- /dev/null +++ b/api/webhdfs_util.py @@ -0,0 +1,77 @@ +"""Shared WebHDFS helpers — resolve datanode redirects via NameNode JMX.""" + +from __future__ import annotations + +import os +import time +from typing import Any +from urllib.parse import quote, urlparse, urlunparse + +import httpx + +HDFS_NN_URL = os.getenv("HDFS_NN_URL", "http://10.0.21.61:9870").rstrip("/") +HDFS_USER = os.getenv("HDFS_USER", "hdfs") +WEBHDFS = f"{HDFS_NN_URL}/webhdfs/v1" + +_dn_cache: dict[str, Any] = {"ts": 0.0, "map": {}} + + +def encode_path(path: str) -> str: + p = path if path.startswith("/") else f"/{path}" + return "/".join(quote(seg, safe="") for seg in p.split("/")) + + +def datanode_host_map() -> dict[str, str]: + now = time.time() + if _dn_cache["map"] and now - _dn_cache["ts"] < 120: + return _dn_cache["map"] + mapping: dict[str, str] = {} + try: + with httpx.Client(timeout=8.0) as client: + r = client.get(f"{HDFS_NN_URL}/jmx?qry=Hadoop:service=NameNode,name=NameNodeInfo") + raw = (r.json().get("beans") or [{}])[0].get("LiveNodes") or "{}" + if isinstance(raw, str): + import json + nodes = json.loads(raw) + else: + nodes = raw + for key, info in nodes.items(): + host = key.split(":")[0] + info_addr = (info or {}).get("infoAddr") or "" + if info_addr: + mapping[host] = info_addr # ip:9864 + except Exception: + pass + _dn_cache["map"] = mapping + _dn_cache["ts"] = now + return mapping + + +def resolve_redirect(url: str) -> str: + parsed = urlparse(url) + host = parsed.hostname or "" + if not host or host.replace(".", "").isdigit(): + return url + dn = datanode_host_map().get(host) + if not dn: + return url + ip, _, port = dn.partition(":") + port = parsed.port or port or "9864" + return urlunparse(parsed._replace(netloc=f"{ip}:{port}")) + + +def open_bytes(path: str, max_bytes: int = 8_000_000) -> bytes: + url = f"{WEBHDFS}{encode_path(path)}" + with httpx.Client(timeout=90.0, follow_redirects=False) as client: + r = client.get(url, params={"op": "OPEN", "user.name": HDFS_USER}) + if r.status_code in (301, 302, 307, 308): + loc = r.headers.get("location") or "" + if loc: + r = client.get(resolve_redirect(loc), follow_redirects=True) + elif r.status_code == 200 and r.headers.get("content-type", "").startswith("application/json"): + loc = r.json().get("Location") or r.json().get("location") or "" + if loc: + r = client.get(resolve_redirect(loc), follow_redirects=True) + if r.status_code >= 400: + raise RuntimeError(r.text[:300]) + return r.content[:max_bytes] diff --git a/ui/index.html b/ui/index.html index 81fa1ec..d4b924e 100644 --- a/ui/index.html +++ b/ui/index.html @@ -1,5 +1,5 @@ - + diff --git a/ui/nginx.conf b/ui/nginx.conf index 018b038..e16597f 100644 --- a/ui/nginx.conf +++ b/ui/nginx.conf @@ -14,6 +14,31 @@ server { proxy_buffering off; } + location /spark-ui/ { + proxy_pass http://10.0.21.50:8080/; + proxy_http_version 1.1; + proxy_set_header Host 10.0.21.50:8080; + proxy_hide_header X-Frame-Options; + proxy_redirect http://10.0.21.50:8080/ /spark-ui/; + sub_filter_once off; + sub_filter 'href="/' 'href="/spark-ui/'; + sub_filter 'src="/' 'src="/spark-ui/'; + sub_filter "href='/" "href='/spark-ui/"; + sub_filter_types text/css application/javascript text/html; + } + + location /kafka-ui/ { + proxy_pass http://10.0.21.36:9000/; + proxy_http_version 1.1; + proxy_set_header Host 10.0.21.36:9000; + proxy_hide_header X-Frame-Options; + proxy_redirect http://10.0.21.36:9000/ /kafka-ui/; + sub_filter_once off; + sub_filter 'href="/' 'href="/kafka-ui/'; + sub_filter 'src="/' 'src="/kafka-ui/'; + sub_filter_types text/html application/javascript text/css; + } + location /assets/ { add_header Cache-Control "public, max-age=31536000, immutable"; try_files $uri =404; diff --git a/ui/src/App.tsx b/ui/src/App.tsx index 3bf6a65..3a86619 100644 --- a/ui/src/App.tsx +++ b/ui/src/App.tsx @@ -11,17 +11,14 @@ import { ChatDrawer } from './components/features/ChatDrawer' import { GpuMonitor } from './components/features/GpuMonitor' import { InfraQuickAccess } from './components/features/InfraQuickAccess' import { InspectorPanel } from './components/features/InspectorPanel' -import { PlatformTopology } from './components/features/PlatformTopology' -import { PresentationView } from './components/features/PresentationView' +import { PlatformView } from './components/features/PlatformView' import { DataQualityView } from './components/features/DataQualityView' import { KnowledgeChatView } from './components/features/KnowledgeChatView' import { StorageView } from './components/features/StorageView' -import { HdfsView } from './components/features/HdfsView' -import { DataGenView } from './components/features/DataGenView' import { ChangesView } from './components/features/ChangesView' import { DataFlowView } from './components/features/DataFlowView' import { SearchView } from './components/features/SearchView' -import { DataSourcesView } from './components/features/DataSourcesView' +import { DataHubView } from './components/features/DataHubView' import { SshTerminal } from './components/features/SshTerminal' import { TerminalDock } from './components/features/TerminalDock' import { WorkbenchPanel } from './components/features/WorkbenchPanel' @@ -39,7 +36,7 @@ export default function App() { const mainScrollRef = useRef(null) const isPlatform = cc.mainView === 'platform' - const isDataSources = cc.mainView === 'datasources' + const isDataSources = cc.mainView === 'datasources' || cc.mainView === 'hdfs' const openApprovals = () => { cc.setMainView('approvals') @@ -123,31 +120,25 @@ export default function App() {
{cc.mainView === 'platform' ? ( - - ) : cc.mainView === 'datasources' ? ( - - ) : cc.mainView === 'datagen' ? ( - cc.setMainView('platform')} /> + ) : cc.mainView === 'datasources' || cc.mainView === 'hdfs' ? ( + ) : cc.mainView === 'changes' ? ( ) : cc.mainView === 'dataflow' ? ( - ) : cc.mainView === 'presentation' ? ( - ) : cc.mainView === 'dataquality' ? ( ) : cc.mainView === 'knowledge' ? ( ) : cc.mainView === 'storage' ? ( - ) : cc.mainView === 'hdfs' ? ( - ) : cc.mainView === 'search' ? ( ) : ( diff --git a/ui/src/components/features/ArchitectureDiagram.tsx b/ui/src/components/features/ArchitectureDiagram.tsx index 9d03168..1370f66 100644 --- a/ui/src/components/features/ArchitectureDiagram.tsx +++ b/ui/src/components/features/ArchitectureDiagram.tsx @@ -146,9 +146,56 @@ const POSITIONS: Record> = { }, } -export function ArchitectureDiagram({ animation }: { animation: string }) { +const COMPACT_POSITIONS: Record> = { + 'full-stack': { + user: { x: 14, y: 18 }, + caddy: { x: 32, y: 18 }, + ui: { x: 50, y: 18 }, + api: { x: 12, y: 48 }, + dq: { x: 34, y: 48 }, + rag: { x: 56, y: 48 }, + docling: { x: 34, y: 78 }, + chroma: { x: 72, y: 48 }, + llm: { x: 88, y: 78 }, + lake: { x: 12, y: 78 }, + }, + 'rag-flow': { + upload: { x: 10, y: 22 }, + store: { x: 10, y: 72 }, + docling: { x: 28, y: 22 }, + chunk: { x: 46, y: 22 }, + embed: { x: 46, y: 47 }, + chroma: { x: 46, y: 72 }, + query: { x: 72, y: 22 }, + retrieve: { x: 72, y: 47 }, + llm: { x: 90, y: 72 }, + }, + 'dq-flow': { + data: { x: 8, y: 50 }, + docling: { x: 24, y: 22 }, + pandas: { x: 24, y: 78 }, + ge: { x: 46, y: 32 }, + soda: { x: 46, y: 68 }, + maturity: { x: 68, y: 50 }, + report: { x: 88, y: 50 }, + }, + 'lakehouse': { + pg: { x: 6, y: 22 }, + mysql: { x: 6, y: 42 }, + mongo: { x: 6, y: 62 }, + debezium: { x: 24, y: 42 }, + kafka: { x: 38, y: 42 }, + spark: { x: 52, y: 42 }, + iceberg: { x: 66, y: 42 }, + trino: { x: 80, y: 42 }, + bi: { x: 92, y: 62 }, + }, +} + +export function ArchitectureDiagram({ animation, compact = false, present = false }: { animation: string; compact?: boolean; present?: boolean }) { const flow = FLOWS[animation] || FLOWS['full-stack'] - const positions = POSITIONS[animation] || POSITIONS['full-stack'] + const positions = (compact ? COMPACT_POSITIONS[animation] : POSITIONS[animation]) + || (compact ? COMPACT_POSITIONS['full-stack'] : POSITIONS['full-stack']) const [tick, setTick] = useState(0) useEffect(() => { @@ -159,7 +206,12 @@ export function ArchitectureDiagram({ animation }: { animation: string }) { const activeEdge = tick % flow.edges.length return ( -
+
{flow.edges.map((edge, i) => { const from = positions[edge.from] @@ -195,15 +247,19 @@ export function ArchitectureDiagram({ animation }: { animation: string }) {
-

+

{node.label}

- {node.sub &&

{node.sub}

} + {node.sub &&

{node.sub}

}
) })} diff --git a/ui/src/components/features/DataBrowserGrid.tsx b/ui/src/components/features/DataBrowserGrid.tsx new file mode 100644 index 0000000..9ce1afa --- /dev/null +++ b/ui/src/components/features/DataBrowserGrid.tsx @@ -0,0 +1,376 @@ +import { useCallback, useMemo, useState } from 'react' +import { Check, Filter, Loader2, Pencil, RotateCcw, Save, Search, X } from 'lucide-react' +import type { SourceEngine } from '../../lib/dataSourceCatalog' +import { cn } from '../../lib/utils' +import { subTabActive, subTabIdle } from '../../lib/tabActive' + +type SampleData = { + ok: boolean + columns?: string[] + rows?: unknown[][] + row_count?: number + elapsed_ms?: number + error?: string + primary_keys?: string[] + cdc?: boolean + editable?: boolean + offset?: number + limit?: number + total_count?: number | null +} + +type DataEngine = SourceEngine | 'hadoop' + +type Props = { + engine: DataEngine + objectFqn: string + sample: SampleData | null + loading: boolean + page: number + pageSize: number + onPageChange: (page: number) => void + onPageSizeChange: (size: number) => void + onReload: () => void +} + +const PAGE_SIZES = [100, 250, 500] + +function cellStr(v: unknown) { + if (v === null || v === undefined) return '' + return String(v) +} + +function rowMatches(row: unknown[], columns: string[], filters: Record, global: string) { + if (global) { + const hay = row.map(cellStr).join(' ').toLowerCase() + if (!hay.includes(global.toLowerCase())) return false + } + for (const col of columns) { + const f = filters[col]?.trim() + if (!f) continue + const idx = columns.indexOf(col) + const val = cellStr(row[idx]).toLowerCase() + if (!val.includes(f.toLowerCase())) return false + } + return true +} + +export function DataBrowserGrid({ + engine, + objectFqn, + sample, + loading, + page, + pageSize, + onPageChange, + onPageSizeChange, + onReload, +}: Props) { + const [globalFilter, setGlobalFilter] = useState('') + const [colFilters, setColFilters] = useState>({}) + const [editMode, setEditMode] = useState(false) + const [drafts, setDrafts] = useState>>({}) + const [savingRow, setSavingRow] = useState(null) + const [msg, setMsg] = useState<{ text: string; ok: boolean } | null>(null) + + const columns = sample?.columns || [] + const rows = sample?.rows || [] + const pks = sample?.primary_keys || [] + const editable = sample?.editable && pks.length > 0 + const totalCount = sample?.total_count ?? null + const offset = sample?.offset ?? (page - 1) * pageSize + const totalPages = totalCount != null ? Math.max(1, Math.ceil(totalCount / pageSize)) : null + const rowFrom = rows.length ? offset + 1 : 0 + const rowTo = offset + rows.length + + const filteredRows = useMemo(() => { + return rows + .map((row, idx) => ({ row, idx })) + .filter(({ row }) => rowMatches(row, columns, colFilters, globalFilter)) + }, [rows, columns, colFilters, globalFilter]) + + const pkValuesForRow = useCallback( + (row: unknown[]) => { + const pk: Record = {} + for (const k of pks) { + const i = columns.indexOf(k) + if (i >= 0) pk[k] = row[i] + } + return pk + }, + [columns, pks], + ) + + const getDraft = (rowIdx: number, col: string, original: unknown) => { + if (drafts[rowIdx]?.[col] !== undefined) return drafts[rowIdx][col] + return cellStr(original) + } + + const setDraft = (rowIdx: number, col: string, val: string) => { + setDrafts((d) => ({ ...d, [rowIdx]: { ...d[rowIdx], [col]: val } })) + } + + const rowDirty = (rowIdx: number, row: unknown[]) => { + const d = drafts[rowIdx] + if (!d) return false + return columns.some((col, j) => { + if (pks.includes(col)) return false + return d[col] !== undefined && d[col] !== cellStr(row[j]) + }) + } + + const saveRow = async (rowIdx: number, row: unknown[]) => { + const pk = pkValuesForRow(row) + if (!Object.keys(pk).length) { + setMsg({ text: 'No primary key — cannot save', ok: false }) + return + } + const changes: Record = {} + const d = drafts[rowIdx] || {} + for (const col of columns) { + if (pks.includes(col)) continue + if (d[col] !== undefined && d[col] !== cellStr(row[columns.indexOf(col)])) { + changes[col] = d[col] === '' ? null : d[col] + } + } + if (!Object.keys(changes).length) return + + setSavingRow(rowIdx) + setMsg(null) + try { + const r = await fetch('/api/sql/row/update', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ engine, object: objectFqn, pk, changes }), + }) + const j = await r.json() + if (!j.ok) { + setMsg({ text: j.error || 'Save failed', ok: false }) + return + } + setMsg({ + text: j.cdc + ? 'Saved — change will appear in Live Changes via CDC' + : 'Saved (this source has no CDC stream)', + ok: true, + }) + setDrafts((d) => { + const next = { ...d } + delete next[rowIdx] + return next + }) + setTimeout(onReload, 600) + } catch { + setMsg({ text: 'API unreachable', ok: false }) + } finally { + setSavingRow(null) + } + } + + const clearFilters = () => { + setGlobalFilter('') + setColFilters({}) + } + + if (!sample?.ok && !loading) { + return ( +

{sample?.error || 'Failed to load data'}

+ ) + } + + return ( +
+ {/* Toolbar */} +
+
+ + setGlobalFilter(e.target.value)} + className="w-full rounded border border-border bg-surface-overlay py-1 pl-7 pr-2 text-[10px] text-foreground" + /> +
+ {(globalFilter || Object.values(colFilters).some(Boolean)) && ( + + )} + {editable && ( + + )} + {sample?.cdc && ( + CDC → Live Changes + )} + {loading && } +
+ + {msg && ( +

{msg.text}

+ )} + +
+ {columns.length > 0 && ( + <> +

+ {filteredRows.length} shown + {totalCount != null + ? ` · rows ${rowFrom.toLocaleString()}–${rowTo.toLocaleString()} of ${totalCount.toLocaleString()}` + : ` · page ${page}${totalPages ? ` of ${totalPages.toLocaleString()}` : ''}`} + {sample?.elapsed_ms != null && ` · ${sample.elapsed_ms}ms`} + {pks.length > 0 && ` · PK: ${pks.join(', ')}`} +

+ + + + {editMode && editable && + ))} + + + + {filteredRows.map(({ row, idx }) => { + const dirty = rowDirty(idx, row) + return ( + + {editMode && editable && ( + + )} + {row.map((cell, j) => { + const col = columns[j] + const isPk = pks.includes(col) + if (editMode && editable && !isPk) { + return ( + + ) + } + return ( + + ) + })} + + ) + })} + +
} + {columns.map((c) => ( + +
+ {c}{pks.includes(c) ? ' (PK)' : ''} +
+ + setColFilters((f) => ({ ...f, [c]: e.target.value }))} + className="w-full min-w-[60px] rounded border border-border/60 bg-surface py-0.5 pl-5 pr-1 text-[9px] font-normal text-foreground" + /> +
+
+
+ {dirty && ( + + )} + + setDraft(idx, col, e.target.value)} + className="w-full rounded border border-border/60 bg-surface-overlay px-1.5 py-0.5 text-[10px] text-foreground" + /> + + {cell === null || cell === undefined ? 'NULL' : String(cell)} +
+ {filteredRows.length === 0 && ( +

No rows match filters

+ )} + + )} + {!columns.length && !loading && ( +

Select a table, collection or label to preview data

+ )} +
+ + {/* Pagination */} + {columns.length > 0 && ( +
+
+ Rows per page + +
+
+ + + + Page {page.toLocaleString()}{totalPages != null ? ` / ${totalPages.toLocaleString()}` : ''} + + + +
+
+ )} + + {editMode && editable && ( +
+ + Edit cells, then click on a row to save. + {sample?.cdc && ' Changes on CDC sources appear in Live Changes within seconds.'} + +
+ )} +
+ ) +} diff --git a/ui/src/components/features/DataFlowView.tsx b/ui/src/components/features/DataFlowView.tsx index 8f7cd8a..8da37a3 100644 --- a/ui/src/components/features/DataFlowView.tsx +++ b/ui/src/components/features/DataFlowView.tsx @@ -1,7 +1,8 @@ import { useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from 'react' -import { GitBranch, Lock, LockOpen, Play, RefreshCw, ShieldAlert, ShieldCheck, Loader2, Bot } from 'lucide-react' +import { GitBranch, Lock, LockOpen, Play, Pause, Square, RefreshCw, ShieldAlert, ShieldCheck, Loader2, Bot } from 'lucide-react' import type { DataflowEdge, DataflowGraph, DataflowNode } from '../../types' -import { fetchDataflow, runDataflowMovement, toggleEtlAgent, fetchAgentOpsStatus, setPiiMask } from '../../lib/api' +import { fetchDataflow, runDataflowMovement, toggleEtlAgent, toggleCustodianOffload, fetchAgentOpsStatus, setPiiMask, setStreamingFlow } from '../../lib/api' +import { SparkKafkaPanel } from './SparkKafkaPanel' import { Badge } from '../ui/Badge' import { cn } from '../../lib/utils' @@ -14,6 +15,7 @@ const NODE_KIND: Record = { stream: { ring: 'border-cyan-400/60', chip: 'bg-cyan-500/15 text-cyan-300 border-cyan-400/40', dot: '#22d3ee' }, sink: { ring: 'border-sky-400/60', chip: 'bg-sky-500/15 text-sky-300 border-sky-400/40', dot: '#38bdf8' }, lakehouse: { ring: 'border-blue-400/60', chip: 'bg-blue-500/15 text-blue-300 border-blue-400/40', dot: '#60a5fa' }, + compute: { ring: 'border-violet-400/60', chip: 'bg-violet-500/15 text-violet-300 border-violet-400/40', dot: '#a78bfa' }, engine: { ring: 'border-violet-400/60', chip: 'bg-violet-500/15 text-violet-300 border-violet-400/40', dot: '#a78bfa' }, governance: { ring: 'border-fuchsia-400/60', chip: 'bg-fuchsia-500/15 text-fuchsia-300 border-fuchsia-400/40', dot: '#d946ef' }, } @@ -70,6 +72,7 @@ export function DataFlowView() { const [selected, setSelected] = useState(null) const [triggering, setTriggering] = useState(null) const [etlEnabled, setEtlEnabled] = useState(null) + const [custEnabled, setCustEnabled] = useState(null) const canvasRef = useRef(null) const nodeRefs = useRef>({}) @@ -91,6 +94,8 @@ export function DataFlowView() { fetchAgentOpsStatus().then((s) => { const etl = (s as { etl?: { enabled?: boolean } })?.etl if (etl) setEtlEnabled(!!etl.enabled) + const cust = (s as { custodian?: { enabled?: boolean } })?.custodian + if (cust) setCustEnabled(!!cust.enabled) }) const iv = setInterval(() => load(), 6000) return () => clearInterval(iv) @@ -152,12 +157,24 @@ export function DataFlowView() { await toggleEtlAgent(next) }, [etlEnabled]) + const onToggleCust = useCallback(async () => { + const next = !custEnabled + setCustEnabled(next) + await toggleCustodianOffload(next) + }, [custEnabled]) + const onRefresh = useCallback(async () => { setRefreshing(true) await load(true) setTimeout(() => setRefreshing(false), 400) }, [load]) + const flowMode = (graph as unknown as { flow?: string })?.flow ?? 'running' + const onFlow = useCallback(async (action: 'pause' | 'resume' | 'stop') => { + await setStreamingFlow(action) + setTimeout(() => load(true), 300) + }, [load]) + const [maskBusy, setMaskBusy] = useState(null) const onToggleMask = useCallback(async (key: string, column: string, masked: boolean) => { setMaskBusy(`${key}.${column}`) @@ -202,7 +219,7 @@ export function DataFlowView() {

Data Flow · live lineage

- Generators → sources → CDC/Kafka → lakehouse · click a node for PII detail · trigger movements below + Generators → sources → CDC/Kafka → Spark → lakehouse · pulses show live flow · Spark/Kafka panel below

@@ -240,6 +257,33 @@ export function DataFlowView() { > ETL agent {etlEnabled == null ? '' : etlEnabled ? 'on' : 'off'} + +
+ + + +
+ {onOpenPlatform && ( + + )} +
+ + )} + + {showSourceTabs && ( +
+ {DATA_GEN_SOURCES.map((s) => { + const Icon = s.icon + return ( + + ) + })} +
+ )} + +
+
+ {embedded && ( +
+ + +
+ )} + +
+
+ +

{meta.label}

+ {meta.cdc ? ( + CDC active + ) : ( + no CDC stream + )} + {agentMap[active] && ( + + {agentMap[active].agent_name} + + )} +
+

{meta.desc} Target: {meta.target}

+ +
+ + + + {active !== 'all' && COUNT_KEYS.includes(active) && ( +
+ Current rows: {counts[active]?.toLocaleString() ?? '…'} +
+ )} +
+ {msg &&

{msg}

} +
+ +
+

Recent runs — {meta.label}

+ {latest ? ( + + + + + + + + + + + {(runs[active] || []).map((run) => ( + + + + + + + ))} + +
StateRowsStartEnd
{stateBadge(run.state)}{run.conf?.rows ?? '—'}{run.start?.slice(11, 19) || '—'}{run.end?.slice(11, 19) || '—'}
+ ) : ( +

No runs yet.

+ )} +
+ + {!embedded && ( + <> +
+

Live counts (Trino)

+
+ {COUNT_KEYS.map((k) => ( +
+
{k}
+
{counts[k]?.toLocaleString() ?? '…'}
+
+ ))} +
+
+ +
+

+ Agent activity +

+ {activity.length === 0 ? ( +

No agent actions logged yet.

+ ) : ( +
    + {activity.map((a) => ( +
  • + {a.ts?.slice(11, 19) || ''} + + {a.agent_name} + + {a.message} +
    • + ))} +
+ )} +
+ + )} + + {embedded && ( +
+
+

Live counts (Trino)

+
+ {COUNT_KEYS.map((k) => ( +
+
{k}
+
{counts[k]?.toLocaleString() ?? '…'}
+
+ ))} +
+
+
+

+ Agent activity +

+ {activity.length === 0 ? ( +

No agent actions yet.

+ ) : ( +
    + {activity.slice(0, 8).map((a) => ( +
  • + {a.ts?.slice(11, 19) || ''} + {a.message} +
    • + ))} +
+ )} +
+
+ )} +
+
+
+ ) +} diff --git a/ui/src/components/features/DataGenView.tsx b/ui/src/components/features/DataGenView.tsx index b35cf9c..0db13a6 100644 --- a/ui/src/components/features/DataGenView.tsx +++ b/ui/src/components/features/DataGenView.tsx @@ -1,303 +1,11 @@ -import { useCallback, useEffect, useRef, useState } from 'react' -import { Database, Boxes, Activity, Cpu, Network, Layers, Play, Loader2, CheckCircle2, XCircle, RefreshCw, Bot, ScrollText } from 'lucide-react' -import { cn } from '../../lib/utils' -import { subTabActive, subTabIdle } from '../../lib/tabActive' - -type SourceKey = 'all' | 'postgres' | 'mysql' | 'mongodb' | 'cassandra' | 'neo4j' - -type SourceMeta = { - key: SourceKey - label: string - icon: typeof Database - accent: string - target: string - cdc: boolean - desc: string - defaultRows: number -} - -const SOURCES: SourceMeta[] = [ - { key: 'all', label: 'All sources', icon: Layers, accent: 'text-violet-400', target: 'alle 5 databases', cdc: true, desc: 'Genereer tegelijk in alle databases.', defaultRows: 5000 }, - { key: 'postgres', label: 'PostgreSQL', icon: Database, accent: 'text-sky-400', target: 'sales_orders', cdc: true, desc: 'Sales orders. CDC via Debezium -> Kafka -> S3.', defaultRows: 5000 }, - { key: 'mysql', label: 'MySQL', icon: Database, accent: 'text-amber-400', target: 'employee_events', cdc: true, desc: 'HR employee events. CDC via Debezium -> Kafka -> S3.', defaultRows: 5000 }, - { key: 'mongodb', label: 'MongoDB', icon: Boxes, accent: 'text-emerald-400', target: 'supplychain.events', cdc: true, desc: 'Supply chain events. CDC via Debezium -> Kafka -> S3.', defaultRows: 5000 }, - { key: 'cassandra', label: 'Cassandra', icon: Activity, accent: 'text-cyan-400', target: 'device_metrics', cdc: false, desc: 'Telemetry metrics. Zichtbaar via Trino.', defaultRows: 5000 }, - { key: 'neo4j', label: 'Neo4j', icon: Network, accent: 'text-pink-400', target: 'Product/Supplier graph', cdc: false, desc: 'Graafdata (producten, leveranciers, relaties).', defaultRows: 2000 }, -] - -type RunInfo = { run_id?: string; state?: string; start?: string; end?: string; conf?: { rows?: number } } -type AgentInfo = { agent_id: string; agent_name: string } -type ActivityItem = { id: string; ts?: string; agent_id: string; agent_name: string; message: string; level: string } +import { DataGenPanel } from './DataGenPanel' type Props = { onPulse: () => void onOpenPlatform: () => void } -const COUNT_KEYS: SourceKey[] = ['postgres', 'mysql', 'mongodb', 'cassandra'] - +/** Standalone page wrapper — prefer Data Sources UI → Generate tab */ export function DataGenView({ onPulse, onOpenPlatform }: Props) { - const [active, setActive] = useState('all') - const [rows, setRows] = useState>( - Object.fromEntries(SOURCES.map((s) => [s.key, s.defaultRows])) as Record, - ) - const [busy, setBusy] = useState>({}) - const [runs, setRuns] = useState>({}) - const [counts, setCounts] = useState>({}) - const [msg, setMsg] = useState(null) - const [agentMap, setAgentMap] = useState>({}) - const [activity, setActivity] = useState([]) - const pollRef = useRef>>({}) - - const meta = SOURCES.find((s) => s.key === active)! - - const loadCounts = useCallback(async () => { - try { - const r = await fetch('/api/pipeline/sync') - const j = await r.json() - if (j.ok) setCounts(j.counts || {}) - } catch { /* */ } - }, []) - - const loadActivity = useCallback(async () => { - try { - const r = await fetch('/api/pipeline/activity?limit=25') - const j = await r.json() - if (j.ok) setActivity(j.activity || []) - } catch { /* */ } - }, []) - - const loadRuns = useCallback(async (source: SourceKey) => { - try { - const r = await fetch(`/api/pipeline/runs/${source}?limit=5`) - const j = await r.json() - if (j.ok) setRuns((prev) => ({ ...prev, [source]: j.runs || [] })) - return (j.runs || [])[0] as RunInfo | undefined - } catch { - return undefined - } - }, []) - - useEffect(() => { - loadCounts() - loadActivity() - SOURCES.forEach((s) => loadRuns(s.key)) - fetch('/api/pipeline/agents').then((r) => r.json()).then((j) => { if (j.ok) setAgentMap(j.agents || {}) }).catch(() => {}) - const act = setInterval(loadActivity, 8000) - const poll = pollRef.current - return () => { Object.values(poll).forEach(clearInterval); clearInterval(act) } - }, [loadCounts, loadRuns, loadActivity]) - - const startPolling = useCallback((source: SourceKey) => { - if (pollRef.current[source]) clearInterval(pollRef.current[source]) - pollRef.current[source] = setInterval(async () => { - const latest = await loadRuns(source) - if (latest && (latest.state === 'success' || latest.state === 'failed')) { - clearInterval(pollRef.current[source]) - delete pollRef.current[source] - setBusy((b) => ({ ...b, [source]: false })) - loadCounts() - if (latest.state === 'success') setMsg(`${source}: klaar — nieuwe data gegenereerd`) - else setMsg(`${source}: run mislukt — check Airflow logs`) - } - }, 3000) - }, [loadRuns, loadCounts]) - - const generate = useCallback(async (source: SourceKey, autonomous = false) => { - setMsg(null) - setBusy((b) => ({ ...b, [source]: true })) - onPulse() - try { - const r = await fetch(`/api/pipeline/generate/${source}`, { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify({ rows: rows[source], autonomous }), - }) - const j = await r.json() - if (!j.ok) { - setMsg(`Fout: ${j.error || 'kon niet triggeren'}`) - setBusy((b) => ({ ...b, [source]: false })) - return - } - const who = autonomous ? `${j.agent_name || 'Agent'} genereert zelf` : 'gestart' - setMsg(`${source}: ${who} (run ${String(j.run_id).slice(-8)})`) - startPolling(source) - setTimeout(loadActivity, 800) - } catch { - setMsg('API niet bereikbaar') - setBusy((b) => ({ ...b, [source]: false })) - } - }, [rows, onPulse, startPolling, loadActivity]) - - const stateBadge = (state?: string) => { - if (state === 'success') return success - if (state === 'failed') return failed - if (state === 'running' || state === 'queued') return {state} - return {state || '—'} - } - - const latest = (runs[active] || [])[0] - - return ( -
-
-
-

- Data Generation -

-

- Genereer per database nieuwe data en pulse de hele flow: bron -> Debezium -> Kafka -> S3 -

-
-
- - -
-
- -
- {SOURCES.map((s) => { - const Icon = s.icon - return ( - - ) - })} -
- -
-
-
-
- -

{meta.label}

- {meta.cdc ? ( - CDC actief - ) : ( - geen CDC-stream - )} - {agentMap[active] && ( - - {agentMap[active].agent_name} - - )} -
-

{meta.desc} Doel: {meta.target}

- -
- - - - {active !== 'all' && COUNT_KEYS.includes(active) && ( -
- Huidige rijen: {counts[active]?.toLocaleString() ?? '…'} -
- )} -
- {msg &&

{msg}

} -
- -
-

Recente runs — {meta.label}

- {latest ? ( - - - - - - - - - - - {(runs[active] || []).map((run) => ( - - - - - - - ))} - -
StateRowsStartEind
{stateBadge(run.state)}{run.conf?.rows ?? '—'}{run.start?.slice(11, 19) || '—'}{run.end?.slice(11, 19) || '—'}
- ) : ( -

Nog geen runs.

- )} -
- -
-

Live tellingen (Trino)

-
- {COUNT_KEYS.map((k) => ( -
-
{k}
-
{counts[k]?.toLocaleString() ?? '…'}
-
- ))} -
-
- -
-

- Agent-activiteit (wat de agents deden) -

- {activity.length === 0 ? ( -

Nog geen agent-acties gelogd.

- ) : ( -
    - {activity.map((a) => ( -
  • - {a.ts?.slice(11, 19) || ''} - - {a.agent_name} - - {a.message} -
    • - ))} -
- )} -
-
-
-
- ) + return } diff --git a/ui/src/components/features/DataHubView.tsx b/ui/src/components/features/DataHubView.tsx new file mode 100644 index 0000000..9c89be1 --- /dev/null +++ b/ui/src/components/features/DataHubView.tsx @@ -0,0 +1,60 @@ +import { useEffect, useState } from 'react' +import { Database, Server } from 'lucide-react' +import { DataSourcesView } from './DataSourcesView' +import { HadoopSourcesView } from './HadoopSourcesView' +import type { SourceEngine } from '../../lib/dataSourceCatalog' +import { cn } from '../../lib/utils' +import { subTabActive, subTabIdle } from '../../lib/tabActive' + +type HubTab = 'sources' | 'hadoop' + +type Props = { + focusEngine?: SourceEngine | null + initialTab?: HubTab + onPulse?: () => void +} + +export function DataHubView({ focusEngine, initialTab = 'sources', onPulse }: Props) { + const [tab, setTab] = useState(initialTab) + + useEffect(() => { + setTab(initialTab) + }, [initialTab]) + + return ( +
+
+ + + + {tab === 'sources' ? 'PostgreSQL · MySQL · MongoDB · Cassandra · Neo4j' : 'HDFS · Hive · Iceberg · Spark · Kafka pipeline'} + +
+
+ {tab === 'sources' ? ( + + ) : ( + + )} +
+
+ ) +} diff --git a/ui/src/components/features/DataSourcesView.tsx b/ui/src/components/features/DataSourcesView.tsx index 7e3edd8..183ee93 100644 --- a/ui/src/components/features/DataSourcesView.tsx +++ b/ui/src/components/features/DataSourcesView.tsx @@ -6,15 +6,20 @@ import { FolderTree, Loader2, Network, + Play, RefreshCw, Server, Table2, TerminalSquare, } from 'lucide-react' import { Badge } from '../ui/Badge' +import { DbBrandIcon, dbBrandColor } from '../ui/DbBrandIcon' +import { DataBrowserGrid } from './DataBrowserGrid' +import { DataGenPanel } from './DataGenPanel' import { DbShell } from './DbShell' import { Neo4jGraphView } from './Neo4jGraphView' import { SqlWorkbench } from './SqlWorkbench' +import { LakehouseWorkbench } from './SparkView' import { getSourceMeta, SOURCE_CATALOG, @@ -39,16 +44,33 @@ type SampleResponse = { row_count?: number elapsed_ms?: number error?: string + primary_keys?: string[] + cdc?: boolean + editable?: boolean + offset?: number + limit?: number + total_count?: number | null } type Props = { focusEngine?: SourceEngine | null + onPulse?: () => void } -const SUB_TABS: { id: SourceSubTab; label: string; icon: typeof FolderTree; neo4jOnly?: boolean }[] = [ +const ENGINE_CATALOG: Record = { + postgres: 'postgres_sales', + mysql: 'mysql_hr', + mongodb: 'mongodb_supplychain', + cassandra: 'cassandra_telemetry', + neo4j: undefined, +} + +const SUB_TABS: { id: SourceSubTab; label: string; icon: typeof FolderTree; neo4jOnly?: boolean; hideNeo4j?: boolean }[] = [ { id: 'browser', label: 'Browser', icon: FolderTree }, { id: 'graph', label: 'Graph', icon: Network, neo4jOnly: true }, { id: 'console', label: 'Query Console', icon: Database }, + { id: 'workbench', label: 'Workbench', icon: Activity, hideNeo4j: true }, + { id: 'generate', label: 'Generate', icon: Play }, { id: 'shell', label: 'Shell', icon: TerminalSquare }, ] @@ -59,7 +81,7 @@ function fmtCount(n?: number | null) { return String(n) } -export function DataSourcesView({ focusEngine }: Props) { +export function DataSourcesView({ focusEngine, onPulse }: Props) { const [active, setActive] = useState(focusEngine || 'postgres') const [subTab, setSubTab] = useState('browser') const [health, setHealth] = useState({}) @@ -68,6 +90,8 @@ export function DataSourcesView({ focusEngine }: Props) { const [selectedObject, setSelectedObject] = useState(null) const [sample, setSample] = useState(null) const [sampleLoading, setSampleLoading] = useState(false) + const [page, setPage] = useState(1) + const [pageSize, setPageSize] = useState(100) const meta = getSourceMeta(active) @@ -104,11 +128,14 @@ export function DataSourcesView({ focusEngine }: Props) { } }, []) - const loadSample = useCallback(async (engine: SourceEngine, obj: CatalogObject) => { + const loadSample = useCallback(async (engine: SourceEngine, obj: CatalogObject, pg = page, ps = pageSize) => { setSampleLoading(true) setSample(null) + const offset = (pg - 1) * ps try { - const r = await fetch(`/api/sql/sample/${engine}?object=${encodeURIComponent(obj.fqn)}&limit=50`) + const r = await fetch( + `/api/sql/sample/${engine}?object=${encodeURIComponent(obj.fqn)}&limit=${ps}&offset=${offset}`, + ) const j = await r.json() setSample(j) } catch { @@ -116,7 +143,7 @@ export function DataSourcesView({ focusEngine }: Props) { } finally { setSampleLoading(false) } - }, []) + }, [page, pageSize]) useEffect(() => { loadHealth() }, [loadHealth]) useEffect(() => { @@ -124,21 +151,26 @@ export function DataSourcesView({ focusEngine }: Props) { }, [active, subTab, loadCatalog]) useEffect(() => { - if (selectedObject && subTab === 'browser') loadSample(active, selectedObject) - }, [selectedObject, active, subTab, loadSample]) + setPage(1) + }, [selectedObject?.fqn, active]) + + useEffect(() => { + if (selectedObject && subTab === 'browser') loadSample(active, selectedObject, page, pageSize) + }, [selectedObject, active, subTab, page, pageSize, loadSample]) const refreshAll = () => { loadHealth() - if (subTab === 'browser') loadCatalog(active) - else if (subTab === 'graph' && active === 'neo4j') { /* Neo4jGraphView reloads itself */ } - else if (selectedObject) loadSample(active, selectedObject) + if (subTab === 'browser') { + loadCatalog(active) + if (selectedObject) loadSample(active, selectedObject, page, pageSize) + } else if (subTab === 'graph' && active === 'neo4j') { /* Neo4jGraphView reloads itself */ } + else if (selectedObject) loadSample(active, selectedObject, page, pageSize) } - const visibleSubTabs = SUB_TABS.filter((t) => !t.neo4jOnly || active === 'neo4j') + const visibleSubTabs = SUB_TABS.filter((t) => (!t.neo4jOnly || active === 'neo4j') && !(t.hideNeo4j && active === 'neo4j')) return (
- {/* Header */}

@@ -146,7 +178,7 @@ export function DataSourcesView({ focusEngine }: Props) { Data Sources UI

- Enterprise data browser — schema exploration, query console & interactive shells for all source databases + Browse, filter, edit & query all source databases — edits on CDC sources flow to Live Changes

-
- {/* Left rail — database cards */} - - - {/* Main panel */} -
- {/* Engine header */} -
-
-

- - {meta.label} - {catalog?.version && ( - v{catalog.version.split(' ')[0]?.slice(0, 20)} + {/* Horizontal database selector */} +
+

Source Databases

+
+ {SOURCE_CATALOG.map((src) => { + const up = health[src.engine]?.ok + const selected = active === src.engine + const brand = dbBrandColor(src.engine) + return ( +

-

- {meta.host}:{meta.port} · {meta.database} · container {meta.container} -

-
-
- {visibleSubTabs.map(({ id, label, icon: Icon }) => ( - - ))} -
-
- - {/* Sub-tab content */} -
- {subTab === 'browser' && ( -
- {/* Object tree */} -
-
- Objects - {catalogLoading && } -
-
- {catalog?.objects?.map((obj) => ( - - ))} - {!catalogLoading && !catalog?.objects?.length && ( -

No objects found

+ style={selected ? { borderColor: brand, backgroundColor: `${brand}18`, boxShadow: `0 0 0 1px ${brand}40` } : undefined} + > +
+ + + {src.label} + + + title={up ? 'Online' : up === false ? 'Offline' : 'Unknown'} + />
+

{src.description}

+
+ {src.host}:{src.port} + {src.cdc && CDC} +
+ + ) + })} +
+
- {/* Sample data grid */} -
-
- - {selectedObject ? ( - <>Sample: {selectedObject.fqn} - ) : 'Select an object'} - - {sampleLoading && } -
-
- {sample?.ok && sample.columns && ( - <> -

- {sample.row_count} rows · {sample.elapsed_ms}ms -

- - - - {sample.columns.map((c) => )} - - - - {sample.rows?.map((row, i) => ( - - {row.map((cell, j) => ( - - ))} - - ))} - -
{c}
- {cell === null || cell === undefined ? 'NULL' : String(cell)} -
- - )} - {sample && !sample.ok && ( -

{sample.error || 'Failed to load sample'}

- )} - {!selectedObject && !sampleLoading && ( -

Select a table, collection or label to preview data

- )} -
+ {/* Main panel — full width */} +
+
+
+

+ + {meta.label} + {catalog?.version && ( + v{catalog.version.split(' ')[0]?.slice(0, 20)} + )} +

+

+ {meta.host}:{meta.port} · {meta.database} · container {meta.container} +

+
+
+ {visibleSubTabs.map(({ id, label, icon: Icon }) => ( + + ))} +
+
+ +
+ {subTab === 'browser' && ( +
+
+
+ Objects + {catalogLoading && } +
+
+ {catalog?.objects?.map((obj) => ( + + ))} + {!catalogLoading && !catalog?.objects?.length && ( +

No objects found

+ )}
- )} - {subTab === 'graph' && active === 'neo4j' && ( - - )} +
+
+ + {selectedObject ? ( + <>Data: {selectedObject.fqn} + ) : 'Select an object'} + +
+ { setPageSize(ps); setPage(1) }} + onReload={() => selectedObject && loadSample(active, selectedObject, page, pageSize)} + /> +
+
+ )} - {subTab === 'console' && ( - - )} + {subTab === 'graph' && active === 'neo4j' && } - {subTab === 'shell' && ( - - )} -
+ {subTab === 'console' && } + + {subTab === 'workbench' && ( +
+ +
+ )} + + {subTab === 'generate' && onPulse && ( + + )} + + {subTab === 'shell' && }
diff --git a/ui/src/components/features/HadoopSourcesView.tsx b/ui/src/components/features/HadoopSourcesView.tsx new file mode 100644 index 0000000..4e27f2c --- /dev/null +++ b/ui/src/components/features/HadoopSourcesView.tsx @@ -0,0 +1,280 @@ +import { useCallback, useEffect, useState } from 'react' +import { + Activity, + ChevronRight, + FolderTree, + GitBranch, + Loader2, + Play, + RefreshCw, + Server, + Table2, + TerminalSquare, + Zap, +} from 'lucide-react' +import { Badge } from '../ui/Badge' +import { DataBrowserGrid } from './DataBrowserGrid' +import { HdfsView } from './HdfsView' +import { SparkView } from './SparkView' +import { SqlWorkbench } from './SqlWorkbench' +import type { CatalogObject } from '../../lib/dataSourceCatalog' +import { cn } from '../../lib/utils' +import { subTabActive, subTabIdle } from '../../lib/tabActive' +import { runDataflowMovement, triggerStreamingPipeline } from '../../lib/api' + +type HadoopSubTab = 'browser' | 'files' | 'console' | 'spark' | 'pipeline' +type CatalogResponse = { engine: string; version?: string; objects: CatalogObject[] } +type SampleResponse = { + ok: boolean + columns?: string[] + rows?: unknown[][] + error?: string + total_count?: number | null + offset?: number + limit?: number + editable?: boolean +} + +const SUB_TABS: { id: HadoopSubTab; label: string; icon: typeof FolderTree }[] = [ + { id: 'browser', label: 'Tables', icon: Table2 }, + { id: 'files', label: 'HDFS Files', icon: FolderTree }, + { id: 'console', label: 'Query Console', icon: TerminalSquare }, + { id: 'spark', label: 'Spark', icon: Activity }, + { id: 'pipeline', label: 'Pipeline', icon: GitBranch }, +] + +export function HadoopSourcesView({ onPulse }: { onPulse?: () => void }) { + const [subTab, setSubTab] = useState('browser') + const [health, setHealth] = useState<{ ok?: boolean; error?: string } | null>(null) + const [catalog, setCatalog] = useState(null) + const [catalogLoading, setCatalogLoading] = useState(false) + const [selectedObject, setSelectedObject] = useState(null) + const [sample, setSample] = useState(null) + const [sampleLoading, setSampleLoading] = useState(false) + const [page, setPage] = useState(1) + const [pageSize, setPageSize] = useState(100) + const [pipelineBusy, setPipelineBusy] = useState(null) + const [pipelineMsg, setPipelineMsg] = useState(null) + + const loadHealth = useCallback(async () => { + try { + const r = await fetch('/api/sql/health/hadoop') + if (r.ok) setHealth(await r.json()) + } catch { /* */ } + }, []) + + const loadCatalog = useCallback(async () => { + setCatalogLoading(true) + try { + const r = await fetch('/api/sql/catalog/hadoop') + if (r.ok) { + const j: CatalogResponse = await r.json() + setCatalog(j) + const first = j.objects?.[0] + if (first) setSelectedObject(first) + } + } catch { /* */ } finally { + setCatalogLoading(false) + } + }, []) + + const loadSample = useCallback(async (obj: CatalogObject, pg = page, ps = pageSize) => { + setSampleLoading(true) + const offset = (pg - 1) * ps + try { + const r = await fetch( + `/api/sql/sample/hadoop?object=${encodeURIComponent(obj.fqn)}&limit=${ps}&offset=${offset}`, + ) + setSample(await r.json()) + } catch { + setSample({ ok: false, error: 'Sample unavailable' }) + } finally { + setSampleLoading(false) + } + }, [page, pageSize]) + + useEffect(() => { loadHealth() }, [loadHealth]) + useEffect(() => { + if (subTab === 'browser') loadCatalog() + }, [subTab, loadCatalog]) + useEffect(() => { setPage(1) }, [selectedObject?.fqn]) + useEffect(() => { + if (selectedObject && subTab === 'browser') loadSample(selectedObject, page, pageSize) + }, [selectedObject, subTab, page, pageSize, loadSample]) + + const refreshAll = () => { + loadHealth() + if (subTab === 'browser') { + loadCatalog() + if (selectedObject) loadSample(selectedObject, page, pageSize) + } + } + + const runPipeline = async (kind: 'full' | 'hdfs_kafka' | 'spark_s3') => { + setPipelineBusy(kind) + setPipelineMsg(null) + try { + if (kind === 'full') { + const r = await triggerStreamingPipeline('hadoop-lake') + const j = await r.json() + setPipelineMsg(j.ok ? `✓ Pipeline started: ${j.steps?.join(' → ') || 'ok'}` : j.error || 'Failed') + } else if (kind === 'hdfs_kafka') { + const r = await fetch('/api/pipeline/streaming/hdfs/to-kafka', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ source: 'trino', table: 'iceberg.hadoop.historical_sales_hdfs', topic: 'hdfs.historical.sales', limit: 2000 }), + }) + const j = await r.json() + setPipelineMsg(j.ok ? `✓ ${j.rows_sent} rows → Kafka topic ${j.topic}` : j.error || 'Failed') + } else { + await runDataflowMovement('spark_to_s3') + setPipelineMsg('✓ Spark → S3 job triggered') + } + onPulse?.() + } catch { + setPipelineMsg('Pipeline failed') + } finally { + setPipelineBusy(null) + } + } + + return ( +
+
+
+

+ + Hadoop Data Lake +

+

+ HDFS · Hive · Iceberg tables · Spark transforms · Kafka bridge → S3 +

+
+
+ + {health?.ok ? 'Hadoop online' : health?.error?.slice(0, 40) || 'Checking…'} + + +
+
+ +
+ {SUB_TABS.map(({ id, label, icon: Icon }) => ( + + ))} +
+ + {subTab === 'browser' && ( +
+ +
+ {selectedObject ? ( + { setPageSize(ps); setPage(1) }} + onReload={() => selectedObject && loadSample(selectedObject, page, pageSize)} + /> + ) : ( +

Select a table or HDFS path

+ )} +
+
+ )} + + {subTab === 'files' && } + {subTab === 'console' && } + {subTab === 'spark' && } + {subTab === 'pipeline' && ( +
+
+

Hadoop → Kafka → Spark → S3

+

+ Full lakehouse pipeline: export HDFS data to Kafka, Spark transforms into Iceberg/S3 curated layer. +

+
+
+ runPipeline('hdfs_kafka')} + /> + runPipeline('spark_s3')} + /> + runPipeline('full')} + /> +
+ {pipelineMsg &&

{pipelineMsg}

} +
+ hdfs:/data/historical/sales_orders → Kafka:hdfs.historical.sales → Spark → iceberg.hadoop → s3://data/hadoop/ +
+
+ )} +
+ ) +} + +function PipelineCard({ title, desc, busy, onRun }: { title: string; desc: string; busy: boolean; onRun: () => void }) { + return ( +
+

{title}

+

{desc}

+ +
+ ) +} diff --git a/ui/src/components/features/PlatformView.tsx b/ui/src/components/features/PlatformView.tsx new file mode 100644 index 0000000..dc6b391 --- /dev/null +++ b/ui/src/components/features/PlatformView.tsx @@ -0,0 +1,66 @@ +import { useState } from 'react' +import { LayoutDashboard, Presentation } from 'lucide-react' +import { PlatformTopology } from './PlatformTopology' +import { PresentationView } from './PresentationView' +import type { AgentAnim, WorkloadData } from '../../types' +import { cn } from '../../lib/utils' +import { subTabActive, subTabIdle } from '../../lib/tabActive' + +type PlatformTab = 'topology' | 'presentation' + +type Props = { + workload: WorkloadData | null + animations: Record + selectedNodeId: string | null + onNodeClick: (nodeId: string) => void + pulse: boolean +} + +const TABS: { id: PlatformTab; label: string; icon: typeof LayoutDashboard }[] = [ + { id: 'topology', label: 'Topology', icon: LayoutDashboard }, + { id: 'presentation', label: 'Presentation', icon: Presentation }, +] + +export function PlatformView({ workload, animations, selectedNodeId, onNodeClick, pulse }: Props) { + const [tab, setTab] = useState('topology') + + return ( +
+
+ {TABS.map(({ id, label, icon: Icon }) => ( + + ))} + {tab === 'presentation' && ( + + Live cluster deck · all running services + + )} +
+ +
+ {tab === 'topology' ? ( + + ) : ( + + )} +
+
+ ) +} diff --git a/ui/src/components/features/PresentationView.tsx b/ui/src/components/features/PresentationView.tsx index 3711b55..4f54f80 100644 --- a/ui/src/components/features/PresentationView.tsx +++ b/ui/src/components/features/PresentationView.tsx @@ -1,8 +1,11 @@ import { useCallback, useEffect, useRef, useState } from 'react' +import { createPortal } from 'react-dom' import { + Expand, ExternalLink, FileUp, ImagePlus, + Maximize2, Monitor, Pencil, Plus, @@ -17,6 +20,136 @@ import { cn } from '../../lib/utils' import { subTabActive, subTabIdle } from '../../lib/tabActive' type DeckSource = 'live' | 'data-maturity' | 'atc-platform' | string +type SlideVariant = 'embedded' | 'normal' | 'present' +type PresentMode = 'popup' | 'fullscreen' | null + +function SlidePanel({ + slide, + slideIdx, + slideCount, + variant, +}: { + slide: PresentationSlide + slideIdx: number + slideCount: number + variant: SlideVariant +}) { + const isPresent = variant === 'present' + const isEmbedded = variant === 'embedded' + + return ( +
+
+
+

+ {slide.kind || 'slide'} · {slideIdx + 1}/{slideCount} +

+

+ {slide.title} +

+ {slide.subtitle && ( +

+ {slide.subtitle} +

+ )} + {'animation' in slide && slide.animation && ( + + )} +
    + {(slide.bullets || []).map((b: string, bi: number) => ( +
  • ▸{b}
    • + ))} +
+
+ {slide.image && !isEmbedded && ( +
+ +
+ )} +
+ {slide.image && isPresent && ( +
+ +
+ )} + {slide.image && !isPresent && ( +
+ +
+ )} +
+ ) +} + +function SlideNav({ + slideIdx, + slideCount, + embedded, + onPrev, + onNext, + onGo, + className, +}: { + slideIdx: number + slideCount: number + embedded?: boolean + onPrev: () => void + onNext: () => void + onGo?: (idx: number) => void + className?: string +}) { + return ( +
+ + {embedded ? ( + {slideIdx + 1} / {slideCount} + ) : ( +
+ {Array.from({ length: slideCount }, (_, i) => ( +
+ )} + +
+ ) +} const KIND_STYLES: Record = { hero: 'from-blue-600/25 via-violet-600/20 to-emerald-600/15', @@ -47,7 +180,7 @@ async function fetchDeck(id: DeckSource): Promise { } } -export function PresentationView() { +export function PresentationView({ embedded = false }: { embedded?: boolean }) { const [source, setSource] = useState('live') const [data, setData] = useState(null) const [slideIdx, setSlideIdx] = useState(0) @@ -63,8 +196,14 @@ export function PresentationView() { const [saving, setSaving] = useState(false) const [imgBusy, setImgBusy] = useState(false) const imgInputRef = useRef(null) + const presentRef = useRef(null) + const suppressFsPopup = useRef(false) + + const [presentMode, setPresentMode] = useState(null) const isCustom = customDecks.some((d) => d.id === source) + const canEditInPlace = source === 'live' || isCustom + const liveEdited = source === 'live' && Boolean(data?.edited) const refreshDeckList = useCallback(async () => { try { @@ -105,20 +244,94 @@ export function PresentationView() { refreshDeckList() }, [source, load, refreshDeckList]) + const slides = data?.slides || [] + const slide: PresentationSlide | undefined = slides[slideIdx] + + const closePresent = useCallback(() => { + suppressFsPopup.current = true + setPresentMode(null) + if (document.fullscreenElement) { + void document.exitFullscreen() + } + window.setTimeout(() => { suppressFsPopup.current = false }, 0) + }, []) + + const openPresent = useCallback((mode: 'popup' | 'fullscreen') => { + if (!data?.slides?.length) return + setPresentMode(mode) + }, [data?.slides?.length]) + + const enterBrowserFullscreen = useCallback(async () => { + const el = presentRef.current + if (!el) return + try { + await el.requestFullscreen() + setPresentMode('fullscreen') + } catch { + setPresentMode('popup') + } + }, []) + useEffect(() => { - if (editing) return + if (editing || presentMode) return const onKey = (e: KeyboardEvent) => { const n = data?.slides.length || 1 if (e.key === 'ArrowRight' || e.key === ' ') { e.preventDefault(); setSlideIdx((i) => Math.min(n - 1, i + 1)) } if (e.key === 'ArrowLeft') setSlideIdx((i) => Math.max(0, i - 1)) - if (e.key === 'f' || e.key === 'F') document.documentElement.requestFullscreen?.() + if ((e.key === 'f' || e.key === 'F') && data?.slides?.length) openPresent('fullscreen') } window.addEventListener('keydown', onKey) return () => window.removeEventListener('keydown', onKey) - }, [data?.slides.length, editing]) + }, [data?.slides.length, editing, presentMode, openPresent]) - const slides = data?.slides || [] - const slide: PresentationSlide | undefined = slides[slideIdx] + useEffect(() => { + if (presentMode !== 'fullscreen') return + const id = window.requestAnimationFrame(() => { void enterBrowserFullscreen() }) + return () => window.cancelAnimationFrame(id) + }, [presentMode, enterBrowserFullscreen]) + + useEffect(() => { + const onFsChange = () => { + if (!document.fullscreenElement && !suppressFsPopup.current) { + setPresentMode((mode) => (mode === 'fullscreen' ? 'popup' : mode)) + } + } + document.addEventListener('fullscreenchange', onFsChange) + return () => document.removeEventListener('fullscreenchange', onFsChange) + }, []) + + useEffect(() => { + if (!presentMode) return + const prev = document.body.style.overflow + document.body.style.overflow = 'hidden' + const onKey = (e: KeyboardEvent) => { + const n = slides.length || 1 + if (e.key === 'Escape') { + e.preventDefault() + closePresent() + return + } + if (e.key === 'ArrowRight' || e.key === ' ') { + e.preventDefault() + setSlideIdx((i) => Math.min(n - 1, i + 1)) + } + if (e.key === 'ArrowLeft') setSlideIdx((i) => Math.max(0, i - 1)) + if (e.key === 'f' || e.key === 'F') { + e.preventDefault() + if (document.fullscreenElement) { + void document.exitFullscreen() + setPresentMode('popup') + } else { + void enterBrowserFullscreen() + } + } + } + window.addEventListener('keydown', onKey) + return () => { + document.body.style.overflow = prev + window.removeEventListener('keydown', onKey) + } + }, [presentMode, slides.length, closePresent, enterBrowserFullscreen]) const editIdx = editing ? slideIdx : null const editSlides = draft?.slides || [] @@ -221,6 +434,26 @@ export function PresentationView() { setDraft(null) } + const resetLive = async () => { + if (!window.confirm('Reset Live Cluster to a fresh cluster snapshot? Your edits will be lost.')) return + setUploadMsg(null) + try { + const r = await fetch('/api/presentation/live/reset', { method: 'POST' }) + const j = await r.json() + if (j.ok && j.deck) { + setData(j.deck) + setSlideIdx(0) + setEditing(false) + setDraft(null) + setUploadMsg('✓ Reset to cluster snapshot') + } else { + setUploadMsg(j.error || 'Reset failed') + } + } catch { + setUploadMsg('Reset failed — check connection') + } + } + const patchSlide = (idx: number, patch: Partial) => { setDraft((d) => { if (!d) return d @@ -279,74 +512,174 @@ export function PresentationView() { ] return ( -
-
-
-

Presentation

-

- Live cluster · HTML templates · PPT upload · editable decks with text & photos -

-
-
+ <> +
+ {embedded ? ( +
+
+ {!editing && ( + <> + + {canEditInPlace ? ( + + ) : ( + + )} + {liveEdited && ( + + )} + + DQ Portal + + + Docling + + + {!loading && slides.length > 0 && ( + <> + + + + )} + + + )} + {editing && ( + <> + Editing + + + + )} +
{!editing && ( - <> - - {isCustom ? ( - - ) : ( - - )} - - DQ Portal - - - Docling - - - - - )} - {editing && ( - <> - Editing - - - + ))} + +
)}
-
+ ) : ( + <> +
+
+

Presentation

+

+ Live cluster · HTML templates · PPT upload · editable decks with text & photos +

+
+
+ {!editing && ( + <> + + {canEditInPlace ? ( + + ) : ( + + )} + {liveEdited && ( + + )} + + DQ Portal + + + Docling + + + {!loading && slides.length > 0 && ( + <> + + + + )} + + + )} + {editing && ( + <> + Editing + + + + )} +
+
- {!editing && ( -
- {tabs.map((t) => ( - - ))} - -
+ {!editing && ( +
+ {tabs.map((t) => ( + + ))} + +
+ )} + )} {uploadMsg &&

{uploadMsg}

} @@ -493,45 +826,103 @@ export function PresentationView() {
) : ( /* ─────────── VIEW MODE ─────────── */ - <> -
-
-
-

{slide.kind || 'slide'} · {slideIdx + 1}/{slides.length}

-

{slide.title}

- {slide.subtitle &&

{slide.subtitle}

} - {'animation' in slide && slide.animation && ( - - )} -
    - {(slide.bullets || []).map((b: string, bi: number) => ( -
  • ▸{b}
    • - ))} -
-
- {slide.image && ( -
- -
- )} -
- {slide.image && ( -
- -
- )} +
+
+
-
- -
- {slides.map((_: PresentationSlide, i: number) => ( -
- -
- + setSlideIdx((i) => Math.max(0, i - 1))} + onNext={() => setSlideIdx((i) => Math.min(slides.length - 1, i + 1))} + onGo={setSlideIdx} + /> +
)}
+ {presentMode && slide && createPortal( + <> + {presentMode === 'popup' && ( + + ) : ( + + )} + +
+
+
+
+ +
+ setSlideIdx((i) => Math.max(0, i - 1))} + onNext={() => setSlideIdx((i) => Math.min(slides.length - 1, i + 1))} + onGo={setSlideIdx} + className="border-t border-border/80 bg-surface-raised/95 px-4 py-2.5" + /> +
+

+ ← → navigate · F toggle fullscreen · Esc close +

+
+ , + document.body, + )} + ) } diff --git a/ui/src/components/features/SparkKafkaPanel.tsx b/ui/src/components/features/SparkKafkaPanel.tsx new file mode 100644 index 0000000..e600d4b --- /dev/null +++ b/ui/src/components/features/SparkKafkaPanel.tsx @@ -0,0 +1,346 @@ +import { useCallback, useEffect, useState } from 'react' +import { + Cpu, + ExternalLink, + Loader2, + Pause, + Play, + RefreshCw, + RotateCcw, + Zap, +} from 'lucide-react' +import type { StreamingStatus } from '../../types' +import { + fetchStreamingStatus, + restartKafkaConnector, + pauseKafkaConnector, + resumeKafkaConnector, + triggerStreamingJob, +} from '../../lib/api' +import { cn } from '../../lib/utils' + +type Tab = 'spark' | 'kafka' | 'jobs' + +export function SparkKafkaPanel({ + embedded, + selectedNodeId, + streaming: initialStreaming, + onRefreshGraph, +}: { + embedded?: boolean + selectedNodeId?: string | null + streaming?: StreamingStatus | null + onRefreshGraph?: () => void +}) { + const [tab, setTab] = useState('spark') + const [streaming, setStreaming] = useState(initialStreaming ?? null) + const [loading, setLoading] = useState(!initialStreaming) + const [busy, setBusy] = useState(null) + const [jobConf, setJobConf] = useState>({}) + const [selectedJob, setSelectedJob] = useState('spark_to_curated') + const [showSparkUi, setShowSparkUi] = useState(false) + + const load = useCallback(async (refresh = false) => { + setLoading(true) + const s = await fetchStreamingStatus(refresh) + if (s) setStreaming(s) + setLoading(false) + }, []) + + useEffect(() => { + if (initialStreaming) setStreaming(initialStreaming) + }, [initialStreaming]) + + useEffect(() => { + if (selectedNodeId === 'spark') setTab('spark') + if (selectedNodeId === 'kafka') setTab('kafka') + }, [selectedNodeId]) + + useEffect(() => { + const iv = setInterval(() => load(), 8000) + return () => clearInterval(iv) + }, [load]) + + const spark = streaming?.spark + const kafka = streaming?.kafka + const jobs = streaming?.jobs ?? [] + + const onJob = async (jobId: string) => { + setBusy(`job:${jobId}`) + try { + let conf: Record = {} + const raw = jobConf[jobId] + if (raw?.trim()) { + conf = JSON.parse(raw) + } + await triggerStreamingJob(jobId, conf) + onRefreshGraph?.() + await load(true) + } catch { + /* ignore */ + } finally { + setBusy(null) + } + } + + const onConnector = async (name: string, action: 'restart' | 'pause' | 'resume') => { + setBusy(`conn:${name}:${action}`) + try { + if (action === 'restart') await restartKafkaConnector(name) + else if (action === 'pause') await pauseKafkaConnector(name) + else await resumeKafkaConnector(name) + await load(true) + onRefreshGraph?.() + } finally { + setBusy(null) + } + } + + return ( +
+
+
+ {(['spark', 'kafka', 'jobs'] as Tab[]).map((t) => ( + + ))} +
+
+ {spark?.ui_ok && ( + + Spark {spark.alive_workers}w · {spark.cores_used}/{spark.cores} cores + + )} + {kafka?.connect_ok && ( + + {kafka.connectors?.length ?? 0} connectors + + )} + +
+
+ +
+ {tab === 'spark' && ( +
+
+ + + + Open full Spark UI + +
+ + {showSparkUi && ( +