feat: Trino federation + Hadoop external tables + LLM data catalog

Trino Federation tab (3 sub-views):
- Federated: catalog landscape + a single cross-source SQL that joins
  PostgreSQL + MySQL + MongoDB (region scorecard) — the federation proof,
  computed in the background and cached (large full scans take ~2 min).
- Hadoop Lake: all federated business data materialized as external Iceberg
  tables on HDFS (iceberg.hadoop.*_ext, ~120k rows each) with live, fast
  business analytics (revenue by region/channel, top customers, HR by
  department, supply by type, telemetry averages). Includes a one-click
  "rebuild external tables" job.
- Data Dictionary: every business table + column with masked / visible PII
  badges and categories.

Backend trino_federated.py: /catalogs, /marquee(+refresh), /lake,
/materialize(+status), /dictionary. Name-based PII detection flags raw PII
in derived/lake tables as visible vs physically-masked curated layer.

LLM context: platform_context now emits a full BUSINESS DATA CATALOG section
(tables, columns, types, source row counts, federated scorecard) with exact
per-column masked/visible status, so the assistant knows the data in detail
and what is masked vs not.
This commit is contained in:
mo
2026-06-28 18:01:25 +00:00
parent 8d28695868
commit ea3e59cf9c
8 changed files with 855 additions and 5 deletions
+1 -1
View File
@@ -4,7 +4,7 @@ WORKDIR /app
RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/* RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/*
COPY requirements.txt . COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt RUN pip install --no-cache-dir -r requirements.txt
COPY main.py lab_context.py agent_terminal.py workload.py node_registry.py node_ops.py topology_views.py supervisor.py approval_service.py db.py dockhand_envs.py presentation.py database_inventory.py presentation_upload.py presentation_static.py storage_s3.py elasticsearch_api.py sql_console.py hdfs_api.py ssh_terminal.py pipeline_ops.py hadoop_analytics.py agent_ops.py cdc_consumer.py movements.py dataflow.py streaming_ops.py spark_workbench.py hadoop_sql.py hdfs_kafka.py webhdfs_util.py pii_catalog.py platform_context.py hive_bench_seed.json . COPY main.py lab_context.py agent_terminal.py workload.py node_registry.py node_ops.py topology_views.py supervisor.py approval_service.py db.py dockhand_envs.py presentation.py database_inventory.py presentation_upload.py presentation_static.py storage_s3.py elasticsearch_api.py sql_console.py hdfs_api.py ssh_terminal.py pipeline_ops.py hadoop_analytics.py agent_ops.py cdc_consumer.py movements.py dataflow.py streaming_ops.py spark_workbench.py hadoop_sql.py hdfs_kafka.py webhdfs_util.py pii_catalog.py platform_context.py trino_federated.py hive_bench_seed.json .
RUN mkdir -p /data RUN mkdir -p /data
ENV DATABASE_URL=sqlite:////data/atc-agents.db ENV DATABASE_URL=sqlite:////data/atc-agents.db
EXPOSE 3201 EXPOSE 3201
+2
View File
@@ -53,6 +53,7 @@ from dataflow import router as dataflow_router
from streaming_ops import router as streaming_router from streaming_ops import router as streaming_router
from spark_workbench import router as spark_workbench_router from spark_workbench import router as spark_workbench_router
from pii_catalog import router as pii_router from pii_catalog import router as pii_router
from trino_federated import router as federated_router
from ssh_terminal import ssh_session from ssh_terminal import ssh_session
from node_registry import NODE_IDS, NODE_AGENT, NODE_REGISTRY, is_node_id from node_registry import NODE_IDS, NODE_AGENT, NODE_REGISTRY, is_node_id
from node_ops import build_node_detail, probe_node, run_node_probe_task from node_ops import build_node_detail, probe_node, run_node_probe_task
@@ -757,6 +758,7 @@ app.include_router(dataflow_router)
app.include_router(streaming_router) app.include_router(streaming_router)
app.include_router(spark_workbench_router) app.include_router(spark_workbench_router)
app.include_router(pii_router) app.include_router(pii_router)
app.include_router(federated_router)
app.add_middleware( app.add_middleware(
CORSMiddleware, CORSMiddleware,
allow_origins=["*"], allow_origins=["*"],
+72 -1
View File
@@ -132,13 +132,84 @@ def build_masking_section() -> str:
return "\n".join(lines) return "\n".join(lines)
def build_business_data_section() -> str:
"""Detailed inventory of the actual business data — tables, columns, types,
row counts and per-column masked/visible status — so the assistant knows the
data in detail and exactly what is masked vs not."""
lines: list[str] = ["=== BUSINESS DATA CATALOG (live, every detail) ==="]
# source row totals (cheap estimates)
try:
import sql_console as s
totals = {
"PostgreSQL sales_orders": s._table_row_count("postgres", "public.sales_orders"),
"MySQL employee_events": s._table_row_count("mysql", "hr.employee_events"),
"MongoDB supplychain.events": s._table_row_count("mongodb", "supplychain.events"),
}
lines.append("Source volumes (live row counts):")
for k, v in totals.items():
lines.append(f" - {k}: {v:,} rows" if isinstance(v, int) else f" - {k}: ~")
except Exception:
pass
# full column dictionary + masking per column
try:
from trino_federated import build_dictionary
d = build_dictionary()
summ = d.get("summary", {})
lines.append(
f"Catalogued tables: {summ.get('tables', 0)} — columns: {summ.get('columns', 0)}, "
f"PII columns: {summ.get('pii_columns', 0)} ({summ.get('masked_columns', 0)} masked)."
)
for t in d.get("tables", []):
lines.append(f"\n{t['engine']} · {t['fqn']}{t['desc']}")
for c in t.get("columns", []):
flag = ""
if c.get("masked"):
flag = f" [MASKED · {c.get('category')}]"
elif c.get("pii"):
flag = f" [PII visible · {c.get('category')}]"
lines.append(f" · {c['name']} ({c['type']}){flag}")
except Exception as exc:
lines.append(f"(data dictionary unavailable: {exc})")
# federated cross-source marquee (region scorecard), if computed
try:
from trino_federated import _marquee, _load_marquee
if _marquee.get("data") is None:
_load_marquee()
m = _marquee.get("data")
if m and m.get("rows"):
lines.append("\nFederated region scorecard (one Trino SQL across PostgreSQL+MySQL+MongoDB):")
for r in m["rows"][:8]:
rev = r.get("revenue")
rev_s = f"{rev:,.0f}" if isinstance(rev, (int, float)) else "?"
lines.append(
f" - {r.get('region')}: orders={r.get('orders')}, revenue={rev_s}, "
f"hr_events={r.get('hr_events')}, supply_events={r.get('supply_events')}"
)
except Exception:
pass
lines.append(
"\nData is also materialized into the Hadoop lake as external Iceberg tables "
"(iceberg.hadoop.*_ext) and exposed through one federated Trino engine "
"(catalogs: postgres_sales, mysql_hr, mongodb_supplychain, cassandra_telemetry, iceberg, kafka)."
)
return "\n".join(lines)
def build_llm_addendum() -> str: def build_llm_addendum() -> str:
try: try:
platform = build_platform_section() platform = build_platform_section()
except Exception as exc: except Exception as exc:
platform = f"(platform section error: {exc})" platform = f"(platform section error: {exc})"
try:
business = build_business_data_section()
except Exception as exc:
business = f"(business data section error: {exc})"
try: try:
masking = build_masking_section() masking = build_masking_section()
except Exception as exc: except Exception as exc:
masking = f"(masking section error: {exc})" masking = f"(masking section error: {exc})"
return "\n\n".join([platform, masking]) return "\n\n".join([platform, business, masking])
+400
View File
@@ -0,0 +1,400 @@
"""Trino federated business analytics + Hadoop lakehouse (external Iceberg tables).
Provides:
- /api/federated/catalogs : the federated catalog landscape (estimates, fast)
- /api/federated/marquee : a single cross-source SQL joining PG+MySQL+Mongo
(the federation proof) — computed in the background, cached
- /api/federated/lake : business analytics over the Hadoop Iceberg lake
(iceberg.hadoop external tables) — live & fast
- /api/federated/materialize: (re)build the Hadoop external tables from the sources
- /api/federated/dictionary : business data dictionary with masked / visible flags
(also fed to the LLM so it knows the data in detail)
"""
from __future__ import annotations
import json
import threading
import time
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from fastapi import APIRouter
from fastapi.responses import JSONResponse
router = APIRouter(prefix="/api/federated", tags=["federated"])
BUSINESS_CATALOGS = ["postgres_sales", "mysql_hr", "mongodb_supplychain", "cassandra_telemetry", "iceberg"]
ALL_CATALOGS = BUSINESS_CATALOGS + ["kafka", "system"]
# Hadoop external tables (Iceberg-on-HDFS) materialized from the federated sources.
# Each: target table name in iceberg.hadoop -> source SELECT (timestamps cast to
# timestamp(6) which Iceberg requires; measures cast to double).
LAKE_SPECS: list[tuple[str, str, str]] = [
("orders_ext", "Sales orders (PostgreSQL)",
"SELECT order_id, customer_id, customer_name, customer_email, product_id, region, "
"sales_channel, order_status, currency, CAST(amount AS double) amount, "
"CAST(order_ts AS timestamp(6)) order_ts "
"FROM postgres_sales.public.sales_orders WHERE customer_name IS NOT NULL LIMIT 120000"),
("employees_ext", "Employee events (MySQL)",
"SELECT employee_id, employee_name, employee_email, employee_phone, department, role_name, "
"region, event_type, CAST(salary_change AS double) salary_change, "
"CAST(event_ts AS timestamp(6)) event_ts "
"FROM mysql_hr.hr.employee_events WHERE employee_name IS NOT NULL LIMIT 120000"),
("supply_events_ext", "Supply chain events (MongoDB)",
"SELECT event_id, type, region, source, CAST(amount AS double) amount, "
"CAST(ts AS timestamp(6)) ts FROM mongodb_supplychain.supplychain.events LIMIT 120000"),
("telemetry_ext", "Device telemetry (Cassandra)",
"SELECT device_id, CAST(metric_ts AS timestamp(6)) metric_ts, metric_type, "
"CAST(metric_value AS double) metric_value FROM cassandra_telemetry.telemetry.device_metrics LIMIT 120000"),
("customers_ext", "Distinct customers (PostgreSQL)",
"SELECT DISTINCT customer_id, customer_name, customer_email, region, sales_channel "
"FROM postgres_sales.public.sales_orders WHERE customer_name IS NOT NULL LIMIT 60000"),
("products_ext", "Distinct products (PostgreSQL)",
"SELECT DISTINCT product_id, region, sales_channel FROM postgres_sales.public.sales_orders "
"WHERE product_id IS NOT NULL LIMIT 20000"),
]
MARQUEE_SQL = (
"WITH o AS (SELECT region, count(*) orders, sum(amount) revenue "
"FROM postgres_sales.public.sales_orders GROUP BY region),\n"
" e AS (SELECT region, count(*) hr_events FROM mysql_hr.hr.employee_events GROUP BY region),\n"
" s AS (SELECT region, count(*) supply_events FROM mongodb_supplychain.supplychain.events GROUP BY region)\n"
"SELECT COALESCE(o.region, e.region, s.region) AS region,\n"
" o.orders, o.revenue, e.hr_events, s.supply_events\n"
"FROM o FULL JOIN e ON o.region = e.region\n"
" FULL JOIN s ON COALESCE(o.region, e.region) = s.region\n"
"ORDER BY o.revenue DESC NULLS LAST"
)
def _trino(sql: str, limit: int = 500) -> dict[str, Any]:
import sql_console as s
return s._run_trino(sql, limit)
def _rows_as_dicts(res: dict) -> list[dict]:
cols = res.get("columns", [])
return [{cols[i]: r[i] for i in range(min(len(cols), len(r)))} for r in res.get("rows", [])]
# ──────────────────────────────────────────────────────────────────────────────
# Marquee federated query (cross-source) — background cached
# ──────────────────────────────────────────────────────────────────────────────
MARQUEE_PATH = Path("/data/federated_marquee.json")
_MARQUEE_TTL = 1800.0
_marquee: dict[str, Any] = {"running": False, "data": None}
_marquee_lock = threading.Lock()
def _load_marquee() -> None:
try:
if MARQUEE_PATH.exists():
_marquee["data"] = json.loads(MARQUEE_PATH.read_text())
except Exception:
pass
def _marquee_worker() -> None:
try:
a = time.time()
res = _trino(MARQUEE_SQL, 50)
elapsed = int((time.time() - a) * 1000)
data = {
"ok": res.get("ok", False),
"sql": MARQUEE_SQL,
"catalogs": ["postgres_sales", "mysql_hr", "mongodb_supplychain"],
"elapsed_ms": elapsed,
"rows": _rows_as_dicts(res) if res.get("ok") else [],
"error": None if res.get("ok") else str(res.get("error", ""))[:300],
"generated_at": datetime.now(timezone.utc).isoformat(),
}
_marquee["data"] = data
try:
MARQUEE_PATH.write_text(json.dumps(data))
except Exception:
pass
finally:
_marquee["running"] = False
def _ensure_marquee(force: bool = False) -> None:
with _marquee_lock:
if _marquee["running"]:
return
data = _marquee["data"]
fresh = data and (time.time() - _ts(data.get("generated_at")) < _MARQUEE_TTL)
if fresh and not force:
return
_marquee["running"] = True
threading.Thread(target=_marquee_worker, daemon=True).start()
def _ts(iso: str | None) -> float:
if not iso:
return 0.0
try:
return datetime.fromisoformat(iso).timestamp()
except Exception:
return 0.0
@router.get("/marquee")
async def get_marquee():
if _marquee["data"] is None:
_load_marquee()
_ensure_marquee()
data = _marquee["data"]
return {
"ok": True,
"running": _marquee["running"],
"marquee": data,
"sql": MARQUEE_SQL,
}
@router.post("/marquee/refresh")
async def refresh_marquee():
_ensure_marquee(force=True)
return {"ok": True, "running": True}
# ──────────────────────────────────────────────────────────────────────────────
# Catalog landscape (fast — SHOW + planner estimates)
# ──────────────────────────────────────────────────────────────────────────────
@router.get("/catalogs")
async def get_catalogs():
import sql_console as s
cat_res = _trino("SHOW CATALOGS", 100)
catalogs = [r[0] for r in cat_res.get("rows", [])] if cat_res.get("ok") else ALL_CATALOGS
totals = {
"orders": s._table_row_count("postgres", "public.sales_orders"),
"hr_events": s._table_row_count("mysql", "hr.employee_events"),
"supply_events": s._table_row_count("mongodb", "supplychain.events"),
}
out = []
labels = {
"postgres_sales": ("PostgreSQL", "Sales / orders OLTP", "#336791"),
"mysql_hr": ("MySQL", "HR / workforce", "#00758f"),
"mongodb_supplychain": ("MongoDB", "Supply chain events", "#4db33d"),
"cassandra_telemetry": ("Cassandra", "Device telemetry", "#1287b1"),
"iceberg": ("Iceberg / Hadoop", "Lakehouse external tables", "#5b8def"),
"kafka": ("Kafka", "CDC + streaming topics", "#231f20"),
"system": ("Trino", "Engine metadata", "#dd00a1"),
}
for c in catalogs:
lbl, desc, color = labels.get(c, (c, "", "#888"))
item: dict[str, Any] = {"catalog": c, "label": lbl, "desc": desc, "color": color, "business": c in BUSINESS_CATALOGS}
if c == "postgres_sales":
item["rows"] = totals["orders"]
elif c == "mysql_hr":
item["rows"] = totals["hr_events"]
elif c == "mongodb_supplychain":
item["rows"] = totals["supply_events"]
out.append(item)
return {"ok": True, "catalogs": out, "source_totals": totals, "count": len(out)}
# ──────────────────────────────────────────────────────────────────────────────
# Hadoop lake analytics (live over the small materialized external tables)
# ──────────────────────────────────────────────────────────────────────────────
def _terms(sql: str, key_col: str, count_col: str = "c", val_col: str | None = None) -> list[dict]:
res = _trino(sql, 100)
if not res.get("ok"):
return []
cols = res.get("columns", [])
idx = {c: i for i, c in enumerate(cols)}
out = []
for r in res.get("rows", []):
item = {"key": r[idx[key_col]] if key_col in idx else None,
"count": r[idx[count_col]] if count_col in idx else 0}
if val_col and val_col in idx:
item["value"] = round(float(r[idx[val_col]] or 0), 2)
out.append(item)
return out
def _lake_table_exists(name: str) -> bool:
res = _trino(f"SELECT count(*) FROM iceberg.hadoop.{name}", 1)
return res.get("ok", False)
@router.get("/lake")
async def get_lake():
# which external tables are present
tbl_res = _trino("SHOW TABLES FROM iceberg.hadoop", 200)
tables = [r[0] for r in tbl_res.get("rows", [])] if tbl_res.get("ok") else []
table_stats = []
for t in tables:
cnt = _trino(f"SELECT count(*) FROM iceberg.hadoop.{t}", 1)
n = cnt.get("rows", [[0]])[0][0] if cnt.get("ok") else 0
table_stats.append({"table": t, "rows": n})
has_orders = "orders_ext" in tables
has_emp = "employees_ext" in tables
has_sup = "supply_events_ext" in tables
has_tel = "telemetry_ext" in tables
orders = {}
if has_orders:
orders = {
"by_region": _terms("SELECT region, count(*) c, sum(amount) rev FROM iceberg.hadoop.orders_ext GROUP BY region ORDER BY rev DESC", "region", "c", "rev"),
"by_channel": _terms("SELECT sales_channel k, count(*) c, sum(amount) rev FROM iceberg.hadoop.orders_ext GROUP BY sales_channel ORDER BY rev DESC", "k", "c", "rev"),
"by_status": _terms("SELECT order_status k, count(*) c FROM iceberg.hadoop.orders_ext GROUP BY order_status ORDER BY c DESC", "k", "c"),
"top_customers": _terms("SELECT customer_name k, count(*) c, sum(amount) rev FROM iceberg.hadoop.orders_ext GROUP BY customer_name ORDER BY rev DESC LIMIT 10", "k", "c", "rev"),
"revenue": (_trino("SELECT sum(amount), avg(amount) FROM iceberg.hadoop.orders_ext", 1).get("rows") or [[0, 0]])[0],
}
hr = {}
if has_emp:
hr = {
"by_department": _terms("SELECT department k, count(*) c FROM iceberg.hadoop.employees_ext GROUP BY department ORDER BY c DESC", "k", "c"),
"by_role": _terms("SELECT role_name k, count(*) c FROM iceberg.hadoop.employees_ext GROUP BY role_name ORDER BY c DESC LIMIT 12", "k", "c"),
}
supply = {}
if has_sup:
supply = {"by_type": _terms("SELECT type k, count(*) c FROM iceberg.hadoop.supply_events_ext GROUP BY type ORDER BY c DESC", "k", "c")}
telemetry = {}
if has_tel:
telemetry = {"by_metric": _terms("SELECT metric_type k, count(*) c, avg(metric_value) v FROM iceberg.hadoop.telemetry_ext GROUP BY metric_type ORDER BY c DESC", "k", "c", "v")}
return {
"ok": True,
"generated_at": datetime.now(timezone.utc).isoformat(),
"tables": table_stats,
"total_rows": sum(t["rows"] or 0 for t in table_stats),
"orders": orders,
"hr": hr,
"supply": supply,
"telemetry": telemetry,
}
# ──────────────────────────────────────────────────────────────────────────────
# Materialize the Hadoop external tables from the sources
# ──────────────────────────────────────────────────────────────────────────────
_mat_state: dict[str, Any] = {"running": False, "current": None, "done": [], "errors": [], "started_at": None, "finished_at": None}
_mat_lock = threading.Lock()
def _materialize_worker() -> None:
try:
for name, label, sql in LAKE_SPECS:
_mat_state["current"] = f"{name} ({label})"
try:
_trino(f"DROP TABLE IF EXISTS iceberg.hadoop.{name}", 1)
res = _trino(f"CREATE TABLE iceberg.hadoop.{name} AS {sql}", 5)
if res.get("ok"):
cnt = _trino(f"SELECT count(*) FROM iceberg.hadoop.{name}", 1)
n = cnt.get("rows", [[0]])[0][0] if cnt.get("ok") else 0
_mat_state["done"].append({"table": name, "rows": n})
else:
_mat_state["errors"].append(f"{name}: {str(res.get('error',''))[:160]}")
except Exception as exc: # noqa: BLE001
_mat_state["errors"].append(f"{name}: {str(exc)[:160]}")
finally:
_mat_state["current"] = None
_mat_state["running"] = False
_mat_state["finished_at"] = datetime.now(timezone.utc).isoformat()
@router.post("/materialize")
async def materialize():
with _mat_lock:
if _mat_state["running"]:
return {"ok": True, "already_running": True, "state": _mat_state}
_mat_state.update({"running": True, "current": "starting…", "done": [], "errors": [],
"started_at": datetime.now(timezone.utc).isoformat(), "finished_at": None})
threading.Thread(target=_materialize_worker, daemon=True).start()
return {"ok": True, "started": True, "tables": [s[0] for s in LAKE_SPECS]}
@router.get("/materialize/status")
async def materialize_status():
return {"ok": True, **_mat_state}
# ──────────────────────────────────────────────────────────────────────────────
# Business data dictionary (columns + masked/visible) — UI + LLM
# ──────────────────────────────────────────────────────────────────────────────
DICT_TABLES = [
("postgres_sales", "public", "sales_orders", "PostgreSQL", "Sales orders (OLTP, CDC source)"),
("mysql_hr", "hr", "employee_events", "MySQL", "Employee / HR events (CDC source)"),
("mongodb_supplychain", "supplychain", "events", "MongoDB", "Supply chain events (CDC source)"),
("cassandra_telemetry", "telemetry", "device_metrics", "Cassandra", "IoT device telemetry"),
("iceberg", "curated_masked", "sales_orders_masked", "Iceberg", "Curated masked layer (physically masked)"),
("iceberg", "hadoop", "orders_ext", "Hadoop", "Lake external table (orders)"),
]
_dict_cache: dict[str, Any] = {"at": 0.0, "data": None}
_DICT_TTL = 120.0
def build_dictionary() -> dict[str, Any]:
if _dict_cache["data"] and time.time() - _dict_cache["at"] < _DICT_TTL:
return _dict_cache["data"]
import sql_console as s
# masking map from the PII catalog
masked_map: dict[tuple[str, str], dict] = {}
pii_names: dict[str, str] = {} # column-name -> category (any dataset)
try:
from pii_catalog import get_pii
pii_data = get_pii()
for d in pii_data.get("datasets", []):
tname = (d.get("table") or "").split(".")[-1]
for c in d.get("pii_columns", []):
masked_map[(tname, c["name"])] = {"masked": c.get("masked"), "category": c.get("category")}
pii_names.setdefault(c["name"], c.get("category"))
except Exception:
pass
tables_out = []
for catalog, schema, table, engine, desc in DICT_TABLES:
cols_res = _trino(f'SHOW COLUMNS FROM "{catalog}"."{schema}"."{table}"', 200)
if not cols_res.get("ok"):
continue
masked_layer = schema in ("curated_masked", "curated")
cols = []
for r in cols_res.get("rows", []):
cname, ctype = r[0], r[1]
direct = masked_map.get((table, cname))
if direct:
is_pii, is_masked, cat = True, bool(direct["masked"]), direct["category"]
elif cname in pii_names:
# PII column name found in a derived/lake table — masked only if it
# is a physically-masked layer; otherwise raw PII is visible there.
is_pii, is_masked, cat = True, masked_layer, pii_names[cname]
else:
is_pii, is_masked, cat = False, False, None
cols.append({
"name": cname, "type": ctype,
"pii": is_pii, "masked": is_masked, "category": cat,
})
tables_out.append({
"catalog": catalog, "schema": schema, "table": table, "engine": engine, "desc": desc,
"fqn": f"{catalog}.{schema}.{table}",
"columns": cols,
"pii_count": sum(1 for c in cols if c["pii"]),
"masked_count": sum(1 for c in cols if c["masked"]),
})
data = {
"ok": True,
"generated_at": datetime.now(timezone.utc).isoformat(),
"tables": tables_out,
"summary": {
"tables": len(tables_out),
"columns": sum(len(t["columns"]) for t in tables_out),
"pii_columns": sum(t["pii_count"] for t in tables_out),
"masked_columns": sum(t["masked_count"] for t in tables_out),
},
}
_dict_cache.update({"at": time.time(), "data": data})
return data
@router.get("/dictionary")
async def get_dictionary():
try:
return build_dictionary()
except Exception as exc: # noqa: BLE001
return JSONResponse({"ok": False, "error": str(exc)[:300]}, status_code=502)
+3
View File
@@ -19,6 +19,7 @@ import { ChangesView } from './components/features/ChangesView'
import { DataFlowView } from './components/features/DataFlowView' import { DataFlowView } from './components/features/DataFlowView'
import { SearchView } from './components/features/SearchView' import { SearchView } from './components/features/SearchView'
import { DataExplorerView } from './components/features/DataExplorerView' import { DataExplorerView } from './components/features/DataExplorerView'
import { TrinoFederationView } from './components/features/TrinoFederationView'
import { DataHubView } from './components/features/DataHubView' import { DataHubView } from './components/features/DataHubView'
import { SshTerminal } from './components/features/SshTerminal' import { SshTerminal } from './components/features/SshTerminal'
import { TerminalDock } from './components/features/TerminalDock' import { TerminalDock } from './components/features/TerminalDock'
@@ -144,6 +145,8 @@ export default function App() {
<SearchView /> <SearchView />
) : cc.mainView === 'dataexplorer' ? ( ) : cc.mainView === 'dataexplorer' ? (
<DataExplorerView /> <DataExplorerView />
) : cc.mainView === 'trino' ? (
<TrinoFederationView />
) : ( ) : (
<ApprovalInbox agents={cc.agents} livePending={cc.approvals} onDecide={cc.decide} /> <ApprovalInbox agents={cc.agents} livePending={cc.approvals} onDecide={cc.decide} />
)} )}
@@ -0,0 +1,373 @@
import { useCallback, useEffect, useState } from 'react'
import {
Network,
Database,
Boxes,
RefreshCw,
Loader2,
DollarSign,
ShoppingCart,
Users,
Activity,
Layers,
ShieldCheck,
ShieldAlert,
Server,
HardDrive,
Play,
} from 'lucide-react'
import { cn } from '../../lib/utils'
type Bucket = { key: string; count: number; value?: number }
type SubTab = 'federated' | 'lake' | 'dictionary'
const COLORS = ['#34d399', '#60a5fa', '#f472b6', '#fbbf24', '#a78bfa', '#22d3ee', '#fb7185']
function fmtNum(n?: number | string | null) {
if (n == null) return '—'
const v = typeof n === 'number' ? n : Number(n)
if (Number.isNaN(v)) return String(n)
if (Math.abs(v) >= 1e9) return `${(v / 1e9).toFixed(2)}B`
if (Math.abs(v) >= 1e6) return `${(v / 1e6).toFixed(1)}M`
if (Math.abs(v) >= 1e3) return `${(v / 1e3).toFixed(1)}K`
return String(v)
}
const fmtMoney = (n?: number | string | null) => (n == null ? '—' : `${fmtNum(n)}`)
function Panel({ title, subtitle, icon: Icon, children }: { title: string; subtitle?: string; icon?: typeof Database; children: React.ReactNode }) {
return (
<div className="panel flex min-h-0 flex-col p-3">
<div className="mb-2 flex items-center gap-1.5">
{Icon && <Icon className="h-3.5 w-3.5 text-docker" />}
<h3 className="text-[11px] font-semibold text-foreground">{title}</h3>
{subtitle && <span className="ml-auto text-[9px] text-foreground-faint">{subtitle}</span>}
</div>
{children}
</div>
)
}
function BarsH({ data, valueKind, colorByIndex }: { data?: Bucket[]; valueKind?: 'money' | 'num'; colorByIndex?: boolean }) {
const d = data || []
const useVal = valueKind != null
const max = Math.max(1, ...d.map((x) => (useVal && x.value != null ? x.value : x.count)))
if (!d.length) return <p className="py-5 text-center text-[10px] text-foreground-faint">No data</p>
return (
<div className="space-y-1.5">
{d.map((x, i) => {
const metric = useVal && x.value != null ? x.value : x.count
const pct = Math.max(2, (metric / max) * 100)
const label = useVal && x.value != null ? (valueKind === 'money' ? fmtMoney(x.value) : fmtNum(x.value)) : fmtNum(x.count)
return (
<div key={x.key ?? i} className="flex items-center gap-2 text-[10px]">
<span className="w-28 shrink-0 truncate text-foreground-muted" title={x.key}>{x.key ?? '—'}</span>
<div className="relative h-3.5 flex-1 overflow-hidden rounded bg-surface-overlay">
<div className="h-full rounded" style={{ width: `${pct}%`, backgroundColor: colorByIndex ? COLORS[i % COLORS.length] : '#38bdf8' }} />
</div>
<span className="w-20 shrink-0 text-right font-mono text-foreground">{label}</span>
</div>
)
})}
</div>
)
}
function Donut({ data }: { data?: Bucket[] }) {
const d = data || []
const total = d.reduce((s, x) => s + x.count, 0) || 1
let acc = 0
const r = 42
const c = 2 * Math.PI * r
if (!d.length) return <p className="py-5 text-center text-[10px] text-foreground-faint">No data</p>
return (
<div className="flex items-center gap-4">
<svg viewBox="0 0 100 100" className="h-24 w-24 shrink-0 -rotate-90">
{d.map((x, i) => {
const dash = (x.count / total) * c
const seg = <circle key={x.key ?? i} cx="50" cy="50" r={r} fill="none" stroke={COLORS[i % COLORS.length]} strokeWidth="14" strokeDasharray={`${dash} ${c - dash}`} strokeDashoffset={-acc} />
acc += dash
return seg
})}
</svg>
<div className="min-w-0 flex-1 space-y-1">
{d.slice(0, 7).map((x, i) => (
<div key={x.key ?? i} className="flex items-center gap-1.5 text-[10px]">
<span className="h-2 w-2 shrink-0 rounded-full" style={{ backgroundColor: COLORS[i % COLORS.length] }} />
<span className="flex-1 truncate text-foreground-muted">{x.key ?? '—'}</span>
<span className="font-mono text-foreground">{((x.count / total) * 100).toFixed(0)}%</span>
</div>
))}
</div>
</div>
)
}
function Kpi({ icon: Icon, label, value, sub, accent }: { icon: typeof Users; label: string; value: string; sub?: string; accent: string }) {
return (
<div className="panel flex items-center gap-3 px-3 py-2.5">
<div className="flex h-9 w-9 shrink-0 items-center justify-center rounded-lg" style={{ backgroundColor: `${accent}1f`, color: accent }}>
<Icon className="h-4 w-4" />
</div>
<div className="min-w-0">
<p className="text-[9px] font-semibold uppercase tracking-wider text-foreground-muted">{label}</p>
<p className="truncate text-lg font-bold leading-tight text-foreground">{value}</p>
{sub && <p className="truncate text-[9px] text-foreground-faint">{sub}</p>}
</div>
</div>
)
}
export function TrinoFederationView() {
const [tab, setTab] = useState<SubTab>('federated')
const [catalogs, setCatalogs] = useState<any>(null)
const [marquee, setMarquee] = useState<any>(null)
const [lake, setLake] = useState<any>(null)
const [dict, setDict] = useState<any>(null)
const [loading, setLoading] = useState(false)
const [matRunning, setMatRunning] = useState(false)
const loadFederated = useCallback(async () => {
setLoading(true)
try {
const [c, m] = await Promise.all([
fetch('/api/federated/catalogs').then((r) => (r.ok ? r.json() : null)),
fetch('/api/federated/marquee').then((r) => (r.ok ? r.json() : null)),
])
setCatalogs(c)
setMarquee(m)
} catch { /* */ } finally { setLoading(false) }
}, [])
const loadLake = useCallback(async () => {
setLoading(true)
try {
const r = await fetch('/api/federated/lake')
if (r.ok) setLake(await r.json())
const ms = await fetch('/api/federated/materialize/status').then((x) => (x.ok ? x.json() : null))
setMatRunning(!!ms?.running)
} catch { /* */ } finally { setLoading(false) }
}, [])
const loadDict = useCallback(async () => {
setLoading(true)
try {
const r = await fetch('/api/federated/dictionary')
if (r.ok) setDict(await r.json())
} catch { /* */ } finally { setLoading(false) }
}, [])
useEffect(() => {
if (tab === 'federated') loadFederated()
else if (tab === 'lake') loadLake()
else loadDict()
}, [tab, loadFederated, loadLake, loadDict])
// poll marquee while it is computing
useEffect(() => {
if (tab !== 'federated' || !marquee?.running) return
const t = setTimeout(loadFederated, 5000)
return () => clearTimeout(t)
}, [tab, marquee, loadFederated])
// poll materialize while running
useEffect(() => {
if (tab !== 'lake' || !matRunning) return
const t = setTimeout(loadLake, 6000)
return () => clearTimeout(t)
}, [tab, matRunning, loadLake])
const rebuildLake = async () => {
await fetch('/api/federated/materialize', { method: 'POST' })
setMatRunning(true)
setTimeout(loadLake, 2000)
}
const refreshMarquee = async () => {
await fetch('/api/federated/marquee/refresh', { method: 'POST' })
setTimeout(loadFederated, 1500)
}
const m = marquee?.marquee
const totals = catalogs?.source_totals || {}
return (
<div className="scrollbar-thin flex h-full min-h-0 flex-col gap-2 overflow-y-auto p-3">
<header className="panel flex shrink-0 flex-wrap items-center justify-between gap-3 px-4 py-3">
<div>
<h1 className="flex items-center gap-2 text-base font-semibold text-foreground">
<Network className="h-5 w-5 text-docker" />
Trino Federation &amp; Hadoop Lakehouse
</h1>
<p className="text-[11px] text-foreground-muted">
One SQL engine over every source federated business analytics, mirrored into Hadoop as external Iceberg tables
</p>
</div>
<div className="flex flex-wrap gap-1">
{([
{ id: 'federated', label: 'Federated', icon: Network },
{ id: 'lake', label: 'Hadoop Lake', icon: HardDrive },
{ id: 'dictionary', label: 'Data Dictionary', icon: ShieldCheck },
] as { id: SubTab; label: string; icon: typeof Network }[]).map(({ id, label, icon: Icon }) => (
<button key={id} type="button" onClick={() => setTab(id)}
className={cn('inline-flex items-center gap-1.5 rounded-md px-3 py-1.5 text-[11px] font-medium transition-colors', tab === id ? 'bg-docker/15 text-docker' : 'text-foreground-muted hover:bg-surface-overlay')}>
<Icon className="h-3.5 w-3.5" /> {label}
</button>
))}
</div>
</header>
{/* ───────── FEDERATED ───────── */}
{tab === 'federated' && (
<>
<div className="grid shrink-0 grid-cols-2 gap-2 md:grid-cols-4">
<Kpi icon={Layers} label="Federated catalogs" value={String(catalogs?.count ?? '—')} sub="one Trino engine" accent="#dd00a1" />
<Kpi icon={ShoppingCart} label="Orders" value={fmtNum(totals.orders)} sub="PostgreSQL (live)" accent="#fbbf24" />
<Kpi icon={Users} label="HR events" value={fmtNum(totals.hr_events)} sub="MySQL (live)" accent="#60a5fa" />
<Kpi icon={Boxes} label="Supply events" value={fmtNum(totals.supply_events)} sub="MongoDB (live)" accent="#a78bfa" />
</div>
<Panel title="Federated catalog landscape" subtitle={`${catalogs?.count ?? 0} catalogs`} icon={Database}>
<div className="flex flex-wrap gap-2">
{(catalogs?.catalogs || []).map((c: any) => (
<div key={c.catalog} className="flex min-w-[150px] flex-col gap-0.5 rounded-lg border px-3 py-2" style={{ borderColor: `${c.color}55`, backgroundColor: `${c.color}12` }}>
<span className="flex items-center gap-1.5 text-[11px] font-semibold text-foreground">
<Server className="h-3 w-3" style={{ color: c.color }} /> {c.label}
</span>
<span className="font-mono text-[9px] text-foreground-muted">{c.catalog}</span>
<span className="text-[9px] text-foreground-faint">{c.desc}</span>
{c.rows != null && <span className="mt-0.5 font-mono text-[10px] text-foreground">{fmtNum(c.rows)} rows</span>}
</div>
))}
</div>
</Panel>
<Panel
title="Cross-source federated query — one SQL, three databases"
subtitle={m?.elapsed_ms != null ? `${(m.elapsed_ms / 1000).toFixed(1)}s` : marquee?.running ? 'computing…' : ''}
icon={Network}
>
<div className="mb-2 flex flex-wrap items-center gap-1.5">
{(m?.catalogs || ['postgres_sales', 'mysql_hr', 'mongodb_supplychain']).map((c: string) => (
<span key={c} className="rounded-full bg-docker/15 px-2 py-0.5 text-[9px] font-medium text-docker">{c}</span>
))}
<button type="button" onClick={refreshMarquee} className="ml-auto inline-flex items-center gap-1 rounded border border-border px-2 py-0.5 text-[9px] text-foreground-muted hover:bg-surface-overlay">
{marquee?.running ? <Loader2 className="h-3 w-3 animate-spin" /> : <RefreshCw className="h-3 w-3" />} re-run
</button>
</div>
<pre className="mb-2 overflow-x-auto rounded-md border border-border bg-surface p-2 font-mono text-[9px] leading-relaxed text-foreground-muted">{marquee?.sql || m?.sql}</pre>
{marquee?.running && !m?.rows?.length ? (
<p className="py-4 text-center text-[10px] text-foreground-muted"><Loader2 className="mr-1 inline h-3 w-3 animate-spin" /> Federating across live sources (~12 min, cached afterwards)</p>
) : m?.rows?.length ? (
<div className="overflow-x-auto">
<table className="w-full text-[10px]">
<thead>
<tr className="border-b border-border text-left text-foreground-muted">
<th className="py-1 pr-3">Region</th>
<th className="py-1 pr-3 text-right">Orders</th>
<th className="py-1 pr-3 text-right">Revenue</th>
<th className="py-1 pr-3 text-right">HR events</th>
<th className="py-1 pr-3 text-right">Supply events</th>
</tr>
</thead>
<tbody>
{m.rows.map((r: any, i: number) => (
<tr key={i} className="border-b border-border/40">
<td className="py-1 pr-3 font-medium text-foreground">{r.region}</td>
<td className="py-1 pr-3 text-right font-mono text-foreground">{fmtNum(r.orders)}</td>
<td className="py-1 pr-3 text-right font-mono text-emerald-400">{fmtMoney(r.revenue)}</td>
<td className="py-1 pr-3 text-right font-mono text-foreground">{fmtNum(r.hr_events)}</td>
<td className="py-1 pr-3 text-right font-mono text-foreground">{fmtNum(r.supply_events)}</td>
</tr>
))}
</tbody>
</table>
{m.generated_at && <p className="mt-1 text-[9px] text-foreground-faint">as of {new Date(m.generated_at).toLocaleString()} · joined live across PostgreSQL + MySQL + MongoDB</p>}
</div>
) : (
<p className="py-4 text-center text-[10px] text-foreground-faint">{m?.error || 'No result yet — click re-run.'}</p>
)}
</Panel>
</>
)}
{/* ───────── HADOOP LAKE ───────── */}
{tab === 'lake' && (
<>
<div className="flex shrink-0 flex-wrap items-center justify-between gap-2 px-1">
<p className="text-[11px] text-foreground-muted">
All federated business data materialized as <span className="text-docker">external Iceberg tables on HDFS</span> queried live (fast).
</p>
<button type="button" onClick={rebuildLake} disabled={matRunning}
className="inline-flex items-center gap-1.5 rounded-md border border-emerald-500/40 bg-emerald-500/10 px-3 py-1.5 text-[11px] font-medium text-emerald-400 hover:bg-emerald-500/20 disabled:opacity-60">
{matRunning ? <Loader2 className="h-3.5 w-3.5 animate-spin" /> : <Play className="h-3.5 w-3.5" />}
{matRunning ? 'Materializing…' : 'Rebuild external tables'}
</button>
</div>
<Panel title="Hadoop external tables (iceberg.hadoop.*)" subtitle={`${fmtNum(lake?.total_rows)} rows total`} icon={HardDrive}>
<div className="grid grid-cols-2 gap-2 md:grid-cols-3 lg:grid-cols-4">
{(lake?.tables || []).map((t: any) => (
<div key={t.table} className="rounded-lg border border-border/60 bg-surface-overlay/40 px-3 py-2">
<p className="flex items-center gap-1 truncate font-mono text-[10px] text-foreground"><Boxes className="h-3 w-3 text-docker" />{t.table}</p>
<p className="font-mono text-[13px] font-bold text-foreground">{fmtNum(t.rows)}</p>
</div>
))}
{!lake?.tables?.length && <p className="col-span-full py-4 text-center text-[10px] text-foreground-faint">No external tables yet click Rebuild external tables.</p>}
</div>
</Panel>
<div className="grid shrink-0 gap-2 lg:grid-cols-2">
<Panel title="Revenue by region" icon={DollarSign}><BarsH data={lake?.orders?.by_region} valueKind="money" colorByIndex /></Panel>
<Panel title="Top customers by spend" icon={Users}><BarsH data={lake?.orders?.top_customers} valueKind="money" /></Panel>
<div className="grid gap-2 sm:grid-cols-2">
<Panel title="Orders by status"><Donut data={lake?.orders?.by_status} /></Panel>
<Panel title="Revenue by channel"><BarsH data={lake?.orders?.by_channel} valueKind="money" colorByIndex /></Panel>
</div>
<Panel title="Employees by department" icon={Users}><BarsH data={lake?.hr?.by_department} colorByIndex /></Panel>
<Panel title="Supply events by type" icon={Boxes}><BarsH data={lake?.supply?.by_type} colorByIndex /></Panel>
<Panel title="Telemetry — avg value by metric" icon={Activity}><BarsH data={lake?.telemetry?.by_metric} valueKind="num" /></Panel>
</div>
</>
)}
{/* ───────── DICTIONARY ───────── */}
{tab === 'dictionary' && (
<>
<div className="grid shrink-0 grid-cols-2 gap-2 md:grid-cols-4">
<Kpi icon={Layers} label="Tables" value={String(dict?.summary?.tables ?? '—')} accent="#5b8def" />
<Kpi icon={Database} label="Columns" value={String(dict?.summary?.columns ?? '—')} accent="#34d399" />
<Kpi icon={ShieldAlert} label="PII columns" value={String(dict?.summary?.pii_columns ?? '—')} accent="#fbbf24" />
<Kpi icon={ShieldCheck} label="Masked" value={String(dict?.summary?.masked_columns ?? '—')} sub="hidden from LLM" accent="#f472b6" />
</div>
<p className="px-1 text-[10px] text-foreground-muted">
This is exactly what the assistant knows about your data every column, its type, and whether it is <span className="text-amber-400">masked</span> or visible.
</p>
<div className="grid gap-2 lg:grid-cols-2">
{(dict?.tables || []).map((t: any) => (
<Panel key={t.fqn} title={t.fqn} subtitle={`${t.masked_count}/${t.pii_count} PII masked`} icon={Database}>
<p className="mb-1.5 text-[9px] text-foreground-faint">{t.engine} · {t.desc}</p>
<div className="flex flex-wrap gap-1">
{t.columns.map((c: any) => (
<span key={c.name}
className={cn('inline-flex items-center gap-1 rounded border px-1.5 py-0.5 font-mono text-[9px]',
c.masked ? 'border-amber-500/40 bg-amber-500/10 text-amber-300'
: c.pii ? 'border-rose-500/40 bg-rose-500/10 text-rose-300'
: 'border-border/60 text-foreground-muted')}
title={`${c.type}${c.category ? ` · ${c.category}` : ''}${c.masked ? ' · MASKED' : c.pii ? ' · PII visible' : ''}`}>
{c.masked && <ShieldCheck className="h-2.5 w-2.5" />}
{!c.masked && c.pii && <ShieldAlert className="h-2.5 w-2.5" />}
{c.name}
</span>
))}
</div>
</Panel>
))}
</div>
</>
)}
{loading && !catalogs && !lake && !dict && (
<div className="flex flex-1 items-center justify-center text-foreground-muted"><Loader2 className="h-5 w-5 animate-spin" /></div>
)}
</div>
)
}
+3 -2
View File
@@ -1,5 +1,5 @@
import { useEffect, useState } from 'react' import { useEffect, useState } from 'react'
import { Database, DatabaseZap, HardDrive, Search, LayoutDashboard, MessageSquare, Server, TerminalSquare, Activity, GitBranch, ExternalLink, LineChart } from 'lucide-react' import { Database, DatabaseZap, HardDrive, Search, LayoutDashboard, MessageSquare, Server, TerminalSquare, Activity, GitBranch, ExternalLink, LineChart, Network } from 'lucide-react'
import type { GpuStatus, WorkloadData } from '../../types' import type { GpuStatus, WorkloadData } from '../../types'
import type { GpuLiveMetrics } from '../../hooks/useLiveMetrics' import type { GpuLiveMetrics } from '../../hooks/useLiveMetrics'
import { cn } from '../../lib/utils' import { cn } from '../../lib/utils'
@@ -7,7 +7,7 @@ import { viewTabActive, viewTabIdle } from '../../lib/tabActive'
import { GpuMatrixPanel } from '../features/GpuMatrixPanel' import { GpuMatrixPanel } from '../features/GpuMatrixPanel'
import { LabHealthPanel } from '../features/LabHealthPanel' import { LabHealthPanel } from '../features/LabHealthPanel'
type MainView = 'platform' | 'dataquality' | 'knowledge' | 'storage' | 'hdfs' | 'search' | 'approvals' | 'changes' | 'dataflow' | 'datasources' | 'dataexplorer' type MainView = 'platform' | 'dataquality' | 'knowledge' | 'storage' | 'hdfs' | 'search' | 'approvals' | 'changes' | 'dataflow' | 'datasources' | 'dataexplorer' | 'trino'
type Props = { type Props = {
workload: WorkloadData | null workload: WorkloadData | null
@@ -27,6 +27,7 @@ const VIEWS: { id: MainView; label: string; icon: typeof LayoutDashboard }[] = [
{ id: 'platform', label: 'Data Platform', icon: LayoutDashboard }, { id: 'platform', label: 'Data Platform', icon: LayoutDashboard },
{ id: 'datasources', label: 'Data Hub', icon: Database }, { id: 'datasources', label: 'Data Hub', icon: Database },
{ id: 'dataexplorer', label: 'Data Explorer', icon: LineChart }, { id: 'dataexplorer', label: 'Data Explorer', icon: LineChart },
{ id: 'trino', label: 'Trino Federation', icon: Network },
{ id: 'changes', label: 'Live Changes', icon: Activity }, { id: 'changes', label: 'Live Changes', icon: Activity },
{ id: 'dataflow', label: 'Data Flow', icon: GitBranch }, { id: 'dataflow', label: 'Data Flow', icon: GitBranch },
{ id: 'dataquality', label: 'Data Quality', icon: DatabaseZap }, { id: 'dataquality', label: 'Data Quality', icon: DatabaseZap },
+1 -1
View File
@@ -60,7 +60,7 @@ export function useCommandCenter() {
const [selectedNode, setSelectedNode] = useState<TopologyNode | null>(null) const [selectedNode, setSelectedNode] = useState<TopologyNode | null>(null)
const [nodeDetail, setNodeDetail] = useState<NodeDetail | null>(null) const [nodeDetail, setNodeDetail] = useState<NodeDetail | null>(null)
const [nodeBusy, setNodeBusy] = useState(false) const [nodeBusy, setNodeBusy] = useState(false)
const [mainView, setMainView] = useState<'platform' | 'approvals' | 'dataquality' | 'knowledge' | 'storage' | 'hdfs' | 'search' | 'changes' | 'dataflow' | 'datasources' | 'dataexplorer'>('platform') const [mainView, setMainView] = useState<'platform' | 'approvals' | 'dataquality' | 'knowledge' | 'storage' | 'hdfs' | 'search' | 'changes' | 'dataflow' | 'datasources' | 'dataexplorer' | 'trino'>('platform')
const [changes, setChanges] = useState<CdcChange[]>([]) const [changes, setChanges] = useState<CdcChange[]>([])
const [genPulse, setGenPulse] = useState(false) const [genPulse, setGenPulse] = useState(false)
const genPulseTimer = useRef<ReturnType<typeof setTimeout> | null>(null) const genPulseTimer = useRef<ReturnType<typeof setTimeout> | null>(null)