feat: Generate-data button + generation-script viewer + vector DB explorer

Data Flow tab:
- Prominent "Generate data" button (500 / 2K / 10K) that inserts a fresh
  burst of business rows into all source DBs on demand via a new
  POST /api/federated/generate (fresh connections, safe alongside the
  background streamer); result toast shows what was inserted, CDC streams it.
- "Scripts" button + a "View generation scripts" action on the Data Generator
  node open a modal listing every generator script with full source, served by
  GET /api/dataflow/scripts. Sources are the real files: the live streaming
  generator (sliced live out of trino_federated.py) and the Airflow per-source
  DAGs + Faker scripts (mounted read-only from infra/airflow into the API).

Knowledge Chat:
- New "Vector DB" explorer modal: shows the ChromaDB chunking config
  (RecursiveCharacterTextSplitter 800/120, all-MiniLM-L6-v2, 384-dim, HNSW),
  collections & documents, and the actual stored chunks with text, metadata and
  an embedding preview (bars + values) so you can see exactly how files are
  split and written as vectors.

Refactor: generator row-builders shared by the streamer and the on-demand burst.
This commit is contained in:
mo
2026-06-28 22:06:27 +00:00
parent 9059006cc2
commit 213350ec75
5 changed files with 528 additions and 25 deletions
+74
View File
@@ -14,6 +14,7 @@ from __future__ import annotations
import os
import time
from pathlib import Path
from typing import Any
import httpx
@@ -300,6 +301,79 @@ async def get_dataflow(refresh: bool = False) -> JSONResponse:
return JSONResponse(data)
GEN_SCRIPTS_DIR = os.getenv("GEN_SCRIPTS_DIR", "/app/gen_scripts")
# Which generation scripts to surface, in display order. `file` is relative to
# GEN_SCRIPTS_DIR (the mounted infra/airflow dir); `live` slices the running
# streamer source straight out of trino_federated.py so it is always in sync.
_SCRIPT_SPECS: list[dict[str, Any]] = [
{"id": "live", "title": "Live streaming generator", "engine": "Command Center API",
"desc": "Runs inside this API. While the Live dashboard is open it streams randomly-sized bursts of rows into PostgreSQL, MySQL, MongoDB & Cassandra every few seconds (and powers the manual 'Generate data' button). CDC propagates everything downstream.",
"live": True},
{"id": "dag", "title": "Airflow per-source DAGs", "engine": "Apache Airflow",
"desc": "One triggerable DAG per database. Each passes a row count via dag_run conf and shells out to the matching generator script below.",
"file": "per_source_gen_dags.py"},
{"id": "postgres", "title": "PostgreSQL — sales orders", "engine": "Faker → psycopg2",
"desc": "Generates realistic customers, products, regions, channels & amounts (incl. the PII columns that the masking layer later protects).",
"file": "scripts/generate_postgres_sales_data.py"},
{"id": "mysql", "title": "MySQL — employee events", "engine": "Faker → PyMySQL",
"desc": "HR lifecycle events (hire, promotion, salary change, …) with employee PII.",
"file": "scripts/generate_mysql_employee_data.py"},
{"id": "mongodb", "title": "MongoDB — supply events", "engine": "Faker → PyMongo",
"desc": "Schemaless supply-chain events with free-form payloads.",
"file": "scripts/generate_mongodb_events_data.py"},
{"id": "cassandra", "title": "Cassandra — device telemetry", "engine": "Faker → cassandra-driver",
"desc": "High-volume IoT device metrics (temperature, voltage, …) on a time-series schema.",
"file": "scripts/generate_cassandra_telemetry_data.py"},
{"id": "neo4j", "title": "Neo4j — product & supplier graph", "engine": "Faker → neo4j driver",
"desc": "Product/supplier nodes and relationships for the graph database.",
"file": "scripts/generate_neo4j_graph_data.py"},
]
def _live_generator_source() -> str:
try:
text = Path("/app/trino_federated.py").read_text(encoding="utf-8")
except Exception:
return "# live generator source unavailable"
start = text.find("# Continuous live generator")
end = text.find('@router.get("/live")', start if start >= 0 else 0)
if start >= 0 and end > start:
return text[start:end].rstrip()
return "# live generator source unavailable"
_scripts_cache: dict[str, Any] = {"ts": 0.0, "data": None}
@router.get("/scripts")
async def get_scripts() -> JSONResponse:
now = time.time()
if _scripts_cache["data"] and now - _scripts_cache["ts"] < 30:
return JSONResponse(_scripts_cache["data"])
base = Path(GEN_SCRIPTS_DIR)
scripts = []
for spec in _SCRIPT_SPECS:
src = ""
if spec.get("live"):
src = _live_generator_source()
else:
p = base / spec["file"]
try:
src = p.read_text(encoding="utf-8")
except Exception as exc:
src = f"# source unavailable ({exc})"
scripts.append({
"id": spec["id"], "title": spec["title"], "engine": spec["engine"],
"desc": spec["desc"], "filename": spec.get("file", "trino_federated.py"),
"language": "python", "lines": src.count("\n") + 1, "source": src,
})
data = {"ok": True, "scripts": scripts}
_scripts_cache["data"] = data
_scripts_cache["ts"] = now
return JSONResponse(data)
@router.post("/{movement_id}/run")
async def run_dataflow_movement(movement_id: str, body: dict[str, Any] = Body(default={})) -> JSONResponse:
try: