feat: Generate-data button + generation-script viewer + vector DB explorer
Data Flow tab: - Prominent "Generate data" button (500 / 2K / 10K) that inserts a fresh burst of business rows into all source DBs on demand via a new POST /api/federated/generate (fresh connections, safe alongside the background streamer); result toast shows what was inserted, CDC streams it. - "Scripts" button + a "View generation scripts" action on the Data Generator node open a modal listing every generator script with full source, served by GET /api/dataflow/scripts. Sources are the real files: the live streaming generator (sliced live out of trino_federated.py) and the Airflow per-source DAGs + Faker scripts (mounted read-only from infra/airflow into the API). Knowledge Chat: - New "Vector DB" explorer modal: shows the ChromaDB chunking config (RecursiveCharacterTextSplitter 800/120, all-MiniLM-L6-v2, 384-dim, HNSW), collections & documents, and the actual stored chunks with text, metadata and an embedding preview (bars + values) so you can see exactly how files are split and written as vectors. Refactor: generator row-builders shared by the streamer and the on-demand burst.
This commit is contained in:
@@ -14,6 +14,7 @@ from __future__ import annotations
|
||||
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
@@ -300,6 +301,79 @@ async def get_dataflow(refresh: bool = False) -> JSONResponse:
|
||||
return JSONResponse(data)
|
||||
|
||||
|
||||
GEN_SCRIPTS_DIR = os.getenv("GEN_SCRIPTS_DIR", "/app/gen_scripts")
|
||||
|
||||
# Which generation scripts to surface, in display order. `file` is relative to
|
||||
# GEN_SCRIPTS_DIR (the mounted infra/airflow dir); `live` slices the running
|
||||
# streamer source straight out of trino_federated.py so it is always in sync.
|
||||
_SCRIPT_SPECS: list[dict[str, Any]] = [
|
||||
{"id": "live", "title": "Live streaming generator", "engine": "Command Center API",
|
||||
"desc": "Runs inside this API. While the Live dashboard is open it streams randomly-sized bursts of rows into PostgreSQL, MySQL, MongoDB & Cassandra every few seconds (and powers the manual 'Generate data' button). CDC propagates everything downstream.",
|
||||
"live": True},
|
||||
{"id": "dag", "title": "Airflow per-source DAGs", "engine": "Apache Airflow",
|
||||
"desc": "One triggerable DAG per database. Each passes a row count via dag_run conf and shells out to the matching generator script below.",
|
||||
"file": "per_source_gen_dags.py"},
|
||||
{"id": "postgres", "title": "PostgreSQL — sales orders", "engine": "Faker → psycopg2",
|
||||
"desc": "Generates realistic customers, products, regions, channels & amounts (incl. the PII columns that the masking layer later protects).",
|
||||
"file": "scripts/generate_postgres_sales_data.py"},
|
||||
{"id": "mysql", "title": "MySQL — employee events", "engine": "Faker → PyMySQL",
|
||||
"desc": "HR lifecycle events (hire, promotion, salary change, …) with employee PII.",
|
||||
"file": "scripts/generate_mysql_employee_data.py"},
|
||||
{"id": "mongodb", "title": "MongoDB — supply events", "engine": "Faker → PyMongo",
|
||||
"desc": "Schemaless supply-chain events with free-form payloads.",
|
||||
"file": "scripts/generate_mongodb_events_data.py"},
|
||||
{"id": "cassandra", "title": "Cassandra — device telemetry", "engine": "Faker → cassandra-driver",
|
||||
"desc": "High-volume IoT device metrics (temperature, voltage, …) on a time-series schema.",
|
||||
"file": "scripts/generate_cassandra_telemetry_data.py"},
|
||||
{"id": "neo4j", "title": "Neo4j — product & supplier graph", "engine": "Faker → neo4j driver",
|
||||
"desc": "Product/supplier nodes and relationships for the graph database.",
|
||||
"file": "scripts/generate_neo4j_graph_data.py"},
|
||||
]
|
||||
|
||||
|
||||
def _live_generator_source() -> str:
|
||||
try:
|
||||
text = Path("/app/trino_federated.py").read_text(encoding="utf-8")
|
||||
except Exception:
|
||||
return "# live generator source unavailable"
|
||||
start = text.find("# Continuous live generator")
|
||||
end = text.find('@router.get("/live")', start if start >= 0 else 0)
|
||||
if start >= 0 and end > start:
|
||||
return text[start:end].rstrip()
|
||||
return "# live generator source unavailable"
|
||||
|
||||
|
||||
_scripts_cache: dict[str, Any] = {"ts": 0.0, "data": None}
|
||||
|
||||
|
||||
@router.get("/scripts")
|
||||
async def get_scripts() -> JSONResponse:
|
||||
now = time.time()
|
||||
if _scripts_cache["data"] and now - _scripts_cache["ts"] < 30:
|
||||
return JSONResponse(_scripts_cache["data"])
|
||||
base = Path(GEN_SCRIPTS_DIR)
|
||||
scripts = []
|
||||
for spec in _SCRIPT_SPECS:
|
||||
src = ""
|
||||
if spec.get("live"):
|
||||
src = _live_generator_source()
|
||||
else:
|
||||
p = base / spec["file"]
|
||||
try:
|
||||
src = p.read_text(encoding="utf-8")
|
||||
except Exception as exc:
|
||||
src = f"# source unavailable ({exc})"
|
||||
scripts.append({
|
||||
"id": spec["id"], "title": spec["title"], "engine": spec["engine"],
|
||||
"desc": spec["desc"], "filename": spec.get("file", "trino_federated.py"),
|
||||
"language": "python", "lines": src.count("\n") + 1, "source": src,
|
||||
})
|
||||
data = {"ok": True, "scripts": scripts}
|
||||
_scripts_cache["data"] = data
|
||||
_scripts_cache["ts"] = now
|
||||
return JSONResponse(data)
|
||||
|
||||
|
||||
@router.post("/{movement_id}/run")
|
||||
async def run_dataflow_movement(movement_id: str, body: dict[str, Any] = Body(default={})) -> JSONResponse:
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user